create_project + update/deploy_project (#3075)

skyvern/forge/sdk/artifact/manager.py

@@ -238,6 +238,43 @@ class ArtifactManager:
             path=path,
         )
 
+    async def create_project_file_artifact(
+        self,
+        *,
+        organization_id: str,
+        project_id: str,
+        project_version: int,
+        file_path: str,
+        data: bytes,
+    ) -> str:
+        """Create an artifact for a project file.
+
+        Args:
+            organization_id: The organization ID
+            project_id: The project ID
+            project_version: The project version
+            file_path: The file path relative to project root
+            data: The file content as bytes
+
+        Returns:
+            The artifact ID
+        """
+        artifact_id = generate_artifact_id()
+        uri = app.STORAGE.build_project_file_uri(
+            organization_id=organization_id,
+            project_id=project_id,
+            project_version=project_version,
+            file_path=file_path,
+        )
+        return await self._create_artifact(
+            aio_task_primary_key=f"{project_id}_{project_version}",
+            artifact_id=artifact_id,
+            artifact_type=ArtifactType.PROJECT_FILE,
+            uri=uri,
+            organization_id=organization_id,
+            data=data,
+        )
+
     async def create_llm_artifact(
         self,
         data: bytes,

skyvern/forge/sdk/artifact/models.py

@@ -49,6 +49,9 @@ class ArtifactType(StrEnum):
     TRACE = "trace"
     HAR = "har"
 
+    # Project files
+    PROJECT_FILE = "project_file"
+
 
 class Artifact(BaseModel):
     created_at: datetime = Field(

skyvern/forge/sdk/artifact/storage/base.py

@@ -81,6 +81,12 @@ class BaseStorage(ABC):
     ) -> str:
         pass
 
+    @abstractmethod
+    def build_project_file_uri(
+        self, *, organization_id: str, project_id: str, project_version: int, file_path: str
+    ) -> str:
+        pass
+
     @abstractmethod
     async def store_artifact(self, artifact: Artifact, data: bytes) -> None:
         pass

skyvern/forge/sdk/artifact/storage/local.py

@@ -78,6 +78,11 @@ class LocalStorage(BaseStorage):
         file_ext = FILE_EXTENTSION_MAP[artifact_type]
         return f"file://{self.artifact_path}/{settings.ENV}/{organization_id}/ai_suggestions/{ai_suggestion.ai_suggestion_id}/{datetime.utcnow().isoformat()}_{artifact_id}_{artifact_type}.{file_ext}"
 
+    def build_project_file_uri(
+        self, *, organization_id: str, project_id: str, project_version: int, file_path: str
+    ) -> str:
+        return f"file://{self.artifact_path}/{settings.ENV}/{organization_id}/projects/{project_id}/{project_version}/{file_path}"
+
     async def store_artifact(self, artifact: Artifact, data: bytes) -> None:
         file_path = None
         try:

skyvern/forge/sdk/artifact/storage/s3.py

@@ -84,6 +84,22 @@ class S3Storage(BaseStorage):
         file_ext = FILE_EXTENTSION_MAP[artifact_type]
         return f"{self._build_base_uri(organization_id)}/ai_suggestions/{ai_suggestion.ai_suggestion_id}/{datetime.utcnow().isoformat()}_{artifact_id}_{artifact_type}.{file_ext}"
 
+    def build_project_file_uri(
+        self, *, organization_id: str, project_id: str, project_version: int, file_path: str
+    ) -> str:
+        """Build the S3 URI for a project file.
+
+        Args:
+            organization_id: The organization ID
+            project_id: The project ID
+            project_version: The project version
+            file_path: The file path relative to project root
+
+        Returns:
+            The S3 URI for the project file
+        """
+        return f"{self._build_base_uri(organization_id)}/projects/{project_id}/{project_version}/{file_path}"
+
     async def store_artifact(self, artifact: Artifact, data: bytes) -> None:
         sc = await self._get_storage_class_for_org(artifact.organization_id)
         tags = await self._get_tags_for_org(artifact.organization_id)

skyvern/forge/sdk/db/client.py

@@ -3,7 +3,7 @@ from datetime import datetime, timedelta
 from typing import Any, List, Sequence
 
 import structlog
-from sqlalchemy import and_, delete, distinct, func, or_, pool, select, tuple_, update
+from sqlalchemy import and_, delete, desc, distinct, func, or_, pool, select, tuple_, update
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker, create_async_engine
 
@@ -29,6 +29,8 @@ from skyvern.forge.sdk.db.models import (
     OrganizationModel,
     OutputParameterModel,
     PersistentBrowserSessionModel,
+    ProjectFileModel,
+    ProjectModel,
     StepModel,
     TaskGenerationModel,
     TaskModel,
@@ -52,6 +54,7 @@ from skyvern.forge.sdk.db.utils import (
     convert_to_organization,
     convert_to_organization_auth_token,
     convert_to_output_parameter,
+    convert_to_project,
     convert_to_step,
     convert_to_task,
     convert_to_workflow,
@@ -96,6 +99,7 @@ from skyvern.forge.sdk.workflow.models.workflow import (
     WorkflowRunStatus,
     WorkflowStatus,
 )
+from skyvern.schemas.projects import Project
 from skyvern.schemas.runs import ProxyLocation, RunEngine, RunType
 from skyvern.webeye.actions.actions import Action
 from skyvern.webeye.actions.models import AgentStepOutput
@@ -3481,3 +3485,228 @@ class AgentDB:
             await session.refresh(debug_session)
 
             return DebugSession.model_validate(debug_session)
+
+    async def update_debug_session(
+        self,
+        *,
+        debug_session_id: str,
+        browser_session_id: str | None = None,
+    ) -> DebugSession:
+        async with self.Session() as session:
+            debug_session = (
+                await session.scalars(select(DebugSessionModel).filter_by(debug_session_id=debug_session_id))
+            ).first()
+
+            if not debug_session:
+                raise NotFoundError(f"Debug session {debug_session_id} not found")
+
+            if browser_session_id:
+                debug_session.browser_session_id = browser_session_id
+
+            await session.commit()
+            await session.refresh(debug_session)
+
+            return DebugSession.model_validate(debug_session)
+
+    async def create_project(
+        self,
+        organization_id: str,
+        workflow_permanent_id: str | None = None,
+        run_id: str | None = None,
+        project_id: str | None = None,
+        version: int | None = None,
+    ) -> Project:
+        try:
+            async with self.Session() as session:
+                project = ProjectModel(
+                    organization_id=organization_id,
+                    workflow_permanent_id=workflow_permanent_id,
+                    run_id=run_id,
+                )
+                if project_id:
+                    project.project_id = project_id
+                if version:
+                    project.version = version
+                session.add(project)
+                await session.commit()
+                await session.refresh(project)
+                return convert_to_project(project)
+        except SQLAlchemyError:
+            LOG.error("SQLAlchemyError", exc_info=True)
+            raise
+
+    async def update_project(
+        self,
+        project_id: str,
+        organization_id: str,
+        artifact_id: str | None = None,
+        workflow_permanent_id: str | None = None,
+        run_id: str | None = None,
+        version: int | None = None,
+    ) -> Project:
+        try:
+            async with self.Session() as session:
+                get_project_query = (
+                    select(ProjectModel)
+                    .filter_by(project_id=project_id)
+                    .filter_by(organization_id=organization_id)
+                    .filter(ProjectModel.deleted_at.is_(None))
+                )
+                if project := (await session.scalars(get_project_query)).first():
+                    if artifact_id:
+                        project.artifact_id = artifact_id
+                    if workflow_permanent_id:
+                        project.workflow_permanent_id = workflow_permanent_id
+                    if run_id:
+                        project.run_id = run_id
+                    if version:
+                        project.version = version
+                    await session.commit()
+                    await session.refresh(project)
+                    return convert_to_project(project)
+                else:
+                    raise NotFoundError("Project not found")
+        except SQLAlchemyError:
+            LOG.error("SQLAlchemyError", exc_info=True)
+            raise
+        except NotFoundError:
+            LOG.error("No project found to update", project_id=project_id)
+            raise
+        except Exception:
+            LOG.error("UnexpectedError", exc_info=True)
+            raise
+
+    async def get_projects(
+        self,
+        organization_id: str,
+        page: int = 1,
+        page_size: int = 10,
+    ) -> list[Project]:
+        try:
+            async with self.Session() as session:
+                # Calculate offset for pagination
+                offset = (page - 1) * page_size
+
+                # Subquery to get the latest version of each project
+                latest_versions_subquery = (
+                    select(ProjectModel.project_id, func.max(ProjectModel.version).label("latest_version"))
+                    .filter_by(organization_id=organization_id)
+                    .filter(ProjectModel.deleted_at.is_(None))
+                    .group_by(ProjectModel.project_id)
+                    .subquery()
+                )
+
+                # Main query to get projects with their latest versions
+                get_projects_query = (
+                    select(ProjectModel)
+                    .join(
+                        latest_versions_subquery,
+                        and_(
+                            ProjectModel.project_id == latest_versions_subquery.c.project_id,
+                            ProjectModel.version == latest_versions_subquery.c.latest_version,
+                        ),
+                    )
+                    .filter_by(organization_id=organization_id)
+                    .filter(ProjectModel.deleted_at.is_(None))
+                    .order_by(ProjectModel.created_at.desc())
+                    .limit(page_size)
+                    .offset(offset)
+                )
+                projects = (await session.scalars(get_projects_query)).all()
+                return [convert_to_project(project) for project in projects]
+        except SQLAlchemyError:
+            LOG.error("SQLAlchemyError", exc_info=True)
+            raise
+
+    async def get_project(
+        self,
+        project_id: str,
+        organization_id: str,
+        version: int | None = None,
+    ) -> Project | None:
+        """Get a specific project by ID and optionally by version."""
+        try:
+            async with self.Session() as session:
+                get_project_query = (
+                    select(ProjectModel)
+                    .filter_by(project_id=project_id)
+                    .filter_by(organization_id=organization_id)
+                    .filter(ProjectModel.deleted_at.is_(None))
+                )
+
+                if version is not None:
+                    get_project_query = get_project_query.filter_by(version=version)
+                else:
+                    # Get the latest version
+                    get_project_query = get_project_query.order_by(desc(ProjectModel.version)).limit(1)
+
+                if project := (await session.scalars(get_project_query)).first():
+                    return convert_to_project(project)
+                return None
+        except SQLAlchemyError:
+            LOG.error("SQLAlchemyError", exc_info=True)
+            raise
+        except Exception:
+            LOG.error("UnexpectedError", exc_info=True)
+            raise
+
+    async def create_project_file(
+        self,
+        project_revision_id: str,
+        project_id: str,
+        organization_id: str,
+        file_path: str,
+        file_name: str,
+        file_type: str,
+        content_hash: str | None = None,
+        file_size: int | None = None,
+        mime_type: str | None = None,
+        encoding: str = "utf-8",
+        artifact_id: str | None = None,
+    ) -> None:
+        """Create a project file record."""
+        try:
+            async with self.Session() as session:
+                project_file = ProjectFileModel(
+                    project_revision_id=project_revision_id,
+                    project_id=project_id,
+                    organization_id=organization_id,
+                    file_path=file_path,
+                    file_name=file_name,
+                    file_type=file_type,
+                    content_hash=content_hash,
+                    file_size=file_size,
+                    mime_type=mime_type,
+                    encoding=encoding,
+                    artifact_id=artifact_id,
+                )
+                session.add(project_file)
+                await session.commit()
+        except SQLAlchemyError:
+            LOG.error("SQLAlchemyError", exc_info=True)
+            raise
+        except Exception:
+            LOG.error("UnexpectedError", exc_info=True)
+            raise
+
+    async def delete_project_files(
+        self,
+        project_revision_id: str,
+        organization_id: str,
+    ) -> None:
+        """Delete all files for a project revision."""
+        try:
+            async with self.Session() as session:
+                delete_files_query = (
+                    delete(ProjectFileModel)
+                    .where(ProjectFileModel.project_revision_id == project_revision_id)
+                    .where(ProjectFileModel.organization_id == organization_id)
+                )
+                await session.execute(delete_files_query)
+                await session.commit()
+        except SQLAlchemyError:
+            LOG.error("SQLAlchemyError", exc_info=True)
+            raise
+        except Exception:
+            LOG.error("UnexpectedError", exc_info=True)
+            raise

skyvern/forge/sdk/db/utils.py

@@ -15,6 +15,7 @@ from skyvern.forge.sdk.db.models import (
     OrganizationAuthTokenModel,
     OrganizationModel,
     OutputParameterModel,
+    ProjectModel,
     StepModel,
     TaskModel,
     WorkflowModel,
@@ -46,6 +47,7 @@ from skyvern.forge.sdk.workflow.models.workflow import (
     WorkflowRunStatus,
     WorkflowStatus,
 )
+from skyvern.schemas.projects import Project
 from skyvern.schemas.runs import ProxyLocation
 from skyvern.webeye.actions.actions import (
     Action,
@@ -492,6 +494,20 @@ def convert_to_workflow_run_block(
     return block
 
 
+def convert_to_project(project_model: ProjectModel) -> Project:
+    return Project(
+        project_revision_id=project_model.project_revision_id,
+        project_id=project_model.project_id,
+        organization_id=project_model.organization_id,
+        workflow_id=project_model.workflow_permanent_id,
+        run_id=project_model.run_id,
+        version=project_model.version,
+        created_at=project_model.created_at,
+        modified_at=project_model.modified_at,
+        deleted_at=project_model.deleted_at,
+    )
+
+
 def hydrate_action(action_model: ActionModel) -> Action:
     """
     Convert ActionModel to the appropriate Action type based on action_type.

skyvern/forge/sdk/routes/__init__.py

@@ -1,6 +1,7 @@
 from skyvern.forge.sdk.routes import agent_protocol  # noqa: F401
 from skyvern.forge.sdk.routes import browser_sessions  # noqa: F401
 from skyvern.forge.sdk.routes import credentials  # noqa: F401
+from skyvern.forge.sdk.routes import projects  # noqa: F401
 from skyvern.forge.sdk.routes import pylon  # noqa: F401
 from skyvern.forge.sdk.routes import run_blocks  # noqa: F401
 from skyvern.forge.sdk.routes import streaming  # noqa: F401

skyvern/forge/sdk/routes/projects.py

@@ -0,0 +1,279 @@
+import base64
+import hashlib
+
+import structlog
+from fastapi import Depends, HTTPException, Path, Query
+
+from skyvern.exceptions import WorkflowNotFound
+from skyvern.forge import app
+from skyvern.forge.sdk.routes.routers import base_router, legacy_base_router
+from skyvern.forge.sdk.schemas.organizations import Organization
+from skyvern.forge.sdk.services import org_auth_service
+from skyvern.schemas.projects import CreateProjectRequest, CreateProjectResponse, DeployProjectRequest, Project
+from skyvern.services import project_service
+
+LOG = structlog.get_logger()
+
+
+@base_router.post(
+    "/projects",
+    response_model=CreateProjectResponse,
+    summary="Create project",
+    description="Create a new project with optional files and metadata",
+    tags=["Projects"],
+    openapi_extra={
+        "x-fern-sdk-method-name": "create_project",
+    },
+)
+@base_router.post(
+    "/projects/",
+    response_model=CreateProjectResponse,
+    include_in_schema=False,
+)
+async def create_project(
+    data: CreateProjectRequest,
+    current_org: Organization = Depends(org_auth_service.get_current_org),
+) -> CreateProjectResponse:
+    """Create a new project with optional files and metadata."""
+    organization_id = current_org.organization_id
+    LOG.info(
+        "Creating project",
+        organization_id=organization_id,
+        file_count=len(data.files) if data.files else 0,
+    )
+    # validate workflow_id and run_id
+    if data.workflow_id:
+        if not await app.DATABASE.get_workflow_by_permanent_id(
+            workflow_permanent_id=data.workflow_id, organization_id=organization_id
+        ):
+            raise WorkflowNotFound(workflow_permanent_id=data.workflow_id)
+    if data.run_id:
+        if not await app.DATABASE.get_run(run_id=data.run_id, organization_id=organization_id):
+            raise HTTPException(status_code=404, detail=f"Run_id {data.run_id} not found")
+    try:
+        # Create the project in the database
+        project = await app.DATABASE.create_project(
+            organization_id=organization_id,
+            workflow_permanent_id=data.workflow_id,
+            run_id=data.run_id,
+        )
+        # Process files if provided
+        file_tree = {}
+        file_count = 0
+        if data.files:
+            file_tree = await project_service.build_file_tree(
+                data.files,
+                organization_id=organization_id,
+                project_id=project.project_id,
+                project_version=project.version,
+                project_revision_id=project.project_revision_id,
+            )
+            file_count = len(data.files)
+        return CreateProjectResponse(
+            project_id=project.project_id,
+            version=project.version,
+            workflow_id=project.workflow_id,
+            run_id=project.run_id,
+            file_count=file_count,
+            created_at=project.created_at,
+            file_tree=file_tree,
+        )
+    except Exception as e:
+        LOG.error("Failed to create project", error=str(e), exc_info=True)
+        raise HTTPException(status_code=500, detail="Failed to create project")
+
+
+@legacy_base_router.get("/projects/{project_id}")
+@legacy_base_router.get("/projects/{project_id}/", include_in_schema=False)
+@base_router.get(
+    "/projects/{project_id}",
+    response_model=Project,
+    summary="Get project by ID",
+    description="Retrieves a specific project by its ID",
+    tags=["Projects"],
+    openapi_extra={
+        "x-fern-sdk-method-name": "get_project",
+    },
+)
+@base_router.get(
+    "/projects/{project_id}/",
+    response_model=Project,
+    include_in_schema=False,
+)
+async def get_project(
+    project_id: str = Path(
+        ...,
+        description="The unique identifier of the project",
+        examples=["proj_abc123"],
+    ),
+    current_org: Organization = Depends(org_auth_service.get_current_org),
+) -> Project:
+    """Get a project by its ID."""
+    LOG.info(
+        "Getting project",
+        organization_id=current_org.organization_id,
+        project_id=project_id,
+    )
+
+    project = await app.DATABASE.get_project(
+        project_id=project_id,
+        organization_id=current_org.organization_id,
+    )
+
+    if not project:
+        raise HTTPException(status_code=404, detail="Project not found")
+
+    return project
+
+
+@legacy_base_router.get("/projects")
+@legacy_base_router.get("/projects/", include_in_schema=False)
+@base_router.get(
+    "/projects",
+    response_model=list[Project],
+    summary="Get all projects",
+    description="Retrieves a paginated list of projects for the current organization",
+    tags=["Projects"],
+    openapi_extra={
+        "x-fern-sdk-method-name": "get_projects",
+    },
+)
+@base_router.get(
+    "/projects/",
+    response_model=list[Project],
+    include_in_schema=False,
+)
+async def get_projects(
+    current_org: Organization = Depends(org_auth_service.get_current_org),
+    page: int = Query(
+        1,
+        ge=1,
+        description="Page number for pagination",
+        examples=[1],
+    ),
+    page_size: int = Query(
+        10,
+        ge=1,
+        description="Number of items per page",
+        examples=[10],
+    ),
+) -> list[Project]:
+    """Get all projects for the current organization."""
+    LOG.info(
+        "Getting projects",
+        organization_id=current_org.organization_id,
+        page=page,
+        page_size=page_size,
+    )
+
+    projects = await app.DATABASE.get_projects(
+        organization_id=current_org.organization_id,
+        page=page,
+        page_size=page_size,
+    )
+
+    return projects
+
+
+@base_router.post(
+    "/projects/{project_id}/deploy",
+    response_model=CreateProjectResponse,
+    summary="Deploy project",
+    description="Deploy a project with updated files, creating a new version",
+    tags=["Projects"],
+    openapi_extra={
+        "x-fern-sdk-method-name": "deploy_project",
+    },
+)
+@base_router.post(
+    "/projects/{project_id}/deploy/",
+    response_model=CreateProjectResponse,
+    include_in_schema=False,
+)
+async def deploy_project(
+    data: DeployProjectRequest,
+    project_id: str = Path(
+        ...,
+        description="The unique identifier of the project",
+        examples=["proj_abc123"],
+    ),
+    current_org: Organization = Depends(org_auth_service.get_current_org),
+) -> CreateProjectResponse:
+    """Deploy a project with updated files, creating a new version."""
+    LOG.info(
+        "Deploying project",
+        organization_id=current_org.organization_id,
+        project_id=project_id,
+        file_count=len(data.files) if data.files else 0,
+    )
+
+    try:
+        # Get the latest version of the project
+        latest_project = await app.DATABASE.get_project(
+            project_id=project_id,
+            organization_id=current_org.organization_id,
+        )
+
+        if not latest_project:
+            raise HTTPException(status_code=404, detail="Project not found")
+
+        # Create a new version of the project
+        new_version = latest_project.version + 1
+        new_project_revision = await app.DATABASE.create_project(
+            organization_id=current_org.organization_id,
+            workflow_permanent_id=latest_project.workflow_id,
+            run_id=latest_project.run_id,
+            project_id=project_id,  # Use the same project_id for versioning
+            version=new_version,
+        )
+
+        # Process files if provided
+        file_tree = {}
+        file_count = 0
+        if data.files:
+            file_tree = await project_service.build_file_tree(
+                data.files,
+                organization_id=current_org.organization_id,
+                project_id=new_project_revision.project_id,
+                project_version=new_project_revision.version,
+                project_revision_id=new_project_revision.project_revision_id,
+            )
+            file_count = len(data.files)
+
+            # Create project file records
+            for file in data.files:
+                content_bytes = base64.b64decode(file.content)
+                content_hash = hashlib.sha256(content_bytes).hexdigest()
+                file_size = len(content_bytes)
+
+                # Extract file name from path
+                file_name = file.path.split("/")[-1]
+
+                await app.DATABASE.create_project_file(
+                    project_revision_id=new_project_revision.project_revision_id,
+                    project_id=new_project_revision.project_id,
+                    organization_id=new_project_revision.organization_id,
+                    file_path=file.path,
+                    file_name=file_name,
+                    file_type="file",
+                    content_hash=f"sha256:{content_hash}",
+                    file_size=file_size,
+                    mime_type=file.mime_type,
+                    encoding=file.encoding,
+                )
+
+        return CreateProjectResponse(
+            project_id=new_project_revision.project_id,
+            version=new_project_revision.version,
+            workflow_id=new_project_revision.workflow_id,
+            run_id=new_project_revision.run_id,
+            file_count=file_count,
+            created_at=new_project_revision.created_at,
+            file_tree=file_tree,
+        )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        LOG.error("Failed to deploy project", error=str(e), exc_info=True)
+        raise HTTPException(status_code=500, detail="Failed to deploy project")