From fe3d5cec1044f18115858a9955084e9ab979a356 Mon Sep 17 00:00:00 2001 From: Shuchang Zheng Date: Mon, 4 Aug 2025 00:33:34 -0700 Subject: [PATCH] project run (#3097) --- skyvern/exceptions.py | 5 ++ skyvern/forge/sdk/db/client.py | 25 ++++++- skyvern/forge/sdk/db/utils.py | 22 +++++- skyvern/forge/sdk/executor/async_executor.py | 29 +++++++- skyvern/forge/sdk/routes/projects.py | 39 +++++++++-- skyvern/schemas/projects.py | 33 ++++++++- skyvern/services/project_service.py | 72 +++++++++++++++++++- 7 files changed, 212 insertions(+), 13 deletions(-) diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py index 6d8118c0..10fdf734 100644 --- a/skyvern/exceptions.py +++ b/skyvern/exceptions.py @@ -744,3 +744,8 @@ class APIKeyNotFound(SkyvernHTTPException): class ElementOutOfCurrentViewport(SkyvernException): def __init__(self, element_id: str): super().__init__(f"Element {element_id} is out of current viewport") + + +class ProjectNotFound(SkyvernHTTPException): + def __init__(self, project_id: str) -> None: + super().__init__(f"Project {project_id} not found") diff --git a/skyvern/forge/sdk/db/client.py b/skyvern/forge/sdk/db/client.py index d9b0d2de..6977a1eb 100644 --- a/skyvern/forge/sdk/db/client.py +++ b/skyvern/forge/sdk/db/client.py @@ -55,6 +55,7 @@ from skyvern.forge.sdk.db.utils import ( convert_to_organization_auth_token, convert_to_output_parameter, convert_to_project, + convert_to_project_file, convert_to_step, convert_to_task, convert_to_workflow, @@ -99,7 +100,7 @@ from skyvern.forge.sdk.workflow.models.workflow import ( WorkflowRunStatus, WorkflowStatus, ) -from skyvern.schemas.projects import Project +from skyvern.schemas.projects import Project, ProjectFile from skyvern.schemas.runs import ProxyLocation, RunEngine, RunType from skyvern.webeye.actions.actions import Action from skyvern.webeye.actions.models import AgentStepOutput @@ -3632,6 +3633,17 @@ class AgentDB: LOG.error("UnexpectedError", exc_info=True) raise + async def get_project_revision(self, project_revision_id: str, organization_id: str) -> Project | None: + async with self.Session() as session: + project = ( + await session.scalars( + select(ProjectModel) + .filter_by(project_revision_id=project_revision_id) + .filter_by(organization_id=organization_id) + ) + ).first() + return convert_to_project(project) if project else None + async def create_project_file( self, project_revision_id: str, @@ -3670,3 +3682,14 @@ class AgentDB: except Exception: LOG.error("UnexpectedError", exc_info=True) raise + + async def get_project_files(self, project_revision_id: str, organization_id: str) -> list[ProjectFile]: + async with self.Session() as session: + project_files = ( + await session.scalars( + select(ProjectFileModel) + .filter_by(project_revision_id=project_revision_id) + .filter_by(organization_id=organization_id) + ) + ).all() + return [convert_to_project_file(project_file) for project_file in project_files] diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.py index 3b494c55..342ba401 100644 --- a/skyvern/forge/sdk/db/utils.py +++ b/skyvern/forge/sdk/db/utils.py @@ -15,6 +15,7 @@ from skyvern.forge.sdk.db.models import ( OrganizationAuthTokenModel, OrganizationModel, OutputParameterModel, + ProjectFileModel, ProjectModel, StepModel, TaskModel, @@ -47,7 +48,7 @@ from skyvern.forge.sdk.workflow.models.workflow import ( WorkflowRunStatus, WorkflowStatus, ) -from skyvern.schemas.projects import Project +from skyvern.schemas.projects import Project, ProjectFile from skyvern.schemas.runs import ProxyLocation from skyvern.webeye.actions.actions import ( Action, @@ -507,6 +508,25 @@ def convert_to_project(project_model: ProjectModel) -> Project: ) +def convert_to_project_file(project_file_model: ProjectFileModel) -> ProjectFile: + return ProjectFile( + file_id=project_file_model.file_id, + project_revision_id=project_file_model.project_revision_id, + project_id=project_file_model.project_id, + organization_id=project_file_model.organization_id, + file_path=project_file_model.file_path, + file_name=project_file_model.file_name, + file_type=project_file_model.file_type, + content_hash=project_file_model.content_hash, + file_size=project_file_model.file_size, + mime_type=project_file_model.mime_type, + encoding=project_file_model.encoding, + artifact_id=project_file_model.artifact_id, + created_at=project_file_model.created_at, + modified_at=project_file_model.modified_at, + ) + + def hydrate_action(action_model: ActionModel) -> Action: """ Convert ActionModel to the appropriate Action type based on action_type. diff --git a/skyvern/forge/sdk/executor/async_executor.py b/skyvern/forge/sdk/executor/async_executor.py index 788222aa..55057f42 100644 --- a/skyvern/forge/sdk/executor/async_executor.py +++ b/skyvern/forge/sdk/executor/async_executor.py @@ -12,7 +12,7 @@ from skyvern.forge.sdk.schemas.task_v2 import TaskV2Status from skyvern.forge.sdk.schemas.tasks import TaskStatus from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus from skyvern.schemas.runs import RunEngine, RunType -from skyvern.services import task_v2_service +from skyvern.services import project_service, task_v2_service from skyvern.utils.files import initialize_skyvern_state_file LOG = structlog.get_logger() @@ -61,6 +61,17 @@ class AsyncExecutor(abc.ABC): ) -> None: pass + @abc.abstractmethod + async def execute_project( + self, + request: Request | None, + project_id: str, + organization_id: str, + background_tasks: BackgroundTasks | None, + **kwargs: dict, + ) -> None: + pass + class BackgroundTaskExecutor(AsyncExecutor): async def execute_task( @@ -197,3 +208,19 @@ class BackgroundTaskExecutor(AsyncExecutor): max_steps_override=max_steps_override, browser_session_id=browser_session_id, ) + + async def execute_project( + self, + request: Request | None, + project_id: str, + organization_id: str, + background_tasks: BackgroundTasks | None, + **kwargs: dict, + ) -> None: + if background_tasks: + background_tasks.add_task( + project_service.execute_project, + project_id=project_id, + organization_id=organization_id, + background_tasks=background_tasks, + ) diff --git a/skyvern/forge/sdk/routes/projects.py b/skyvern/forge/sdk/routes/projects.py index 67ff188f..0ec1dc8d 100644 --- a/skyvern/forge/sdk/routes/projects.py +++ b/skyvern/forge/sdk/routes/projects.py @@ -2,10 +2,11 @@ import base64 import hashlib import structlog -from fastapi import Depends, HTTPException, Path, Query +from fastapi import BackgroundTasks, Depends, HTTPException, Path, Query, Request from skyvern.forge import app -from skyvern.forge.sdk.routes.routers import base_router, legacy_base_router +from skyvern.forge.sdk.executor.factory import AsyncExecutorFactory +from skyvern.forge.sdk.routes.routers import base_router from skyvern.forge.sdk.schemas.organizations import Organization from skyvern.forge.sdk.services import org_auth_service from skyvern.schemas.projects import CreateProjectRequest, CreateProjectResponse, DeployProjectRequest, Project @@ -74,8 +75,6 @@ async def create_project( raise HTTPException(status_code=500, detail="Failed to create project") -@legacy_base_router.get("/projects/{project_id}") -@legacy_base_router.get("/projects/{project_id}/", include_in_schema=False) @base_router.get( "/projects/{project_id}", response_model=Project, @@ -117,8 +116,6 @@ async def get_project( return project -@legacy_base_router.get("/projects") -@legacy_base_router.get("/projects/", include_in_schema=False) @base_router.get( "/projects", response_model=list[Project], @@ -266,3 +263,33 @@ async def deploy_project( except Exception as e: LOG.error("Failed to deploy project", error=str(e), exc_info=True) raise HTTPException(status_code=500, detail="Failed to deploy project") + + +@base_router.post( + "/projects/{project_id}/run", + summary="Run project", + description="Run a project", + tags=["Projects"], +) +async def run_project( + request: Request, + background_tasks: BackgroundTasks, + project_id: str = Path( + ..., + description="The unique identifier of the project", + examples=["proj_abc123"], + ), + current_org: Organization = Depends(org_auth_service.get_current_org), +) -> None: + """Run a project.""" + # await project_service.execute_project( + # project_id=project_id, + # organization_id=current_org.organization_id, + # background_tasks=background_tasks, + # ) + await AsyncExecutorFactory.get_executor().execute_project( + request=request, + project_id=project_id, + organization_id=current_org.organization_id, + background_tasks=background_tasks, + ) diff --git a/skyvern/schemas/projects.py b/skyvern/schemas/projects.py index a38eaff8..518733a2 100644 --- a/skyvern/schemas/projects.py +++ b/skyvern/schemas/projects.py @@ -1,5 +1,6 @@ from __future__ import annotations +import base64 from datetime import datetime from enum import StrEnum @@ -10,9 +11,37 @@ class FileEncoding(StrEnum): """Supported file content encodings.""" BASE64 = "base64" + UTF8 = "utf-8" class ProjectFile(BaseModel): + file_id: str + project_revision_id: str + project_id: str + organization_id: str + + file_path: str # e.g., "src/utils.py" + file_name: str # e.g., "utils.py" + file_type: str # "file" or "directory" + + # File content and metadata + content_hash: str | None = None # SHA-256 hash for deduplication + file_size: int | None = None # Size in bytes + mime_type: str | None = None # e.g., "text/x-python" + encoding: FileEncoding | None = Field(default=None, description="Content encoding") + + artifact_id: str | None = None + created_at: datetime + modified_at: datetime + + async def get_content(self) -> str: + # get the content from the artifact + if self.encoding == FileEncoding.BASE64: + return base64.b64decode(self.content).decode("utf-8") + return self.content + + +class ProjectFileCreate(BaseModel): """Model representing a file in a project.""" path: str = Field(..., description="File path relative to project root", examples=["src/main.py"]) @@ -24,7 +53,7 @@ class ProjectFile(BaseModel): class CreateProjectRequest(BaseModel): workflow_id: str | None = Field(default=None, description="Associated workflow ID") run_id: str | None = Field(default=None, description="Associated run ID") - files: list[ProjectFile] | None = Field( + files: list[ProjectFileCreate] | None = Field( default=None, description="Array of files to include in the project", examples=[ @@ -58,7 +87,7 @@ class FileNode(BaseModel): class DeployProjectRequest(BaseModel): """Request model for deploying a project with updated files.""" - files: list[ProjectFile] = Field( + files: list[ProjectFileCreate] = Field( ..., description="Array of files to include in the project", examples=[ diff --git a/skyvern/services/project_service.py b/skyvern/services/project_service.py index 6a536176..09bf557f 100644 --- a/skyvern/services/project_service.py +++ b/skyvern/services/project_service.py @@ -1,17 +1,21 @@ import base64 import hashlib +import os +import subprocess from datetime import datetime import structlog +from fastapi import BackgroundTasks +from skyvern.exceptions import ProjectNotFound from skyvern.forge import app -from skyvern.schemas.projects import FileNode, ProjectFile +from skyvern.schemas.projects import FileNode, ProjectFileCreate LOG = structlog.get_logger(__name__) async def build_file_tree( - files: list[ProjectFile], + files: list[ProjectFileCreate], organization_id: str, project_id: str, project_version: int, @@ -90,3 +94,67 @@ async def build_file_tree( ) return file_tree + + +async def execute_project( + project_id: str, + organization_id: str, + background_tasks: BackgroundTasks | None = None, +) -> None: + # TODO: assume the project only has one ProjectFile called main.py + # step 1: get the project revision + # step 2: get the project files + # step 3: copy the project files to the local directory + # step 4: execute the project + + # step 1: get the project revision + project = await app.DATABASE.get_project( + project_id=project_id, + organization_id=organization_id, + ) + if not project: + raise ProjectNotFound(project_id=project_id) + + # step 2: get the project files + project_files = await app.DATABASE.get_project_files( + project_revision_id=project.project_revision_id, organization_id=organization_id + ) + + # step 3: copy the project files to the local directory + for file in project_files: + # retrieve the artifact + if not file.artifact_id: + continue + artifact = await app.DATABASE.get_artifact_by_id(file.artifact_id, organization_id) + if not artifact: + LOG.error("Artifact not found", artifact_id=file.artifact_id, project_id=project_id) + continue + file_content = await app.ARTIFACT_MANAGER.retrieve_artifact(artifact) + if not file_content: + continue + file_path = os.path.join(project.project_id, file.file_path) + # create the directory if it doesn't exist + os.makedirs(os.path.dirname(file_path), exist_ok=True) + + # Determine the encoding to use + encoding = "utf-8" + + try: + # Try to decode as text + if file.mime_type and file.mime_type.startswith("text/"): + # Text file - decode as string + with open(file_path, "w", encoding=encoding) as f: + f.write(file_content.decode(encoding)) + else: + # Binary file - write as bytes + with open(file_path, "wb") as f: + f.write(file_content) + except UnicodeDecodeError: + # Fallback to binary mode if text decoding fails + with open(file_path, "wb") as f: + f.write(file_content) + + # step 4: execute the project + if background_tasks: + background_tasks.add_task(subprocess.run, ["python", f"{project.project_id}/main.py"]) + LOG.info("Project executed successfully", project_id=project_id)