shu/add project files (#3072)

alembic/versions/2025_07_31_1609-1b2a8b62de61_add_project_files_table.py

@@ -0,0 +1,54 @@
+"""add project_files table
+
+Revision ID: 1b2a8b62de61
+Revises: 0ecb03206fc6
+Create Date: 2025-07-31 16:09:22.454667+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "1b2a8b62de61"
+down_revision: Union[str, None] = "0ecb03206fc6"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "project_files",
+        sa.Column("file_id", sa.String(), nullable=False),
+        sa.Column("project_revision_id", sa.String(), nullable=False),
+        sa.Column("project_id", sa.String(), nullable=False),
+        sa.Column("organization_id", sa.String(), nullable=False),
+        sa.Column("file_path", sa.String(), nullable=False),
+        sa.Column("file_name", sa.String(), nullable=False),
+        sa.Column("file_type", sa.String(), nullable=False),
+        sa.Column("content_hash", sa.String(), nullable=True),
+        sa.Column("file_size", sa.Integer(), nullable=True),
+        sa.Column("mime_type", sa.String(), nullable=True),
+        sa.Column("encoding", sa.String(), nullable=True),
+        sa.Column("artifact_id", sa.String(), nullable=True),
+        sa.Column("created_at", sa.DateTime(), nullable=False),
+        sa.Column("modified_at", sa.DateTime(), nullable=False),
+        sa.Column("deleted_at", sa.DateTime(), nullable=True),
+        sa.PrimaryKeyConstraint("file_id"),
+        sa.UniqueConstraint("project_revision_id", "file_path", name="unique_project_file_path"),
+    )
+    op.create_index("file_project_path_index", "project_files", ["project_revision_id", "file_path"], unique=False)
+    op.drop_column("projects", "artifact_id")
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("projects", sa.Column("artifact_id", sa.VARCHAR(), autoincrement=False, nullable=True))
+    op.drop_index("file_project_path_index", table_name="project_files")
+    op.drop_table("project_files")
+    # ### end Alembic commands ###

skyvern/forge/sdk/db/id.py

@@ -45,6 +45,7 @@ ORGANIZATION_AUTH_TOKEN_PREFIX = "oat"
 ORG_PREFIX = "o"
 OUTPUT_PARAMETER_PREFIX = "op"
 PERSISTENT_BROWSER_SESSION_ID = "pbs"
+PROJECT_FILE_PREFIX = "pf"
 PROJECT_REVISION_PREFIX = "pv"
 PROJECT_PREFIX = "p"
 STEP_PREFIX = "stp"
@@ -215,6 +216,11 @@ def generate_project_revision_id() -> str:
     return f"{PROJECT_REVISION_PREFIX}_{int_id}"
 
 
+def generate_project_file_id() -> str:
+    int_id = generate_id()
+    return f"{PROJECT_FILE_PREFIX}_{int_id}"
+
+
 ############# Helper functions below ##############
 def generate_id() -> int:
     """

skyvern/forge/sdk/db/models.py

@@ -36,6 +36,7 @@ from skyvern.forge.sdk.db.id import (
     generate_organization_bitwarden_collection_id,
     generate_output_parameter_id,
     generate_persistent_browser_session_id,
+    generate_project_file_id,
     generate_project_id,
     generate_project_revision_id,
     generate_step_id,
@@ -785,8 +786,6 @@ class ProjectModel(Base):
     project_revision_id = Column(String, primary_key=True, default=generate_project_revision_id)
     project_id = Column(String, default=generate_project_id, nullable=False)  # User-facing, consistent across versions
     organization_id = Column(String, nullable=False)
-    # the artifact id for the code
-    artifact_id = Column(String, nullable=True)
     # the wpid that this project is associated with
     workflow_permanent_id = Column(String, nullable=True)
     # The workflow run or task run id that this project is generated
@@ -800,3 +799,32 @@ class ProjectModel(Base):
         nullable=False,
     )
     deleted_at = Column(DateTime, nullable=True)
+
+
+class ProjectFileModel(Base):
+    __tablename__ = "project_files"
+    __table_args__ = (
+        Index("file_project_path_index", "project_revision_id", "file_path"),
+        UniqueConstraint("project_revision_id", "file_path", name="unique_project_file_path"),
+    )
+
+    file_id = Column(String, primary_key=True, default=generate_project_file_id)
+    project_revision_id = Column(String, nullable=False)
+    project_id = Column(String, nullable=False)
+    organization_id = Column(String, nullable=False)
+
+    file_path = Column(String, nullable=False)  # e.g., "src/utils.py"
+    file_name = Column(String, nullable=False)  # e.g., "utils.py"
+    file_type = Column(String, nullable=False)  # "file" or "directory"
+
+    # File content and metadata
+    content_hash = Column(String, nullable=True)  # SHA-256 hash for deduplication
+    file_size = Column(Integer, nullable=True)  # Size in bytes
+    mime_type = Column(String, nullable=True)  # e.g., "text/python"
+    encoding = Column(String, default="utf-8", nullable=True)
+
+    # Storage reference (could be S3 key, artifact_id, etc.)
+    artifact_id = Column(String, nullable=True)
+    created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)
+    modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, nullable=False)
+    deleted_at = Column(DateTime, nullable=True)

skyvern/schemas/peojects.py

@@ -13,4 +13,3 @@ class Project(BaseModel):
     version: int | None = None
     created_at: datetime
     modified_at: datetime
-    deleted_at: datetime | None = None