From fe0f9718427010f400d3afd9cd0bdb26e748b327 Mon Sep 17 00:00:00 2001 From: LawyZheng Date: Fri, 29 Nov 2024 17:06:04 +0800 Subject: [PATCH] adjust s3 prefix for downloaded files (#1290) --- skyvern/config.py | 1 - skyvern/constants.py | 1 + skyvern/forge/sdk/artifact/storage/s3.py | 7 ++++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/skyvern/config.py b/skyvern/config.py index 2dc982b7..bce59736 100644 --- a/skyvern/config.py +++ b/skyvern/config.py @@ -52,7 +52,6 @@ class Settings(BaseSettings): GENERATE_PRESIGNED_URLS: bool = False AWS_S3_BUCKET_ARTIFACTS: str = "skyvern-artifacts" AWS_S3_BUCKET_SCREENSHOTS: str = "skyvern-screenshots" - AWS_S3_BUCKET_DOWNLOADS: str = "skyvern-uploads" AWS_S3_BUCKET_BROWSER_SESSIONS: str = "skyvern-browser-sessions" # Supported storage types: local, s3 diff --git a/skyvern/constants.py b/skyvern/constants.py index c9a19871..15705093 100644 --- a/skyvern/constants.py +++ b/skyvern/constants.py @@ -11,6 +11,7 @@ PAGE_CONTENT_TIMEOUT = 300 # 5 mins BUILDING_ELEMENT_TREE_TIMEOUT_MS = 60 * 1000 # 1 minute BROWSER_CLOSE_TIMEOUT = 180 # 3 minute BROWSER_DOWNLOAD_TIMEOUT = 600 # 10 minute +DOWNLOAD_FILE_PREFIX = "downloads" SAVE_DOWNLOADED_FILES_TIMEOUT = 180 GET_DOWNLOADED_FILES_TIMEOUT = 30 diff --git a/skyvern/forge/sdk/artifact/storage/s3.py b/skyvern/forge/sdk/artifact/storage/s3.py index ece1c072..17b587f0 100644 --- a/skyvern/forge/sdk/artifact/storage/s3.py +++ b/skyvern/forge/sdk/artifact/storage/s3.py @@ -3,6 +3,7 @@ import shutil from datetime import datetime from skyvern.config import settings +from skyvern.constants import DOWNLOAD_FILE_PREFIX from skyvern.forge.sdk.api.aws import AsyncAWSClient from skyvern.forge.sdk.api.files import ( create_named_temporary_file, @@ -79,20 +80,20 @@ class S3Storage(BaseStorage): for file in files: fpath = os.path.join(download_dir, file) if os.path.isfile(fpath): - uri = f"s3://{settings.AWS_S3_BUCKET_DOWNLOADS}/{settings.ENV}/{organization_id}/{workflow_run_id or task_id}/{file}" + uri = f"s3://{settings.AWS_S3_BUCKET_UPLOADS}/{DOWNLOAD_FILE_PREFIX}/{settings.ENV}/{organization_id}/{workflow_run_id or task_id}/{file}" # TODO: use coroutine to speed up uploading if too many files await self.async_client.upload_file_from_path(uri, fpath) async def get_downloaded_files( self, organization_id: str, task_id: str | None, workflow_run_id: str | None ) -> list[str]: - uri = f"s3://{settings.AWS_S3_BUCKET_DOWNLOADS}/{settings.ENV}/{organization_id}/{workflow_run_id or task_id}" + uri = f"s3://{settings.AWS_S3_BUCKET_UPLOADS}/{DOWNLOAD_FILE_PREFIX}/{settings.ENV}/{organization_id}/{workflow_run_id or task_id}" object_keys = await self.async_client.list_files(uri=uri) if len(object_keys) == 0: return [] object_uris: list[str] = [] for key in object_keys: - object_uri = f"s3://{settings.AWS_S3_BUCKET_DOWNLOADS}/{key}" + object_uri = f"s3://{settings.AWS_S3_BUCKET_UPLOADS}/{key}" object_uris.append(object_uri) presigned_urils = await self.async_client.create_presigned_urls(object_uris) if presigned_urils is None: