vodorod/Dorod-Sky
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

adjust s3 prefix for downloaded files (#1290)

Browse Source
This commit is contained in:
LawyZheng
2024-11-29 17:06:04 +08:00
committed by GitHub
parent 87061f5bb6
commit fe0f971842
3 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
skyvern/config.py
View File

@@ -52,7 +52,6 @@ class Settings(BaseSettings):
GENERATE_PRESIGNED_URLS: bool = False GENERATE_PRESIGNED_URLS: bool = False
AWS_S3_BUCKET_ARTIFACTS: str = "skyvern-artifacts" AWS_S3_BUCKET_ARTIFACTS: str = "skyvern-artifacts"
AWS_S3_BUCKET_SCREENSHOTS: str = "skyvern-screenshots" AWS_S3_BUCKET_SCREENSHOTS: str = "skyvern-screenshots"
AWS_S3_BUCKET_DOWNLOADS: str = "skyvern-uploads"
AWS_S3_BUCKET_BROWSER_SESSIONS: str = "skyvern-browser-sessions" AWS_S3_BUCKET_BROWSER_SESSIONS: str = "skyvern-browser-sessions"
# Supported storage types: local, s3 # Supported storage types: local, s3

1
skyvern/constants.py
View File

@@ -11,6 +11,7 @@ PAGE_CONTENT_TIMEOUT = 300 # 5 mins
BUILDING_ELEMENT_TREE_TIMEOUT_MS = 60 * 1000 # 1 minute BUILDING_ELEMENT_TREE_TIMEOUT_MS = 60 * 1000 # 1 minute
BROWSER_CLOSE_TIMEOUT = 180 # 3 minute BROWSER_CLOSE_TIMEOUT = 180 # 3 minute
BROWSER_DOWNLOAD_TIMEOUT = 600 # 10 minute BROWSER_DOWNLOAD_TIMEOUT = 600 # 10 minute
DOWNLOAD_FILE_PREFIX = "downloads"
SAVE_DOWNLOADED_FILES_TIMEOUT = 180 SAVE_DOWNLOADED_FILES_TIMEOUT = 180
GET_DOWNLOADED_FILES_TIMEOUT = 30 GET_DOWNLOADED_FILES_TIMEOUT = 30

7
skyvern/forge/sdk/artifact/storage/s3.py
View File

@@ -3,6 +3,7 @@ import shutil
from datetime import datetime from datetime import datetime
from skyvern.config import settings from skyvern.config import settings
from skyvern.constants import DOWNLOAD_FILE_PREFIX
from skyvern.forge.sdk.api.aws import AsyncAWSClient from skyvern.forge.sdk.api.aws import AsyncAWSClient
from skyvern.forge.sdk.api.files import ( from skyvern.forge.sdk.api.files import (
create_named_temporary_file, create_named_temporary_file,
@@ -79,20 +80,20 @@ class S3Storage(BaseStorage):
for file in files: for file in files:
fpath = os.path.join(download_dir, file) fpath = os.path.join(download_dir, file)
if os.path.isfile(fpath): if os.path.isfile(fpath):
uri = f"s3://{settings.AWS_S3_BUCKET_DOWNLOADS}/{settings.ENV}/{organization_id}/{workflow_run_id or task_id}/{file}" uri = f"s3://{settings.AWS_S3_BUCKET_UPLOADS}/{DOWNLOAD_FILE_PREFIX}/{settings.ENV}/{organization_id}/{workflow_run_id or task_id}/{file}"
# TODO: use coroutine to speed up uploading if too many files # TODO: use coroutine to speed up uploading if too many files
await self.async_client.upload_file_from_path(uri, fpath) await self.async_client.upload_file_from_path(uri, fpath)
async def get_downloaded_files( async def get_downloaded_files(
self, organization_id: str, task_id: str | None, workflow_run_id: str | None self, organization_id: str, task_id: str | None, workflow_run_id: str | None
) -> list[str]: ) -> list[str]:
uri = f"s3://{settings.AWS_S3_BUCKET_DOWNLOADS}/{settings.ENV}/{organization_id}/{workflow_run_id or task_id}" uri = f"s3://{settings.AWS_S3_BUCKET_UPLOADS}/{DOWNLOAD_FILE_PREFIX}/{settings.ENV}/{organization_id}/{workflow_run_id or task_id}"
object_keys = await self.async_client.list_files(uri=uri) object_keys = await self.async_client.list_files(uri=uri)
if len(object_keys) == 0: if len(object_keys) == 0:
return [] return []
object_uris: list[str] = [] object_uris: list[str] = []
for key in object_keys: for key in object_keys:
object_uri = f"s3://{settings.AWS_S3_BUCKET_DOWNLOADS}/{key}" object_uri = f"s3://{settings.AWS_S3_BUCKET_UPLOADS}/{key}"
object_uris.append(object_uri) object_uris.append(object_uri)
presigned_urils = await self.async_client.create_presigned_urls(object_uris) presigned_urils = await self.async_client.create_presigned_urls(object_uris)
if presigned_urils is None: if presigned_urils is None: