From de8e887e0fcdcf21013e686e09cb5881c601c905 Mon Sep 17 00:00:00 2001 From: LawyZheng Date: Thu, 31 Oct 2024 23:10:11 +0800 Subject: [PATCH] add browser console log artifact (#1097) --- .pre-commit-config.yaml | 1 + Dockerfile | 1 + docker-compose.yml | 1 + skyvern/config.py | 1 + skyvern/forge/agent.py | 10 +++++ skyvern/forge/sdk/artifact/models.py | 1 + skyvern/forge/sdk/artifact/storage/base.py | 1 + skyvern/forge/sdk/workflow/service.py | 19 ++++++++++ skyvern/webeye/browser_factory.py | 43 +++++++++++++++++++++- skyvern/webeye/browser_manager.py | 18 +++++++++ 10 files changed, 94 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9daa711f..2bcae6fd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,6 +48,7 @@ repos: - alembic - "sqlalchemy[mypy]" - types-PyYAML + - types-aiofiles exclude: | (?x)( ^tests.*| diff --git a/Dockerfile b/Dockerfile index baca3163..1364616b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,7 @@ COPY . /app ENV PYTHONPATH="/app:$PYTHONPATH" ENV VIDEO_PATH=/data/videos ENV HAR_PATH=/data/har +ENV LOG_PATH=/data/log ENV ARTIFACT_STORAGE_PATH=/data/artifacts COPY ./entrypoint-skyvern.sh /app/entrypoint-skyvern.sh diff --git a/docker-compose.yml b/docker-compose.yml index f3303184..83844785 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,7 @@ services: - ./artifacts:/data/artifacts - ./videos:/data/videos - ./har:/data/har + - ./log:/data/log - ./.streamlit:/app/.streamlit environment: - DATABASE_STRING=postgresql+psycopg://skyvern:skyvern@postgres:5432/skyvern diff --git a/skyvern/config.py b/skyvern/config.py index fb6356f8..aed4b70b 100644 --- a/skyvern/config.py +++ b/skyvern/config.py @@ -13,6 +13,7 @@ class Settings(BaseSettings): MAX_SCRAPING_RETRIES: int = 0 VIDEO_PATH: str | None = None HAR_PATH: str | None = "./har" + LOG_PATH: str = "./log" BROWSER_ACTION_TIMEOUT_MS: int = 5000 BROWSER_SCREENSHOT_TIMEOUT_MS: int = 20000 BROWSER_LOADING_TIMEOUT_MS: int = 120000 diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py index f6577ffb..f19dcb0e 100644 --- a/skyvern/forge/agent.py +++ b/skyvern/forge/agent.py @@ -1517,6 +1517,16 @@ class ForgeAgent: data=har_data, ) + browser_log = await app.BROWSER_MANAGER.get_browser_console_log( + task_id=task.task_id, browser_state=browser_state + ) + if browser_log: + await app.ARTIFACT_MANAGER.create_artifact( + step=last_step, + artifact_type=ArtifactType.BROWSER_CONSOLE_LOG, + data=browser_log, + ) + if browser_state.browser_context and browser_state.browser_artifacts.traces_dir: trace_path = f"{browser_state.browser_artifacts.traces_dir}/{task.task_id}.zip" await app.ARTIFACT_MANAGER.create_artifact( diff --git a/skyvern/forge/sdk/artifact/models.py b/skyvern/forge/sdk/artifact/models.py index 3ea021bf..261788e5 100644 --- a/skyvern/forge/sdk/artifact/models.py +++ b/skyvern/forge/sdk/artifact/models.py @@ -8,6 +8,7 @@ from pydantic import BaseModel, Field, field_serializer class ArtifactType(StrEnum): RECORDING = "recording" + BROWSER_CONSOLE_LOG = "browser_console_log" # DEPRECATED. pls use SCREENSHOT_LLM, SCREENSHOT_ACTION or SCREENSHOT_FINAL SCREENSHOT = "screenshot" diff --git a/skyvern/forge/sdk/artifact/storage/base.py b/skyvern/forge/sdk/artifact/storage/base.py index 34704779..4e5e6118 100644 --- a/skyvern/forge/sdk/artifact/storage/base.py +++ b/skyvern/forge/sdk/artifact/storage/base.py @@ -6,6 +6,7 @@ from skyvern.forge.sdk.models import Step # TODO: This should be a part of the ArtifactType model FILE_EXTENTSION_MAP: dict[ArtifactType, str] = { ArtifactType.RECORDING: "webm", + ArtifactType.BROWSER_CONSOLE_LOG: "log", ArtifactType.SCREENSHOT_LLM: "png", ArtifactType.SCREENSHOT_ACTION: "png", ArtifactType.SCREENSHOT_FINAL: "png", diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py index 776e573f..3584169f 100644 --- a/skyvern/forge/sdk/workflow/service.py +++ b/skyvern/forge/sdk/workflow/service.py @@ -843,6 +843,25 @@ class WorkflowService: data=har_data, ) + async def persist_browser_console_log( + self, + browser_state: BrowserState, + last_step: Step, + workflow: Workflow, + workflow_run: WorkflowRun, + ) -> None: + browser_log = await app.BROWSER_MANAGER.get_browser_console_log( + workflow_id=workflow.workflow_id, + workflow_run_id=workflow_run.workflow_run_id, + browser_state=browser_state, + ) + if browser_log: + await app.ARTIFACT_MANAGER.create_artifact( + step=last_step, + artifact_type=ArtifactType.BROWSER_CONSOLE_LOG, + data=browser_log, + ) + async def persist_tracing_data( self, browser_state: BrowserState, last_step: Step, workflow_run: WorkflowRun ) -> None: diff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.py index 791de3af..8eda8194 100644 --- a/skyvern/webeye/browser_factory.py +++ b/skyvern/webeye/browser_factory.py @@ -8,9 +8,10 @@ import uuid from datetime import datetime from typing import Any, Awaitable, Callable, Protocol +import aiofiles import structlog -from playwright.async_api import BrowserContext, Error, Page, Playwright, async_playwright -from pydantic import BaseModel +from playwright.async_api import BrowserContext, ConsoleMessage, Error, Page, Playwright, async_playwright +from pydantic import BaseModel, PrivateAttr from skyvern.config import settings from skyvern.constants import REPO_ROOT_DIR @@ -40,6 +41,23 @@ def get_download_dir(workflow_run_id: str | None, task_id: str | None) -> str: return download_dir +def set_browser_console_log(browser_context: BrowserContext, browser_artifacts: BrowserArtifacts) -> str: + if browser_artifacts.browser_console_log_path is None: + log_path = f"{settings.LOG_PATH}/{datetime.utcnow().strftime('%Y-%m-%d')}/{uuid.uuid4()}.log" + os.makedirs(os.path.dirname(log_path), exist_ok=True) + browser_artifacts.browser_console_log_path = log_path + + async def browser_console_log(msg: ConsoleMessage) -> None: + current_time = datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ") + key_values = " ".join([f"{key}={value}" for key, value in msg.location.items()]) + format_log = f"{current_time}[{msg.type}]{msg.text} {key_values}\n" + await browser_artifacts.append_browser_console_log(format_log) + + LOG.info("browser console log is saved", log_path=browser_artifacts.browser_console_log_path) + browser_context.on("console", browser_console_log) + return browser_artifacts.browser_console_log_path + + class BrowserContextCreator(Protocol): def __call__( self, playwright: Playwright, **kwargs: dict[str, Any] @@ -91,12 +109,14 @@ class BrowserContextFactory: har_path: str | None = None, traces_dir: str | None = None, browser_session_dir: str | None = None, + browser_console_log_path: str | None = None, ) -> BrowserArtifacts: return BrowserArtifacts( video_artifacts=video_artifacts or [], har_path=har_path, traces_dir=traces_dir, browser_session_dir=browser_session_dir, + browser_console_log_path=browser_console_log_path, ) @classmethod @@ -113,6 +133,7 @@ class BrowserContextFactory: if not creator: raise UnknownBrowserType(browser_type) browser_context, browser_artifacts, cleanup_func = await creator(playwright, **kwargs) + set_browser_console_log(browser_context=browser_context, browser_artifacts=browser_artifacts) return browser_context, browser_artifacts, cleanup_func except UnknownBrowserType as e: raise e @@ -141,6 +162,24 @@ class BrowserArtifacts(BaseModel): har_path: str | None = None traces_dir: str | None = None browser_session_dir: str | None = None + browser_console_log_path: str | None = None + _browser_console_log_lock: asyncio.Lock = PrivateAttr(default_factory=asyncio.Lock) + + async def append_browser_console_log(self, msg: str) -> int: + if self.browser_console_log_path is None: + return 0 + + async with self._browser_console_log_lock: + async with aiofiles.open(self.browser_console_log_path, "a") as f: + return await f.write(msg) + + async def read_browser_console_log(self) -> bytes: + if self.browser_console_log_path is None: + return b"" + + async with self._browser_console_log_lock: + async with aiofiles.open(self.browser_console_log_path, "rb") as f: + return await f.read() async def _create_headless_chromium( diff --git a/skyvern/webeye/browser_manager.py b/skyvern/webeye/browser_manager.py index 055eb4e4..5f54a43d 100644 --- a/skyvern/webeye/browser_manager.py +++ b/skyvern/webeye/browser_manager.py @@ -169,6 +169,24 @@ class BrowserManager: ) return b"" + async def get_browser_console_log( + self, + browser_state: BrowserState, + task_id: str = "", + workflow_id: str = "", + workflow_run_id: str = "", + ) -> bytes: + if browser_state.browser_artifacts.browser_console_log_path is None: + LOG.warning( + "browser console log not found for task", + task_id=task_id, + workflow_id=workflow_id, + workflow_run_id=workflow_run_id, + ) + return b"" + + return await browser_state.browser_artifacts.read_browser_console_log() + @classmethod async def close(cls) -> None: LOG.info("Closing BrowserManager")