diff --git a/skyvern/config.py b/skyvern/config.py index fbf18753..9b0db312 100644 --- a/skyvern/config.py +++ b/skyvern/config.py @@ -238,6 +238,8 @@ class Settings(BaseSettings): ENABLE_LOG_ARTIFACTS: bool = False ENABLE_CODE_BLOCK: bool = False + TASK_BLOCKED_SITE_FALLBACK_URL: str = "https://www.google.com" + # SkyvernClient Settings SKYVERN_BASE_URL: str = "https://api.skyvern.com" SKYVERN_API_KEY: str = "PLACEHOLDER" diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.py index 18f60cbe..2facdd53 100644 --- a/skyvern/services/task_v2_service.py +++ b/skyvern/services/task_v2_service.py @@ -10,7 +10,13 @@ from playwright.async_api import Page from sqlalchemy.exc import OperationalError from skyvern.config import settings -from skyvern.exceptions import FailedToSendWebhook, TaskTerminationError, TaskV2NotFound, UrlGenerationFailure +from skyvern.exceptions import ( + FailedToSendWebhook, + MissingBrowserState, + TaskTerminationError, + TaskV2NotFound, + UrlGenerationFailure, +) from skyvern.forge import app from skyvern.forge.prompts import prompt_engine from skyvern.forge.sdk.artifact.models import ArtifactType @@ -481,31 +487,86 @@ async def run_task_v2_helper( task_history_record: dict[str, Any] = {} context = skyvern_context.ensure_context() + # Always ensure browser_state is available at the start of the loop current_url: str | None = None page: Page | None = None - browser_state = app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id, workflow_run.parent_workflow_run_id) - if browser_state: + fallback_url = settings.TASK_BLOCKED_SITE_FALLBACK_URL + browser_state = app.BROWSER_MANAGER.get_for_workflow_run( + workflow_run_id=workflow_run_id, parent_workflow_run_id=workflow_run.parent_workflow_run_id + ) + if browser_state is None: + fallback_occurred = False + try: + browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run( + workflow_run=workflow_run, + url=url, + browser_session_id=browser_session_id, + ) + except Exception: + LOG.warning("Failed to get or create browser state, fallback to Google", exc_info=True, url=url) + + browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run( + workflow_run=workflow_run, + url=fallback_url, + browser_session_id=browser_session_id, + ) + + fallback_occurred = True + + if browser_state is None: + LOG.error("Failed to create browser state even after fallback", workflow_run_id=workflow_run_id) + raise MissingBrowserState(workflow_run_id=workflow_run_id) + + page = await browser_state.get_working_page() + + page_loaded = False + if page: + try: + # Check if the page has a body element to verify it loaded + # page will always be None if browser state failed to load + page_loaded = await browser_state.validate_browser_context(page) + except Exception: + page_loaded = False + LOG.warning( + "Page failed to load properly, fallback to Google", + exc_info=True, + url=url, + current_url=current_url, + ) + + if not page_loaded: + # Page failed to load properly, fallback to Google + if page: + try: + await page.goto(fallback_url, timeout=15000) + fallback_occurred = True + except Exception: + LOG.exception("Failed to load Google fallback", exc_info=True, url=url, current_url=current_url) + else: page = await browser_state.get_working_page() if page: current_url = await SkyvernFrame.get_url(page) if i == 0 and current_url != url: - # The first iteration is always a GOTO_URL task - task_type = "goto_url" - plan = f"Go to this website: {url}" - task_history_record = {"type": task_type, "task": plan} - block, block_yaml_list, parameter_yaml_list = await _generate_goto_url_task( - workflow_id=workflow_id, - url=url, - ) + if fallback_occurred: + plan = f"Go to Google because the intended website ({url}) failed to load properly." + task_type = "goto_url" + task_history_record = {"type": task_type, "task": plan} + block, block_yaml_list, parameter_yaml_list = await _generate_goto_url_task( + workflow_id=workflow_id, + url=fallback_url, + ) + else: + # Page loaded successfully, proceed with original URL + plan = f"Go to this website: {url}" + task_type = "goto_url" + task_history_record = {"type": task_type, "task": plan} + block, block_yaml_list, parameter_yaml_list = await _generate_goto_url_task( + workflow_id=workflow_id, + url=url, + ) else: try: - if browser_state is None: - browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run( - workflow_run=workflow_run, - url=url, - browser_session_id=browser_session_id, - ) scraped_page = await scrape_website( browser_state, url, diff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.py index bdba7297..be53b433 100644 --- a/skyvern/webeye/browser_factory.py +++ b/skyvern/webeye/browser_factory.py @@ -274,16 +274,6 @@ class BrowserContextFactory: raise UnknownErrorWhileCreatingBrowserContext(browser_type, e) from e - @classmethod - def set_validate_browser_context(cls, validator: Callable[[Page], Awaitable[bool]]) -> None: - cls._validator = validator - - @classmethod - async def validate_browser_context(cls, page: Page) -> bool: - if cls._validator is None: - return True - return await cls._validator(page) - class VideoArtifact(BaseModel): video_path: str | None = None @@ -603,6 +593,7 @@ class BrowserState: loading_time=end_time - start_time, url=url, ) + # Do we need this? await asyncio.sleep(5) LOG.info(f"Successfully went to {url}", url=url, retry_time=retry_time) return @@ -639,6 +630,30 @@ class BrowserState: await self.set_working_page(last_page, len(self.browser_context.pages) - 1) return last_page + async def validate_browser_context(self, page: Page) -> bool: + # validate the content + try: + skyvern_frame = await SkyvernFrame.create_instance(frame=page) + html = await skyvern_frame.get_content() + except Exception: + LOG.error( + "Error happened while getting the first page content", + exc_info=True, + ) + return False + + if "Bad gateway error" in html: + LOG.warning("Bad gateway error on the page, recreate a new browser context with another proxy node") + return False + + if "client_connect_forbidden_host" in html: + LOG.warning( + "capture the client_connect_forbidden_host error on the page, recreate a new browser context with another proxy node" + ) + return False + + return True + async def must_get_working_page(self) -> Page: page = await self.get_working_page() assert page is not None @@ -710,7 +725,7 @@ class BrowserState: ) page = await self.__assert_page() - if not await BrowserContextFactory.validate_browser_context(await self.get_working_page()): + if not await self.validate_browser_context(await self.get_working_page()): if not await self.close_current_open_page(): LOG.warning("Failed to close the current open page, going to skip the browser context validation") return page