From cca2772765de9633e93fe1264df095eaece0a85a Mon Sep 17 00:00:00 2001 From: Shuchang Zheng Date: Thu, 22 May 2025 22:18:42 -0700 Subject: [PATCH] fix new tab a issue (#2437) --- skyvern/webeye/actions/handler.py | 17 +++++++---- skyvern/webeye/scraper/domUtils.js | 17 ++++++----- skyvern/webeye/scraper/scraper.py | 3 ++ skyvern/webeye/utils/dom.py | 47 ++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 14 deletions(-) diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py index 4006e30e..b0037c84 100644 --- a/skyvern/webeye/actions/handler.py +++ b/skyvern/webeye/actions/handler.py @@ -479,10 +479,11 @@ async def handle_click_action( task: Task, step: Step, ) -> list[ActionResult]: + dom = DomUtil(scraped_page=scraped_page, page=page) original_url = page.url if action.x is not None and action.y is not None: # Find the element at the clicked location using JavaScript evaluation - element_id = await page.evaluate( + element_id: str | None = await page.evaluate( """data => { const element = document.elementFromPoint(data.x, data.y); if (!element) return null; @@ -506,6 +507,10 @@ async def handle_click_action( {"x": action.x, "y": action.y}, ) LOG.info("Clicked element at location", x=action.x, y=action.y, element_id=element_id, button=action.button) + if element_id: + skyvern_element = await dom.get_skyvern_element_by_id(element_id) + if await skyvern_element.navigate_to_a_href(page=page): + return [ActionSuccess()] if action.repeat == 1: await page.mouse.click(x=action.x, y=action.y, button=action.button) @@ -518,7 +523,6 @@ async def handle_click_action( return [ActionSuccess()] - dom = DomUtil(scraped_page=scraped_page, page=page) skyvern_element = await dom.get_skyvern_element_by_id(action.element_id) await asyncio.sleep(0.3) @@ -694,7 +698,8 @@ async def handle_click_to_download_file_action( ) try: - await locator.click(timeout=settings.BROWSER_ACTION_TIMEOUT_MS) + if not await skyvern_element.navigate_to_a_href(page=page): + await locator.click(timeout=settings.BROWSER_ACTION_TIMEOUT_MS) await page.wait_for_load_state(timeout=settings.BROWSER_LOADING_TIMEOUT_MS) except Exception as e: LOG.exception( @@ -1862,9 +1867,9 @@ async def chain_click( :param css: css of the element to click """ try: - await locator.click(timeout=timeout) - - LOG.info("Chain click: main element click succeeded", action=action, locator=locator) + if not await skyvern_element.navigate_to_a_href(page=page): + await locator.click(timeout=timeout) + LOG.info("Chain click: main element click succeeded", action=action, locator=locator) return [ActionSuccess()] except Exception as e: diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js index 7b6d82e4..1da9185b 100644 --- a/skyvern/webeye/scraper/domUtils.js +++ b/skyvern/webeye/scraper/domUtils.js @@ -1378,6 +1378,15 @@ async function buildElementObject( isSelect2MultiChoice(element), }; + // if element is an "a" tag and has a target="_blank" attribute, remove the target attribute but keep it in the elementObj + // We're doing this so that skyvern can do all the navigation in a single page/tab and not open new tab + if (elementTagNameLower === "a") { + if (element.getAttribute("target") === "_blank") { + elementObj.target = "_blank"; + element.removeAttribute("target"); + } + } + let isInShadowRoot = element.getRootNode() instanceof ShadowRoot; if (isInShadowRoot) { let shadowHostEle = element.getRootNode().host; @@ -1472,14 +1481,6 @@ async function buildElementTree( "]"; } - // if element is an "a" tag and has a target="_blank" attribute, remove the target attribute - // We're doing this so that skyvern can do all the navigation in a single page/tab and not open new tab - if (tagName === "a") { - if (element.getAttribute("target") === "_blank") { - element.removeAttribute("target"); - } - } - let shadowDOMchildren = []; // sometimes the shadowRoot is not visible, but the elemnets in the shadowRoot are visible if (element.shadowRoot) { diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py index 1163d68d..e4c07960 100644 --- a/skyvern/webeye/scraper/scraper.py +++ b/skyvern/webeye/scraper/scraper.py @@ -869,6 +869,9 @@ def trim_element(element: dict) -> dict: if "afterPseudoText" in queue_ele and not queue_ele.get("afterPseudoText"): del queue_ele["afterPseudoText"] + if "target" in queue_ele: + del queue_ele["target"] + return element diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py index d6fed140..0ecf2aa5 100644 --- a/skyvern/webeye/utils/dom.py +++ b/skyvern/webeye/utils/dom.py @@ -5,6 +5,7 @@ import copy import typing from enum import StrEnum from random import uniform +from urllib.parse import urlparse import structlog from playwright.async_api import ElementHandle, Frame, FrameLocator, Locator, Page, TimeoutError @@ -350,6 +351,27 @@ class SkyvernElement: assert handler is not None return handler + async def should_use_navigation_instead_click(self, page: Page) -> str | None: + if self.__static_element.get("target") != "_blank": + return None + + href: str | None = await self.get_attr("href", mode="static") + if not href: + return None + + href_url = urlparse(href) + if href_url.scheme.lower() not in ["http", "https"]: + return None + + if not href_url.netloc: + return None + + cur_url = urlparse(page.url) + if href_url.netloc.lower() == cur_url.netloc.lower(): + return None + + return href + async def find_blocking_element( self, dom: DomUtil, incremental_page: IncrementalScrapePage | None = None ) -> tuple[SkyvernElement | None, bool]: @@ -739,6 +761,31 @@ class SkyvernElement: return True + async def navigate_to_a_href(self, page: Page) -> str | None: + if self.get_tag_name() != InteractiveElement.A: + return None + + href = await self.should_use_navigation_instead_click(page) + if not href: + return None + + LOG.info( + "Trying to navigate to the href link instead of clicking", + href=href, + current_url=page.url, + ) + try: + await page.goto(href, timeout=settings.BROWSER_LOADING_TIMEOUT_MS) + return href + except Exception as e: + # some cases use this method to download a file. but it will be redirected away soon + # and agent will run into ABORTED error. + if "net::ERR_ABORTED" in str(e): + return href + + LOG.warning("Failed to navigate to the href link", exc_info=True, href=href, current_url=page.url) + raise + class DomUtil: """