fix new tab a issue (#2437)

skyvern/webeye/actions/handler.py

@@ -479,10 +479,11 @@ async def handle_click_action(
     task: Task,
     step: Step,
 ) -> list[ActionResult]:
+    dom = DomUtil(scraped_page=scraped_page, page=page)
     original_url = page.url
     if action.x is not None and action.y is not None:
         # Find the element at the clicked location using JavaScript evaluation
-        element_id = await page.evaluate(
+        element_id: str | None = await page.evaluate(
             """data => {
             const element = document.elementFromPoint(data.x, data.y);
             if (!element) return null;
@@ -506,6 +507,10 @@ async def handle_click_action(
             {"x": action.x, "y": action.y},
         )
         LOG.info("Clicked element at location", x=action.x, y=action.y, element_id=element_id, button=action.button)
+        if element_id:
+            skyvern_element = await dom.get_skyvern_element_by_id(element_id)
+            if await skyvern_element.navigate_to_a_href(page=page):
+                return [ActionSuccess()]
 
         if action.repeat == 1:
             await page.mouse.click(x=action.x, y=action.y, button=action.button)
@@ -518,7 +523,6 @@ async def handle_click_action(
 
         return [ActionSuccess()]
 
-    dom = DomUtil(scraped_page=scraped_page, page=page)
     skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)
     await asyncio.sleep(0.3)
 
@@ -694,7 +698,8 @@ async def handle_click_to_download_file_action(
     )
 
     try:
-        await locator.click(timeout=settings.BROWSER_ACTION_TIMEOUT_MS)
+        if not await skyvern_element.navigate_to_a_href(page=page):
+            await locator.click(timeout=settings.BROWSER_ACTION_TIMEOUT_MS)
         await page.wait_for_load_state(timeout=settings.BROWSER_LOADING_TIMEOUT_MS)
     except Exception as e:
         LOG.exception(
@@ -1862,9 +1867,9 @@ async def chain_click(
     :param css: css of the element to click
     """
     try:
-        await locator.click(timeout=timeout)
+        if not await skyvern_element.navigate_to_a_href(page=page):
-
+            await locator.click(timeout=timeout)
-        LOG.info("Chain click: main element click succeeded", action=action, locator=locator)
+            LOG.info("Chain click: main element click succeeded", action=action, locator=locator)
         return [ActionSuccess()]
 
     except Exception as e:

skyvern/webeye/scraper/domUtils.js

@@ -1378,6 +1378,15 @@ async function buildElementObject(
       isSelect2MultiChoice(element),
   };
 
+  // if element is an "a" tag and has a target="_blank" attribute, remove the target attribute but keep it in the elementObj
+  // We're doing this so that skyvern can do all the navigation in a single page/tab and not open new tab
+  if (elementTagNameLower === "a") {
+    if (element.getAttribute("target") === "_blank") {
+      elementObj.target = "_blank";
+      element.removeAttribute("target");
+    }
+  }
+
   let isInShadowRoot = element.getRootNode() instanceof ShadowRoot;
   if (isInShadowRoot) {
     let shadowHostEle = element.getRootNode().host;
@@ -1472,14 +1481,6 @@ async function buildElementTree(
         "]";
     }
 
-    // if element is an "a" tag and has a target="_blank" attribute, remove the target attribute
-    // We're doing this so that skyvern can do all the navigation in a single page/tab and not open new tab
-    if (tagName === "a") {
-      if (element.getAttribute("target") === "_blank") {
-        element.removeAttribute("target");
-      }
-    }
-
     let shadowDOMchildren = [];
     // sometimes the shadowRoot is not visible, but the elemnets in the shadowRoot are visible
     if (element.shadowRoot) {

skyvern/webeye/scraper/scraper.py

@@ -869,6 +869,9 @@ def trim_element(element: dict) -> dict:
         if "afterPseudoText" in queue_ele and not queue_ele.get("afterPseudoText"):
             del queue_ele["afterPseudoText"]
 
+        if "target" in queue_ele:
+            del queue_ele["target"]
+
     return element
 
 

skyvern/webeye/utils/dom.py

@@ -5,6 +5,7 @@ import copy
 import typing
 from enum import StrEnum
 from random import uniform
+from urllib.parse import urlparse
 
 import structlog
 from playwright.async_api import ElementHandle, Frame, FrameLocator, Locator, Page, TimeoutError
@@ -350,6 +351,27 @@ class SkyvernElement:
         assert handler is not None
         return handler
 
+    async def should_use_navigation_instead_click(self, page: Page) -> str | None:
+        if self.__static_element.get("target") != "_blank":
+            return None
+
+        href: str | None = await self.get_attr("href", mode="static")
+        if not href:
+            return None
+
+        href_url = urlparse(href)
+        if href_url.scheme.lower() not in ["http", "https"]:
+            return None
+
+        if not href_url.netloc:
+            return None
+
+        cur_url = urlparse(page.url)
+        if href_url.netloc.lower() == cur_url.netloc.lower():
+            return None
+
+        return href
+
     async def find_blocking_element(
         self, dom: DomUtil, incremental_page: IncrementalScrapePage | None = None
     ) -> tuple[SkyvernElement | None, bool]:
@@ -739,6 +761,31 @@ class SkyvernElement:
 
         return True
 
+    async def navigate_to_a_href(self, page: Page) -> str | None:
+        if self.get_tag_name() != InteractiveElement.A:
+            return None
+
+        href = await self.should_use_navigation_instead_click(page)
+        if not href:
+            return None
+
+        LOG.info(
+            "Trying to navigate to the <a> href link instead of clicking",
+            href=href,
+            current_url=page.url,
+        )
+        try:
+            await page.goto(href, timeout=settings.BROWSER_LOADING_TIMEOUT_MS)
+            return href
+        except Exception as e:
+            # some cases use this method to download a file. but it will be redirected away soon
+            # and agent will run into ABORTED error.
+            if "net::ERR_ABORTED" in str(e):
+                return href
+
+            LOG.warning("Failed to navigate to the <a> href link", exc_info=True, href=href, current_url=page.url)
+            raise
+
 
 class DomUtil:
     """