diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 4006e30e..b0037c84 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -479,10 +479,11 @@ async def handle_click_action(
task: Task,
step: Step,
) -> list[ActionResult]:
+ dom = DomUtil(scraped_page=scraped_page, page=page)
original_url = page.url
if action.x is not None and action.y is not None:
# Find the element at the clicked location using JavaScript evaluation
- element_id = await page.evaluate(
+ element_id: str | None = await page.evaluate(
"""data => {
const element = document.elementFromPoint(data.x, data.y);
if (!element) return null;
@@ -506,6 +507,10 @@ async def handle_click_action(
{"x": action.x, "y": action.y},
)
LOG.info("Clicked element at location", x=action.x, y=action.y, element_id=element_id, button=action.button)
+ if element_id:
+ skyvern_element = await dom.get_skyvern_element_by_id(element_id)
+ if await skyvern_element.navigate_to_a_href(page=page):
+ return [ActionSuccess()]
if action.repeat == 1:
await page.mouse.click(x=action.x, y=action.y, button=action.button)
@@ -518,7 +523,6 @@ async def handle_click_action(
return [ActionSuccess()]
- dom = DomUtil(scraped_page=scraped_page, page=page)
skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)
await asyncio.sleep(0.3)
@@ -694,7 +698,8 @@ async def handle_click_to_download_file_action(
)
try:
- await locator.click(timeout=settings.BROWSER_ACTION_TIMEOUT_MS)
+ if not await skyvern_element.navigate_to_a_href(page=page):
+ await locator.click(timeout=settings.BROWSER_ACTION_TIMEOUT_MS)
await page.wait_for_load_state(timeout=settings.BROWSER_LOADING_TIMEOUT_MS)
except Exception as e:
LOG.exception(
@@ -1862,9 +1867,9 @@ async def chain_click(
:param css: css of the element to click
"""
try:
- await locator.click(timeout=timeout)
-
- LOG.info("Chain click: main element click succeeded", action=action, locator=locator)
+ if not await skyvern_element.navigate_to_a_href(page=page):
+ await locator.click(timeout=timeout)
+ LOG.info("Chain click: main element click succeeded", action=action, locator=locator)
return [ActionSuccess()]
except Exception as e:
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
index 7b6d82e4..1da9185b 100644
--- a/skyvern/webeye/scraper/domUtils.js
+++ b/skyvern/webeye/scraper/domUtils.js
@@ -1378,6 +1378,15 @@ async function buildElementObject(
isSelect2MultiChoice(element),
};
+ // if element is an "a" tag and has a target="_blank" attribute, remove the target attribute but keep it in the elementObj
+ // We're doing this so that skyvern can do all the navigation in a single page/tab and not open new tab
+ if (elementTagNameLower === "a") {
+ if (element.getAttribute("target") === "_blank") {
+ elementObj.target = "_blank";
+ element.removeAttribute("target");
+ }
+ }
+
let isInShadowRoot = element.getRootNode() instanceof ShadowRoot;
if (isInShadowRoot) {
let shadowHostEle = element.getRootNode().host;
@@ -1472,14 +1481,6 @@ async function buildElementTree(
"]";
}
- // if element is an "a" tag and has a target="_blank" attribute, remove the target attribute
- // We're doing this so that skyvern can do all the navigation in a single page/tab and not open new tab
- if (tagName === "a") {
- if (element.getAttribute("target") === "_blank") {
- element.removeAttribute("target");
- }
- }
-
let shadowDOMchildren = [];
// sometimes the shadowRoot is not visible, but the elemnets in the shadowRoot are visible
if (element.shadowRoot) {
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index 1163d68d..e4c07960 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -869,6 +869,9 @@ def trim_element(element: dict) -> dict:
if "afterPseudoText" in queue_ele and not queue_ele.get("afterPseudoText"):
del queue_ele["afterPseudoText"]
+ if "target" in queue_ele:
+ del queue_ele["target"]
+
return element
diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py
index d6fed140..0ecf2aa5 100644
--- a/skyvern/webeye/utils/dom.py
+++ b/skyvern/webeye/utils/dom.py
@@ -5,6 +5,7 @@ import copy
import typing
from enum import StrEnum
from random import uniform
+from urllib.parse import urlparse
import structlog
from playwright.async_api import ElementHandle, Frame, FrameLocator, Locator, Page, TimeoutError
@@ -350,6 +351,27 @@ class SkyvernElement:
assert handler is not None
return handler
+ async def should_use_navigation_instead_click(self, page: Page) -> str | None:
+ if self.__static_element.get("target") != "_blank":
+ return None
+
+ href: str | None = await self.get_attr("href", mode="static")
+ if not href:
+ return None
+
+ href_url = urlparse(href)
+ if href_url.scheme.lower() not in ["http", "https"]:
+ return None
+
+ if not href_url.netloc:
+ return None
+
+ cur_url = urlparse(page.url)
+ if href_url.netloc.lower() == cur_url.netloc.lower():
+ return None
+
+ return href
+
async def find_blocking_element(
self, dom: DomUtil, incremental_page: IncrementalScrapePage | None = None
) -> tuple[SkyvernElement | None, bool]:
@@ -739,6 +761,31 @@ class SkyvernElement:
return True
+ async def navigate_to_a_href(self, page: Page) -> str | None:
+ if self.get_tag_name() != InteractiveElement.A:
+ return None
+
+ href = await self.should_use_navigation_instead_click(page)
+ if not href:
+ return None
+
+ LOG.info(
+ "Trying to navigate to the href link instead of clicking",
+ href=href,
+ current_url=page.url,
+ )
+ try:
+ await page.goto(href, timeout=settings.BROWSER_LOADING_TIMEOUT_MS)
+ return href
+ except Exception as e:
+ # some cases use this method to download a file. but it will be redirected away soon
+ # and agent will run into ABORTED error.
+ if "net::ERR_ABORTED" in str(e):
+ return href
+
+ LOG.warning("Failed to navigate to the href link", exc_info=True, href=href, current_url=page.url)
+ raise
+
class DomUtil:
"""