From 48e4984deb7fe453d567975cb62e5b4536b0d909 Mon Sep 17 00:00:00 2001
From: LawyZheng <lawyzheng1106@gmail.com>
Date: Wed, 27 Nov 2024 22:44:05 +0800
Subject: [PATCH] fallback to use js click (#1274)

---
 skyvern/webeye/actions/handler.py  | 22 +++++++++++++++++++---
 skyvern/webeye/scraper/domUtils.js |  4 ++--
 skyvern/webeye/utils/dom.py        | 12 ++++++++----
 skyvern/webeye/utils/page.py       |  6 +++++-
 4 files changed, 34 insertions(+), 10 deletions(-)

diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index c15e61f5..e2b2885b 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -1262,18 +1262,34 @@ async def chain_click(
             )
             return action_results
 
-        blocking_element = await skyvern_element.find_blocking_element(
+        blocking_element, blocked = await skyvern_element.find_blocking_element(
             dom=DomUtil(scraped_page=scraped_page, page=page)
         )
         if blocking_element is None:
+            if not blocked:
+                LOG.info(
+                    "Chain click: exit since the element is not blocking by any element",
+                    task_id=task.task_id,
+                    action=action,
+                    element=str(skyvern_element),
+                    locator=locator,
+                )
+                return action_results
+
             LOG.info(
-                "Chain click: exit since the element is not blocking by any skyvern element",
+                "Chain click: element is blocked by an non-interactable element, going to use javascript click instead of playwright click",
                 task_id=task.task_id,
                 action=action,
                 element=str(skyvern_element),
                 locator=locator,
             )
-            return action_results
+            try:
+                await skyvern_element.click_in_javascript()
+                action_results.append(ActionSuccess())
+                return action_results
+            except Exception as e:
+                action_results.append(ActionFailure(FailToClick(action.element_id, anchor="self_js", msg=str(e))))
+                return action_results
 
         try:
             LOG.debug(
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
index af79abd1..16c8798b 100644
--- a/skyvern/webeye/scraper/domUtils.js
+++ b/skyvern/webeye/scraper/domUtils.js
@@ -427,10 +427,10 @@ function getBlockElementUniqueID(element) {
   );
 
   if (!hitElement) {
-    return "";
+    return ["", false];
   }
 
-  return hitElement.getAttribute("unique_id") ?? "";
+  return [hitElement.getAttribute("unique_id") ?? "", true];
 }
 
 function isHidden(element) {
diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py
index cd117ec0..b44b89d1 100644
--- a/skyvern/webeye/utils/dom.py
+++ b/skyvern/webeye/utils/dom.py
@@ -320,12 +320,12 @@ class SkyvernElement:
         assert handler is not None
         return handler
 
-    async def find_blocking_element(self, dom: DomUtil) -> SkyvernElement | None:
+    async def find_blocking_element(self, dom: DomUtil) -> tuple[SkyvernElement | None, bool]:
         skyvern_frame = await SkyvernFrame.create_instance(self.get_frame())
-        blocking_element_id = await skyvern_frame.get_blocking_element_id(await self.get_element_handler())
+        blocking_element_id, blocked = await skyvern_frame.get_blocking_element_id(await self.get_element_handler())
         if not blocking_element_id:
-            return None
-        return await dom.get_skyvern_element_by_id(blocking_element_id)
+            return None, blocked
+        return await dom.get_skyvern_element_by_id(blocking_element_id), blocked
 
     async def find_element_in_label_children(
         self, dom: DomUtil, element_type: InteractiveElement
@@ -576,6 +576,10 @@ class SkyvernElement:
 
         return dest_x, dest_y
 
+    async def click_in_javascript(self) -> None:
+        skyvern_frame = await SkyvernFrame.create_instance(self.get_frame())
+        await skyvern_frame.click_element_in_javascript(await self.get_element_handler())
+
     async def coordinate_click(
         self, page: Page, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
     ) -> None:
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index 85dbf803..c5e6d883 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -182,7 +182,7 @@ class SkyvernFrame:
         js_script = "(element) => checkDisabledFromStyle(element)"
         return await self.evaluate(frame=self.frame, expression=js_script, arg=element)
 
-    async def get_blocking_element_id(self, element: ElementHandle) -> str:
+    async def get_blocking_element_id(self, element: ElementHandle) -> tuple[str, bool]:
         js_script = "(element) => getBlockElementUniqueID(element)"
         return await self.evaluate(frame=self.frame, expression=js_script, arg=element)
 
@@ -239,3 +239,7 @@ class SkyvernFrame:
     async def has_ASP_client_control(self) -> bool:
         js_script = "() => hasASPClientControl()"
         return await self.evaluate(frame=self.frame, expression=js_script)
+
+    async def click_element_in_javascript(self, element: ElementHandle) -> None:
+        js_script = "(element) => element.click()"
+        return await self.evaluate(frame=self.frame, expression=js_script, arg=element)