current view coordinate click (#3001)

skyvern/exceptions.py

@@ -739,3 +739,8 @@ class BrowserSessionNotFound(SkyvernHTTPException):
 class APIKeyNotFound(SkyvernHTTPException):
     def __init__(self, organization_id: str) -> None:
         super().__init__(f"No valid API key token found for organization {organization_id}")
+
+
+class ElementOutOfCurrentViewport(SkyvernException):
+    def __init__(self, element_id: str):
+        super().__init__(f"Element {element_id} is out of current viewport")

skyvern/webeye/actions/handler.py

@@ -3246,7 +3246,7 @@ async def scroll_down_to_load_all_options(
     else:
         await dropdown_menu_element_handle.scroll_into_view_if_needed(timeout=timeout)
 
-    await scrollable_element.move_mouse_to(page=page)
+    await scrollable_element.move_mouse_to_safe(page=page)
 
     scroll_pace = 0
     previous_num = await incremental_scraped.get_incremental_elements_num()

skyvern/webeye/utils/dom.py

@@ -14,6 +14,7 @@ from skyvern.config import settings
 from skyvern.constants import SKYVERN_ID_ATTR, TEXT_INPUT_DELAY
 from skyvern.exceptions import (
     ElementIsNotLabel,
+    ElementOutOfCurrentViewport,
     InteractWithDisabledElement,
     MissingElement,
     MissingElementDict,
@@ -644,6 +645,14 @@ class SkyvernElement:
                 element_id=element_id,
                 exc_info=True,
             )
+        except ElementOutOfCurrentViewport:
+            LOG.warning(
+                "Failed to move mouse to the element - ElementOutOfCurrentViewport",
+                task_id=task_id,
+                step_id=step_id,
+                element_id=element_id,
+                exc_info=True,
+            )
         except Exception:
             LOG.warning(
                 "Failed to move mouse to the element - unexpectd exception",
@@ -666,6 +675,12 @@ class SkyvernElement:
         epsilon = 0.01
         dest_x = uniform(x + epsilon, x + width - epsilon) if width > 2 * epsilon else (x + width) / 2
         dest_y = uniform(y + epsilon, y + height - epsilon) if height > 2 * epsilon else (y + height) / 2
+
+        # TODO: a better way to check if the element is out of current viewport
+        # eg: x > window.innerWidth or y > window.innerHeight; part of the element is out of the viewport
+        if dest_x < 0 or dest_y < 0:
+            raise ElementOutOfCurrentViewport(element_id=self.get_id())
+
         await page.mouse.move(dest_x, dest_y)
 
         return dest_x, dest_y