diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py index 2d78a158..91b06320 100644 --- a/skyvern/webeye/actions/handler.py +++ b/skyvern/webeye/actions/handler.py @@ -1680,6 +1680,7 @@ async def handle_select_option_action( ) try: + await skyvern_element.scroll_into_view() blocking_element, exist = await skyvern_element.find_blocking_element(dom=dom) except Exception: LOG.warning( diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py index 8b64d6d3..5a889070 100644 --- a/skyvern/webeye/utils/dom.py +++ b/skyvern/webeye/utils/dom.py @@ -8,7 +8,7 @@ from random import uniform from urllib.parse import urlparse import structlog -from playwright.async_api import ElementHandle, Frame, FrameLocator, Locator, Page, TimeoutError +from playwright.async_api import ElementHandle, FloatRect, Frame, FrameLocator, Locator, Page, TimeoutError from skyvern.config import settings from skyvern.constants import SKYVERN_ID_ATTR, TEXT_INPUT_DELAY @@ -133,6 +133,7 @@ class SkyvernElement: self._selectable = static_element.get("isSelectable", False) self._frame_id = static_element.get("frame", "") self._attributes = static_element.get("attributes", {}) + self._rect: FloatRect | None = None def __repr__(self) -> str: return f"SkyvernElement({str(self.__static_element)})" @@ -414,6 +415,12 @@ class SkyvernElement: def get_locator(self) -> Locator: return self.locator + async def get_rect(self, timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS) -> FloatRect | None: + if self._rect is not None: + return self._rect + self._rect = await self.get_locator().bounding_box(timeout=timeout) + return self._rect + async def get_element_handler(self, timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS) -> ElementHandle: handler = await self.locator.element_handle(timeout=timeout) assert handler is not None @@ -820,6 +827,33 @@ class SkyvernElement: async def scroll_into_view(self, timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS) -> None: if not await self.is_visible(): return + + try: + target_x: int | None = None + target_y: int | None = None + + rect = await self.get_rect(timeout=timeout) + if rect is not None: + element_x = rect["x"] if rect["x"] > 0 else None + element_y = rect["y"] if rect["y"] > 0 else None + + # calculating y to move the element to the middle of the viewport + if element_y is not None: + target_y = max(int(element_y - (settings.BROWSER_HEIGHT / 2)), 0) + + if element_x is not None: + target_x = max(int(element_x - (settings.BROWSER_WIDTH / 2)), 0) + + skyvern_frame = await SkyvernFrame.create_instance(self.get_frame()) + if target_x is not None and target_y is not None: + await skyvern_frame.safe_scroll_to_x_y(target_x, target_y) + except Exception: + LOG.info( + "Failed to calculate the y to move the element to the middle of the viewport, ignore it", + exc_info=True, + element_id=self.get_id(), + ) + try: element_handler = await self.get_element_handler(timeout=timeout) await element_handler.scroll_into_view_if_needed(timeout=timeout)