fix custom selection bug (#871)

skyvern/webeye/actions/handler.py

@@ -1740,9 +1740,12 @@ async def locate_dropdown_menu(
                     exc_info=True,
                 )
 
+        # sometimes taking screenshot might scroll away, need to scroll back after the screenshot
+        x, y = await skyvern_frame.get_scroll_x_y()
         screenshot = await head_element.get_locator().screenshot(
             timeout=SettingsManager.get_settings().BROWSER_SCREENSHOT_TIMEOUT_MS
         )
+        await skyvern_frame.scroll_to_x_y(x, y)
 
         # TODO: better to send untrimmed HTML without skyvern attributes in the future
         dropdown_confirm_prompt = prompt_engine.load_prompt("opened-dropdown-confirm")

skyvern/webeye/scraper/domUtils.js

@@ -1476,6 +1476,14 @@ function scrollToTop(draw_boxes) {
   return window.scrollY;
 }
 
+function getScrollXY() {
+  return [window.scrollX, window.scrollY];
+}
+
+function scrollToXY(x, y) {
+  window.scroll({ left: x, top: y, behavior: "instant" });
+}
+
 function scrollToNextPage(draw_boxes) {
   // remove bounding boxes, scroll to next page with 200px overlap, then draw bounding boxes again
   // return true if there is a next page, false otherwise

skyvern/webeye/utils/page.py

@@ -160,6 +160,14 @@ class SkyvernFrame:
         async with asyncio.timeout(timeout):
             return await self.frame.content()
 
+    async def get_scroll_x_y(self) -> tuple[int, int]:
+        js_script = "() => getScrollXY()"
+        return await self.frame.evaluate(js_script)
+
+    async def scroll_to_x_y(self, x: int, y: int) -> None:
+        js_script = "([x, y]) => scrollToXY(x, y)"
+        return await self.frame.evaluate(js_script, [x, y])
+
     async def scroll_to_element_bottom(self, element: ElementHandle, page_by_page: bool = False) -> None:
         js_script = "([element, page_by_page]) => scrollToElementBottom(element, page_by_page)"
         return await self.frame.evaluate(js_script, [element, page_by_page])