From 839320dd42433c769c2f7dd006075f0c5af235ea Mon Sep 17 00:00:00 2001
From: LawyZheng <lawyzheng1106@gmail.com>
Date: Sat, 21 Sep 2024 21:05:40 +0800
Subject: [PATCH] fix custom selection bug (#871)

---
 skyvern/webeye/actions/handler.py  | 3 +++
 skyvern/webeye/scraper/domUtils.js | 8 ++++++++
 skyvern/webeye/utils/page.py       | 8 ++++++++
 3 files changed, 19 insertions(+)

diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 337307e7..3200f4e6 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -1740,9 +1740,12 @@ async def locate_dropdown_menu(
                     exc_info=True,
                 )
 
+        # sometimes taking screenshot might scroll away, need to scroll back after the screenshot
+        x, y = await skyvern_frame.get_scroll_x_y()
         screenshot = await head_element.get_locator().screenshot(
             timeout=SettingsManager.get_settings().BROWSER_SCREENSHOT_TIMEOUT_MS
         )
+        await skyvern_frame.scroll_to_x_y(x, y)
 
         # TODO: better to send untrimmed HTML without skyvern attributes in the future
         dropdown_confirm_prompt = prompt_engine.load_prompt("opened-dropdown-confirm")
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
index b6b41879..e7cf969c 100644
--- a/skyvern/webeye/scraper/domUtils.js
+++ b/skyvern/webeye/scraper/domUtils.js
@@ -1476,6 +1476,14 @@ function scrollToTop(draw_boxes) {
   return window.scrollY;
 }
 
+function getScrollXY() {
+  return [window.scrollX, window.scrollY];
+}
+
+function scrollToXY(x, y) {
+  window.scroll({ left: x, top: y, behavior: "instant" });
+}
+
 function scrollToNextPage(draw_boxes) {
   // remove bounding boxes, scroll to next page with 200px overlap, then draw bounding boxes again
   // return true if there is a next page, false otherwise
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index a1415aa1..a8f6c9f1 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -160,6 +160,14 @@ class SkyvernFrame:
         async with asyncio.timeout(timeout):
             return await self.frame.content()
 
+    async def get_scroll_x_y(self) -> tuple[int, int]:
+        js_script = "() => getScrollXY()"
+        return await self.frame.evaluate(js_script)
+
+    async def scroll_to_x_y(self, x: int, y: int) -> None:
+        js_script = "([x, y]) => scrollToXY(x, y)"
+        return await self.frame.evaluate(js_script, [x, y])
+
     async def scroll_to_element_bottom(self, element: ElementHandle, page_by_page: bool = False) -> None:
         js_script = "([element, page_by_page]) => scrollToElementBottom(element, page_by_page)"
         return await self.frame.evaluate(js_script, [element, page_by_page])