support shadow dom mutation observer (#2206)

Co-authored-by: lawyzheng <lawyzheng1106@gmail.com>

skyvern/webeye/actions/handler.py

@@ -711,7 +711,7 @@ async def handle_input_text_action(
     if skyvern_element.get_tag_name() == InteractiveElement.INPUT and not await skyvern_element.is_raw_input():
         await skyvern_element.scroll_into_view()
         # press arrowdown to watch if there's any options popping up
-        await incremental_scraped.start_listen_dom_increment()
+        await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())
         try:
             await skyvern_element.input_clear()
         except Exception:
@@ -925,7 +925,7 @@ async def handle_input_text_action(
                     auto_complete_hacky_flag = False
                     return [result]
 
-        await incremental_scraped.start_listen_dom_increment()
+        await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())
 
         try:
             await skyvern_element.input_sequentially(text=text)
@@ -1263,7 +1263,7 @@ async def handle_select_option_action(
     results: list[ActionResult] = []
 
     try:
-        await incremental_scraped.start_listen_dom_increment()
+        await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())
         await skyvern_element.scroll_into_view()
 
         await skyvern_element.click(page=page, dom=dom, timeout=timeout)
@@ -1343,7 +1343,7 @@ async def handle_select_option_action(
         step_id=step.step_id,
     )
     try:
-        await incremental_scraped.start_listen_dom_increment()
+        await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())
         timeout = settings.BROWSER_ACTION_TIMEOUT_MS
         await skyvern_element.scroll_into_view()
 
@@ -1879,7 +1879,7 @@ async def choose_auto_completion_dropdown(
     current_frame = skyvern_element.get_frame()
     skyvern_frame = await SkyvernFrame.create_instance(current_frame)
     incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame)
-    await incremental_scraped.start_listen_dom_increment()
+    await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())
 
     try:
         await skyvern_element.press_fill(text)

skyvern/webeye/scraper/domUtils.js

@@ -545,6 +545,7 @@ function hasWidgetRole(element) {
     "spinbutton",
     "switch",
     "gridcell",
+    "option",
   ];
   return widgetRoles.includes(role.toLowerCase().trim());
 }
@@ -1423,6 +1424,9 @@ async function buildElementTree(
       const interactable = isInteractable(element, hoverStylesMap);
       let elementObj = null;
       let isParentSVG = null;
+      if (element.shadowRoot) {
+        children = getChildElements(element.shadowRoot);
+      }
       if (interactable) {
         elementObj = await buildElementObject(frame, element, interactable);
       } else if (
@@ -1433,7 +1437,6 @@ async function buildElementTree(
         elementObj = await buildElementObject(frame, element, interactable);
       } else if (element.shadowRoot) {
         elementObj = await buildElementObject(frame, element, interactable);
-        children = getChildElements(element.shadowRoot);
       } else if (isTableRelatedElement(element)) {
         // build all table related elements into skyvern element
         // we need these elements to preserve the DOM structure
@@ -2341,7 +2344,7 @@ if (window.globalObserverForDOMIncrement === undefined) {
   });
 }
 
-function startGlobalIncrementalObserver() {
+function startGlobalIncrementalObserver(element = null) {
   window.globalListnerFlag = true;
   window.globalDomDepthMap = new Map();
   window.globalOneTimeIncrementElements = [];
@@ -2354,6 +2357,17 @@ function startGlobalIncrementalObserver() {
     subtree: true,
     characterData: true,
   });
+
+  // if the element is in shadow DOM, we need to observe the shadow DOM as well
+  if (element && element.getRootNode() instanceof ShadowRoot) {
+    window.globalObserverForDOMIncrement.observe(element.getRootNode(), {
+      attributes: true,
+      attributeOldValue: true,
+      childList: true,
+      subtree: true,
+      characterData: true,
+    });
+  }
 }
 
 async function stopGlobalIncrementalObserver() {
@@ -2397,6 +2411,10 @@ async function getIncrementElements(wait_until_finished = true) {
       let children = element.children;
       for (let i = 0; i < children.length; i++) {
         const child = children[i];
+        // FIXME: skip to update the element if it is in shadow DOM, since document.querySelector will not work
+        if (child.shadowHost) {
+          continue;
+        }
         const domElement = document.querySelector(`[unique_id="${child.id}"]`);
         // if the element is still on the page, we rebuild the element to update the information
         if (domElement) {
@@ -2430,21 +2448,24 @@ async function getIncrementElements(wait_until_finished = true) {
     };
 
     for (let treeHeadElement of treeList) {
-      const domElement = document.querySelector(
+      // FIXME: skip to update the element if it is in shadow DOM, since document.querySelector will not work
-        `[unique_id="${treeHeadElement.id}"]`,
+      if (!treeHeadElement.shadowHost) {
-      );
+        const domElement = document.querySelector(
-      // if the element is still on the page, we rebuild the element to update the information
+          `[unique_id="${treeHeadElement.id}"]`,
-      if (domElement) {
-        let newHead = await buildElementObject(
-          "",
-          domElement,
-          treeHeadElement.interactable,
-          treeHeadElement.purgeable,
         );
-        newHead.children = treeHeadElement.children;
+        // if the element is still on the page, we rebuild the element to update the information
-        treeHeadElement = newHead;
+        if (domElement) {
-      } else {
+          let newHead = await buildElementObject(
-        treeHeadElement.interactable = false;
+            "",
+            domElement,
+            treeHeadElement.interactable,
+            treeHeadElement.purgeable,
+          );
+          newHead.children = treeHeadElement.children;
+          treeHeadElement = newHead;
+        } else {
+          treeHeadElement.interactable = false;
+        }
       }
 
       // check if the element is existed

skyvern/webeye/scraper/scraper.py

@@ -7,7 +7,7 @@ from typing import Any, Awaitable, Callable, Self
 
 import structlog
 from playwright._impl._errors import TimeoutError
-from playwright.async_api import Frame, Locator, Page
+from playwright.async_api import ElementHandle, Frame, Locator, Page
 from pydantic import BaseModel, PrivateAttr
 
 from skyvern.config import settings
@@ -712,9 +712,9 @@ class IncrementalScrapePage:
 
         return self.element_tree_trimmed
 
-    async def start_listen_dom_increment(self) -> None:
+    async def start_listen_dom_increment(self, element: ElementHandle | None = None) -> None:
-        js_script = "() => startGlobalIncrementalObserver()"
+        js_script = "(element) => startGlobalIncrementalObserver(element)"
-        await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script)
+        await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script, arg=element)
 
     async def stop_listen_dom_increment(self) -> None:
         # check if the DOM has navigated away or refreshed