From f2eef5aa6dc380ccbab7324d5121df2f8a3685de Mon Sep 17 00:00:00 2001 From: Shuchang Zheng Date: Mon, 21 Apr 2025 22:36:56 +0800 Subject: [PATCH] support shadow dom mutation observer (#2206) Co-authored-by: lawyzheng --- skyvern/webeye/actions/handler.py | 10 +++--- skyvern/webeye/scraper/domUtils.js | 53 +++++++++++++++++++++--------- skyvern/webeye/scraper/scraper.py | 8 ++--- 3 files changed, 46 insertions(+), 25 deletions(-) diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py index da72296d..d7c6e2de 100644 --- a/skyvern/webeye/actions/handler.py +++ b/skyvern/webeye/actions/handler.py @@ -711,7 +711,7 @@ async def handle_input_text_action( if skyvern_element.get_tag_name() == InteractiveElement.INPUT and not await skyvern_element.is_raw_input(): await skyvern_element.scroll_into_view() # press arrowdown to watch if there's any options popping up - await incremental_scraped.start_listen_dom_increment() + await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler()) try: await skyvern_element.input_clear() except Exception: @@ -925,7 +925,7 @@ async def handle_input_text_action( auto_complete_hacky_flag = False return [result] - await incremental_scraped.start_listen_dom_increment() + await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler()) try: await skyvern_element.input_sequentially(text=text) @@ -1263,7 +1263,7 @@ async def handle_select_option_action( results: list[ActionResult] = [] try: - await incremental_scraped.start_listen_dom_increment() + await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler()) await skyvern_element.scroll_into_view() await skyvern_element.click(page=page, dom=dom, timeout=timeout) @@ -1343,7 +1343,7 @@ async def handle_select_option_action( step_id=step.step_id, ) try: - await incremental_scraped.start_listen_dom_increment() + await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler()) timeout = settings.BROWSER_ACTION_TIMEOUT_MS await skyvern_element.scroll_into_view() @@ -1879,7 +1879,7 @@ async def choose_auto_completion_dropdown( current_frame = skyvern_element.get_frame() skyvern_frame = await SkyvernFrame.create_instance(current_frame) incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame) - await incremental_scraped.start_listen_dom_increment() + await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler()) try: await skyvern_element.press_fill(text) diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js index 5dec00fc..bd168dd8 100644 --- a/skyvern/webeye/scraper/domUtils.js +++ b/skyvern/webeye/scraper/domUtils.js @@ -545,6 +545,7 @@ function hasWidgetRole(element) { "spinbutton", "switch", "gridcell", + "option", ]; return widgetRoles.includes(role.toLowerCase().trim()); } @@ -1423,6 +1424,9 @@ async function buildElementTree( const interactable = isInteractable(element, hoverStylesMap); let elementObj = null; let isParentSVG = null; + if (element.shadowRoot) { + children = getChildElements(element.shadowRoot); + } if (interactable) { elementObj = await buildElementObject(frame, element, interactable); } else if ( @@ -1433,7 +1437,6 @@ async function buildElementTree( elementObj = await buildElementObject(frame, element, interactable); } else if (element.shadowRoot) { elementObj = await buildElementObject(frame, element, interactable); - children = getChildElements(element.shadowRoot); } else if (isTableRelatedElement(element)) { // build all table related elements into skyvern element // we need these elements to preserve the DOM structure @@ -2341,7 +2344,7 @@ if (window.globalObserverForDOMIncrement === undefined) { }); } -function startGlobalIncrementalObserver() { +function startGlobalIncrementalObserver(element = null) { window.globalListnerFlag = true; window.globalDomDepthMap = new Map(); window.globalOneTimeIncrementElements = []; @@ -2354,6 +2357,17 @@ function startGlobalIncrementalObserver() { subtree: true, characterData: true, }); + + // if the element is in shadow DOM, we need to observe the shadow DOM as well + if (element && element.getRootNode() instanceof ShadowRoot) { + window.globalObserverForDOMIncrement.observe(element.getRootNode(), { + attributes: true, + attributeOldValue: true, + childList: true, + subtree: true, + characterData: true, + }); + } } async function stopGlobalIncrementalObserver() { @@ -2397,6 +2411,10 @@ async function getIncrementElements(wait_until_finished = true) { let children = element.children; for (let i = 0; i < children.length; i++) { const child = children[i]; + // FIXME: skip to update the element if it is in shadow DOM, since document.querySelector will not work + if (child.shadowHost) { + continue; + } const domElement = document.querySelector(`[unique_id="${child.id}"]`); // if the element is still on the page, we rebuild the element to update the information if (domElement) { @@ -2430,21 +2448,24 @@ async function getIncrementElements(wait_until_finished = true) { }; for (let treeHeadElement of treeList) { - const domElement = document.querySelector( - `[unique_id="${treeHeadElement.id}"]`, - ); - // if the element is still on the page, we rebuild the element to update the information - if (domElement) { - let newHead = await buildElementObject( - "", - domElement, - treeHeadElement.interactable, - treeHeadElement.purgeable, + // FIXME: skip to update the element if it is in shadow DOM, since document.querySelector will not work + if (!treeHeadElement.shadowHost) { + const domElement = document.querySelector( + `[unique_id="${treeHeadElement.id}"]`, ); - newHead.children = treeHeadElement.children; - treeHeadElement = newHead; - } else { - treeHeadElement.interactable = false; + // if the element is still on the page, we rebuild the element to update the information + if (domElement) { + let newHead = await buildElementObject( + "", + domElement, + treeHeadElement.interactable, + treeHeadElement.purgeable, + ); + newHead.children = treeHeadElement.children; + treeHeadElement = newHead; + } else { + treeHeadElement.interactable = false; + } } // check if the element is existed diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py index a1d95d75..329337f3 100644 --- a/skyvern/webeye/scraper/scraper.py +++ b/skyvern/webeye/scraper/scraper.py @@ -7,7 +7,7 @@ from typing import Any, Awaitable, Callable, Self import structlog from playwright._impl._errors import TimeoutError -from playwright.async_api import Frame, Locator, Page +from playwright.async_api import ElementHandle, Frame, Locator, Page from pydantic import BaseModel, PrivateAttr from skyvern.config import settings @@ -712,9 +712,9 @@ class IncrementalScrapePage: return self.element_tree_trimmed - async def start_listen_dom_increment(self) -> None: - js_script = "() => startGlobalIncrementalObserver()" - await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script) + async def start_listen_dom_increment(self, element: ElementHandle | None = None) -> None: + js_script = "(element) => startGlobalIncrementalObserver(element)" + await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script, arg=element) async def stop_listen_dom_increment(self) -> None: # check if the DOM has navigated away or refreshed