diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py index 7ea12976..381f2751 100644 --- a/skyvern/webeye/actions/handler.py +++ b/skyvern/webeye/actions/handler.py @@ -1101,22 +1101,9 @@ async def handle_select_option_action( await incremental_scraped.start_listen_dom_increment() await skyvern_element.scroll_into_view() - try: - await skyvern_element.get_locator().click(timeout=timeout) - except Exception: - LOG.info( - "fail to open dropdown by clicking, try to press ArrowDown to open", - exc_info=True, - element_id=skyvern_element.get_id(), - task_id=task.task_id, - step_id=step.step_id, - ) - await skyvern_element.scroll_into_view() - await skyvern_element.press_key("ArrowDown") - + await skyvern_element.click(page=page, dom=dom, timeout=timeout) # wait 5s for options to load await asyncio.sleep(5) - is_open = True incremental_element = await incremental_scraped.get_incremental_element_tree( clean_and_remove_element_tree_factory(task=task, step=step, check_filter_funcs=[dom.check_id_in_dom]), @@ -1140,6 +1127,7 @@ async def handle_select_option_action( if len(incremental_element) == 0: raise NoIncrementalElementFoundForCustomSelection(element_id=skyvern_element.get_id()) + is_open = True # TODO: support sequetially select from dropdown by value, just support single select now result = await sequentially_select_from_dropdown( action=action, @@ -2222,7 +2210,7 @@ async def select_from_dropdown( return single_select_result await selected_element.scroll_into_view() - await selected_element.get_locator().click(timeout=timeout) + await selected_element.click(page=page, timeout=timeout) single_select_result.action_result = ActionSuccess() return single_select_result except (MissingElement, MissingElementDict, MissingElementInCSSMap, MultipleElementsFound): diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js index b7e28d23..b0c1b755 100644 --- a/skyvern/webeye/scraper/domUtils.js +++ b/skyvern/webeye/scraper/domUtils.js @@ -920,6 +920,19 @@ function hasNgAttribute(element) { return false; } +function isAngularMaterial(element) { + if (!element.attributes[Symbol.iterator]) { + return false; + } + + for (let attr of element.attributes) { + if (attr.name.startsWith("mat")) { + return true; + } + } + return false; +} + const isAngularDropdown = (element) => { if (!hasNgAttribute(element)) { return false; @@ -936,6 +949,20 @@ const isAngularDropdown = (element) => { return false; }; +const isAngularMaterialDatePicker = (element) => { + if (!isAngularMaterial(element)) { + return false; + } + + const tagName = element.tagName.toLowerCase(); + if (tagName !== "input") return false; + + return ( + (element.closest("mat-datepicker") || + element.closest("mat-formio-date")) !== null + ); +}; + function getPseudoContent(element, pseudo) { const pseudoStyle = getElementComputedStyle(element, pseudo); if (!pseudoStyle) { @@ -1325,6 +1352,7 @@ async function buildElementObject( isDivComboboxDropdown(element) || isDropdownButton(element) || isAngularDropdown(element) || + isAngularMaterialDatePicker(element) || isSelect2Dropdown(element) || isSelect2MultiChoice(element), isCheckable: isCheckableDiv(element), diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py index b487b41c..b027eb3b 100644 --- a/skyvern/webeye/utils/dom.py +++ b/skyvern/webeye/utils/dom.py @@ -13,6 +13,7 @@ from skyvern.config import settings from skyvern.constants import SKYVERN_ID_ATTR from skyvern.exceptions import ( ElementIsNotLabel, + InteractWithDisabledElement, MissingElement, MissingElementDict, MissingElementInCSSMap, @@ -580,6 +581,45 @@ class SkyvernElement: return dest_x, dest_y + async def click( + self, + page: Page, + dom: DomUtil | None = None, + incremental_page: IncrementalScrapePage | None = None, + timeout: float = settings.BROWSER_ACTION_TIMEOUT_MS, + ) -> None: + if await self.is_disabled(dynamic=True): + raise InteractWithDisabledElement(element_id=self.get_id()) + + try: + await self.get_locator().click(timeout=timeout) + return + except Exception: + LOG.info("Failed to click by playwright", exc_info=True, element_id=self.get_id()) + + if dom is not None: + # try to click on the blocking element + try: + await self.scroll_into_view(timeout=timeout) + blocking_element, _ = await self.find_blocking_element(dom=dom, incremental_page=incremental_page) + if blocking_element: + LOG.debug("Find the blocking element", element_id=blocking_element.get_id()) + await blocking_element.get_locator().click(timeout=timeout) + return + except Exception: + LOG.info("Failed to click on the blocking element", exc_info=True, element_id=self.get_id()) + + try: + await self.scroll_into_view(timeout=timeout) + await self.coordinate_click(page=page, timeout=timeout) + return + except Exception: + LOG.info("Failed to click by coordinate", exc_info=True, element_id=self.get_id()) + + await self.scroll_into_view(timeout=timeout) + await self.click_in_javascript() + return + async def click_in_javascript(self) -> None: skyvern_frame = await SkyvernFrame.create_instance(self.get_frame()) await skyvern_frame.click_element_in_javascript(await self.get_element_handler())