diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py index 376185f7..e6b89029 100644 --- a/skyvern/webeye/actions/handler.py +++ b/skyvern/webeye/actions/handler.py @@ -1102,6 +1102,7 @@ async def chain_click( # This automatically dismisses the dialog # File choosers are impossible to close if you don't expect one. Instead of dealing with it, close it! + dom = DomUtil(scraped_page=scraped_page, page=page) locator = skyvern_element.locator # TODO (suchintan): This should likely result in an ActionFailure -- we can figure out how to do this later! LOG.info("Chain click starts", action=action, locator=locator) @@ -1142,31 +1143,51 @@ async def chain_click( action_results: list[ActionResult] = [ActionFailure(FailToClick(action.element_id, msg=str(e)))] if skyvern_element.get_tag_name() == "label": - LOG.info( - "Chain click: it's a label element. going to try for-click", - task_id=task.task_id, - action=action, - element=str(skyvern_element), - locator=locator, - ) try: - if bound_element := await skyvern_element.find_label_for( - dom=DomUtil(scraped_page=scraped_page, page=page) - ): + LOG.info( + "Chain click: it's a label element. going to try for-click", + task_id=task.task_id, + action=action, + element=str(skyvern_element), + locator=locator, + ) + if bound_element := await skyvern_element.find_label_for(dom=dom): await bound_element.get_locator().click(timeout=timeout) action_results.append(ActionSuccess()) return action_results except Exception as e: action_results.append(ActionFailure(FailToClick(action.element_id, anchor="for", msg=str(e)))) - else: - LOG.info( - "Chain click: it's a non-label element. going to find the bound label element by attribute id and click", - task_id=task.task_id, - action=action, - element=str(skyvern_element), - locator=locator, - ) + try: + # sometimes the element is the direct chidren of the label, instead of using for="xx" attribute + # since it's a click action, the target element we're searching should only be INPUT + LOG.info( + "Chain click: it's a label element. going to check for input of the direct chidren", + task_id=task.task_id, + action=action, + element=str(skyvern_element), + locator=locator, + ) + if bound_element := await skyvern_element.find_element_in_label_children( + dom=dom, element_type=InteractiveElement.INPUT + ): + await bound_element.get_locator().click(timeout=timeout) + action_results.append(ActionSuccess()) + return action_results + except Exception as e: + action_results.append( + ActionFailure(FailToClick(action.element_id, anchor="direct_children", msg=str(e))) + ) + + else: + try: + LOG.info( + "Chain click: it's a non-label element. going to find the bound label element by attribute id and click", + task_id=task.task_id, + action=action, + element=str(skyvern_element), + locator=locator, + ) if bound_locator := await skyvern_element.find_bound_label_by_attr_id(): await bound_locator.click(timeout=timeout) action_results.append(ActionSuccess()) @@ -1174,10 +1195,27 @@ async def chain_click( except Exception as e: action_results.append(ActionFailure(FailToClick(action.element_id, anchor="attr_id", msg=str(e)))) + try: + # sometimes the element is the direct chidren of the label, instead of using for="xx" attribute + # so we check the direct parent if it's a label element + LOG.info( + "Chain click: it's a non-label element. going to find the bound label element by direct parent", + task_id=task.task_id, + action=action, + element=str(skyvern_element), + locator=locator, + ) + if bound_locator := await skyvern_element.find_bound_label_by_direct_parent(): + await bound_locator.click(timeout=timeout) + action_results.append(ActionSuccess()) + return action_results + except Exception as e: + action_results.append(ActionFailure(FailToClick(action.element_id, anchor="direct_parent", msg=str(e)))) + if not await skyvern_element.is_visible(): LOG.info( "Chain click: exit since the element is not visible on the page anymore", - taks_id=task.task_id, + task_id=task.task_id, action=action, element=str(skyvern_element), locator=locator, @@ -1190,7 +1228,7 @@ async def chain_click( if blocking_element is None: LOG.info( "Chain click: exit since the element is not blocking by any skyvern element", - taks_id=task.task_id, + task_id=task.task_id, action=action, element=str(skyvern_element), locator=locator, @@ -1200,7 +1238,7 @@ async def chain_click( try: LOG.debug( "Chain click: verifying the blocking element is parent or sibling of the target element", - taks_id=task.task_id, + task_id=task.task_id, action=action, element=str(blocking_element), locator=locator, @@ -1210,7 +1248,7 @@ async def chain_click( ) or await blocking_element.is_sibling_of(await skyvern_element.get_element_handler()): LOG.info( "Chain click: element is blocked by other elements, going to click on the blocking element", - taks_id=task.task_id, + task_id=task.task_id, action=action, element=str(blocking_element), locator=locator, diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py index a28ce39e..cd117ec0 100644 --- a/skyvern/webeye/utils/dom.py +++ b/skyvern/webeye/utils/dom.py @@ -327,6 +327,14 @@ class SkyvernElement: return None return await dom.get_skyvern_element_by_id(blocking_element_id) + async def find_element_in_label_children( + self, dom: DomUtil, element_type: InteractiveElement + ) -> SkyvernElement | None: + element_id = self.find_element_id_in_label_children(element_type=element_type) + if not element_id: + return None + return await dom.get_skyvern_element_by_id(element_id=element_id) + def find_element_id_in_label_children(self, element_type: InteractiveElement) -> str | None: tag_name = self.get_tag_name() if tag_name != "label": @@ -397,6 +405,28 @@ class SkyvernElement: return None + async def find_bound_label_by_direct_parent( + self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS + ) -> Locator | None: + if self.get_tag_name() == "label": + return None + + parent_locator = self.get_locator().locator("..") + cnt = await parent_locator.count() + if cnt != 1: + return None + + timeout_sec = timeout / 1000 + async with asyncio.timeout(timeout_sec): + tag_name: str | None = await parent_locator.evaluate("el => el.tagName") + if not tag_name: + return None + + if tag_name.lower() != "label": + return None + + return parent_locator + async def find_selectable_child(self, dom: DomUtil) -> SkyvernElement | None: # BFS to find the first selectable child index = 0