diff --git a/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2 b/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2 index ea3a7da9..12b8d9a2 100644 --- a/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2 +++ b/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2 @@ -3,6 +3,7 @@ There is a screenshot from a part of a web HTML page. Help me confirm if it is a An open dropdown menu can be defined as: - At least one option is visible in the screenshot. - Do not consider it an open dropdown menu if the only visible option displays a message like "No results" or "No match". + - Do not consider it an open dropdown menu if the only visible element displays a placeholder like "Please select", "-", or "Select...". MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc. diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py index f3979815..4c486258 100644 --- a/skyvern/webeye/actions/handler.py +++ b/skyvern/webeye/actions/handler.py @@ -417,12 +417,12 @@ async def handle_input_text_action( # check if it's selectable if skyvern_element.get_tag_name() == InteractiveElement.INPUT and not await skyvern_element.is_raw_input(): await skyvern_element.scroll_into_view() - select_action = SelectOptionAction( - reasoning=action.reasoning, - element_id=skyvern_element.get_id(), - option=SelectOption(label=text), - ) if skyvern_element.get_selectable(): + select_action = SelectOptionAction( + reasoning=action.reasoning, + element_id=skyvern_element.get_id(), + option=SelectOption(label=text), + ) LOG.info( "Input element is selectable, doing select actions", task_id=task.task_id, @@ -434,8 +434,7 @@ async def handle_input_text_action( # press arrowdown to watch if there's any options popping up await incremental_scraped.start_listen_dom_increment() - await skyvern_element.get_locator().focus(timeout=timeout) - await skyvern_element.get_locator().press("ArrowDown", timeout=timeout) + await skyvern_element.press_key("ArrowDown") await asyncio.sleep(5) incremental_element = await incremental_scraped.get_incremental_element_tree( @@ -726,46 +725,25 @@ async def handle_select_option_action( timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame()) incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame) - dropdown_element: SkyvernElement | None = None is_open = False suggested_value: str | None = None results: list[ActionResult] = [] try: await incremental_scraped.start_listen_dom_increment() - await skyvern_element.focus() - # wait 2s for options to load - await asyncio.sleep(2) + await skyvern_element.scroll_into_view() - incremental_element = await incremental_scraped.get_incremental_element_tree( - clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]), - ) - if len(incremental_element) > 0: + try: + await skyvern_element.get_locator().click(timeout=timeout) + except Exception: LOG.info( - "Focus trigger DOM element change, confirm if it's a dropdown showing up", + "fail to open dropdown by clicking, try to press ArrowDown to open", + element_id=skyvern_element.get_id(), task_id=task.task_id, step_id=step.step_id, ) - dropdown_element = await locate_dropdown_menu( - incremental_scraped=incremental_scraped, - step=step, - task=task, - ) - if dropdown_element is not None: - is_open = True - - if not is_open: - try: - await skyvern_element.get_locator().click(timeout=timeout) - except Exception: - LOG.info( - "fail to open dropdown by clicking, try to press ArrowDown to open", - element_id=skyvern_element.get_id(), - task_id=task.task_id, - step_id=step.step_id, - ) - await skyvern_element.focus() - await skyvern_element.press_key("ArrowDown") + await skyvern_element.scroll_into_view() + await skyvern_element.press_key("ArrowDown") # wait 5s for options to load await asyncio.sleep(5) @@ -786,7 +764,6 @@ async def handle_select_option_action( incremental_scraped=incremental_scraped, step=step, task=task, - dropdown_menu_element=dropdown_element, force_select=True, ) # force_select won't return None result @@ -805,7 +782,6 @@ async def handle_select_option_action( await skyvern_element.coordinate_click(page=page) await skyvern_element.press_key("Escape") is_open = False - dropdown_element = None await skyvern_element.blur() await incremental_scraped.stop_listen_dom_increment() @@ -819,41 +795,21 @@ async def handle_select_option_action( try: await incremental_scraped.start_listen_dom_increment() timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS - await skyvern_element.focus() - # wait 2s for options to load - await asyncio.sleep(2) + await skyvern_element.scroll_into_view() - incremental_element = await incremental_scraped.get_incremental_element_tree( - clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]), - ) - if len(incremental_element) > 0: + try: + await skyvern_element.get_locator().click(timeout=timeout) + except Exception: LOG.info( - "Focus trigger DOM element change, confirm if it's a dropdown showing up", + "fail to open dropdown by clicking, try to press arrow down to open", + element_id=skyvern_element.get_id(), task_id=task.task_id, step_id=step.step_id, ) - dropdown_element = await locate_dropdown_menu( - incremental_scraped=incremental_scraped, - step=step, - task=task, - ) - if dropdown_element is not None: - is_open = True - - if not is_open: - try: - await skyvern_element.get_locator().click(timeout=timeout) - except Exception: - LOG.info( - "fail to open dropdown by clicking, try to press arrow down to open", - element_id=skyvern_element.get_id(), - task_id=task.task_id, - step_id=step.step_id, - ) - await skyvern_element.focus() - await skyvern_element.press_key("ArrowDown") - await asyncio.sleep(5) - is_open = True + await skyvern_element.scroll_into_view() + await skyvern_element.press_key("ArrowDown") + await asyncio.sleep(5) + is_open = True result = await select_from_dropdown_by_value( value=suggested_value, @@ -863,7 +819,6 @@ async def handle_select_option_action( incremental_scraped=incremental_scraped, task=task, step=step, - dropdown_menu_element=dropdown_element, ) results.append(result) return results