fix dropdown menu detect bug (#874)

skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2

@@ -3,6 +3,7 @@ There is a screenshot from a part of a web HTML page. Help me confirm if it is a
 An open dropdown menu can be defined as:
   - At least one option is visible in the screenshot.
   - Do not consider it an open dropdown menu if the only visible option displays a message like "No results" or "No match".
+  - Do not consider it an open dropdown menu if the only visible element displays a placeholder like "Please select", "-", or "Select...".
 
 MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
 

skyvern/webeye/actions/handler.py

@@ -417,12 +417,12 @@ async def handle_input_text_action(
     # check if it's selectable
     if skyvern_element.get_tag_name() == InteractiveElement.INPUT and not await skyvern_element.is_raw_input():
         await skyvern_element.scroll_into_view()
-        select_action = SelectOptionAction(
-            reasoning=action.reasoning,
-            element_id=skyvern_element.get_id(),
-            option=SelectOption(label=text),
-        )
         if skyvern_element.get_selectable():
+            select_action = SelectOptionAction(
+                reasoning=action.reasoning,
+                element_id=skyvern_element.get_id(),
+                option=SelectOption(label=text),
+            )
             LOG.info(
                 "Input element is selectable, doing select actions",
                 task_id=task.task_id,
@@ -434,8 +434,7 @@ async def handle_input_text_action(
 
         # press arrowdown to watch if there's any options popping up
         await incremental_scraped.start_listen_dom_increment()
-        await skyvern_element.get_locator().focus(timeout=timeout)
+        await skyvern_element.press_key("ArrowDown")
-        await skyvern_element.get_locator().press("ArrowDown", timeout=timeout)
         await asyncio.sleep(5)
 
         incremental_element = await incremental_scraped.get_incremental_element_tree(
@@ -726,46 +725,25 @@ async def handle_select_option_action(
     timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
     skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame())
     incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame)
-    dropdown_element: SkyvernElement | None = None
     is_open = False
     suggested_value: str | None = None
     results: list[ActionResult] = []
 
     try:
         await incremental_scraped.start_listen_dom_increment()
-        await skyvern_element.focus()
+        await skyvern_element.scroll_into_view()
-        # wait 2s for options to load
-        await asyncio.sleep(2)
 
-        incremental_element = await incremental_scraped.get_incremental_element_tree(
+        try:
-            clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),
+            await skyvern_element.get_locator().click(timeout=timeout)
-        )
+        except Exception:
-        if len(incremental_element) > 0:
             LOG.info(
-                "Focus trigger DOM element change, confirm if it's a dropdown showing up",
+                "fail to open dropdown by clicking, try to press ArrowDown to open",
+                element_id=skyvern_element.get_id(),
                 task_id=task.task_id,
                 step_id=step.step_id,
             )
-            dropdown_element = await locate_dropdown_menu(
+            await skyvern_element.scroll_into_view()
-                incremental_scraped=incremental_scraped,
+            await skyvern_element.press_key("ArrowDown")
-                step=step,
-                task=task,
-            )
-            if dropdown_element is not None:
-                is_open = True
-
-        if not is_open:
-            try:
-                await skyvern_element.get_locator().click(timeout=timeout)
-            except Exception:
-                LOG.info(
-                    "fail to open dropdown by clicking, try to press ArrowDown to open",
-                    element_id=skyvern_element.get_id(),
-                    task_id=task.task_id,
-                    step_id=step.step_id,
-                )
-                await skyvern_element.focus()
-                await skyvern_element.press_key("ArrowDown")
 
             # wait 5s for options to load
             await asyncio.sleep(5)
@@ -786,7 +764,6 @@ async def handle_select_option_action(
             incremental_scraped=incremental_scraped,
             step=step,
             task=task,
-            dropdown_menu_element=dropdown_element,
             force_select=True,
         )
         # force_select won't return None result
@@ -805,7 +782,6 @@ async def handle_select_option_action(
             await skyvern_element.coordinate_click(page=page)
             await skyvern_element.press_key("Escape")
         is_open = False
-        dropdown_element = None
         await skyvern_element.blur()
         await incremental_scraped.stop_listen_dom_increment()
 
@@ -819,41 +795,21 @@ async def handle_select_option_action(
     try:
         await incremental_scraped.start_listen_dom_increment()
         timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
-        await skyvern_element.focus()
+        await skyvern_element.scroll_into_view()
-        # wait 2s for options to load
-        await asyncio.sleep(2)
 
-        incremental_element = await incremental_scraped.get_incremental_element_tree(
+        try:
-            clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),
+            await skyvern_element.get_locator().click(timeout=timeout)
-        )
+        except Exception:
-        if len(incremental_element) > 0:
             LOG.info(
-                "Focus trigger DOM element change, confirm if it's a dropdown showing up",
+                "fail to open dropdown by clicking, try to press arrow down to open",
+                element_id=skyvern_element.get_id(),
                 task_id=task.task_id,
                 step_id=step.step_id,
             )
-            dropdown_element = await locate_dropdown_menu(
+            await skyvern_element.scroll_into_view()
-                incremental_scraped=incremental_scraped,
+            await skyvern_element.press_key("ArrowDown")
-                step=step,
+        await asyncio.sleep(5)
-                task=task,
+        is_open = True
-            )
-            if dropdown_element is not None:
-                is_open = True
-
-        if not is_open:
-            try:
-                await skyvern_element.get_locator().click(timeout=timeout)
-            except Exception:
-                LOG.info(
-                    "fail to open dropdown by clicking, try to press arrow down to open",
-                    element_id=skyvern_element.get_id(),
-                    task_id=task.task_id,
-                    step_id=step.step_id,
-                )
-                await skyvern_element.focus()
-                await skyvern_element.press_key("ArrowDown")
-            await asyncio.sleep(5)
-            is_open = True
 
         result = await select_from_dropdown_by_value(
             value=suggested_value,
@@ -863,7 +819,6 @@ async def handle_select_option_action(
             incremental_scraped=incremental_scraped,
             task=task,
             step=step,
-            dropdown_menu_element=dropdown_element,
         )
         results.append(result)
         return results