fix dropdown menu detect bug (#874)

2024-09-24 12:44:06 +08:00
parent e59eb2b2bc
commit c27c2a55f9
2 changed files with 25 additions and 69 deletions
--- a/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2
+++ b/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2
@@ -3,6 +3,7 @@ There is a screenshot from a part of a web HTML page. Help me confirm if it is a
 An open dropdown menu can be defined as:
  - At least one option is visible in the screenshot.
  - Do not consider it an open dropdown menu if the only visible option displays a message like "No results" or "No match".
+  - Do not consider it an open dropdown menu if the only visible element displays a placeholder like "Please select", "-", or "Select...".

 MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.

--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -417,12 +417,12 @@ async def handle_input_text_action(
    # check if it's selectable
    if skyvern_element.get_tag_name() == InteractiveElement.INPUT and not await skyvern_element.is_raw_input():
        await skyvern_element.scroll_into_view()
-        select_action = SelectOptionAction(
-            reasoning=action.reasoning,
-            element_id=skyvern_element.get_id(),
-            option=SelectOption(label=text),
-        )
        if skyvern_element.get_selectable():
+            select_action = SelectOptionAction(
+                reasoning=action.reasoning,
+                element_id=skyvern_element.get_id(),
+                option=SelectOption(label=text),
+            )
            LOG.info(
                "Input element is selectable, doing select actions",
                task_id=task.task_id,
@@ -434,8 +434,7 @@ async def handle_input_text_action(

        # press arrowdown to watch if there's any options popping up
        await incremental_scraped.start_listen_dom_increment()
-        await skyvern_element.get_locator().focus(timeout=timeout)
-        await skyvern_element.get_locator().press("ArrowDown", timeout=timeout)
+        await skyvern_element.press_key("ArrowDown")
        await asyncio.sleep(5)

        incremental_element = await incremental_scraped.get_incremental_element_tree(
@@ -726,46 +725,25 @@ async def handle_select_option_action(
    timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
    skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame())
    incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame)
-    dropdown_element: SkyvernElement | None = None
    is_open = False
    suggested_value: str | None = None
    results: list[ActionResult] = []

    try:
        await incremental_scraped.start_listen_dom_increment()
-        await skyvern_element.focus()
-        # wait 2s for options to load
-        await asyncio.sleep(2)
+        await skyvern_element.scroll_into_view()

-        incremental_element = await incremental_scraped.get_incremental_element_tree(
-            clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),
-        )
-        if len(incremental_element) > 0:
+        try:
+            await skyvern_element.get_locator().click(timeout=timeout)
+        except Exception:
            LOG.info(
-                "Focus trigger DOM element change, confirm if it's a dropdown showing up",
+                "fail to open dropdown by clicking, try to press ArrowDown to open",
+                element_id=skyvern_element.get_id(),
                task_id=task.task_id,
                step_id=step.step_id,
            )
-            dropdown_element = await locate_dropdown_menu(
-                incremental_scraped=incremental_scraped,
-                step=step,
-                task=task,
-            )
-            if dropdown_element is not None:
-                is_open = True
-
-        if not is_open:
-            try:
-                await skyvern_element.get_locator().click(timeout=timeout)
-            except Exception:
-                LOG.info(
-                    "fail to open dropdown by clicking, try to press ArrowDown to open",
-                    element_id=skyvern_element.get_id(),
-                    task_id=task.task_id,
-                    step_id=step.step_id,
-                )
-                await skyvern_element.focus()
-                await skyvern_element.press_key("ArrowDown")
+            await skyvern_element.scroll_into_view()
+            await skyvern_element.press_key("ArrowDown")

            # wait 5s for options to load
            await asyncio.sleep(5)
@@ -786,7 +764,6 @@ async def handle_select_option_action(
            incremental_scraped=incremental_scraped,
            step=step,
            task=task,
-            dropdown_menu_element=dropdown_element,
            force_select=True,
        )
        # force_select won't return None result
@@ -805,7 +782,6 @@ async def handle_select_option_action(
            await skyvern_element.coordinate_click(page=page)
            await skyvern_element.press_key("Escape")
        is_open = False
-        dropdown_element = None
        await skyvern_element.blur()
        await incremental_scraped.stop_listen_dom_increment()

@@ -819,41 +795,21 @@ async def handle_select_option_action(
    try:
        await incremental_scraped.start_listen_dom_increment()
        timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
-        await skyvern_element.focus()
-        # wait 2s for options to load
-        await asyncio.sleep(2)
+        await skyvern_element.scroll_into_view()

-        incremental_element = await incremental_scraped.get_incremental_element_tree(
-            clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),
-        )
-        if len(incremental_element) > 0:
+        try:
+            await skyvern_element.get_locator().click(timeout=timeout)
+        except Exception:
            LOG.info(
-                "Focus trigger DOM element change, confirm if it's a dropdown showing up",
+                "fail to open dropdown by clicking, try to press arrow down to open",
+                element_id=skyvern_element.get_id(),
                task_id=task.task_id,
                step_id=step.step_id,
            )
-            dropdown_element = await locate_dropdown_menu(
-                incremental_scraped=incremental_scraped,
-                step=step,
-                task=task,
-            )
-            if dropdown_element is not None:
-                is_open = True
-
-        if not is_open:
-            try:
-                await skyvern_element.get_locator().click(timeout=timeout)
-            except Exception:
-                LOG.info(
-                    "fail to open dropdown by clicking, try to press arrow down to open",
-                    element_id=skyvern_element.get_id(),
-                    task_id=task.task_id,
-                    step_id=step.step_id,
-                )
-                await skyvern_element.focus()
-                await skyvern_element.press_key("ArrowDown")
-            await asyncio.sleep(5)
-            is_open = True
+            await skyvern_element.scroll_into_view()
+            await skyvern_element.press_key("ArrowDown")
+        await asyncio.sleep(5)
+        is_open = True

        result = await select_from_dropdown_by_value(
            value=suggested_value,
@@ -863,7 +819,6 @@ async def handle_select_option_action(
            incremental_scraped=incremental_scraped,
            task=task,
            step=step,
-            dropdown_menu_element=dropdown_element,
        )
        results.append(result)
        return results