diff --git a/skyvern/forge/prompts/skyvern/custom-select.j2 b/skyvern/forge/prompts/skyvern/custom-select.j2 index fc611f75..c408e251 100644 --- a/skyvern/forge/prompts/skyvern/custom-select.j2 +++ b/skyvern/forge/prompts/skyvern/custom-select.j2 @@ -1,4 +1,9 @@ You are doing a select action on HTML page. Help to click the best match element for the target value among HTML elements based on the context. +You can find the match element based on the following attempts: + 1. Find the semantically most similar element + 2. Reconsider if target value is reasonable based on context and the options in the HTML elements. If it doesn't make sense, you can tweak the target value into a reasonable one. + 3. Find the element, which semantically is the superset of target value. Like "Others", "None of them matched" + MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc. Each interactable element is tagged with an ID. diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py index c0015385..4566f2e0 100644 --- a/skyvern/webeye/actions/handler.py +++ b/skyvern/webeye/actions/handler.py @@ -886,8 +886,18 @@ async def select_from_dropdown( task=task, ) - await incremental_scraped.get_incremental_element_tree(app.AGENT_FUNCTION.cleanup_element_tree) - html = incremental_scraped.build_html_tree() + trimmed_element_tree = await incremental_scraped.get_incremental_element_tree( + app.AGENT_FUNCTION.cleanup_element_tree + ) + if dropdown_menu_element: + # if there's a dropdown menu detected, only elements in the dropdown should be sent to LLM + dropdown_id = dropdown_menu_element.get_id() + for head_node in trimmed_element_tree: + if head_node.get("id") == dropdown_id: + trimmed_element_tree = [head_node] + break + + html = incremental_scraped.build_html_tree(element_tree=trimmed_element_tree) target_value = action.option.label or action.option.value if target_value is None: diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py index 791b6b13..4d099b23 100644 --- a/skyvern/webeye/scraper/scraper.py +++ b/skyvern/webeye/scraper/scraper.py @@ -442,8 +442,8 @@ class IncrementalScrapePage: js_script = "() => window.globalOneTimeIncrementElements.length" return await self.skyvern_frame.get_frame().evaluate(js_script) - def build_html_tree(self) -> str: - return "".join([json_to_html(element) for element in self.element_tree_trimmed]) + def build_html_tree(self, element_tree: list[dict] = []) -> str: + return "".join([json_to_html(element) for element in (element_tree or self.element_tree_trimmed)]) def trim_element_tree(elements: list[dict]) -> list[dict]: