From b12f09c535393d15cbe2facf95bfcea16030c2b9 Mon Sep 17 00:00:00 2001 From: LawyZheng Date: Tue, 10 Sep 2024 17:10:47 +0800 Subject: [PATCH] fix custom selection bugs (#800) --- .../skyvern/auto-completion-choose-option.j2 | 8 ++--- .../forge/prompts/skyvern/custom-select.j2 | 6 ++-- .../skyvern/opened-dropdown-confirm.j2 | 2 +- skyvern/webeye/actions/handler.py | 9 ++--- skyvern/webeye/scraper/domUtils.js | 35 +++++++++++++++++-- 5 files changed, 46 insertions(+), 14 deletions(-) diff --git a/skyvern/forge/prompts/skyvern/auto-completion-choose-option.j2 b/skyvern/forge/prompts/skyvern/auto-completion-choose-option.j2 index c3f20810..396f3c5e 100644 --- a/skyvern/forge/prompts/skyvern/auto-completion-choose-option.j2 +++ b/skyvern/forge/prompts/skyvern/auto-completion-choose-option.j2 @@ -1,14 +1,14 @@ There is an input element on an HTML page. Based on the context and information provided, you have two goals: - Confirm if an auto-completion attempt appears after the user inputs the current value. - - If auto-completion suggestions appear, assist the user in selecting the most appropriate element based on the user’s goal, details, and the context. + - If auto-completion suggestions appear, assist the user in selecting the most appropriate element based on the user's goal, details, and the context. You can confirm an auto-completion attempt based on the following rules: - Several auto-completion suggestions appear for the input value. - - Although messages like “No results” and “No match” mean no option was matched, they still indicate an attempt to generate auto-completion suggestions. + - Although messages like "No results" and "No match" mean no option was matched, they still indicate an attempt to generate auto-completion suggestions. You must identify a potential auto-completion suggestion based on the following rules: - - The option must be an element with an ID from the provided “HTML elements”. Do not create or assume options outside of these elements. - - The content of the option must be meaningful. Do not consider non-message indicators like “No results” or “No match” as valid options. + - The option must be an element with an ID from the provided "HTML elements". Do not create or assume options outside of these elements. + - The content of the option must be meaningful. Do not consider non-message indicators like "No results" or "No match" as valid options. MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc. Each interactable element is tagged with an ID. diff --git a/skyvern/forge/prompts/skyvern/custom-select.j2 b/skyvern/forge/prompts/skyvern/custom-select.j2 index 3b0cd2bf..34f108d8 100644 --- a/skyvern/forge/prompts/skyvern/custom-select.j2 +++ b/skyvern/forge/prompts/skyvern/custom-select.j2 @@ -2,10 +2,10 @@ You are performing a selection action on an HTML page. Assist the user in select You can identify the matching element based on the following guidelines: 1. Select the most suitable element based on the user goal, user details, and the context. - 2. If no option is a perfect match, choose a fallback option such as “Others” or “None of the above”. + 2. If no option is a perfect match, and there is a fallback option such as "Others" or "None of the above" in the DOM elements, you can consider it a match. 3. If a field is required, do not leave it blank. - 4. If a field is required, do not select a placeholder value, such as “Please select”, “-”, or “Select…”. - 5. Exclude loading indicators like “loading more results” as valid options. + 4. If a field is required, do not select a placeholder value, such as "Please select", "-", or "Select…". + 5. Exclude loading indicators like "loading more results" as valid options. MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc. Each interactable element is tagged with an ID. diff --git a/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2 b/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2 index cf6f9900..4bbaef11 100644 --- a/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2 +++ b/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2 @@ -2,7 +2,7 @@ There is a screenshot from a part of a web HTML page. Help me confirm if it is a An open dropdown menu can be defined as: - At least one option is visible in the screenshot. - - Do not consider it an open dropdown menu if the only visible option displays a message like “No results” or “No match”. + - Do not consider it an open dropdown menu if the only visible option displays a message like "No results" or "No match". MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc. diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py index 6c38df5f..a632a79f 100644 --- a/skyvern/webeye/actions/handler.py +++ b/skyvern/webeye/actions/handler.py @@ -82,10 +82,11 @@ COMMON_INPUT_TAGS = {"input", "textarea", "select"} class CustomSingleSelectResult: - def __init__(self) -> None: + def __init__(self, skyvern_frame: SkyvernFrame) -> None: self.action_result: ActionResult | None = None self.value: str | None = None self.dropdown_menu: SkyvernElement | None = None + self.skyvern_frame = skyvern_frame async def is_done(self) -> bool: # check if the dropdown menu is still on the page @@ -100,7 +101,7 @@ class CustomSingleSelectResult: if await self.dropdown_menu.get_locator().count() == 0: return True - return False + return not await self.skyvern_frame.get_element_visible(await self.dropdown_menu.get_element_handler()) def is_ul_or_listbox_element_factory( @@ -1336,7 +1337,7 @@ async def sequentially_select_from_dropdown( # TODO: only suport the third-level dropdown selection now MAX_SELECT_DEPTH = 3 values: list[str | None] = [] - single_select_result = CustomSingleSelectResult() + single_select_result = CustomSingleSelectResult(skyvern_frame=skyvern_frame) check_exist_funcs: list[CheckExistIDFunc] = [dom.check_id_in_dom] for i in range(MAX_SELECT_DEPTH): @@ -1418,7 +1419,7 @@ async def select_from_dropdown( 1. force_select is false and no dropdown menu popped 2. force_select is false and match value is not relevant to the target value """ - single_select_result = CustomSingleSelectResult() + single_select_result = CustomSingleSelectResult(skyvern_frame=skyvern_frame) timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js index c0d6dc7e..1e2708d2 100644 --- a/skyvern/webeye/scraper/domUtils.js +++ b/skyvern/webeye/scraper/domUtils.js @@ -1798,7 +1798,23 @@ function getIncrementElements() { const treeList = window.globalDomDepthMap.get(depth); const removeDupAndConcatChildren = (element) => { - const children = element.children; + let children = element.children; + for (let i = 0; i < children.length; i++) { + const child = children[i]; + const domElement = document.querySelector(`[unique_id="${child.id}"]`); + // if the element is still on the page, we rebuild the element to update the information + if (domElement) { + let newChild = buildElementObject( + "", + domElement, + child.interactable, + child.purgeable, + ); + newChild.children = child.children; + children[i] = newChild; + } + } + if (idToElement.has(element.id)) { element = idToElement.get(element.id); for (let i = 0; i < children.length; i++) { @@ -1815,7 +1831,22 @@ function getIncrementElements() { } }; - for (const treeHeadElement of treeList) { + for (let treeHeadElement of treeList) { + const domElement = document.querySelector( + `[unique_id="${treeHeadElement.id}"]`, + ); + // if the element is still on the page, we rebuild the element to update the information + if (domElement) { + let newHead = buildElementObject( + "", + domElement, + treeHeadElement.interactable, + treeHeadElement.purgeable, + ); + newHead.children = treeHeadElement.children; + treeHeadElement = newHead; + } + // check if the element is existed if (!idToElement.has(treeHeadElement.id)) { cleanedTreeList.push(treeHeadElement);