fix custom selection bugs (#800)

skyvern/forge/prompts/skyvern/auto-completion-choose-option.j2

@@ -1,14 +1,14 @@
 There is an input element on an HTML page. Based on the context and information provided, you have two goals:
   - Confirm if an auto-completion attempt appears after the user inputs the current value.
-  - If auto-completion suggestions appear, assist the user in selecting the most appropriate element based on the user’s goal, details, and the context.
+  - If auto-completion suggestions appear, assist the user in selecting the most appropriate element based on the user's goal, details, and the context.
 
 You can confirm an auto-completion attempt based on the following rules:
   - Several auto-completion suggestions appear for the input value.
-  - Although messages like “No results” and “No match” mean no option was matched, they still indicate an attempt to generate auto-completion suggestions.
+  - Although messages like "No results" and "No match" mean no option was matched, they still indicate an attempt to generate auto-completion suggestions.
 
 You must identify a potential auto-completion suggestion based on the following rules:
-  - The option must be an element with an ID from the provided “HTML elements”. Do not create or assume options outside of these elements.
+  - The option must be an element with an ID from the provided "HTML elements". Do not create or assume options outside of these elements.
-  - The content of the option must be meaningful. Do not consider non-message indicators like “No results” or “No match” as valid options.
+  - The content of the option must be meaningful. Do not consider non-message indicators like "No results" or "No match" as valid options.
 
 MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
 Each interactable element is tagged with an ID.

skyvern/forge/prompts/skyvern/custom-select.j2

@@ -2,10 +2,10 @@ You are performing a selection action on an HTML page. Assist the user in select
 
 You can identify the matching element based on the following guidelines:
   1. Select the most suitable element based on the user goal, user details, and the context.
-  2. If no option is a perfect match, choose a fallback option such as “Others” or “None of the above”.
+  2. If no option is a perfect match, and there is a fallback option such as "Others" or "None of the above" in the DOM elements, you can consider it a match.
   3. If a field is required, do not leave it blank.
-  4. If a field is required, do not select a placeholder value, such as “Please select”, “-”, or “Select…”.
+  4. If a field is required, do not select a placeholder value, such as "Please select", "-", or "Select…".
-  5. Exclude loading indicators like “loading more results” as valid options.
+  5. Exclude loading indicators like "loading more results" as valid options.
 
 MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
 Each interactable element is tagged with an ID.

skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2

@@ -2,7 +2,7 @@ There is a screenshot from a part of a web HTML page. Help me confirm if it is a
 
 An open dropdown menu can be defined as:
   - At least one option is visible in the screenshot.
-  - Do not consider it an open dropdown menu if the only visible option displays a message like “No results” or “No match”.
+  - Do not consider it an open dropdown menu if the only visible option displays a message like "No results" or "No match".
 
 MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
 

skyvern/webeye/actions/handler.py

@@ -82,10 +82,11 @@ COMMON_INPUT_TAGS = {"input", "textarea", "select"}
 
 
 class CustomSingleSelectResult:
-    def __init__(self) -> None:
+    def __init__(self, skyvern_frame: SkyvernFrame) -> None:
         self.action_result: ActionResult | None = None
         self.value: str | None = None
         self.dropdown_menu: SkyvernElement | None = None
+        self.skyvern_frame = skyvern_frame
 
     async def is_done(self) -> bool:
         # check if the dropdown menu is still on the page
@@ -100,7 +101,7 @@ class CustomSingleSelectResult:
         if await self.dropdown_menu.get_locator().count() == 0:
             return True
 
-        return False
+        return not await self.skyvern_frame.get_element_visible(await self.dropdown_menu.get_element_handler())
 
 
 def is_ul_or_listbox_element_factory(
@@ -1336,7 +1337,7 @@ async def sequentially_select_from_dropdown(
     # TODO: only suport the third-level dropdown selection now
     MAX_SELECT_DEPTH = 3
     values: list[str | None] = []
-    single_select_result = CustomSingleSelectResult()
+    single_select_result = CustomSingleSelectResult(skyvern_frame=skyvern_frame)
 
     check_exist_funcs: list[CheckExistIDFunc] = [dom.check_id_in_dom]
     for i in range(MAX_SELECT_DEPTH):
@@ -1418,7 +1419,7 @@ async def select_from_dropdown(
         1. force_select is false and no dropdown menu popped
         2. force_select is false and match value is not relevant to the target value
     """
-    single_select_result = CustomSingleSelectResult()
+    single_select_result = CustomSingleSelectResult(skyvern_frame=skyvern_frame)
 
     timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
 

skyvern/webeye/scraper/domUtils.js

@@ -1798,7 +1798,23 @@ function getIncrementElements() {
     const treeList = window.globalDomDepthMap.get(depth);
 
     const removeDupAndConcatChildren = (element) => {
-      const children = element.children;
+      let children = element.children;
+      for (let i = 0; i < children.length; i++) {
+        const child = children[i];
+        const domElement = document.querySelector(`[unique_id="${child.id}"]`);
+        // if the element is still on the page, we rebuild the element to update the information
+        if (domElement) {
+          let newChild = buildElementObject(
+            "",
+            domElement,
+            child.interactable,
+            child.purgeable,
+          );
+          newChild.children = child.children;
+          children[i] = newChild;
+        }
+      }
+
       if (idToElement.has(element.id)) {
         element = idToElement.get(element.id);
         for (let i = 0; i < children.length; i++) {
@@ -1815,7 +1831,22 @@ function getIncrementElements() {
       }
     };
 
-    for (const treeHeadElement of treeList) {
+    for (let treeHeadElement of treeList) {
+      const domElement = document.querySelector(
+        `[unique_id="${treeHeadElement.id}"]`,
+      );
+      // if the element is still on the page, we rebuild the element to update the information
+      if (domElement) {
+        let newHead = buildElementObject(
+          "",
+          domElement,
+          treeHeadElement.interactable,
+          treeHeadElement.purgeable,
+        );
+        newHead.children = treeHeadElement.children;
+        treeHeadElement = newHead;
+      }
+
       // check if the element is existed
       if (!idToElement.has(treeHeadElement.id)) {
         cleanedTreeList.push(treeHeadElement);