fix search on auto completion (#1544)

skyvern/webeye/actions/handler.py

@@ -788,12 +788,21 @@ async def handle_input_text_action(
             await incremental_scraped.stop_listen_dom_increment()
 
         return [ActionSuccess()]
+    except Exception as e:
+        LOG.exception(
+            "Failed to input the value or finish the auto completion",
+            task_id=task.task_id,
+            step_id=step.step_id,
+        )
+        raise e
     finally:
         # HACK: force to finish missing auto completion input
-        if auto_complete_hacky_flag and not await skyvern_element.is_raw_input():
+        if auto_complete_hacky_flag and await skyvern_element.is_visible() and not await skyvern_element.is_raw_input():
             LOG.debug(
                 "Trigger input-selection hack, pressing Tab to choose one",
                 action=action,
+                task_id=task.task_id,
+                step_id=step.step_id,
             )
             await skyvern_element.press_key("Tab")
 
@@ -1624,6 +1633,7 @@ async def choose_auto_completion_dropdown(
         html = incremental_scraped.build_html_tree(cleaned_incremental_element)
         auto_completion_confirm_prompt = prompt_engine.load_prompt(
             "auto-completion-choose-option",
+            is_search=context.is_search_bar,
             field_information=context.field,
             filled_value=text,
             navigation_goal=task.navigation_goal,
@@ -1638,6 +1648,16 @@ async def choose_auto_completion_dropdown(
         json_response = await app.SECONDARY_LLM_API_HANDLER(prompt=auto_completion_confirm_prompt, step=step)
         element_id = json_response.get("id", "")
         relevance_float = json_response.get("relevance_float", 0)
+        if json_response.get("direct_searching", False):
+            LOG.info(
+                "Decided to directly search with the current value",
+                value=text,
+                step_id=step.step_id,
+                task_id=task.task_id,
+            )
+            await skyvern_element.press_key("Enter")
+            return result
+
         if not element_id:
             reasoning = json_response.get("reasoning")
             raise NoSuitableAutoCompleteOption(reasoning=reasoning, target_value=text)
@@ -1682,7 +1702,7 @@ async def choose_auto_completion_dropdown(
         return result
     finally:
         await incremental_scraped.stop_listen_dom_increment()
-        if clear_input:
+        if clear_input and await skyvern_element.is_visible():
             await skyvern_element.input_clear()
 
 

skyvern/webeye/scraper/domUtils.js

@@ -2132,15 +2132,26 @@ if (window.globalObserverForDOMIncrement === undefined) {
       }
 
       if (mutation.type === "childList") {
+        if (mutation.target.nodeType === Node.TEXT_NODE) continue;
+        const node = mutation.target;
         let changedNode = {
-          targetNode: mutation.target, // TODO: for future usage, when we want to parse new elements into a tree
+          targetNode: node, // TODO: for future usage, when we want to parse new elements into a tree
         };
         let newNodes = [];
-        if (mutation.addedNodes && mutation.addedNodes.length > 0) {
-          for (const node of mutation.addedNodes) {
-            // skip the text nodes, they won't be interactable
-            if (node.nodeType === Node.TEXT_NODE) continue;
-            newNodes.push(node);
+        if (
+          node.tagName.toLowerCase() === "ul" ||
+          (node.tagName.toLowerCase() === "div" &&
+            node.hasAttribute("role") &&
+            node.getAttribute("role").toLowerCase() === "listbox")
+        ) {
+          newNodes.push(node);
+        } else {
+          if (mutation.addedNodes && mutation.addedNodes.length > 0) {
+            for (const node of mutation.addedNodes) {
+              // skip the text nodes, they won't be interactable
+              if (node.nodeType === Node.TEXT_NODE) continue;
+              newNodes.push(node);
+            }
           }
         }
         if (newNodes.length > 0) {

skyvern/webeye/scraper/scraper.py

@@ -575,6 +575,10 @@ class IncrementalScrapePage:
         await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script)
 
     async def stop_listen_dom_increment(self) -> None:
+        # check if the DOM has navigated away or refreshed
+        js_script = "() => window.globalObserverForDOMIncrement === undefined"
+        if await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script):
+            return
         js_script = "() => stopGlobalIncrementalObserver()"
         await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script)
 

skyvern/webeye/utils/dom.py

@@ -267,6 +267,8 @@ class SkyvernElement:
         return self.get_selectable() or self.get_tag_name() in SELECTABLE_ELEMENT
 
     async def is_visible(self) -> bool:
+        if not await self.get_locator().count():
+            return False
         skyvern_frame = await SkyvernFrame.create_instance(self.get_frame())
         return await skyvern_frame.get_element_visible(await self.get_element_handler())