better detect dropdown menu (#778)

skyvern/webeye/scraper/domUtils.js

@@ -888,7 +888,7 @@ function uniqueId() {
   return result;
 }
 
-function buildElementObject(frame, element, interactable) {
+function buildElementObject(frame, element, interactable, purgeable = false) {
   var element_id = element.getAttribute("unique_id") ?? uniqueId();
   var elementTagNameLower = element.tagName.toLowerCase();
   element.setAttribute("unique_id", element_id);
@@ -940,6 +940,8 @@ function buildElementObject(frame, element, interactable) {
     text: getElementContent(element),
     children: [],
     rect: DomUtils.getVisibleClientRect(element, true),
+    // if purgeable is True, which means this element is only used for building the tree relationship
+    purgeable: purgeable,
     // don't trim any attr of this element if keepAllAttr=True
     keepAllAttr:
       elementTagNameLower === "svg" || element.closest("svg") !== null,
@@ -979,11 +981,11 @@ function buildElementObject(frame, element, interactable) {
   return elementObj;
 }
 
-function buildTreeFromBody(frame = "main.frame", open_select = false) {
-  return buildElementTree(document.body, frame, open_select);
+function buildTreeFromBody(frame = "main.frame") {
+  return buildElementTree(document.body, frame);
 }
 
-function buildElementTree(starter = document.body, frame = "main.frame") {
+function buildElementTree(starter = document.body, frame, full_tree = false) {
   var elements = [];
   var resultArray = [];
 
@@ -1078,6 +1080,23 @@ function buildElementTree(starter = document.body, frame = "main.frame") {
           // build all table related elements into skyvern element
           // we need these elements to preserve the DOM structure
           elementObj = buildElementObject(frame, element, false);
+        } else if (full_tree) {
+          // when building full tree, we only get text from element itself
+          // elements without text are purgeable
+          elementObj = buildElementObject(frame, element, false, true);
+          let textContent = "";
+          if (isElementVisible(element)) {
+            for (let i = 0; i < element.childNodes.length; i++) {
+              var node = element.childNodes[i];
+              if (node.nodeType === Node.TEXT_NODE) {
+                textContent += node.data.trim();
+              }
+            }
+          }
+          elementObj.text = textContent;
+          if (textContent.length > 0) {
+            elementObj.purgeable = false;
+          }
         } else {
           // character length limit for non-interactable elements should be 5000
           // we don't use element context in HTML format,
@@ -1673,7 +1692,7 @@ function addIncrementalNodeToMap(parentNode, childrenNode) {
   }
 
   for (const child of childrenNode) {
-    const [_, newNodeTree] = buildElementTree(child, "", false);
+    const [_, newNodeTree] = buildElementTree(child, "", true);
     if (newNodeTree.length > 0) {
       newNodesTreeList.push(...newNodeTree);
     }

skyvern/webeye/scraper/scraper.py

@@ -109,6 +109,9 @@ def json_to_html(element: dict) -> str:
         for option in element.get("options", [])
     )
 
+    if element.get("purgeable", False):
+        return children_html + option_html
+
     # Check if the element is self-closing
     if tag in ["img", "input", "br", "hr", "meta", "link"] and not option_html and not children_html:
         return f'<{tag}{attributes_html if not attributes_html else " "+attributes_html}>'
@@ -338,7 +341,7 @@ async def get_interactable_element_tree_in_frame(
 
         unique_id = await frame_element.get_attribute("unique_id")
 
-        frame_js_script = f"() => buildTreeFromBody('{unique_id}', true)"
+        frame_js_script = f"() => buildTreeFromBody('{unique_id}')"
 
         await frame.evaluate(JS_FUNCTION_DEFS)
         frame_elements, frame_element_tree = await frame.evaluate(frame_js_script)
@@ -374,7 +377,7 @@ async def get_interactable_element_tree(
     :return: Tuple containing the element tree and a map of element IDs to elements.
     """
     await page.evaluate(JS_FUNCTION_DEFS)
-    main_frame_js_script = "() => buildTreeFromBody('main.frame', true)"
+    main_frame_js_script = "() => buildTreeFromBody()"
     elements, element_tree = await page.evaluate(main_frame_js_script)
 
     if len(page.main_frame.child_frames) > 0:
@@ -504,8 +507,7 @@ def trim_element_tree(elements: list[dict]) -> list[dict]:
                 del queue_ele["attributes"]
 
         if "attributes" in queue_ele and not queue_ele.get("keepAllAttr", False):
-            tag_name = queue_ele["tagName"] if "tagName" in queue_ele else ""
-            new_attributes = _trimmed_attributes(tag_name, queue_ele["attributes"])
+            new_attributes = _trimmed_attributes(queue_ele["attributes"])
             if new_attributes:
                 queue_ele["attributes"] = new_attributes
             else:
@@ -536,13 +538,10 @@ def _trimmed_base64_data(attributes: dict) -> dict:
     return new_attributes
 
 
-def _trimmed_attributes(tag_name: str, attributes: dict) -> dict:
+def _trimmed_attributes(attributes: dict) -> dict:
     new_attributes: dict = {}
 
     for key in attributes:
-        if key == "id" and tag_name in ["input", "textarea", "select"]:
-            # We don't want to remove the id attribute any of these elements in case there's a label for it
-            new_attributes[key] = attributes[key]
         if key == "role" and attributes[key] in ["listbox", "option"]:
             new_attributes[key] = attributes[key]
         if key in RESERVED_ATTRIBUTES and attributes[key]: