fix selection issues (#1515)

skyvern/forge/agent_functions.py

@@ -267,6 +267,16 @@ async def _convert_css_shape_to_string(
 
         try:
             LOG.debug("call LLM to convert css shape to string shape", element_id=element_id)
+            if not await locater.is_visible(timeout=settings.BROWSER_ACTION_TIMEOUT_MS):
+                LOG.info(
+                    "element is not visible on the page, going to abort conversion",
+                    task_id=task_id,
+                    step_id=step_id,
+                    element_id=element_id,
+                    key=shape_key,
+                )
+                return None
+
             screenshot = await locater.screenshot(timeout=settings.BROWSER_SCREENSHOT_TIMEOUT_MS)
             prompt = prompt_engine.load_prompt("css-shape-convert")
 

skyvern/forge/prompts/skyvern/custom-select.j2

@@ -2,7 +2,7 @@ You are performing a {{ "multi-level selection" if select_history else "selectio
 
 You can identify the matching element based on the following guidelines:
   1. Select the most suitable element based on the user goal, user details, and the context.
-  2. If no option is a perfect match, and there is a fallback option such as "Others" or "None of the above" in the DOM elements, you can consider it a match.
+  2. If none of the options perfectly match, and there is no search box for input, but there is a fallback option such as "Others" or "None of the above" in the DOM elements, you can consider it a match.
   3. If a field is required, do not leave it blank.
   4. If a field is required, do not select a placeholder value, such as "Please select", "-", or "Select...".
   5. Exclude loading indicators like "loading more results" as valid options.{% if select_history %}

skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2

@@ -2,7 +2,7 @@ There is a screenshot from a part of a web HTML page. Help me confirm if it is a
 
 An open dropdown menu can be defined as:
   - At least one option is visible in the screenshot.
-  - A calendar view or date picker could be considered as an open dropdown menu.
+  - A calendar view could be considered as an open dropdown menu. But DO NOT consider an calendar icon as the dropdown menu.
   - Do not consider it an open dropdown menu if the only visible option displays a message like "No results" or "No match".
   - Do not consider it an open dropdown menu if the only visible element displays a placeholder like "Please select", "-", or "Select...".