diff --git a/skyvern/forge/prompts/skyvern/custom-select.j2 b/skyvern/forge/prompts/skyvern/custom-select.j2 index 3427a0ed..0fe91049 100644 --- a/skyvern/forge/prompts/skyvern/custom-select.j2 +++ b/skyvern/forge/prompts/skyvern/custom-select.j2 @@ -19,14 +19,14 @@ Reply in JSON format with the following keys: "reasoning": str, // The reasoning behind the current single action. Be specific, referencing the value and the element id in your reasoning. Mention why you chose the element id. Keep the reasoning short and to the point. "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence "id": str, // The id of the element to take action on. The id has to be one from {{ "the emerging HTML elements list" if new_elements_ids else "the HTML elements list" }}. - "action_type": str, // It's a string enum: "CLICK", "INPUT_TEXT". "CLICK" is an option you'd like to click to choose. "INPUT_TEXT" is an element you'd like to input text into for searching, but it only should be used when there's no valid option to click. + "action_type": str, // It's a string enum: "CLICK", "INPUT_TEXT"{{' ,"COMPLETE"' if support_complete_action else ""}}. "CLICK" is an option you'd like to click to choose. "INPUT_TEXT" is an element you'd like to input text into for searching, but it only should be used when there's no valid option to click.{{ ' "COMPLETE" means the user has completed user goal.' if support_complete_action else "" }} "value": str, // The value to select.{% if target_value %} "relevant": bool, // True if the value you select is relevant to the target value, otherwise False. If the value is a fallback option according to the guidelines, it's still relevant.{% endif %} } Context: ``` -Select an option for "{{ field_information }}". It's {{ "a required" if required_field else "an optional" }} field. +Select an option for "{{ field_information }}"{{" if user goal has not been completed" if support_complete_action else ""}}. It's {{ "a required" if required_field else "an optional" }} field. ``` {% if target_value %} Target value: diff --git a/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2 b/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2 index 959c09aa..86525761 100644 --- a/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2 +++ b/skyvern/forge/prompts/skyvern/opened-dropdown-confirm.j2 @@ -6,6 +6,7 @@ An open dropdown menu can be defined as: - If the screenshot contains multiple input fields for selecting quantities, consider as the dropdown menu. - Do not consider it an open dropdown menu if the only visible option displays a message like "No results" or "No match". - Do not consider it an open dropdown menu if the only visible element displays a placeholder like "Please select", "-", or "Select...". + - It can be a selection dropdown or a date picker. But DO NOT consider it an open dropdown if it's a popup modal window. MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc. diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py index aff9b019..4660343d 100644 --- a/skyvern/webeye/actions/handler.py +++ b/skyvern/webeye/actions/handler.py @@ -724,6 +724,7 @@ async def handle_sequential_click_for_dropdown( scraped_page=scraped_page, step=step, task=task, + support_complete_action=True, ) @@ -2675,6 +2676,7 @@ async def select_from_emerging_elements( scraped_page: ScrapedPage, step: Step, task: Task, + support_complete_action: bool = False, ) -> ActionResult: """ This is the function to select an element from the new showing elements. @@ -2705,6 +2707,7 @@ async def select_from_emerging_elements( target_value=options.target_value, navigation_goal=task.navigation_goal, new_elements_ids=new_interactable_element_ids, + support_complete_action=support_complete_action, navigation_payload_str=json.dumps(task.navigation_payload), local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(), ) @@ -2728,9 +2731,17 @@ async def select_from_emerging_elements( action_type_str: str = json_response.get("action_type", "") or "" action_type = ActionType(action_type_str.lower()) element_id: str | None = json_response.get("id", None) - if not element_id or action_type not in [ActionType.CLICK, ActionType.INPUT_TEXT]: + if not element_id or action_type not in [ActionType.CLICK, ActionType.INPUT_TEXT, ActionType.COMPLETE]: raise NoAvailableOptionFoundForCustomSelection(reason=json_response.get("reasoning")) + if action_type == ActionType.COMPLETE: + LOG.info( + "The user has completed the user goal in the current opened dropdown, although the dropdown might not be closed", + step_id=step.step_id, + task_id=task.task_id, + ) + return ActionSuccess() + if value is not None and action_type == ActionType.INPUT_TEXT: LOG.info( "No clickable option found, but found input element to search",