vodorod/Dorod-Sky
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

support custom select serach (#1085)

Browse Source
This commit is contained in:
LawyZheng
2024-10-29 16:04:41 +08:00
committed by GitHub
parent e4a834d158
commit aaaacf3ec7
2 changed files with 28 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
skyvern/forge/prompts/skyvern/custom-select.j2
View File

@@ -1,4 +1,4 @@
You are performing a {{ "multi-level selection" if select_history else "selection" }} action on an HTML page. Assist the user in selecting the most appropriate option to advance toward their goal, considering the context, user details, and the DOM elements provided in the list.
You are performing a {{ "multi-level selection" if select_history else "selection" }} action on an HTML page. Assist the user in selecting the most appropriate option(or typing some values to search if neccesary) to advance toward their goal, considering the context, user details, and the DOM elements provided in the list.
You can identify the matching element based on the following guidelines:
1. Select the most suitable element based on the user goal, user details, and the context.
@@ -16,6 +16,7 @@ Reply in JSON format with the following keys:
"reasoning": str, // The reasoning behind the action. Be specific, referencing the value and the element id in your reasoning. Mention why you chose the element id. Keep the reasoning short and to the point.
"confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence
"id": str, // The id of the element to take action on. The id has to be one from the elements list
"action_type": str, // It's a string enum: "CLICK", "INPUT_TEXT". "CLICK" is an option you'd like to click to choose. "INPUT_TEXT" is an element you'd like to input text into for searching, but it only should be used when there's no valid option to click.
"value": str, // The value to select.{% if target_value %}
"relevant": bool, // True if the value you select is relevant to the target value, otherwise False.{% endif %}
}

27
skyvern/webeye/actions/handler.py
View File

@@ -1718,8 +1718,10 @@ async def select_from_dropdown(
task_id=task.task_id,
)
action_type: str = json_response.get("action_type", "")
action_type = action_type.upper()
element_id: str | None = json_response.get("id", None)
if not element_id:
if not element_id or action_type not in ["CLICK", "INPUT_TEXT"]:
raise NoAvailableOptionFoundForCustomSelection(reason=json_response.get("reasoning"))
if not force_select and target_value:
@@ -1732,6 +1734,29 @@ async def select_from_dropdown(
)
return single_select_result
if value is not None and action_type == "INPUT_TEXT":
LOG.info(
"No clickable option found, but found input element to search",
element_id=element_id,
task_id=task.task_id,
step_id=step.step_id,
)
try:
input_element = await SkyvernElement.create_from_incremental(incremental_scraped, element_id)
await input_element.scroll_into_view()
current_text = await get_input_value(input_element.get_tag_name(), input_element.get_locator())
if current_text == value:
single_select_result.action_result = ActionSuccess()
return single_select_result
await input_element.input_clear()
await input_element.input_sequentially(value)
single_select_result.action_result = ActionSuccess()
return single_select_result
except Exception as e:
single_select_result.action_result = ActionFailure(exception=e)
return single_select_result
try:
selected_element = await SkyvernElement.create_from_incremental(incremental_scraped, element_id)
await selected_element.scroll_into_view()