vodorod/Dorod-Sky
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix date picker issue (#1523)

Browse Source
This commit is contained in:
LawyZheng
2025-01-09 16:14:31 +08:00
committed by GitHub
parent d4ffcdbfda
commit dc53a49749
9 changed files with 117 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
skyvern/webeye/actions/handler.py
View File

@@ -142,25 +142,43 @@ def is_ul_or_listbox_element_factory(
return wrapper
CheckExistIDFunc = Callable[[str], bool]
CheckFilterOutElementIDFunc = Callable[[str], Awaitable[bool]]
def check_id_in_dict_factory(id_dict: dict[str, Any]) -> CheckExistIDFunc:
def helper(element_id: str) -> bool:
if id_dict.get(element_id, ""):
def check_disappeared_element_id_in_incremental_factory(
incremental_scraped: IncrementalScrapePage,
) -> CheckFilterOutElementIDFunc:
current_element_to_dict = copy.deepcopy(incremental_scraped.id_to_css_dict)
async def helper(element_id: str) -> bool:
if not current_element_to_dict.get(element_id, ""):
return False
try:
skyvern_element = await SkyvernElement.create_from_incremental(
incre_page=incremental_scraped, element_id=element_id
)
except Exception:
LOG.info(
"Failed to create skyvern element, going to drop the element from incremental tree",
exc_info=True,
element_id=element_id,
)
return True
return False
skyvern_frame = incremental_scraped.skyvern_frame
return not await skyvern_frame.get_element_visible(await skyvern_element.get_element_handler())
return helper
def remove_exist_elements(element_tree: list[dict], check_exist: CheckExistIDFunc) -> list[dict]:
async def filter_out_elements(element_tree: list[dict], check_filter: CheckFilterOutElementIDFunc) -> list[dict]:
new_element_tree = []
for element in element_tree:
children_elements = element.get("children", [])
if len(children_elements) > 0:
children_elements = remove_exist_elements(element_tree=children_elements, check_exist=check_exist)
if check_exist(element.get("id", "")):
children_elements = await filter_out_elements(element_tree=children_elements, check_filter=check_filter)
if await check_filter(element.get("id", "")):
new_element_tree.extend(children_elements)
else:
element["children"] = children_elements
@@ -169,14 +187,14 @@ def remove_exist_elements(element_tree: list[dict], check_exist: CheckExistIDFun
def clean_and_remove_element_tree_factory(
task: Task, step: Step, check_exist_funcs: list[CheckExistIDFunc]
task: Task, step: Step, check_filter_funcs: list[CheckFilterOutElementIDFunc]
) -> CleanupElementTreeFunc:
async def helper_func(frame: Page | Frame, url: str, element_tree: list[dict]) -> list[dict]:
element_tree = await app.AGENT_FUNCTION.cleanup_element_tree_factory(task=task, step=step)(
frame, url, element_tree
)
for check_exist in check_exist_funcs:
element_tree = remove_exist_elements(element_tree=element_tree, check_exist=check_exist)
for check_filter in check_filter_funcs:
element_tree = await filter_out_elements(element_tree=element_tree, check_filter=check_filter)
return element_tree
@@ -557,27 +575,26 @@ async def handle_input_text_action(
)
return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]
select_action = SelectOptionAction(
reasoning=action.reasoning,
element_id=skyvern_element.get_id(),
option=SelectOption(label=text),
)
if skyvern_element.get_selectable():
LOG.info(
"Input element is selectable, doing select actions",
task_id=task.task_id,
step_id=step.step_id,
element_id=skyvern_element.get_id(),
action=action,
)
return await handle_select_option_action(select_action, page, scraped_page, task, step)
incremental_element: list[dict] = []
auto_complete_hacky_flag: bool = False
# check if it's selectable
if skyvern_element.get_tag_name() == InteractiveElement.INPUT and not await skyvern_element.is_raw_input():
select_action = SelectOptionAction(
reasoning=action.reasoning,
element_id=skyvern_element.get_id(),
option=SelectOption(label=text),
)
await skyvern_element.scroll_into_view()
if skyvern_element.get_selectable():
LOG.info(
"Input element is selectable, doing select actions",
task_id=task.task_id,
step_id=step.step_id,
element_id=skyvern_element.get_id(),
action=action,
)
return await handle_select_option_action(select_action, page, scraped_page, task, step)
# press arrowdown to watch if there's any options popping up
await incremental_scraped.start_listen_dom_increment()
try:
@@ -605,7 +622,7 @@ async def handle_input_text_action(
await asyncio.sleep(5)
incremental_element = await incremental_scraped.get_incremental_element_tree(
clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),
clean_and_remove_element_tree_factory(task=task, step=step, check_filter_funcs=[dom.check_id_in_dom]),
)
if len(incremental_element) == 0:
LOG.info(
@@ -755,6 +772,7 @@ async def handle_input_text_action(
step=step,
task=task,
):
auto_complete_hacky_flag = False
return [result]
await incremental_scraped.start_listen_dom_increment()
@@ -763,7 +781,7 @@ async def handle_input_text_action(
await skyvern_element.input_sequentially(text=text)
finally:
incremental_element = await incremental_scraped.get_incremental_element_tree(
clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),
clean_and_remove_element_tree_factory(task=task, step=step, check_filter_funcs=[dom.check_id_in_dom]),
)
if len(incremental_element) > 0:
auto_complete_hacky_flag = True
@@ -1052,7 +1070,7 @@ async def handle_select_option_action(
is_open = True
incremental_element = await incremental_scraped.get_incremental_element_tree(
clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),
clean_and_remove_element_tree_factory(task=task, step=step, check_filter_funcs=[dom.check_id_in_dom]),
)
if len(incremental_element) == 0 and skyvern_element.get_tag_name() == InteractiveElement.INPUT:
@@ -1067,7 +1085,7 @@ async def handle_select_option_action(
# wait 5s for options to load
await asyncio.sleep(5)
incremental_element = await incremental_scraped.get_incremental_element_tree(
clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),
clean_and_remove_element_tree_factory(task=task, step=step, check_filter_funcs=[dom.check_id_in_dom]),
)
if len(incremental_element) == 0:
@@ -1539,7 +1557,7 @@ async def choose_auto_completion_dropdown(
# wait for new elemnts to load
await asyncio.sleep(5)
incremental_element = await incremental_scraped.get_incremental_element_tree(
clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),
clean_and_remove_element_tree_factory(task=task, step=step, check_filter_funcs=[dom.check_id_in_dom]),
)
# check if elements in preserve list are still on the page
@@ -1856,7 +1874,7 @@ async def sequentially_select_from_dropdown(
values: list[str | None] = []
select_history: list[CustomSingleSelectResult] = []
check_exist_funcs: list[CheckExistIDFunc] = [dom.check_id_in_dom]
check_filter_funcs: list[CheckFilterOutElementIDFunc] = [dom.check_id_in_dom]
for i in range(MAX_SELECT_DEPTH):
single_select_result = await select_from_dropdown(
context=input_or_select_context,
@@ -1864,7 +1882,7 @@ async def sequentially_select_from_dropdown(
skyvern_element=skyvern_element,
skyvern_frame=skyvern_frame,
incremental_scraped=incremental_scraped,
check_exist_funcs=check_exist_funcs,
check_filter_funcs=check_filter_funcs,
step=step,
task=task,
dropdown_menu_element=dropdown_menu_element,
@@ -1877,6 +1895,18 @@ async def sequentially_select_from_dropdown(
# wait 1s until DOM finished updating
await asyncio.sleep(1)
# HACK: if agent took mini actions 2 times, stop executing the rest actions
# this is a hack to fix some date picker issues.
if input_or_select_context.is_date_related and i >= 1 and single_select_result.action_result:
LOG.warning(
"It's a date picker, going to skip reamaining actions",
depth=i,
task_id=task.task_id,
step_id=step.step_id,
)
single_select_result.action_result.skip_remaining_actions = True
break
if await single_select_result.is_done():
return single_select_result.action_result, values[-1] if len(values) > 0 else None
@@ -1898,14 +1928,15 @@ async def sequentially_select_from_dropdown(
# wait for 3s to load new options
await asyncio.sleep(3)
current_element_to_dict = copy.deepcopy(incremental_scraped.id_to_css_dict)
check_exist_funcs.append(check_id_in_dict_factory(current_element_to_dict))
check_filter_funcs.append(
check_disappeared_element_id_in_incremental_factory(incremental_scraped=incremental_scraped)
)
secondary_increment_element = await incremental_scraped.get_incremental_element_tree(
clean_and_remove_element_tree_factory(
task=task,
step=step,
check_exist_funcs=check_exist_funcs,
check_filter_funcs=check_filter_funcs,
)
)
if len(secondary_increment_element) == 0:
@@ -1940,7 +1971,7 @@ async def select_from_dropdown(
skyvern_element: SkyvernElement,
skyvern_frame: SkyvernFrame,
incremental_scraped: IncrementalScrapePage,
check_exist_funcs: list[CheckExistIDFunc],
check_filter_funcs: list[CheckFilterOutElementIDFunc],
step: Step,
task: Task,
dropdown_menu_element: SkyvernElement | None = None,
@@ -1991,7 +2022,7 @@ async def select_from_dropdown(
)
trimmed_element_tree = await incremental_scraped.get_incremental_element_tree(
clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=check_exist_funcs),
clean_and_remove_element_tree_factory(task=task, step=step, check_filter_funcs=check_filter_funcs),
)
html = incremental_scraped.build_html_tree(element_tree=trimmed_element_tree)
@@ -1999,6 +2030,7 @@ async def select_from_dropdown(
skyvern_context = ensure_context()
prompt = prompt_engine.load_prompt(
"custom-select",
is_date_related=context.is_date_related,
field_information=context.field,
required_field=context.is_required,
target_value="" if force_select else target_value,
@@ -2014,7 +2046,12 @@ async def select_from_dropdown(
step_id=step.step_id,
task_id=task.task_id,
)
json_response = await app.SECONDARY_LLM_API_HANDLER(prompt=prompt, step=step)
if context.is_date_related:
# HACK: according to the test, secondary LLM is not doing well on the date picker
# using the main LLM to handle the case
json_response = await app.LLM_API_HANDLER(prompt=prompt, step=step)
else:
json_response = await app.SECONDARY_LLM_API_HANDLER(prompt=prompt, step=step)
value: str | None = json_response.get("value", None)
single_select_result.value = value
select_reason: str | None = json_response.get("reasoning", None)
@@ -2121,7 +2158,7 @@ async def select_from_dropdown_by_value(
) -> ActionResult:
timeout = settings.BROWSER_ACTION_TIMEOUT_MS
await incremental_scraped.get_incremental_element_tree(
clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),
clean_and_remove_element_tree_factory(task=task, step=step, check_filter_funcs=[dom.check_id_in_dom]),
)
element_locator = await incremental_scraped.select_one_element_by_value(value=value)
@@ -2155,7 +2192,7 @@ async def select_from_dropdown_by_value(
async def continue_callback(incre_scraped: IncrementalScrapePage) -> bool:
await incre_scraped.get_incremental_element_tree(
clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),
clean_and_remove_element_tree_factory(task=task, step=step, check_filter_funcs=[dom.check_id_in_dom]),
)
element_locator = await incre_scraped.select_one_element_by_value(value=value)