fix sequential click bug (#2494)

Co-authored-by: lawyzheng <lawyzheng1106@gmail.com>
This commit is contained in:
Shuchang Zheng
2025-05-28 00:33:29 -07:00
committed by GitHub
parent 1d1827059c
commit 91a666f705

View File

@@ -597,6 +597,7 @@ async def handle_click_action(
workflow_run_id=task.workflow_run_id, workflow_run_id=task.workflow_run_id,
) )
else: else:
incremental_scraped: IncrementalScrapePage | None = None
try: try:
skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame()) skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame())
incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame) incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame)
@@ -616,9 +617,50 @@ async def handle_click_action(
if results and not isinstance(results[-1], ActionSuccess): if results and not isinstance(results[-1], ActionSuccess):
return results return results
if await incremental_scraped.get_incremental_elements_num() == 0: try:
if sequential_click_result := await handle_sequential_click_for_dropdown(
action=action,
anchor_element=skyvern_element,
dom=dom,
page=page,
scraped_page=scraped_page,
incremental_scraped=incremental_scraped,
task=task,
step=step,
):
results.append(sequential_click_result)
return results return results
except Exception:
LOG.warning(
"Failed to do sequential logic for the click action, skipping",
exc_info=True,
step_id=step.step_id,
task_id=task.task_id,
element_id=skyvern_element.get_id(),
)
return results
finally:
if incremental_scraped:
await incremental_scraped.stop_listen_dom_increment()
return results
async def handle_sequential_click_for_dropdown(
action: actions.ClickAction,
anchor_element: SkyvernElement,
dom: DomUtil,
page: Page,
scraped_page: ScrapedPage,
incremental_scraped: IncrementalScrapePage,
task: Task,
step: Step,
) -> ActionResult | None:
if await incremental_scraped.get_incremental_elements_num() == 0:
return None
incremental_elements = await incremental_scraped.get_incremental_element_tree( incremental_elements = await incremental_scraped.get_incremental_element_tree(
clean_and_remove_element_tree_factory( clean_and_remove_element_tree_factory(
task=task, step=step, check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)] task=task, step=step, check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)]
@@ -626,18 +668,18 @@ async def handle_click_action(
) )
if len(incremental_elements) == 0: if len(incremental_elements) == 0:
return results return None
LOG.info("Detected new element after clicking", action=action) LOG.info("Detected new element after clicking", action=action)
dropdown_menu_element = await locate_dropdown_menu( dropdown_menu_element = await locate_dropdown_menu(
current_anchor_element=skyvern_element, current_anchor_element=anchor_element,
incremental_scraped=incremental_scraped, incremental_scraped=incremental_scraped,
step=step, step=step,
task=task, task=task,
) )
if dropdown_menu_element is None: if dropdown_menu_element is None:
return results return None
LOG.info( LOG.info(
"Found the dropdown menu element after clicking, triggering the sequential click logic", "Found the dropdown menu element after clicking, triggering the sequential click logic",
@@ -646,8 +688,8 @@ async def handle_click_action(
element_id=dropdown_menu_element.get_id(), element_id=dropdown_menu_element.get_id(),
) )
action_result = await select_from_emerging_elements( return await select_from_emerging_elements(
current_element_id=skyvern_element.get_id(), current_element_id=anchor_element.get_id(),
options=CustomSelectPromptOptions( options=CustomSelectPromptOptions(
field_information=action.intention if action.intention else action.reasoning, field_information=action.intention if action.intention else action.reasoning,
), # FIXME: need a better options data ), # FIXME: need a better options data
@@ -657,23 +699,6 @@ async def handle_click_action(
task=task, task=task,
) )
results.append(action_result)
return results
except NoIncrementalElementFoundForCustomSelection:
LOG.info(
"No incremental element found, skip the sequential click logic",
step_id=step.step_id,
task_id=task.task_id,
element_id=skyvern_element.get_id(),
)
return results
finally:
await incremental_scraped.stop_listen_dom_increment()
return results
async def handle_click_to_download_file_action( async def handle_click_to_download_file_action(
action: actions.ClickAction, action: actions.ClickAction,