vodorod/Dorod-Sky
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix focus triggers dropdown menu (#873)

Browse Source
This commit is contained in:
LawyZheng
2024-09-23 22:36:37 +08:00
committed by GitHub
parent ae9ca27919
commit e59eb2b2bc
1 changed files with 85 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
skyvern/webeye/actions/handler.py
View File

@@ -44,7 +44,6 @@ from skyvern.forge.sdk.api.files import (
get_number_of_files_in_directory, get_number_of_files_in_directory,
get_path_for_workflow_download_directory, get_path_for_workflow_download_directory,
) )
from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandler
from skyvern.forge.sdk.core.aiohttp_helper import aiohttp_post from skyvern.forge.sdk.core.aiohttp_helper import aiohttp_post
from skyvern.forge.sdk.core.security import generate_skyvern_signature from skyvern.forge.sdk.core.security import generate_skyvern_signature
from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType
@@ -460,7 +459,6 @@ async def handle_input_text_action(
dom=dom, dom=dom,
skyvern_frame=skyvern_frame, skyvern_frame=skyvern_frame,
incremental_scraped=incremental_scraped, incremental_scraped=incremental_scraped,
llm_handler=app.SECONDARY_LLM_API_HANDLER,
step=step, step=step,
task=task, task=task,
target_value=text, target_value=text,
@@ -728,6 +726,7 @@ async def handle_select_option_action(
timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame()) skyvern_frame = await SkyvernFrame.create_instance(skyvern_element.get_frame())
incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame) incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame)
dropdown_element: SkyvernElement | None = None
is_open = False is_open = False
suggested_value: str | None = None suggested_value: str | None = None
results: list[ActionResult] = [] results: list[ActionResult] = []
@@ -735,22 +734,42 @@ async def handle_select_option_action(
try: try:
await incremental_scraped.start_listen_dom_increment() await incremental_scraped.start_listen_dom_increment()
await skyvern_element.focus() await skyvern_element.focus()
# wait 2s for options to load
await asyncio.sleep(2)
try: incremental_element = await incremental_scraped.get_incremental_element_tree(
await skyvern_element.get_locator().click(timeout=timeout) clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),
except Exception: )
if len(incremental_element) > 0:
LOG.info( LOG.info(
"fail to open dropdown by clicking, try to press ArrowDown to open", "Focus trigger DOM element change, confirm if it's a dropdown showing up",
element_id=skyvern_element.get_id(),
task_id=task.task_id, task_id=task.task_id,
step_id=step.step_id, step_id=step.step_id,
) )
await skyvern_element.focus() dropdown_element = await locate_dropdown_menu(
await skyvern_element.press_key("ArrowDown") incremental_scraped=incremental_scraped,
step=step,
task=task,
)
if dropdown_element is not None:
is_open = True
# wait 5s for options to load if not is_open:
await asyncio.sleep(5) try:
is_open = True await skyvern_element.get_locator().click(timeout=timeout)
except Exception:
LOG.info(
"fail to open dropdown by clicking, try to press ArrowDown to open",
element_id=skyvern_element.get_id(),
task_id=task.task_id,
step_id=step.step_id,
)
await skyvern_element.focus()
await skyvern_element.press_key("ArrowDown")
# wait 5s for options to load
await asyncio.sleep(5)
is_open = True
incremental_element = await incremental_scraped.get_incremental_element_tree( incremental_element = await incremental_scraped.get_incremental_element_tree(
clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]), clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),
@@ -765,9 +784,9 @@ async def handle_select_option_action(
dom=dom, dom=dom,
skyvern_frame=skyvern_frame, skyvern_frame=skyvern_frame,
incremental_scraped=incremental_scraped, incremental_scraped=incremental_scraped,
llm_handler=app.SECONDARY_LLM_API_HANDLER,
step=step, step=step,
task=task, task=task,
dropdown_menu_element=dropdown_element,
force_select=True, force_select=True,
) )
# force_select won't return None result # force_select won't return None result
@@ -785,6 +804,8 @@ async def handle_select_option_action(
await skyvern_element.scroll_into_view() await skyvern_element.scroll_into_view()
await skyvern_element.coordinate_click(page=page) await skyvern_element.coordinate_click(page=page)
await skyvern_element.press_key("Escape") await skyvern_element.press_key("Escape")
is_open = False
dropdown_element = None
await skyvern_element.blur() await skyvern_element.blur()
await incremental_scraped.stop_listen_dom_increment() await incremental_scraped.stop_listen_dom_increment()
@@ -799,20 +820,40 @@ async def handle_select_option_action(
await incremental_scraped.start_listen_dom_increment() await incremental_scraped.start_listen_dom_increment()
timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
await skyvern_element.focus() await skyvern_element.focus()
# wait 2s for options to load
await asyncio.sleep(2)
try: incremental_element = await incremental_scraped.get_incremental_element_tree(
await skyvern_element.get_locator().click(timeout=timeout) clean_and_remove_element_tree_factory(task=task, step=step, check_exist_funcs=[dom.check_id_in_dom]),
except Exception: )
if len(incremental_element) > 0:
LOG.info( LOG.info(
"fail to open dropdown by clicking, try to press arrow down to open", "Focus trigger DOM element change, confirm if it's a dropdown showing up",
element_id=skyvern_element.get_id(),
task_id=task.task_id, task_id=task.task_id,
step_id=step.step_id, step_id=step.step_id,
) )
await skyvern_element.focus() dropdown_element = await locate_dropdown_menu(
await skyvern_element.press_key("ArrowDown") incremental_scraped=incremental_scraped,
await asyncio.sleep(5) step=step,
is_open = True task=task,
)
if dropdown_element is not None:
is_open = True
if not is_open:
try:
await skyvern_element.get_locator().click(timeout=timeout)
except Exception:
LOG.info(
"fail to open dropdown by clicking, try to press arrow down to open",
element_id=skyvern_element.get_id(),
task_id=task.task_id,
step_id=step.step_id,
)
await skyvern_element.focus()
await skyvern_element.press_key("ArrowDown")
await asyncio.sleep(5)
is_open = True
result = await select_from_dropdown_by_value( result = await select_from_dropdown_by_value(
value=suggested_value, value=suggested_value,
@@ -820,9 +861,9 @@ async def handle_select_option_action(
dom=dom, dom=dom,
skyvern_frame=skyvern_frame, skyvern_frame=skyvern_frame,
incremental_scraped=incremental_scraped, incremental_scraped=incremental_scraped,
llm_handler=app.SECONDARY_LLM_API_HANDLER,
task=task, task=task,
step=step, step=step,
dropdown_menu_element=dropdown_element,
) )
results.append(result) results.append(result)
return results return results
@@ -1350,9 +1391,9 @@ async def sequentially_select_from_dropdown(
dom: DomUtil, dom: DomUtil,
skyvern_frame: SkyvernFrame, skyvern_frame: SkyvernFrame,
incremental_scraped: IncrementalScrapePage, incremental_scraped: IncrementalScrapePage,
llm_handler: LLMAPIHandler,
step: Step, step: Step,
task: Task, task: Task,
dropdown_menu_element: SkyvernElement | None = None,
force_select: bool = False, force_select: bool = False,
target_value: str = "", target_value: str = "",
) -> tuple[ActionResult | None, str | None]: ) -> tuple[ActionResult | None, str | None]:
@@ -1367,7 +1408,7 @@ async def sequentially_select_from_dropdown(
element_id=action.element_id, element_id=action.element_id,
elements=dom.scraped_page.build_element_tree(ElementTreeFormat.HTML), elements=dom.scraped_page.build_element_tree(ElementTreeFormat.HTML),
) )
json_response = await llm_handler(prompt=prompt, step=step) json_response = await app.SECONDARY_LLM_API_HANDLER(prompt=prompt, step=step)
input_or_select_context = InputOrSelectContext.model_validate(json_response) input_or_select_context = InputOrSelectContext.model_validate(json_response)
LOG.info( LOG.info(
"Parsed input/select context", "Parsed input/select context",
@@ -1388,10 +1429,10 @@ async def sequentially_select_from_dropdown(
page=page, page=page,
skyvern_frame=skyvern_frame, skyvern_frame=skyvern_frame,
incremental_scraped=incremental_scraped, incremental_scraped=incremental_scraped,
llm_handler=llm_handler,
check_exist_funcs=check_exist_funcs, check_exist_funcs=check_exist_funcs,
step=step, step=step,
task=task, task=task,
dropdown_menu_element=dropdown_menu_element,
select_history=select_history, select_history=select_history,
force_select=force_select, force_select=force_select,
target_value=target_value, target_value=target_value,
@@ -1463,10 +1504,10 @@ async def select_from_dropdown(
page: Page, page: Page,
skyvern_frame: SkyvernFrame, skyvern_frame: SkyvernFrame,
incremental_scraped: IncrementalScrapePage, incremental_scraped: IncrementalScrapePage,
llm_handler: LLMAPIHandler,
check_exist_funcs: list[CheckExistIDFunc], check_exist_funcs: list[CheckExistIDFunc],
step: Step, step: Step,
task: Task, task: Task,
dropdown_menu_element: SkyvernElement | None = None,
select_history: list[CustomSingleSelectResult] | None = None, select_history: list[CustomSingleSelectResult] | None = None,
force_select: bool = False, force_select: bool = False,
target_value: str = "", target_value: str = "",
@@ -1483,12 +1524,12 @@ async def select_from_dropdown(
timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
dropdown_menu_element = await locate_dropdown_menu( if dropdown_menu_element is None:
incremental_scraped=incremental_scraped, dropdown_menu_element = await locate_dropdown_menu(
llm_handler=llm_handler, incremental_scraped=incremental_scraped,
step=step, step=step,
task=task, task=task,
) )
single_select_result.dropdown_menu = dropdown_menu_element single_select_result.dropdown_menu = dropdown_menu_element
if not force_select and dropdown_menu_element is None: if not force_select and dropdown_menu_element is None:
@@ -1534,7 +1575,7 @@ async def select_from_dropdown(
step_id=step.step_id, step_id=step.step_id,
task_id=task.task_id, task_id=task.task_id,
) )
json_response = await llm_handler(prompt=prompt, step=step) json_response = await app.SECONDARY_LLM_API_HANDLER(prompt=prompt, step=step)
value: str | None = json_response.get("value", None) value: str | None = json_response.get("value", None)
single_select_result.value = value single_select_result.value = value
select_reason: str | None = json_response.get("reasoning", None) select_reason: str | None = json_response.get("reasoning", None)
@@ -1603,9 +1644,9 @@ async def select_from_dropdown_by_value(
skyvern_frame: SkyvernFrame, skyvern_frame: SkyvernFrame,
dom: DomUtil, dom: DomUtil,
incremental_scraped: IncrementalScrapePage, incremental_scraped: IncrementalScrapePage,
llm_handler: LLMAPIHandler,
task: Task, task: Task,
step: Step, step: Step,
dropdown_menu_element: SkyvernElement | None = None,
) -> ActionResult: ) -> ActionResult:
timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
await incremental_scraped.get_incremental_element_tree( await incremental_scraped.get_incremental_element_tree(
@@ -1617,12 +1658,12 @@ async def select_from_dropdown_by_value(
await element_locator.click(timeout=timeout) await element_locator.click(timeout=timeout)
return ActionSuccess() return ActionSuccess()
dropdown_menu_element = await locate_dropdown_menu( if dropdown_menu_element is None:
incremental_scraped=incremental_scraped, dropdown_menu_element = await locate_dropdown_menu(
llm_handler=llm_handler, incremental_scraped=incremental_scraped,
step=step, step=step,
task=task, task=task,
) )
if not dropdown_menu_element: if not dropdown_menu_element:
raise NoElementMatchedForTargetOption(target=value, reason="No value matched") raise NoElementMatchedForTargetOption(target=value, reason="No value matched")
@@ -1673,7 +1714,6 @@ async def select_from_dropdown_by_value(
async def locate_dropdown_menu( async def locate_dropdown_menu(
incremental_scraped: IncrementalScrapePage, incremental_scraped: IncrementalScrapePage,
llm_handler: LLMAPIHandler,
step: Step, step: Step,
task: Task, task: Task,
) -> SkyvernElement | None: ) -> SkyvernElement | None:
@@ -1755,7 +1795,9 @@ async def locate_dropdown_menu(
task_id=task.task_id, task_id=task.task_id,
element=element_dict, element=element_dict,
) )
json_response = await llm_handler(prompt=dropdown_confirm_prompt, screenshots=[screenshot], step=step) json_response = await app.SECONDARY_LLM_API_HANDLER(
prompt=dropdown_confirm_prompt, screenshots=[screenshot], step=step
)
is_opened_dropdown_menu = json_response.get("is_opened_dropdown_menu") is_opened_dropdown_menu = json_response.get("is_opened_dropdown_menu")
if is_opened_dropdown_menu: if is_opened_dropdown_menu:
LOG.info( LOG.info(