fix chain click bug (#1189)
This commit is contained in:
LawyZheng
@@ -1102,6 +1102,7 @@ async def chain_click(
|
||||
# This automatically dismisses the dialog
|
||||
# File choosers are impossible to close if you don't expect one. Instead of dealing with it, close it!
|
||||
|
||||
dom = DomUtil(scraped_page=scraped_page, page=page)
|
||||
locator = skyvern_element.locator
|
||||
# TODO (suchintan): This should likely result in an ActionFailure -- we can figure out how to do this later!
|
||||
LOG.info("Chain click starts", action=action, locator=locator)
|
||||
@@ -1142,31 +1143,51 @@ async def chain_click(
|
||||
action_results: list[ActionResult] = [ActionFailure(FailToClick(action.element_id, msg=str(e)))]
|
||||
|
||||
if skyvern_element.get_tag_name() == "label":
|
||||
LOG.info(
|
||||
"Chain click: it's a label element. going to try for-click",
|
||||
task_id=task.task_id,
|
||||
action=action,
|
||||
element=str(skyvern_element),
|
||||
locator=locator,
|
||||
)
|
||||
try:
|
||||
if bound_element := await skyvern_element.find_label_for(
|
||||
dom=DomUtil(scraped_page=scraped_page, page=page)
|
||||
):
|
||||
LOG.info(
|
||||
"Chain click: it's a label element. going to try for-click",
|
||||
task_id=task.task_id,
|
||||
action=action,
|
||||
element=str(skyvern_element),
|
||||
locator=locator,
|
||||
)
|
||||
if bound_element := await skyvern_element.find_label_for(dom=dom):
|
||||
await bound_element.get_locator().click(timeout=timeout)
|
||||
action_results.append(ActionSuccess())
|
||||
return action_results
|
||||
except Exception as e:
|
||||
action_results.append(ActionFailure(FailToClick(action.element_id, anchor="for", msg=str(e))))
|
||||
else:
|
||||
LOG.info(
|
||||
"Chain click: it's a non-label element. going to find the bound label element by attribute id and click",
|
||||
task_id=task.task_id,
|
||||
action=action,
|
||||
element=str(skyvern_element),
|
||||
locator=locator,
|
||||
)
|
||||
|
||||
try:
|
||||
# sometimes the element is the direct chidren of the label, instead of using for="xx" attribute
|
||||
# since it's a click action, the target element we're searching should only be INPUT
|
||||
LOG.info(
|
||||
"Chain click: it's a label element. going to check for input of the direct chidren",
|
||||
task_id=task.task_id,
|
||||
action=action,
|
||||
element=str(skyvern_element),
|
||||
locator=locator,
|
||||
)
|
||||
if bound_element := await skyvern_element.find_element_in_label_children(
|
||||
dom=dom, element_type=InteractiveElement.INPUT
|
||||
):
|
||||
await bound_element.get_locator().click(timeout=timeout)
|
||||
action_results.append(ActionSuccess())
|
||||
return action_results
|
||||
except Exception as e:
|
||||
action_results.append(
|
||||
ActionFailure(FailToClick(action.element_id, anchor="direct_children", msg=str(e)))
|
||||
)
|
||||
|
||||
else:
|
||||
try:
|
||||
LOG.info(
|
||||
"Chain click: it's a non-label element. going to find the bound label element by attribute id and click",
|
||||
task_id=task.task_id,
|
||||
action=action,
|
||||
element=str(skyvern_element),
|
||||
locator=locator,
|
||||
)
|
||||
if bound_locator := await skyvern_element.find_bound_label_by_attr_id():
|
||||
await bound_locator.click(timeout=timeout)
|
||||
action_results.append(ActionSuccess())
|
||||
@@ -1174,10 +1195,27 @@ async def chain_click(
|
||||
except Exception as e:
|
||||
action_results.append(ActionFailure(FailToClick(action.element_id, anchor="attr_id", msg=str(e))))
|
||||
|
||||
try:
|
||||
# sometimes the element is the direct chidren of the label, instead of using for="xx" attribute
|
||||
# so we check the direct parent if it's a label element
|
||||
LOG.info(
|
||||
"Chain click: it's a non-label element. going to find the bound label element by direct parent",
|
||||
task_id=task.task_id,
|
||||
action=action,
|
||||
element=str(skyvern_element),
|
||||
locator=locator,
|
||||
)
|
||||
if bound_locator := await skyvern_element.find_bound_label_by_direct_parent():
|
||||
await bound_locator.click(timeout=timeout)
|
||||
action_results.append(ActionSuccess())
|
||||
return action_results
|
||||
except Exception as e:
|
||||
action_results.append(ActionFailure(FailToClick(action.element_id, anchor="direct_parent", msg=str(e))))
|
||||
|
||||
if not await skyvern_element.is_visible():
|
||||
LOG.info(
|
||||
"Chain click: exit since the element is not visible on the page anymore",
|
||||
taks_id=task.task_id,
|
||||
task_id=task.task_id,
|
||||
action=action,
|
||||
element=str(skyvern_element),
|
||||
locator=locator,
|
||||
@@ -1190,7 +1228,7 @@ async def chain_click(
|
||||
if blocking_element is None:
|
||||
LOG.info(
|
||||
"Chain click: exit since the element is not blocking by any skyvern element",
|
||||
taks_id=task.task_id,
|
||||
task_id=task.task_id,
|
||||
action=action,
|
||||
element=str(skyvern_element),
|
||||
locator=locator,
|
||||
@@ -1200,7 +1238,7 @@ async def chain_click(
|
||||
try:
|
||||
LOG.debug(
|
||||
"Chain click: verifying the blocking element is parent or sibling of the target element",
|
||||
taks_id=task.task_id,
|
||||
task_id=task.task_id,
|
||||
action=action,
|
||||
element=str(blocking_element),
|
||||
locator=locator,
|
||||
@@ -1210,7 +1248,7 @@ async def chain_click(
|
||||
) or await blocking_element.is_sibling_of(await skyvern_element.get_element_handler()):
|
||||
LOG.info(
|
||||
"Chain click: element is blocked by other elements, going to click on the blocking element",
|
||||
taks_id=task.task_id,
|
||||
task_id=task.task_id,
|
||||
action=action,
|
||||
element=str(blocking_element),
|
||||
locator=locator,
|
||||
|
||||
@@ -327,6 +327,14 @@ class SkyvernElement:
|
||||
return None
|
||||
return await dom.get_skyvern_element_by_id(blocking_element_id)
|
||||
|
||||
async def find_element_in_label_children(
|
||||
self, dom: DomUtil, element_type: InteractiveElement
|
||||
) -> SkyvernElement | None:
|
||||
element_id = self.find_element_id_in_label_children(element_type=element_type)
|
||||
if not element_id:
|
||||
return None
|
||||
return await dom.get_skyvern_element_by_id(element_id=element_id)
|
||||
|
||||
def find_element_id_in_label_children(self, element_type: InteractiveElement) -> str | None:
|
||||
tag_name = self.get_tag_name()
|
||||
if tag_name != "label":
|
||||
@@ -397,6 +405,28 @@ class SkyvernElement:
|
||||
|
||||
return None
|
||||
|
||||
async def find_bound_label_by_direct_parent(
|
||||
self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
|
||||
) -> Locator | None:
|
||||
if self.get_tag_name() == "label":
|
||||
return None
|
||||
|
||||
parent_locator = self.get_locator().locator("..")
|
||||
cnt = await parent_locator.count()
|
||||
if cnt != 1:
|
||||
return None
|
||||
|
||||
timeout_sec = timeout / 1000
|
||||
async with asyncio.timeout(timeout_sec):
|
||||
tag_name: str | None = await parent_locator.evaluate("el => el.tagName")
|
||||
if not tag_name:
|
||||
return None
|
||||
|
||||
if tag_name.lower() != "label":
|
||||
return None
|
||||
|
||||
return parent_locator
|
||||
|
||||
async def find_selectable_child(self, dom: DomUtil) -> SkyvernElement | None:
|
||||
# BFS to find the first selectable child
|
||||
index = 0
|
||||
|
||||
Reference in New Issue
Block a user