refactor select handler (#565)

This commit is contained in:
LawyZheng
2024-07-09 02:22:16 +08:00
committed by GitHub
parent 7a75bb3741
commit bcbd738326
4 changed files with 309 additions and 113 deletions

View File

@@ -313,6 +313,11 @@ class ElementIsNotSelect2Dropdown(SkyvernException):
super().__init__(f"element[{element}] is not select2 dropdown. element_id={element_id}")
class ElementIsNotComboboxDropdown(SkyvernException):
def __init__(self, element_id: str, element: dict):
super().__init__(f"element[{element}] is not combobox dropdown. element_id={element_id}")
class NoneFrameError(SkyvernException):
def __init__(self, frame_id: str):
super().__init__(f"frame content is none. frame_id={frame_id}")
@@ -382,3 +387,32 @@ class TaskAlreadyCanceled(SkyvernHTTPException):
class InvalidTaskStatusTransition(SkyvernHTTPException):
def __init__(self, old_status: str, new_status: str, task_id: str):
super().__init__(f"Invalid task status transition from {old_status} to {new_status} for {task_id}")
class ErrFoundSelectableElement(SkyvernException):
def __init__(self, element_id: str, err: Exception):
super().__init__(
f"error when selecting elements in the children list. element_id={element_id}, error={repr(err)}"
)
class NoSelectableElementFound(SkyvernException):
def __init__(self, element_id: str):
super().__init__(f"No selectable elements found in the children list. element_id={element_id}")
class NoDropdownAnchorErr(SkyvernException):
def __init__(self, dropdowm_type: str, element_id: str):
super().__init__(f"No {dropdowm_type} dropdown found. element_id={element_id}")
class MultipleDropdownAnchorErr(SkyvernException):
def __init__(self, dropdowm_type: str, element_id: str):
super().__init__(f"Multiple {dropdowm_type} dropdown found. element_id={element_id}")
class FailedToGetCurrentValueOfDropdown(SkyvernException):
def __init__(self, dropdowm_type: str, element_id: str, fail_reason: str):
super().__init__(
f"Failed to get current value of {dropdowm_type} dropdown. element_id={element_id}, failure_reason={fail_reason}"
)

View File

@@ -11,6 +11,7 @@ from playwright.async_api import Locator, Page, TimeoutError
from skyvern.constants import INPUT_TEXT_TIMEOUT, REPO_ROOT_DIR
from skyvern.exceptions import (
EmptySelect,
ErrFoundSelectableElement,
FailToClick,
FailToSelectByIndex,
FailToSelectByLabel,
@@ -21,6 +22,7 @@ from skyvern.exceptions import (
MissingElement,
MissingFileUrl,
MultipleElementsFound,
NoSelectableElementFound,
OptionIndexOutOfBound,
)
from skyvern.forge import app
@@ -48,7 +50,7 @@ from skyvern.webeye.actions.actions import (
from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccess
from skyvern.webeye.browser_factory import BrowserState
from skyvern.webeye.scraper.scraper import ScrapedPage
from skyvern.webeye.utils.dom import DomUtil, InteractiveElement, SkyvernElement
from skyvern.webeye.utils.dom import AbstractSelectDropdown, DomUtil, SkyvernElement
LOG = structlog.get_logger()
TEXT_INPUT_DELAY = 10 # 10ms between each character input
@@ -427,79 +429,73 @@ async def handle_select_option_action(
element_dict=element_dict,
)
# if element is not a select option, prioritize clicking the linked element if any
if tag_name != "select" and "linked_element" in element_dict:
if not await skyvern_element.is_selectable():
# 1. find from children
# TODO: 2. find from siblings and their chidren
LOG.info(
"SelectOptionAction is not on a select tag and found a linked element",
"Element is not selectable, try to find the selectable element in the chidren",
tag_name=tag_name,
action=action,
linked_element=element_dict["linked_element"],
)
listbox_click_success = await click_listbox_option(scraped_page, page, action, element_dict["linked_element"])
if listbox_click_success:
LOG.info(
"Successfully clicked linked element",
action=action,
linked_element=element_dict["linked_element"],
)
return [ActionSuccess()]
LOG.warning(
"Failed to click linked element",
action=action,
linked_element=element_dict["linked_element"],
)
# check if the element is an a tag first. If yes, click it instead of selecting the option
if tag_name == "label":
# label pointed to select2 <a> element
select2_element_id: str | None = None
# search <a> anchor first and then search <input> anchor
select2_element_id = skyvern_element.find_element_id_in_label_children(InteractiveElement.A)
if select2_element_id is None:
select2_element_id = skyvern_element.find_element_id_in_label_children(InteractiveElement.INPUT)
if select2_element_id is not None:
select2_skyvern_element = await dom.get_skyvern_element_by_id(element_id=select2_element_id)
if await select2_skyvern_element.is_select2_dropdown():
LOG.info(
"SelectOptionAction is on <label>. take the action on the real select2 element",
action=action,
select2_element_id=select2_element_id,
)
select_action = SelectOptionAction(element_id=select2_element_id, option=action.option)
return await handle_select_option_action(select_action, page, scraped_page, task, step)
# handler the select action on <label>
if select_element_id := get_select_id_in_label_children(scraped_page, action.element_id):
LOG.info(
"SelectOptionAction is on <label>. take the action on the real <select>",
selectable_child: SkyvernElement | None = None
try:
selectable_child = await skyvern_element.find_selectable_child(dom=dom)
except Exception as e:
LOG.error(
"Failed to find selectable element in chidren",
exc_info=True,
tag_name=tag_name,
action=action,
select_element_id=select_element_id,
)
select_action = SelectOptionAction(element_id=select_element_id, option=action.option)
return await handle_select_option_action(select_action, page, scraped_page, task, step)
return [ActionFailure(ErrFoundSelectableElement(action.element_id, e))]
# handle the select action on <label> of checkbox/radio
if checkbox_element_id := get_checkbox_id_in_label_children(scraped_page, action.element_id):
LOG.info(
"SelectOptionAction is on <label> of <input> checkbox/radio. take the action on the real <input> checkbox/radio",
if selectable_child is None:
LOG.error(
"No selectable element found in chidren",
tag_name=tag_name,
action=action,
checkbox_element_id=checkbox_element_id,
)
select_action = SelectOptionAction(element_id=checkbox_element_id, option=action.option)
return await handle_select_option_action(select_action, page, scraped_page, task, step)
return [ActionFailure(NoSelectableElementFound(action.element_id))]
return [ActionFailure(Exception("No element pointed by the label found"))]
elif await skyvern_element.is_select2_dropdown():
LOG.info(
"Found selectable element in the children",
tag_name=selectable_child.get_tag_name(),
element_id=selectable_child.get_id(),
)
select_action = SelectOptionAction(element_id=selectable_child.get_id(), option=action.option)
return await handle_select_option_action(select_action, page, scraped_page, task, step)
select_framework: AbstractSelectDropdown | None = None
if await skyvern_element.is_combobox_dropdown():
LOG.info(
"This is a combobox dropdown",
action=action,
)
select_framework = await skyvern_element.get_combobox_dropdown()
if await skyvern_element.is_select2_dropdown():
LOG.info(
"This is a select2 dropdown",
action=action,
)
select_framework = await skyvern_element.get_select2_dropdown()
if select_framework is not None:
timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
select2_element = await skyvern_element.get_select2_dropdown()
try:
current_value = await select_framework.get_current_value()
if current_value == action.option.label or current_value == action.option.value:
return [ActionSuccess()]
except Exception:
LOG.info(
"failed to confirm if the select option has been done, force to take the action again.",
exc_info=True,
)
await select2_element.open()
options = await select2_element.get_options()
await select_framework.open()
options = await select_framework.get_options()
result: List[ActionResult] = []
# select by label first, then by index
@@ -510,22 +506,22 @@ async def handle_select_option_action(
option_index = option.get("optionIndex", None)
if option_index is None:
LOG.warning(
"Select2 option index is None",
f"{select_framework.name()} option index is None",
option=option,
)
continue
if action.option.label == option_content or action.option.value == option_content:
await select2_element.select_by_index(index=option_index, timeout=timeout)
await select_framework.select_by_index(index=option_index, timeout=timeout)
result.append(ActionSuccess())
return result
LOG.info(
"no target select2 option matched by label, try to select by index",
f"no target {select_framework.name()} option matched by label, try to select by index",
action=action,
)
except Exception as e:
result.append(ActionFailure(e))
LOG.info(
"failed to select by label in select2, try to select by index",
f"failed to select by label in {select_framework.name()}, try to select by index",
exc_info=True,
action=action,
)
@@ -542,13 +538,13 @@ async def handle_select_option_action(
option_content=option_content,
action=action,
)
await select2_element.select_by_index(index=action.option.index, timeout=timeout)
await select_framework.select_by_index(index=action.option.index, timeout=timeout)
result.append(ActionSuccess())
return result
except Exception:
result.append(ActionFailure(FailToSelectByIndex(action.element_id)))
LOG.info(
"failed to select by index in select2",
f"failed to select by index in {select_framework.name()}",
exc_info=True,
action=action,
)
@@ -558,57 +554,22 @@ async def handle_select_option_action(
if isinstance(result[-1], ActionFailure):
LOG.info(
"Failed to select a select2 option, close the dropdown",
f"Failed to select a {select_framework.name()} option, close the dropdown",
action=action,
)
await select2_element.close()
await select_framework.close()
return result
elif tag_name == "ul" or tag_name == "div" or tag_name == "li":
# DEPRECATED: This was used for handle select2 dropdown, and isn't used anymore.
# if the role is listbox, find the option with the "label" or "value" and click that option element
# references:
# https://developer.mozilla.org/en-US/docs/Web/Accessibility/ARIA/Roles/listbox_role
# https://developer.mozilla.org/en-US/docs/Web/Accessibility/ARIA/Roles/option_role
role_attribute = await locator.get_attribute("role")
if role_attribute == "listbox":
LOG.info(
"SelectOptionAction on a listbox element. Searching for the option and click it",
action=action,
)
# use playwright to click the option
# clickOption is defined in domUtils.js
option_locator = locator.locator('[role="option"]')
option_num = await option_locator.count()
if action.option.index and action.option.index < option_num:
try:
await option_locator.nth(action.option.index).click(timeout=2000)
return [ActionSuccess()]
except Exception as e:
LOG.error("Failed to click option", action=action, exc_info=True)
return [ActionFailure(e)]
return [ActionFailure(Exception("SelectOption option index is missing"))]
elif role_attribute == "option":
LOG.info(
"SelectOptionAction on an option element. Clicking the option",
action=action,
)
# click the option element
click_action = ClickAction(element_id=action.element_id)
return await chain_click(task, scraped_page, page, click_action, skyvern_element)
else:
LOG.error(
"SelectOptionAction on a non-listbox element. Cannot handle this action",
)
return [ActionFailure(Exception("Cannot handle SelectOptionAction on a non-listbox element"))]
elif await skyvern_element.is_checkbox():
if await skyvern_element.is_checkbox():
LOG.info(
"SelectOptionAction is on <input> checkbox",
action=action,
)
check_action = CheckboxAction(element_id=action.element_id, is_checked=True)
return await handle_checkbox_action(check_action, page, scraped_page, task, step)
elif await skyvern_element.is_radio():
if await skyvern_element.is_radio():
LOG.info(
"SelectOptionAction is on <input> radio",
action=action,

View File

@@ -324,6 +324,12 @@ async def get_select2_options(frame: Page | Frame, element: ElementHandle) -> li
return await frame.evaluate(js_script, element)
async def get_combobox_options(frame: Page | Frame, element: ElementHandle) -> list[dict[str, Any]]:
await frame.evaluate(JS_FUNCTION_DEFS)
js_script = "async (element) => await getListboxOptions(element)"
return await frame.evaluate(js_script, element)
async def get_interactable_element_tree_in_frame(
frames: list[Frame],
elements: list[dict],

View File

@@ -1,7 +1,7 @@
from __future__ import annotations
import asyncio
import typing
from abc import ABC, abstractmethod
from enum import StrEnum
import structlog
@@ -9,18 +9,22 @@ from playwright.async_api import Frame, FrameLocator, Locator, Page
from skyvern.constants import INPUT_TEXT_TIMEOUT, SKYVERN_ID_ATTR
from skyvern.exceptions import (
ElementIsNotComboboxDropdown,
ElementIsNotLabel,
ElementIsNotSelect2Dropdown,
FailedToGetCurrentValueOfDropdown,
MissingElement,
MissingElementDict,
MissingElementInCSSMap,
MissingElementInIframe,
MultipleDropdownAnchorErr,
MultipleElementsFound,
NoDropdownAnchorErr,
NoneFrameError,
SkyvernException,
)
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.webeye.scraper.scraper import ScrapedPage, get_select2_options
from skyvern.webeye.scraper.scraper import ScrapedPage, get_combobox_options, get_select2_options
LOG = structlog.get_logger()
@@ -68,6 +72,9 @@ class InteractiveElement(StrEnum):
BUTTON = "button"
SELECTABLE_ELEMENT = [InteractiveElement.INPUT, InteractiveElement.SELECT]
class SkyvernOptionType(typing.TypedDict):
optionIndex: int
text: str
@@ -96,6 +103,12 @@ class SkyvernElement:
or (tag_name == "input" and "select2-input" in element_class)
)
async def is_combobox_dropdown(self) -> bool:
tag_name = self.get_tag_name()
role = await self.get_attr("role")
haspopup = await self.get_attr("aria-haspopup")
return tag_name == InteractiveElement.INPUT and role == "combobox" and haspopup == "listbox"
async def is_checkbox(self) -> bool:
tag_name = self.get_tag_name()
if tag_name != "input":
@@ -112,6 +125,16 @@ class SkyvernElement:
button_type = await self.get_attr("type")
return button_type == "radio"
def is_interactable(self) -> bool:
return self.__static_element.get("interactable", False)
async def is_selectable(self) -> bool:
return (
await self.is_select2_dropdown()
or await self.is_combobox_dropdown()
or self.get_tag_name() in SELECTABLE_ELEMENT
)
def get_tag_name(self) -> str:
return self.__static_element.get("tagName", "")
@@ -140,6 +163,12 @@ class SkyvernElement:
return Select2Dropdown(self.get_frame(), self)
async def get_combobox_dropdown(self) -> ComboboxDropdown:
if not await self.is_combobox_dropdown():
raise ElementIsNotComboboxDropdown(self.get_id(), self.__static_element)
return ComboboxDropdown(self.get_frame(), self)
def find_element_id_in_label_children(self, element_type: InteractiveElement) -> str | None:
tag_name = self.get_tag_name()
if tag_name != "label":
@@ -155,6 +184,56 @@ class SkyvernElement:
return None
async def find_label_for(
self, dom: DomUtil, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
) -> SkyvernElement | None:
if self.get_tag_name() != "label":
return None
for_id = await self.get_attr("for")
if for_id == "":
return None
locator = self.get_frame().locator(f"#{for_id}")
# supposed to be only one element, since id is unique in the whole DOM
if await locator.count() != 1:
return None
unique_id = await locator.get_attribute(SKYVERN_ID_ATTR, timeout=timeout)
if unique_id is None:
return None
return await dom.get_skyvern_element_by_id(unique_id)
async def find_selectable_child(self, dom: DomUtil) -> SkyvernElement | None:
# BFS to find the first selectable child
index = 0
queue = [self]
while index < len(queue):
item = queue[index]
if item.is_interactable() and await item.is_selectable():
return item
try:
for_element = await item.find_label_for(dom=dom)
if for_element is not None and await for_element.is_selectable():
return for_element
except Exception:
LOG.error(
"Failed to find element by label-for",
element=item.__static_element,
exc_info=True,
)
children: list[dict] = item.__static_element.get("children", [])
for child in children:
child_id = child.get("id", "")
child_element = await dom.get_skyvern_element_by_id(child_id)
queue.append(child_element)
index += 1
return None
async def get_attr(
self,
attr_name: str,
@@ -216,28 +295,144 @@ class DomUtil:
return SkyvernElement(locator, frame_content, element)
class Select2Dropdown:
class AbstractSelectDropdown(ABC):
@abstractmethod
def name(self) -> str:
pass
@abstractmethod
async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
pass
@abstractmethod
async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
pass
@abstractmethod
async def get_current_value(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> str:
pass
@abstractmethod
async def get_options(
self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
) -> typing.List[SkyvernOptionType]:
pass
@abstractmethod
async def select_by_index(
self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
) -> None:
pass
class Select2Dropdown(AbstractSelectDropdown):
def __init__(self, frame: Page | Frame, skyvern_element: SkyvernElement) -> None:
self.skyvern_element = skyvern_element
self.frame = frame
async def __find_anchor(self, timeout: float) -> Locator:
locator = self.frame.locator("#select2-drop")
await locator.wait_for(state="visible", timeout=timeout)
cnt = await locator.count()
if cnt == 0:
raise NoDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
if cnt > 1:
raise MultipleDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
return locator
def name(self) -> str:
return "select2"
async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
await self.skyvern_element.get_locator().click(timeout=timeout)
# wait for the options to load
await asyncio.sleep(3)
await self.__find_anchor(timeout=timeout)
async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
await self.frame.locator("#select2-drop").press("Escape", timeout=timeout)
anchor = await self.__find_anchor(timeout=timeout)
await anchor.press("Escape", timeout=timeout)
async def get_current_value(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> str:
tag_name = self.skyvern_element.get_tag_name()
if tag_name == "input":
# TODO: this is multiple options case, we haven't fully supported it yet.
return ""
# check SkyvernElement.is_select2_dropdown() method, only <a> and <span> element left
# we should make sure the locator is on <a>, so we're able to find the [class="select2-chosen"] child
locator = self.skyvern_element.get_locator()
if tag_name == "span":
locator = locator.locator("..")
elif tag_name == "a":
pass
else:
raise FailedToGetCurrentValueOfDropdown(
self.name(), self.skyvern_element.get_id(), "invalid element of select2"
)
try:
return await locator.locator("span[class='select2-chosen']").text_content(timeout=timeout)
except Exception as e:
raise FailedToGetCurrentValueOfDropdown(self.name(), self.skyvern_element.get_id(), repr(e))
async def get_options(
self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
) -> typing.List[SkyvernOptionType]:
element_handler = await self.skyvern_element.get_locator().element_handle(timeout=timeout)
anchor = await self.__find_anchor(timeout=timeout)
element_handler = await anchor.element_handle(timeout=timeout)
options = await get_select2_options(self.frame, element_handler)
return typing.cast(typing.List[SkyvernOptionType], options)
async def select_by_index(
self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
) -> None:
anchor = self.frame.locator("#select2-drop li[role='option']")
await anchor.nth(index).click(timeout=timeout)
anchor = await self.__find_anchor(timeout=timeout)
options = anchor.locator("li[role='option']")
await options.nth(index).click(timeout=timeout)
class ComboboxDropdown(AbstractSelectDropdown):
def __init__(self, frame: Page | Frame, skyvern_element: SkyvernElement) -> None:
self.skyvern_element = skyvern_element
self.frame = frame
async def __find_anchor(self, timeout: float) -> Locator:
control_id = await self.skyvern_element.get_attr("aria-controls", timeout=timeout)
locator = self.frame.locator(f"#{control_id}")
await locator.wait_for(state="visible", timeout=timeout)
cnt = await locator.count()
if cnt == 0:
raise NoDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
if cnt > 1:
raise MultipleDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
return locator
def name(self) -> str:
return "combobox"
async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
await self.skyvern_element.get_locator().click(timeout=timeout)
await self.__find_anchor(timeout=timeout)
async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
await self.skyvern_element.get_locator().press("Tab", timeout=timeout)
async def get_current_value(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> str:
try:
return await self.skyvern_element.get_attr("value", dynamic=True, timeout=timeout)
except Exception as e:
raise FailedToGetCurrentValueOfDropdown(self.name(), self.skyvern_element.get_id(), repr(e))
async def get_options(
self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
) -> typing.List[SkyvernOptionType]:
anchor = await self.__find_anchor(timeout=timeout)
element_handler = await anchor.element_handle()
options = await get_combobox_options(self.frame, element_handler)
return typing.cast(typing.List[SkyvernOptionType], options)
async def select_by_index(
self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
) -> None:
anchor = await self.__find_anchor(timeout=timeout)
options = anchor.locator("li[role='option']")
await options.nth(index).click(timeout=timeout)