refactor select handler (#565)
This commit is contained in:
@@ -313,6 +313,11 @@ class ElementIsNotSelect2Dropdown(SkyvernException):
|
|||||||
super().__init__(f"element[{element}] is not select2 dropdown. element_id={element_id}")
|
super().__init__(f"element[{element}] is not select2 dropdown. element_id={element_id}")
|
||||||
|
|
||||||
|
|
||||||
|
class ElementIsNotComboboxDropdown(SkyvernException):
|
||||||
|
def __init__(self, element_id: str, element: dict):
|
||||||
|
super().__init__(f"element[{element}] is not combobox dropdown. element_id={element_id}")
|
||||||
|
|
||||||
|
|
||||||
class NoneFrameError(SkyvernException):
|
class NoneFrameError(SkyvernException):
|
||||||
def __init__(self, frame_id: str):
|
def __init__(self, frame_id: str):
|
||||||
super().__init__(f"frame content is none. frame_id={frame_id}")
|
super().__init__(f"frame content is none. frame_id={frame_id}")
|
||||||
@@ -382,3 +387,32 @@ class TaskAlreadyCanceled(SkyvernHTTPException):
|
|||||||
class InvalidTaskStatusTransition(SkyvernHTTPException):
|
class InvalidTaskStatusTransition(SkyvernHTTPException):
|
||||||
def __init__(self, old_status: str, new_status: str, task_id: str):
|
def __init__(self, old_status: str, new_status: str, task_id: str):
|
||||||
super().__init__(f"Invalid task status transition from {old_status} to {new_status} for {task_id}")
|
super().__init__(f"Invalid task status transition from {old_status} to {new_status} for {task_id}")
|
||||||
|
|
||||||
|
|
||||||
|
class ErrFoundSelectableElement(SkyvernException):
|
||||||
|
def __init__(self, element_id: str, err: Exception):
|
||||||
|
super().__init__(
|
||||||
|
f"error when selecting elements in the children list. element_id={element_id}, error={repr(err)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class NoSelectableElementFound(SkyvernException):
|
||||||
|
def __init__(self, element_id: str):
|
||||||
|
super().__init__(f"No selectable elements found in the children list. element_id={element_id}")
|
||||||
|
|
||||||
|
|
||||||
|
class NoDropdownAnchorErr(SkyvernException):
|
||||||
|
def __init__(self, dropdowm_type: str, element_id: str):
|
||||||
|
super().__init__(f"No {dropdowm_type} dropdown found. element_id={element_id}")
|
||||||
|
|
||||||
|
|
||||||
|
class MultipleDropdownAnchorErr(SkyvernException):
|
||||||
|
def __init__(self, dropdowm_type: str, element_id: str):
|
||||||
|
super().__init__(f"Multiple {dropdowm_type} dropdown found. element_id={element_id}")
|
||||||
|
|
||||||
|
|
||||||
|
class FailedToGetCurrentValueOfDropdown(SkyvernException):
|
||||||
|
def __init__(self, dropdowm_type: str, element_id: str, fail_reason: str):
|
||||||
|
super().__init__(
|
||||||
|
f"Failed to get current value of {dropdowm_type} dropdown. element_id={element_id}, failure_reason={fail_reason}"
|
||||||
|
)
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ from playwright.async_api import Locator, Page, TimeoutError
|
|||||||
from skyvern.constants import INPUT_TEXT_TIMEOUT, REPO_ROOT_DIR
|
from skyvern.constants import INPUT_TEXT_TIMEOUT, REPO_ROOT_DIR
|
||||||
from skyvern.exceptions import (
|
from skyvern.exceptions import (
|
||||||
EmptySelect,
|
EmptySelect,
|
||||||
|
ErrFoundSelectableElement,
|
||||||
FailToClick,
|
FailToClick,
|
||||||
FailToSelectByIndex,
|
FailToSelectByIndex,
|
||||||
FailToSelectByLabel,
|
FailToSelectByLabel,
|
||||||
@@ -21,6 +22,7 @@ from skyvern.exceptions import (
|
|||||||
MissingElement,
|
MissingElement,
|
||||||
MissingFileUrl,
|
MissingFileUrl,
|
||||||
MultipleElementsFound,
|
MultipleElementsFound,
|
||||||
|
NoSelectableElementFound,
|
||||||
OptionIndexOutOfBound,
|
OptionIndexOutOfBound,
|
||||||
)
|
)
|
||||||
from skyvern.forge import app
|
from skyvern.forge import app
|
||||||
@@ -48,7 +50,7 @@ from skyvern.webeye.actions.actions import (
|
|||||||
from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccess
|
from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccess
|
||||||
from skyvern.webeye.browser_factory import BrowserState
|
from skyvern.webeye.browser_factory import BrowserState
|
||||||
from skyvern.webeye.scraper.scraper import ScrapedPage
|
from skyvern.webeye.scraper.scraper import ScrapedPage
|
||||||
from skyvern.webeye.utils.dom import DomUtil, InteractiveElement, SkyvernElement
|
from skyvern.webeye.utils.dom import AbstractSelectDropdown, DomUtil, SkyvernElement
|
||||||
|
|
||||||
LOG = structlog.get_logger()
|
LOG = structlog.get_logger()
|
||||||
TEXT_INPUT_DELAY = 10 # 10ms between each character input
|
TEXT_INPUT_DELAY = 10 # 10ms between each character input
|
||||||
@@ -427,79 +429,73 @@ async def handle_select_option_action(
|
|||||||
element_dict=element_dict,
|
element_dict=element_dict,
|
||||||
)
|
)
|
||||||
|
|
||||||
# if element is not a select option, prioritize clicking the linked element if any
|
if not await skyvern_element.is_selectable():
|
||||||
if tag_name != "select" and "linked_element" in element_dict:
|
# 1. find from children
|
||||||
|
# TODO: 2. find from siblings and their chidren
|
||||||
LOG.info(
|
LOG.info(
|
||||||
"SelectOptionAction is not on a select tag and found a linked element",
|
"Element is not selectable, try to find the selectable element in the chidren",
|
||||||
|
tag_name=tag_name,
|
||||||
action=action,
|
action=action,
|
||||||
linked_element=element_dict["linked_element"],
|
|
||||||
)
|
|
||||||
listbox_click_success = await click_listbox_option(scraped_page, page, action, element_dict["linked_element"])
|
|
||||||
if listbox_click_success:
|
|
||||||
LOG.info(
|
|
||||||
"Successfully clicked linked element",
|
|
||||||
action=action,
|
|
||||||
linked_element=element_dict["linked_element"],
|
|
||||||
)
|
|
||||||
return [ActionSuccess()]
|
|
||||||
LOG.warning(
|
|
||||||
"Failed to click linked element",
|
|
||||||
action=action,
|
|
||||||
linked_element=element_dict["linked_element"],
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# check if the element is an a tag first. If yes, click it instead of selecting the option
|
selectable_child: SkyvernElement | None = None
|
||||||
if tag_name == "label":
|
try:
|
||||||
# label pointed to select2 <a> element
|
selectable_child = await skyvern_element.find_selectable_child(dom=dom)
|
||||||
select2_element_id: str | None = None
|
except Exception as e:
|
||||||
# search <a> anchor first and then search <input> anchor
|
LOG.error(
|
||||||
select2_element_id = skyvern_element.find_element_id_in_label_children(InteractiveElement.A)
|
"Failed to find selectable element in chidren",
|
||||||
if select2_element_id is None:
|
exc_info=True,
|
||||||
select2_element_id = skyvern_element.find_element_id_in_label_children(InteractiveElement.INPUT)
|
tag_name=tag_name,
|
||||||
|
|
||||||
if select2_element_id is not None:
|
|
||||||
select2_skyvern_element = await dom.get_skyvern_element_by_id(element_id=select2_element_id)
|
|
||||||
if await select2_skyvern_element.is_select2_dropdown():
|
|
||||||
LOG.info(
|
|
||||||
"SelectOptionAction is on <label>. take the action on the real select2 element",
|
|
||||||
action=action,
|
|
||||||
select2_element_id=select2_element_id,
|
|
||||||
)
|
|
||||||
select_action = SelectOptionAction(element_id=select2_element_id, option=action.option)
|
|
||||||
return await handle_select_option_action(select_action, page, scraped_page, task, step)
|
|
||||||
|
|
||||||
# handler the select action on <label>
|
|
||||||
if select_element_id := get_select_id_in_label_children(scraped_page, action.element_id):
|
|
||||||
LOG.info(
|
|
||||||
"SelectOptionAction is on <label>. take the action on the real <select>",
|
|
||||||
action=action,
|
action=action,
|
||||||
select_element_id=select_element_id,
|
|
||||||
)
|
)
|
||||||
select_action = SelectOptionAction(element_id=select_element_id, option=action.option)
|
return [ActionFailure(ErrFoundSelectableElement(action.element_id, e))]
|
||||||
return await handle_select_option_action(select_action, page, scraped_page, task, step)
|
|
||||||
|
|
||||||
# handle the select action on <label> of checkbox/radio
|
if selectable_child is None:
|
||||||
if checkbox_element_id := get_checkbox_id_in_label_children(scraped_page, action.element_id):
|
LOG.error(
|
||||||
LOG.info(
|
"No selectable element found in chidren",
|
||||||
"SelectOptionAction is on <label> of <input> checkbox/radio. take the action on the real <input> checkbox/radio",
|
tag_name=tag_name,
|
||||||
action=action,
|
action=action,
|
||||||
checkbox_element_id=checkbox_element_id,
|
|
||||||
)
|
)
|
||||||
select_action = SelectOptionAction(element_id=checkbox_element_id, option=action.option)
|
return [ActionFailure(NoSelectableElementFound(action.element_id))]
|
||||||
return await handle_select_option_action(select_action, page, scraped_page, task, step)
|
|
||||||
|
|
||||||
return [ActionFailure(Exception("No element pointed by the label found"))]
|
LOG.info(
|
||||||
elif await skyvern_element.is_select2_dropdown():
|
"Found selectable element in the children",
|
||||||
|
tag_name=selectable_child.get_tag_name(),
|
||||||
|
element_id=selectable_child.get_id(),
|
||||||
|
)
|
||||||
|
select_action = SelectOptionAction(element_id=selectable_child.get_id(), option=action.option)
|
||||||
|
return await handle_select_option_action(select_action, page, scraped_page, task, step)
|
||||||
|
|
||||||
|
select_framework: AbstractSelectDropdown | None = None
|
||||||
|
|
||||||
|
if await skyvern_element.is_combobox_dropdown():
|
||||||
|
LOG.info(
|
||||||
|
"This is a combobox dropdown",
|
||||||
|
action=action,
|
||||||
|
)
|
||||||
|
select_framework = await skyvern_element.get_combobox_dropdown()
|
||||||
|
if await skyvern_element.is_select2_dropdown():
|
||||||
LOG.info(
|
LOG.info(
|
||||||
"This is a select2 dropdown",
|
"This is a select2 dropdown",
|
||||||
action=action,
|
action=action,
|
||||||
)
|
)
|
||||||
|
select_framework = await skyvern_element.get_select2_dropdown()
|
||||||
|
|
||||||
|
if select_framework is not None:
|
||||||
timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
|
timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
|
||||||
|
|
||||||
select2_element = await skyvern_element.get_select2_dropdown()
|
try:
|
||||||
|
current_value = await select_framework.get_current_value()
|
||||||
|
if current_value == action.option.label or current_value == action.option.value:
|
||||||
|
return [ActionSuccess()]
|
||||||
|
except Exception:
|
||||||
|
LOG.info(
|
||||||
|
"failed to confirm if the select option has been done, force to take the action again.",
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
|
||||||
await select2_element.open()
|
await select_framework.open()
|
||||||
options = await select2_element.get_options()
|
options = await select_framework.get_options()
|
||||||
|
|
||||||
result: List[ActionResult] = []
|
result: List[ActionResult] = []
|
||||||
# select by label first, then by index
|
# select by label first, then by index
|
||||||
@@ -510,22 +506,22 @@ async def handle_select_option_action(
|
|||||||
option_index = option.get("optionIndex", None)
|
option_index = option.get("optionIndex", None)
|
||||||
if option_index is None:
|
if option_index is None:
|
||||||
LOG.warning(
|
LOG.warning(
|
||||||
"Select2 option index is None",
|
f"{select_framework.name()} option index is None",
|
||||||
option=option,
|
option=option,
|
||||||
)
|
)
|
||||||
continue
|
continue
|
||||||
if action.option.label == option_content or action.option.value == option_content:
|
if action.option.label == option_content or action.option.value == option_content:
|
||||||
await select2_element.select_by_index(index=option_index, timeout=timeout)
|
await select_framework.select_by_index(index=option_index, timeout=timeout)
|
||||||
result.append(ActionSuccess())
|
result.append(ActionSuccess())
|
||||||
return result
|
return result
|
||||||
LOG.info(
|
LOG.info(
|
||||||
"no target select2 option matched by label, try to select by index",
|
f"no target {select_framework.name()} option matched by label, try to select by index",
|
||||||
action=action,
|
action=action,
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
result.append(ActionFailure(e))
|
result.append(ActionFailure(e))
|
||||||
LOG.info(
|
LOG.info(
|
||||||
"failed to select by label in select2, try to select by index",
|
f"failed to select by label in {select_framework.name()}, try to select by index",
|
||||||
exc_info=True,
|
exc_info=True,
|
||||||
action=action,
|
action=action,
|
||||||
)
|
)
|
||||||
@@ -542,13 +538,13 @@ async def handle_select_option_action(
|
|||||||
option_content=option_content,
|
option_content=option_content,
|
||||||
action=action,
|
action=action,
|
||||||
)
|
)
|
||||||
await select2_element.select_by_index(index=action.option.index, timeout=timeout)
|
await select_framework.select_by_index(index=action.option.index, timeout=timeout)
|
||||||
result.append(ActionSuccess())
|
result.append(ActionSuccess())
|
||||||
return result
|
return result
|
||||||
except Exception:
|
except Exception:
|
||||||
result.append(ActionFailure(FailToSelectByIndex(action.element_id)))
|
result.append(ActionFailure(FailToSelectByIndex(action.element_id)))
|
||||||
LOG.info(
|
LOG.info(
|
||||||
"failed to select by index in select2",
|
f"failed to select by index in {select_framework.name()}",
|
||||||
exc_info=True,
|
exc_info=True,
|
||||||
action=action,
|
action=action,
|
||||||
)
|
)
|
||||||
@@ -558,57 +554,22 @@ async def handle_select_option_action(
|
|||||||
|
|
||||||
if isinstance(result[-1], ActionFailure):
|
if isinstance(result[-1], ActionFailure):
|
||||||
LOG.info(
|
LOG.info(
|
||||||
"Failed to select a select2 option, close the dropdown",
|
f"Failed to select a {select_framework.name()} option, close the dropdown",
|
||||||
action=action,
|
action=action,
|
||||||
)
|
)
|
||||||
await select2_element.close()
|
await select_framework.close()
|
||||||
|
|
||||||
return result
|
return result
|
||||||
elif tag_name == "ul" or tag_name == "div" or tag_name == "li":
|
|
||||||
# DEPRECATED: This was used for handle select2 dropdown, and isn't used anymore.
|
if await skyvern_element.is_checkbox():
|
||||||
# if the role is listbox, find the option with the "label" or "value" and click that option element
|
|
||||||
# references:
|
|
||||||
# https://developer.mozilla.org/en-US/docs/Web/Accessibility/ARIA/Roles/listbox_role
|
|
||||||
# https://developer.mozilla.org/en-US/docs/Web/Accessibility/ARIA/Roles/option_role
|
|
||||||
role_attribute = await locator.get_attribute("role")
|
|
||||||
if role_attribute == "listbox":
|
|
||||||
LOG.info(
|
|
||||||
"SelectOptionAction on a listbox element. Searching for the option and click it",
|
|
||||||
action=action,
|
|
||||||
)
|
|
||||||
# use playwright to click the option
|
|
||||||
# clickOption is defined in domUtils.js
|
|
||||||
option_locator = locator.locator('[role="option"]')
|
|
||||||
option_num = await option_locator.count()
|
|
||||||
if action.option.index and action.option.index < option_num:
|
|
||||||
try:
|
|
||||||
await option_locator.nth(action.option.index).click(timeout=2000)
|
|
||||||
return [ActionSuccess()]
|
|
||||||
except Exception as e:
|
|
||||||
LOG.error("Failed to click option", action=action, exc_info=True)
|
|
||||||
return [ActionFailure(e)]
|
|
||||||
return [ActionFailure(Exception("SelectOption option index is missing"))]
|
|
||||||
elif role_attribute == "option":
|
|
||||||
LOG.info(
|
|
||||||
"SelectOptionAction on an option element. Clicking the option",
|
|
||||||
action=action,
|
|
||||||
)
|
|
||||||
# click the option element
|
|
||||||
click_action = ClickAction(element_id=action.element_id)
|
|
||||||
return await chain_click(task, scraped_page, page, click_action, skyvern_element)
|
|
||||||
else:
|
|
||||||
LOG.error(
|
|
||||||
"SelectOptionAction on a non-listbox element. Cannot handle this action",
|
|
||||||
)
|
|
||||||
return [ActionFailure(Exception("Cannot handle SelectOptionAction on a non-listbox element"))]
|
|
||||||
elif await skyvern_element.is_checkbox():
|
|
||||||
LOG.info(
|
LOG.info(
|
||||||
"SelectOptionAction is on <input> checkbox",
|
"SelectOptionAction is on <input> checkbox",
|
||||||
action=action,
|
action=action,
|
||||||
)
|
)
|
||||||
check_action = CheckboxAction(element_id=action.element_id, is_checked=True)
|
check_action = CheckboxAction(element_id=action.element_id, is_checked=True)
|
||||||
return await handle_checkbox_action(check_action, page, scraped_page, task, step)
|
return await handle_checkbox_action(check_action, page, scraped_page, task, step)
|
||||||
elif await skyvern_element.is_radio():
|
|
||||||
|
if await skyvern_element.is_radio():
|
||||||
LOG.info(
|
LOG.info(
|
||||||
"SelectOptionAction is on <input> radio",
|
"SelectOptionAction is on <input> radio",
|
||||||
action=action,
|
action=action,
|
||||||
|
|||||||
@@ -324,6 +324,12 @@ async def get_select2_options(frame: Page | Frame, element: ElementHandle) -> li
|
|||||||
return await frame.evaluate(js_script, element)
|
return await frame.evaluate(js_script, element)
|
||||||
|
|
||||||
|
|
||||||
|
async def get_combobox_options(frame: Page | Frame, element: ElementHandle) -> list[dict[str, Any]]:
|
||||||
|
await frame.evaluate(JS_FUNCTION_DEFS)
|
||||||
|
js_script = "async (element) => await getListboxOptions(element)"
|
||||||
|
return await frame.evaluate(js_script, element)
|
||||||
|
|
||||||
|
|
||||||
async def get_interactable_element_tree_in_frame(
|
async def get_interactable_element_tree_in_frame(
|
||||||
frames: list[Frame],
|
frames: list[Frame],
|
||||||
elements: list[dict],
|
elements: list[dict],
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import typing
|
import typing
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
from enum import StrEnum
|
from enum import StrEnum
|
||||||
|
|
||||||
import structlog
|
import structlog
|
||||||
@@ -9,18 +9,22 @@ from playwright.async_api import Frame, FrameLocator, Locator, Page
|
|||||||
|
|
||||||
from skyvern.constants import INPUT_TEXT_TIMEOUT, SKYVERN_ID_ATTR
|
from skyvern.constants import INPUT_TEXT_TIMEOUT, SKYVERN_ID_ATTR
|
||||||
from skyvern.exceptions import (
|
from skyvern.exceptions import (
|
||||||
|
ElementIsNotComboboxDropdown,
|
||||||
ElementIsNotLabel,
|
ElementIsNotLabel,
|
||||||
ElementIsNotSelect2Dropdown,
|
ElementIsNotSelect2Dropdown,
|
||||||
|
FailedToGetCurrentValueOfDropdown,
|
||||||
MissingElement,
|
MissingElement,
|
||||||
MissingElementDict,
|
MissingElementDict,
|
||||||
MissingElementInCSSMap,
|
MissingElementInCSSMap,
|
||||||
MissingElementInIframe,
|
MissingElementInIframe,
|
||||||
|
MultipleDropdownAnchorErr,
|
||||||
MultipleElementsFound,
|
MultipleElementsFound,
|
||||||
|
NoDropdownAnchorErr,
|
||||||
NoneFrameError,
|
NoneFrameError,
|
||||||
SkyvernException,
|
SkyvernException,
|
||||||
)
|
)
|
||||||
from skyvern.forge.sdk.settings_manager import SettingsManager
|
from skyvern.forge.sdk.settings_manager import SettingsManager
|
||||||
from skyvern.webeye.scraper.scraper import ScrapedPage, get_select2_options
|
from skyvern.webeye.scraper.scraper import ScrapedPage, get_combobox_options, get_select2_options
|
||||||
|
|
||||||
LOG = structlog.get_logger()
|
LOG = structlog.get_logger()
|
||||||
|
|
||||||
@@ -68,6 +72,9 @@ class InteractiveElement(StrEnum):
|
|||||||
BUTTON = "button"
|
BUTTON = "button"
|
||||||
|
|
||||||
|
|
||||||
|
SELECTABLE_ELEMENT = [InteractiveElement.INPUT, InteractiveElement.SELECT]
|
||||||
|
|
||||||
|
|
||||||
class SkyvernOptionType(typing.TypedDict):
|
class SkyvernOptionType(typing.TypedDict):
|
||||||
optionIndex: int
|
optionIndex: int
|
||||||
text: str
|
text: str
|
||||||
@@ -96,6 +103,12 @@ class SkyvernElement:
|
|||||||
or (tag_name == "input" and "select2-input" in element_class)
|
or (tag_name == "input" and "select2-input" in element_class)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
async def is_combobox_dropdown(self) -> bool:
|
||||||
|
tag_name = self.get_tag_name()
|
||||||
|
role = await self.get_attr("role")
|
||||||
|
haspopup = await self.get_attr("aria-haspopup")
|
||||||
|
return tag_name == InteractiveElement.INPUT and role == "combobox" and haspopup == "listbox"
|
||||||
|
|
||||||
async def is_checkbox(self) -> bool:
|
async def is_checkbox(self) -> bool:
|
||||||
tag_name = self.get_tag_name()
|
tag_name = self.get_tag_name()
|
||||||
if tag_name != "input":
|
if tag_name != "input":
|
||||||
@@ -112,6 +125,16 @@ class SkyvernElement:
|
|||||||
button_type = await self.get_attr("type")
|
button_type = await self.get_attr("type")
|
||||||
return button_type == "radio"
|
return button_type == "radio"
|
||||||
|
|
||||||
|
def is_interactable(self) -> bool:
|
||||||
|
return self.__static_element.get("interactable", False)
|
||||||
|
|
||||||
|
async def is_selectable(self) -> bool:
|
||||||
|
return (
|
||||||
|
await self.is_select2_dropdown()
|
||||||
|
or await self.is_combobox_dropdown()
|
||||||
|
or self.get_tag_name() in SELECTABLE_ELEMENT
|
||||||
|
)
|
||||||
|
|
||||||
def get_tag_name(self) -> str:
|
def get_tag_name(self) -> str:
|
||||||
return self.__static_element.get("tagName", "")
|
return self.__static_element.get("tagName", "")
|
||||||
|
|
||||||
@@ -140,6 +163,12 @@ class SkyvernElement:
|
|||||||
|
|
||||||
return Select2Dropdown(self.get_frame(), self)
|
return Select2Dropdown(self.get_frame(), self)
|
||||||
|
|
||||||
|
async def get_combobox_dropdown(self) -> ComboboxDropdown:
|
||||||
|
if not await self.is_combobox_dropdown():
|
||||||
|
raise ElementIsNotComboboxDropdown(self.get_id(), self.__static_element)
|
||||||
|
|
||||||
|
return ComboboxDropdown(self.get_frame(), self)
|
||||||
|
|
||||||
def find_element_id_in_label_children(self, element_type: InteractiveElement) -> str | None:
|
def find_element_id_in_label_children(self, element_type: InteractiveElement) -> str | None:
|
||||||
tag_name = self.get_tag_name()
|
tag_name = self.get_tag_name()
|
||||||
if tag_name != "label":
|
if tag_name != "label":
|
||||||
@@ -155,6 +184,56 @@ class SkyvernElement:
|
|||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
async def find_label_for(
|
||||||
|
self, dom: DomUtil, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
|
||||||
|
) -> SkyvernElement | None:
|
||||||
|
if self.get_tag_name() != "label":
|
||||||
|
return None
|
||||||
|
|
||||||
|
for_id = await self.get_attr("for")
|
||||||
|
if for_id == "":
|
||||||
|
return None
|
||||||
|
|
||||||
|
locator = self.get_frame().locator(f"#{for_id}")
|
||||||
|
# supposed to be only one element, since id is unique in the whole DOM
|
||||||
|
if await locator.count() != 1:
|
||||||
|
return None
|
||||||
|
|
||||||
|
unique_id = await locator.get_attribute(SKYVERN_ID_ATTR, timeout=timeout)
|
||||||
|
if unique_id is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return await dom.get_skyvern_element_by_id(unique_id)
|
||||||
|
|
||||||
|
async def find_selectable_child(self, dom: DomUtil) -> SkyvernElement | None:
|
||||||
|
# BFS to find the first selectable child
|
||||||
|
index = 0
|
||||||
|
queue = [self]
|
||||||
|
while index < len(queue):
|
||||||
|
item = queue[index]
|
||||||
|
if item.is_interactable() and await item.is_selectable():
|
||||||
|
return item
|
||||||
|
|
||||||
|
try:
|
||||||
|
for_element = await item.find_label_for(dom=dom)
|
||||||
|
if for_element is not None and await for_element.is_selectable():
|
||||||
|
return for_element
|
||||||
|
except Exception:
|
||||||
|
LOG.error(
|
||||||
|
"Failed to find element by label-for",
|
||||||
|
element=item.__static_element,
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
children: list[dict] = item.__static_element.get("children", [])
|
||||||
|
for child in children:
|
||||||
|
child_id = child.get("id", "")
|
||||||
|
child_element = await dom.get_skyvern_element_by_id(child_id)
|
||||||
|
queue.append(child_element)
|
||||||
|
|
||||||
|
index += 1
|
||||||
|
return None
|
||||||
|
|
||||||
async def get_attr(
|
async def get_attr(
|
||||||
self,
|
self,
|
||||||
attr_name: str,
|
attr_name: str,
|
||||||
@@ -216,28 +295,144 @@ class DomUtil:
|
|||||||
return SkyvernElement(locator, frame_content, element)
|
return SkyvernElement(locator, frame_content, element)
|
||||||
|
|
||||||
|
|
||||||
class Select2Dropdown:
|
class AbstractSelectDropdown(ABC):
|
||||||
|
@abstractmethod
|
||||||
|
def name(self) -> str:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def get_current_value(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> str:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def get_options(
|
||||||
|
self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
|
||||||
|
) -> typing.List[SkyvernOptionType]:
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
async def select_by_index(
|
||||||
|
self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
|
||||||
|
) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class Select2Dropdown(AbstractSelectDropdown):
|
||||||
def __init__(self, frame: Page | Frame, skyvern_element: SkyvernElement) -> None:
|
def __init__(self, frame: Page | Frame, skyvern_element: SkyvernElement) -> None:
|
||||||
self.skyvern_element = skyvern_element
|
self.skyvern_element = skyvern_element
|
||||||
self.frame = frame
|
self.frame = frame
|
||||||
|
|
||||||
|
async def __find_anchor(self, timeout: float) -> Locator:
|
||||||
|
locator = self.frame.locator("#select2-drop")
|
||||||
|
await locator.wait_for(state="visible", timeout=timeout)
|
||||||
|
cnt = await locator.count()
|
||||||
|
if cnt == 0:
|
||||||
|
raise NoDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
|
||||||
|
if cnt > 1:
|
||||||
|
raise MultipleDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
|
||||||
|
return locator
|
||||||
|
|
||||||
|
def name(self) -> str:
|
||||||
|
return "select2"
|
||||||
|
|
||||||
async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
|
async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
|
||||||
await self.skyvern_element.get_locator().click(timeout=timeout)
|
await self.skyvern_element.get_locator().click(timeout=timeout)
|
||||||
# wait for the options to load
|
await self.__find_anchor(timeout=timeout)
|
||||||
await asyncio.sleep(3)
|
|
||||||
|
|
||||||
async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
|
async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
|
||||||
await self.frame.locator("#select2-drop").press("Escape", timeout=timeout)
|
anchor = await self.__find_anchor(timeout=timeout)
|
||||||
|
await anchor.press("Escape", timeout=timeout)
|
||||||
|
|
||||||
|
async def get_current_value(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> str:
|
||||||
|
tag_name = self.skyvern_element.get_tag_name()
|
||||||
|
if tag_name == "input":
|
||||||
|
# TODO: this is multiple options case, we haven't fully supported it yet.
|
||||||
|
return ""
|
||||||
|
|
||||||
|
# check SkyvernElement.is_select2_dropdown() method, only <a> and <span> element left
|
||||||
|
# we should make sure the locator is on <a>, so we're able to find the [class="select2-chosen"] child
|
||||||
|
locator = self.skyvern_element.get_locator()
|
||||||
|
if tag_name == "span":
|
||||||
|
locator = locator.locator("..")
|
||||||
|
elif tag_name == "a":
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
raise FailedToGetCurrentValueOfDropdown(
|
||||||
|
self.name(), self.skyvern_element.get_id(), "invalid element of select2"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
return await locator.locator("span[class='select2-chosen']").text_content(timeout=timeout)
|
||||||
|
except Exception as e:
|
||||||
|
raise FailedToGetCurrentValueOfDropdown(self.name(), self.skyvern_element.get_id(), repr(e))
|
||||||
|
|
||||||
async def get_options(
|
async def get_options(
|
||||||
self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
|
self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
|
||||||
) -> typing.List[SkyvernOptionType]:
|
) -> typing.List[SkyvernOptionType]:
|
||||||
element_handler = await self.skyvern_element.get_locator().element_handle(timeout=timeout)
|
anchor = await self.__find_anchor(timeout=timeout)
|
||||||
|
element_handler = await anchor.element_handle(timeout=timeout)
|
||||||
options = await get_select2_options(self.frame, element_handler)
|
options = await get_select2_options(self.frame, element_handler)
|
||||||
return typing.cast(typing.List[SkyvernOptionType], options)
|
return typing.cast(typing.List[SkyvernOptionType], options)
|
||||||
|
|
||||||
async def select_by_index(
|
async def select_by_index(
|
||||||
self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
|
self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
|
||||||
) -> None:
|
) -> None:
|
||||||
anchor = self.frame.locator("#select2-drop li[role='option']")
|
anchor = await self.__find_anchor(timeout=timeout)
|
||||||
await anchor.nth(index).click(timeout=timeout)
|
options = anchor.locator("li[role='option']")
|
||||||
|
await options.nth(index).click(timeout=timeout)
|
||||||
|
|
||||||
|
|
||||||
|
class ComboboxDropdown(AbstractSelectDropdown):
|
||||||
|
def __init__(self, frame: Page | Frame, skyvern_element: SkyvernElement) -> None:
|
||||||
|
self.skyvern_element = skyvern_element
|
||||||
|
self.frame = frame
|
||||||
|
|
||||||
|
async def __find_anchor(self, timeout: float) -> Locator:
|
||||||
|
control_id = await self.skyvern_element.get_attr("aria-controls", timeout=timeout)
|
||||||
|
locator = self.frame.locator(f"#{control_id}")
|
||||||
|
await locator.wait_for(state="visible", timeout=timeout)
|
||||||
|
cnt = await locator.count()
|
||||||
|
if cnt == 0:
|
||||||
|
raise NoDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
|
||||||
|
if cnt > 1:
|
||||||
|
raise MultipleDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
|
||||||
|
return locator
|
||||||
|
|
||||||
|
def name(self) -> str:
|
||||||
|
return "combobox"
|
||||||
|
|
||||||
|
async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
|
||||||
|
await self.skyvern_element.get_locator().click(timeout=timeout)
|
||||||
|
await self.__find_anchor(timeout=timeout)
|
||||||
|
|
||||||
|
async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
|
||||||
|
await self.skyvern_element.get_locator().press("Tab", timeout=timeout)
|
||||||
|
|
||||||
|
async def get_current_value(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> str:
|
||||||
|
try:
|
||||||
|
return await self.skyvern_element.get_attr("value", dynamic=True, timeout=timeout)
|
||||||
|
except Exception as e:
|
||||||
|
raise FailedToGetCurrentValueOfDropdown(self.name(), self.skyvern_element.get_id(), repr(e))
|
||||||
|
|
||||||
|
async def get_options(
|
||||||
|
self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
|
||||||
|
) -> typing.List[SkyvernOptionType]:
|
||||||
|
anchor = await self.__find_anchor(timeout=timeout)
|
||||||
|
element_handler = await anchor.element_handle()
|
||||||
|
options = await get_combobox_options(self.frame, element_handler)
|
||||||
|
return typing.cast(typing.List[SkyvernOptionType], options)
|
||||||
|
|
||||||
|
async def select_by_index(
|
||||||
|
self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
|
||||||
|
) -> None:
|
||||||
|
anchor = await self.__find_anchor(timeout=timeout)
|
||||||
|
options = anchor.locator("li[role='option']")
|
||||||
|
await options.nth(index).click(timeout=timeout)
|
||||||
|
|||||||
Reference in New Issue
Block a user