vodorod/Dorod-Sky
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

support react select (#652)

Browse Source
This commit is contained in:
LawyZheng
2024-07-27 01:32:35 +08:00
committed by GitHub
parent be75055fd8
commit 1629d84375
5 changed files with 197 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
skyvern/exceptions.py
View File

@@ -313,6 +313,11 @@ class ElementIsNotSelect2Dropdown(SkyvernException):
super().__init__(f"element[{element}] is not select2 dropdown. element_id={element_id}")
class ElementIsNotReactSelectDropdown(SkyvernException):
def __init__(self, element_id: str, element: dict):
super().__init__(f"element[{element}] is not react select dropdown. element_id={element_id}")
class ElementIsNotComboboxDropdown(SkyvernException):
def __init__(self, element_id: str, element: dict):
super().__init__(f"element[{element}] is not combobox dropdown. element_id={element_id}")

6
skyvern/webeye/actions/handler.py
View File

@@ -499,6 +499,12 @@ async def handle_select_option_action(
action=action,
)
select_framework = await skyvern_element.get_select2_dropdown()
if await skyvern_element.is_react_select_dropdown():
LOG.info(
"This is a react select dropdown",
action=action,
)
select_framework = await skyvern_element.get_react_select_dropdown()
if select_framework is not None:
timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS

97
skyvern/webeye/scraper/domUtils.js
View File

@@ -448,6 +448,14 @@ const isSelect2MultiChoice = (element) => {
);
};
const isReactSelectDropdown = (element) => {
return (
element.tagName.toLowerCase() === "input" &&
element.className.toString().includes("select__input") &&
element.getAttribute("role") === "combobox"
);
};
const checkParentClass = (className) => {
const targetParentClasses = ["field", "entry"];
for (let i = 0; i < targetParentClasses.length; i++) {
@@ -687,6 +695,78 @@ async function getSelect2Options(element) {
return selectOptions;
}
async function getReactSelectOptionElements(element) {
let optionList = [];
// wait for 2s until the element is updated with `aria-controls`
console.log("wait 2s for the dropdown being updated.");
await sleep(2000);
dropdownId = element.getAttribute("aria-controls");
if (!dropdownId) {
return optionList;
}
const document = element.getRootNode();
dropdownDiv = document.querySelector(`div[id="${dropdownId}"]`);
let previousOptionCount = null;
while (true) {
// sometimes need more time to load the options
console.log("wait 5s to load all options");
await sleep(5000); // wait 5s
optionList = dropdownDiv.querySelectorAll("div[role='option']");
if (optionList.length === 0) {
break;
}
if (
previousOptionCount !== null &&
previousOptionCount == optionList.length
) {
break;
}
previousOptionCount = optionList.length;
lastOption = optionList[optionList.length - 1];
lastOption.scrollIntoView({ behavior: "instant" });
lastOption.dispatchEvent(
new WheelEvent("wheel", {
bubbles: true,
cancelable: true,
deltaX: 0,
deltaY: -20,
deltaZ: 0,
}),
);
lastOption.dispatchEvent(
new WheelEvent("wheel", {
bubbles: true,
cancelable: true,
deltaX: 0,
deltaY: 20,
deltaZ: 0,
}),
);
}
return optionList;
}
async function getReactSelectOptions(element) {
const optionList = await getReactSelectOptionElements(element);
let selectOptions = [];
for (let i = 0; i < optionList.length; i++) {
let ele = optionList[i];
selectOptions.push({
optionIndex: i,
text: removeMultipleSpaces(ele.textContent),
});
}
return selectOptions;
}
function getDOMElementBySkyvenElement(elementObj) {
// if element has shadowHost set, we need to find the shadowHost element first then find the element
if (elementObj.shadowHost) {
@@ -800,6 +880,23 @@ async function buildTreeFromBody(frame = "main.frame", open_select = false) {
} else if (attrs["role"] && attrs["role"].toLowerCase() === "listbox") {
// if "role" key is inside attrs, then get all the elements with role "option" and get their text
selectOptions = getListboxOptions(element);
} else if (open_select && isReactSelectDropdown(element)) {
element.dispatchEvent(
new MouseEvent("mouseup", {
bubbles: true,
view: window,
}),
);
selectOptions = await getReactSelectOptions(element);
// click again to close
element.dispatchEvent(
new MouseEvent("mouseup", {
bubbles: true,
view: window,
}),
);
} else if (open_select && isComboboxDropdown(element)) {
// open combobox dropdown to get options
element.click();

84
skyvern/webeye/utils/dom.py
View File

@@ -11,6 +11,7 @@ from skyvern.constants import SKYVERN_ID_ATTR
from skyvern.exceptions import (
ElementIsNotComboboxDropdown,
ElementIsNotLabel,
ElementIsNotReactSelectDropdown,
ElementIsNotSelect2Dropdown,
FailedToGetCurrentValueOfDropdown,
MissingElement,
@@ -107,6 +108,16 @@ class SkyvernElement:
or (tag_name == "input" and "select2-input" in element_class)
)
async def is_react_select_dropdown(self) -> bool:
tag_name = self.get_tag_name()
element_class = await self.get_attr("class")
if element_class is None:
return False
return (tag_name == InteractiveElement.INPUT and "select__input" in element_class) or (
tag_name == InteractiveElement.BUTTON and await self.get_attr("aria-label") == "Toggle flyout"
)
async def is_combobox_dropdown(self) -> bool:
tag_name = self.get_tag_name()
role = await self.get_attr("role")
@@ -135,6 +146,7 @@ class SkyvernElement:
async def is_selectable(self) -> bool:
return (
await self.is_select2_dropdown()
or await self.is_react_select_dropdown()
or await self.is_combobox_dropdown()
or self.get_tag_name() in SELECTABLE_ELEMENT
)
@@ -168,6 +180,13 @@ class SkyvernElement:
frame = await SkyvernFrame.create_instance(self.get_frame())
return Select2Dropdown(frame, self)
async def get_react_select_dropdown(self) -> ReactSelectDropdown:
if not await self.is_react_select_dropdown():
raise ElementIsNotReactSelectDropdown(self.get_id(), self.__static_element)
frame = await SkyvernFrame.create_instance(self.get_frame())
return ReactSelectDropdown(frame, self)
async def get_combobox_dropdown(self) -> ComboboxDropdown:
if not await self.is_combobox_dropdown():
raise ElementIsNotComboboxDropdown(self.get_id(), self.__static_element)
@@ -411,6 +430,71 @@ class Select2Dropdown(AbstractSelectDropdown):
await options.nth(index).click(timeout=timeout)
class ReactSelectDropdown(AbstractSelectDropdown):
def __init__(self, skyvern_frame: SkyvernFrame, skyvern_element: SkyvernElement) -> None:
self.skyvern_element = skyvern_element
self.skyvern_frame = skyvern_frame
def __find_input_locator(self) -> Locator:
tag_name = self.skyvern_element.get_tag_name()
locator = self.skyvern_element.get_locator()
if tag_name == InteractiveElement.BUTTON:
return locator.locator("..").locator("..").locator("input[class~='select__input']")
return locator
async def __find_anchor(self, timeout: float) -> Locator:
input_locator = self.__find_input_locator()
anchor_id = await input_locator.get_attribute("aria-controls", timeout=timeout)
locator = self.skyvern_element.get_frame().locator(f"div[id='{anchor_id}']")
await locator.wait_for(state="visible", timeout=timeout)
cnt = await locator.count()
if cnt == 0:
raise NoDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
if cnt > 1:
raise MultipleDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
return locator
def name(self) -> str:
return "react-select"
async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
await self.skyvern_element.get_locator().click(timeout=timeout)
await self.__find_anchor(timeout=timeout)
async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
await self.__find_anchor(timeout=timeout)
await self.skyvern_element.get_locator().click(timeout=timeout)
async def get_current_value(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> str:
input_locator = self.__find_input_locator()
# TODO: only support single value now
value_locator = input_locator.locator("..").locator("..").locator("div[class~='select__single-value']")
if await value_locator.count() == 0:
return ""
try:
return await value_locator.text_content(timeout=timeout)
except Exception as e:
raise FailedToGetCurrentValueOfDropdown(self.name(), self.skyvern_element.get_id(), repr(e))
async def get_options(
self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
) -> typing.List[SkyvernOptionType]:
input_locator = self.__find_input_locator()
element_handler = await input_locator.element_handle(timeout=timeout)
options = await self.skyvern_frame.get_react_select_options(element_handler)
return typing.cast(typing.List[SkyvernOptionType], options)
async def select_by_index(
self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
) -> None:
anchor = await self.__find_anchor(timeout=timeout)
options = anchor.locator("div[role='option']")
await options.nth(index).click(timeout=timeout)
class ComboboxDropdown(AbstractSelectDropdown):
def __init__(self, skyvern_frame: SkyvernFrame, skyvern_element: SkyvernElement) -> None:
self.skyvern_element = skyvern_element

5
skyvern/webeye/utils/page.py
View File

@@ -150,6 +150,11 @@ class SkyvernFrame:
js_script = "async (element) => await getSelect2Options(element)"
return await self.frame.evaluate(js_script, element)
async def get_react_select_options(self, element: ElementHandle) -> List[Dict[str, Any]]:
await self.frame.evaluate(JS_FUNCTION_DEFS)
js_script = "async (element) => await getReactSelectOptions(element)"
return await self.frame.evaluate(js_script, element)
async def get_combobox_options(self, element: ElementHandle) -> List[Dict[str, Any]]:
await self.frame.evaluate(JS_FUNCTION_DEFS)
js_script = "async (element) => await getListboxOptions(element)"