From 1629d84375bcb852e38c2a8b65c45b3b99eb1860 Mon Sep 17 00:00:00 2001 From: LawyZheng Date: Sat, 27 Jul 2024 01:32:35 +0800 Subject: [PATCH] support react select (#652) --- skyvern/exceptions.py | 5 ++ skyvern/webeye/actions/handler.py | 6 ++ skyvern/webeye/scraper/domUtils.js | 97 ++++++++++++++++++++++++++++++ skyvern/webeye/utils/dom.py | 84 ++++++++++++++++++++++++++ skyvern/webeye/utils/page.py | 5 ++ 5 files changed, 197 insertions(+) diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py index 069337ee..761cd7d6 100644 --- a/skyvern/exceptions.py +++ b/skyvern/exceptions.py @@ -313,6 +313,11 @@ class ElementIsNotSelect2Dropdown(SkyvernException): super().__init__(f"element[{element}] is not select2 dropdown. element_id={element_id}") +class ElementIsNotReactSelectDropdown(SkyvernException): + def __init__(self, element_id: str, element: dict): + super().__init__(f"element[{element}] is not react select dropdown. element_id={element_id}") + + class ElementIsNotComboboxDropdown(SkyvernException): def __init__(self, element_id: str, element: dict): super().__init__(f"element[{element}] is not combobox dropdown. element_id={element_id}") diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py index b355d1f2..dbf0f554 100644 --- a/skyvern/webeye/actions/handler.py +++ b/skyvern/webeye/actions/handler.py @@ -499,6 +499,12 @@ async def handle_select_option_action( action=action, ) select_framework = await skyvern_element.get_select2_dropdown() + if await skyvern_element.is_react_select_dropdown(): + LOG.info( + "This is a react select dropdown", + action=action, + ) + select_framework = await skyvern_element.get_react_select_dropdown() if select_framework is not None: timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js index 05e29993..dbf5c068 100644 --- a/skyvern/webeye/scraper/domUtils.js +++ b/skyvern/webeye/scraper/domUtils.js @@ -448,6 +448,14 @@ const isSelect2MultiChoice = (element) => { ); }; +const isReactSelectDropdown = (element) => { + return ( + element.tagName.toLowerCase() === "input" && + element.className.toString().includes("select__input") && + element.getAttribute("role") === "combobox" + ); +}; + const checkParentClass = (className) => { const targetParentClasses = ["field", "entry"]; for (let i = 0; i < targetParentClasses.length; i++) { @@ -687,6 +695,78 @@ async function getSelect2Options(element) { return selectOptions; } +async function getReactSelectOptionElements(element) { + let optionList = []; + // wait for 2s until the element is updated with `aria-controls` + console.log("wait 2s for the dropdown being updated."); + await sleep(2000); + + dropdownId = element.getAttribute("aria-controls"); + if (!dropdownId) { + return optionList; + } + + const document = element.getRootNode(); + dropdownDiv = document.querySelector(`div[id="${dropdownId}"]`); + let previousOptionCount = null; + + while (true) { + // sometimes need more time to load the options + console.log("wait 5s to load all options"); + await sleep(5000); // wait 5s + optionList = dropdownDiv.querySelectorAll("div[role='option']"); + if (optionList.length === 0) { + break; + } + + if ( + previousOptionCount !== null && + previousOptionCount == optionList.length + ) { + break; + } + previousOptionCount = optionList.length; + + lastOption = optionList[optionList.length - 1]; + lastOption.scrollIntoView({ behavior: "instant" }); + + lastOption.dispatchEvent( + new WheelEvent("wheel", { + bubbles: true, + cancelable: true, + deltaX: 0, + deltaY: -20, + deltaZ: 0, + }), + ); + lastOption.dispatchEvent( + new WheelEvent("wheel", { + bubbles: true, + cancelable: true, + deltaX: 0, + deltaY: 20, + deltaZ: 0, + }), + ); + } + + return optionList; +} + +async function getReactSelectOptions(element) { + const optionList = await getReactSelectOptionElements(element); + + let selectOptions = []; + for (let i = 0; i < optionList.length; i++) { + let ele = optionList[i]; + selectOptions.push({ + optionIndex: i, + text: removeMultipleSpaces(ele.textContent), + }); + } + return selectOptions; +} + function getDOMElementBySkyvenElement(elementObj) { // if element has shadowHost set, we need to find the shadowHost element first then find the element if (elementObj.shadowHost) { @@ -800,6 +880,23 @@ async function buildTreeFromBody(frame = "main.frame", open_select = false) { } else if (attrs["role"] && attrs["role"].toLowerCase() === "listbox") { // if "role" key is inside attrs, then get all the elements with role "option" and get their text selectOptions = getListboxOptions(element); + } else if (open_select && isReactSelectDropdown(element)) { + element.dispatchEvent( + new MouseEvent("mouseup", { + bubbles: true, + view: window, + }), + ); + + selectOptions = await getReactSelectOptions(element); + + // click again to close + element.dispatchEvent( + new MouseEvent("mouseup", { + bubbles: true, + view: window, + }), + ); } else if (open_select && isComboboxDropdown(element)) { // open combobox dropdown to get options element.click(); diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py index 611df7b3..10cf2004 100644 --- a/skyvern/webeye/utils/dom.py +++ b/skyvern/webeye/utils/dom.py @@ -11,6 +11,7 @@ from skyvern.constants import SKYVERN_ID_ATTR from skyvern.exceptions import ( ElementIsNotComboboxDropdown, ElementIsNotLabel, + ElementIsNotReactSelectDropdown, ElementIsNotSelect2Dropdown, FailedToGetCurrentValueOfDropdown, MissingElement, @@ -107,6 +108,16 @@ class SkyvernElement: or (tag_name == "input" and "select2-input" in element_class) ) + async def is_react_select_dropdown(self) -> bool: + tag_name = self.get_tag_name() + element_class = await self.get_attr("class") + if element_class is None: + return False + + return (tag_name == InteractiveElement.INPUT and "select__input" in element_class) or ( + tag_name == InteractiveElement.BUTTON and await self.get_attr("aria-label") == "Toggle flyout" + ) + async def is_combobox_dropdown(self) -> bool: tag_name = self.get_tag_name() role = await self.get_attr("role") @@ -135,6 +146,7 @@ class SkyvernElement: async def is_selectable(self) -> bool: return ( await self.is_select2_dropdown() + or await self.is_react_select_dropdown() or await self.is_combobox_dropdown() or self.get_tag_name() in SELECTABLE_ELEMENT ) @@ -168,6 +180,13 @@ class SkyvernElement: frame = await SkyvernFrame.create_instance(self.get_frame()) return Select2Dropdown(frame, self) + async def get_react_select_dropdown(self) -> ReactSelectDropdown: + if not await self.is_react_select_dropdown(): + raise ElementIsNotReactSelectDropdown(self.get_id(), self.__static_element) + + frame = await SkyvernFrame.create_instance(self.get_frame()) + return ReactSelectDropdown(frame, self) + async def get_combobox_dropdown(self) -> ComboboxDropdown: if not await self.is_combobox_dropdown(): raise ElementIsNotComboboxDropdown(self.get_id(), self.__static_element) @@ -411,6 +430,71 @@ class Select2Dropdown(AbstractSelectDropdown): await options.nth(index).click(timeout=timeout) +class ReactSelectDropdown(AbstractSelectDropdown): + def __init__(self, skyvern_frame: SkyvernFrame, skyvern_element: SkyvernElement) -> None: + self.skyvern_element = skyvern_element + self.skyvern_frame = skyvern_frame + + def __find_input_locator(self) -> Locator: + tag_name = self.skyvern_element.get_tag_name() + locator = self.skyvern_element.get_locator() + + if tag_name == InteractiveElement.BUTTON: + return locator.locator("..").locator("..").locator("input[class~='select__input']") + + return locator + + async def __find_anchor(self, timeout: float) -> Locator: + input_locator = self.__find_input_locator() + anchor_id = await input_locator.get_attribute("aria-controls", timeout=timeout) + + locator = self.skyvern_element.get_frame().locator(f"div[id='{anchor_id}']") + await locator.wait_for(state="visible", timeout=timeout) + cnt = await locator.count() + if cnt == 0: + raise NoDropdownAnchorErr(self.name(), self.skyvern_element.get_id()) + if cnt > 1: + raise MultipleDropdownAnchorErr(self.name(), self.skyvern_element.get_id()) + return locator + + def name(self) -> str: + return "react-select" + + async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None: + await self.skyvern_element.get_locator().click(timeout=timeout) + await self.__find_anchor(timeout=timeout) + + async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None: + await self.__find_anchor(timeout=timeout) + await self.skyvern_element.get_locator().click(timeout=timeout) + + async def get_current_value(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> str: + input_locator = self.__find_input_locator() + # TODO: only support single value now + value_locator = input_locator.locator("..").locator("..").locator("div[class~='select__single-value']") + if await value_locator.count() == 0: + return "" + try: + return await value_locator.text_content(timeout=timeout) + except Exception as e: + raise FailedToGetCurrentValueOfDropdown(self.name(), self.skyvern_element.get_id(), repr(e)) + + async def get_options( + self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS + ) -> typing.List[SkyvernOptionType]: + input_locator = self.__find_input_locator() + element_handler = await input_locator.element_handle(timeout=timeout) + options = await self.skyvern_frame.get_react_select_options(element_handler) + return typing.cast(typing.List[SkyvernOptionType], options) + + async def select_by_index( + self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS + ) -> None: + anchor = await self.__find_anchor(timeout=timeout) + options = anchor.locator("div[role='option']") + await options.nth(index).click(timeout=timeout) + + class ComboboxDropdown(AbstractSelectDropdown): def __init__(self, skyvern_frame: SkyvernFrame, skyvern_element: SkyvernElement) -> None: self.skyvern_element = skyvern_element diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py index 42e85ff6..410489e7 100644 --- a/skyvern/webeye/utils/page.py +++ b/skyvern/webeye/utils/page.py @@ -150,6 +150,11 @@ class SkyvernFrame: js_script = "async (element) => await getSelect2Options(element)" return await self.frame.evaluate(js_script, element) + async def get_react_select_options(self, element: ElementHandle) -> List[Dict[str, Any]]: + await self.frame.evaluate(JS_FUNCTION_DEFS) + js_script = "async (element) => await getReactSelectOptions(element)" + return await self.frame.evaluate(js_script, element) + async def get_combobox_options(self, element: ElementHandle) -> List[Dict[str, Any]]: await self.frame.evaluate(JS_FUNCTION_DEFS) js_script = "async (element) => await getListboxOptions(element)"