support react select (#652)

skyvern/exceptions.py

@@ -313,6 +313,11 @@ class ElementIsNotSelect2Dropdown(SkyvernException):
         super().__init__(f"element[{element}] is not select2 dropdown. element_id={element_id}")
 
 
+class ElementIsNotReactSelectDropdown(SkyvernException):
+    def __init__(self, element_id: str, element: dict):
+        super().__init__(f"element[{element}] is not react select dropdown. element_id={element_id}")
+
+
 class ElementIsNotComboboxDropdown(SkyvernException):
     def __init__(self, element_id: str, element: dict):
         super().__init__(f"element[{element}] is not combobox dropdown. element_id={element_id}")

skyvern/webeye/actions/handler.py

@@ -499,6 +499,12 @@ async def handle_select_option_action(
             action=action,
         )
         select_framework = await skyvern_element.get_select2_dropdown()
+    if await skyvern_element.is_react_select_dropdown():
+        LOG.info(
+            "This is a react select dropdown",
+            action=action,
+        )
+        select_framework = await skyvern_element.get_react_select_dropdown()
 
     if select_framework is not None:
         timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS

skyvern/webeye/scraper/domUtils.js

@@ -448,6 +448,14 @@ const isSelect2MultiChoice = (element) => {
   );
 };
 
+const isReactSelectDropdown = (element) => {
+  return (
+    element.tagName.toLowerCase() === "input" &&
+    element.className.toString().includes("select__input") &&
+    element.getAttribute("role") === "combobox"
+  );
+};
+
 const checkParentClass = (className) => {
   const targetParentClasses = ["field", "entry"];
   for (let i = 0; i < targetParentClasses.length; i++) {
@@ -687,6 +695,78 @@ async function getSelect2Options(element) {
   return selectOptions;
 }
 
+async function getReactSelectOptionElements(element) {
+  let optionList = [];
+  // wait for 2s until the element is updated with `aria-controls`
+  console.log("wait 2s for the dropdown being updated.");
+  await sleep(2000);
+
+  dropdownId = element.getAttribute("aria-controls");
+  if (!dropdownId) {
+    return optionList;
+  }
+
+  const document = element.getRootNode();
+  dropdownDiv = document.querySelector(`div[id="${dropdownId}"]`);
+  let previousOptionCount = null;
+
+  while (true) {
+    // sometimes need more time to load the options
+    console.log("wait 5s to load all options");
+    await sleep(5000); // wait 5s
+    optionList = dropdownDiv.querySelectorAll("div[role='option']");
+    if (optionList.length === 0) {
+      break;
+    }
+
+    if (
+      previousOptionCount !== null &&
+      previousOptionCount == optionList.length
+    ) {
+      break;
+    }
+    previousOptionCount = optionList.length;
+
+    lastOption = optionList[optionList.length - 1];
+    lastOption.scrollIntoView({ behavior: "instant" });
+
+    lastOption.dispatchEvent(
+      new WheelEvent("wheel", {
+        bubbles: true,
+        cancelable: true,
+        deltaX: 0,
+        deltaY: -20,
+        deltaZ: 0,
+      }),
+    );
+    lastOption.dispatchEvent(
+      new WheelEvent("wheel", {
+        bubbles: true,
+        cancelable: true,
+        deltaX: 0,
+        deltaY: 20,
+        deltaZ: 0,
+      }),
+    );
+  }
+
+  return optionList;
+}
+
+async function getReactSelectOptions(element) {
+  const optionList = await getReactSelectOptionElements(element);
+
+  let selectOptions = [];
+  for (let i = 0; i < optionList.length; i++) {
+    let ele = optionList[i];
+    selectOptions.push({
+      optionIndex: i,
+      text: removeMultipleSpaces(ele.textContent),
+    });
+  }
+  return selectOptions;
+}
+
 function getDOMElementBySkyvenElement(elementObj) {
   // if element has shadowHost set, we need to find the shadowHost element first then find the element
   if (elementObj.shadowHost) {
@@ -800,6 +880,23 @@ async function buildTreeFromBody(frame = "main.frame", open_select = false) {
     } else if (attrs["role"] && attrs["role"].toLowerCase() === "listbox") {
       // if "role" key is inside attrs, then get all the elements with role "option" and get their text
       selectOptions = getListboxOptions(element);
+    } else if (open_select && isReactSelectDropdown(element)) {
+      element.dispatchEvent(
+        new MouseEvent("mouseup", {
+          bubbles: true,
+          view: window,
+        }),
+      );
+
+      selectOptions = await getReactSelectOptions(element);
+
+      // click again to close
+      element.dispatchEvent(
+        new MouseEvent("mouseup", {
+          bubbles: true,
+          view: window,
+        }),
+      );
     } else if (open_select && isComboboxDropdown(element)) {
       // open combobox dropdown to get options
       element.click();

skyvern/webeye/utils/dom.py

@@ -11,6 +11,7 @@ from skyvern.constants import SKYVERN_ID_ATTR
 from skyvern.exceptions import (
     ElementIsNotComboboxDropdown,
     ElementIsNotLabel,
+    ElementIsNotReactSelectDropdown,
     ElementIsNotSelect2Dropdown,
     FailedToGetCurrentValueOfDropdown,
     MissingElement,
@@ -107,6 +108,16 @@ class SkyvernElement:
             or (tag_name == "input" and "select2-input" in element_class)
         )
 
+    async def is_react_select_dropdown(self) -> bool:
+        tag_name = self.get_tag_name()
+        element_class = await self.get_attr("class")
+        if element_class is None:
+            return False
+
+        return (tag_name == InteractiveElement.INPUT and "select__input" in element_class) or (
+            tag_name == InteractiveElement.BUTTON and await self.get_attr("aria-label") == "Toggle flyout"
+        )
+
     async def is_combobox_dropdown(self) -> bool:
         tag_name = self.get_tag_name()
         role = await self.get_attr("role")
@@ -135,6 +146,7 @@ class SkyvernElement:
     async def is_selectable(self) -> bool:
         return (
             await self.is_select2_dropdown()
+            or await self.is_react_select_dropdown()
             or await self.is_combobox_dropdown()
             or self.get_tag_name() in SELECTABLE_ELEMENT
         )
@@ -168,6 +180,13 @@ class SkyvernElement:
         frame = await SkyvernFrame.create_instance(self.get_frame())
         return Select2Dropdown(frame, self)
 
+    async def get_react_select_dropdown(self) -> ReactSelectDropdown:
+        if not await self.is_react_select_dropdown():
+            raise ElementIsNotReactSelectDropdown(self.get_id(), self.__static_element)
+
+        frame = await SkyvernFrame.create_instance(self.get_frame())
+        return ReactSelectDropdown(frame, self)
+
     async def get_combobox_dropdown(self) -> ComboboxDropdown:
         if not await self.is_combobox_dropdown():
             raise ElementIsNotComboboxDropdown(self.get_id(), self.__static_element)
@@ -411,6 +430,71 @@ class Select2Dropdown(AbstractSelectDropdown):
         await options.nth(index).click(timeout=timeout)
 
 
+class ReactSelectDropdown(AbstractSelectDropdown):
+    def __init__(self, skyvern_frame: SkyvernFrame, skyvern_element: SkyvernElement) -> None:
+        self.skyvern_element = skyvern_element
+        self.skyvern_frame = skyvern_frame
+
+    def __find_input_locator(self) -> Locator:
+        tag_name = self.skyvern_element.get_tag_name()
+        locator = self.skyvern_element.get_locator()
+
+        if tag_name == InteractiveElement.BUTTON:
+            return locator.locator("..").locator("..").locator("input[class~='select__input']")
+
+        return locator
+
+    async def __find_anchor(self, timeout: float) -> Locator:
+        input_locator = self.__find_input_locator()
+        anchor_id = await input_locator.get_attribute("aria-controls", timeout=timeout)
+
+        locator = self.skyvern_element.get_frame().locator(f"div[id='{anchor_id}']")
+        await locator.wait_for(state="visible", timeout=timeout)
+        cnt = await locator.count()
+        if cnt == 0:
+            raise NoDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
+        if cnt > 1:
+            raise MultipleDropdownAnchorErr(self.name(), self.skyvern_element.get_id())
+        return locator
+
+    def name(self) -> str:
+        return "react-select"
+
+    async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
+        await self.skyvern_element.get_locator().click(timeout=timeout)
+        await self.__find_anchor(timeout=timeout)
+
+    async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
+        await self.__find_anchor(timeout=timeout)
+        await self.skyvern_element.get_locator().click(timeout=timeout)
+
+    async def get_current_value(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> str:
+        input_locator = self.__find_input_locator()
+        # TODO: only support single value now
+        value_locator = input_locator.locator("..").locator("..").locator("div[class~='select__single-value']")
+        if await value_locator.count() == 0:
+            return ""
+        try:
+            return await value_locator.text_content(timeout=timeout)
+        except Exception as e:
+            raise FailedToGetCurrentValueOfDropdown(self.name(), self.skyvern_element.get_id(), repr(e))
+
+    async def get_options(
+        self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
+    ) -> typing.List[SkyvernOptionType]:
+        input_locator = self.__find_input_locator()
+        element_handler = await input_locator.element_handle(timeout=timeout)
+        options = await self.skyvern_frame.get_react_select_options(element_handler)
+        return typing.cast(typing.List[SkyvernOptionType], options)
+
+    async def select_by_index(
+        self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
+    ) -> None:
+        anchor = await self.__find_anchor(timeout=timeout)
+        options = anchor.locator("div[role='option']")
+        await options.nth(index).click(timeout=timeout)
+
+
 class ComboboxDropdown(AbstractSelectDropdown):
     def __init__(self, skyvern_frame: SkyvernFrame, skyvern_element: SkyvernElement) -> None:
         self.skyvern_element = skyvern_element

skyvern/webeye/utils/page.py

@@ -150,6 +150,11 @@ class SkyvernFrame:
         js_script = "async (element) => await getSelect2Options(element)"
         return await self.frame.evaluate(js_script, element)
 
+    async def get_react_select_options(self, element: ElementHandle) -> List[Dict[str, Any]]:
+        await self.frame.evaluate(JS_FUNCTION_DEFS)
+        js_script = "async (element) => await getReactSelectOptions(element)"
+        return await self.frame.evaluate(js_script, element)
+
     async def get_combobox_options(self, element: ElementHandle) -> List[Dict[str, Any]]:
         await self.frame.evaluate(JS_FUNCTION_DEFS)
         js_script = "async (element) => await getListboxOptions(element)"