fix hidden input element (#2314)

skyvern/exceptions.py

@@ -619,6 +619,13 @@ class InteractWithDisabledElement(SkyvernException):
         )
 
 
+class InputToInvisibleElement(SkyvernException):
+    def __init__(self, element_id: str):
+        super().__init__(
+            f"The element(id={element_id}) now is not visible. Try to interact with other elements, or try to interact with it later when it's visible."
+        )
+
+
 class FailedToParseActionInstruction(SkyvernException):
     def __init__(self, reason: str | None, error_type: str | None):
         super().__init__(

skyvern/webeye/actions/handler.py

@@ -32,6 +32,7 @@ from skyvern.exceptions import (
     FailToSelectByValue,
     IllegitComplete,
     ImaginaryFileUrl,
+    InputToInvisibleElement,
     InteractWithDisabledElement,
     InteractWithDropdownContainer,
     InvalidElementForTextInput,
@@ -876,6 +877,11 @@ async def handle_input_text_action(
                     await skyvern_element.blur()
                 await incremental_scraped.stop_listen_dom_increment()
 
+    ### Start filling text logic
+    # check if the element has hidden attribute
+    if await skyvern_element.has_hidden_attr():
+        return [ActionFailure(InputToInvisibleElement(skyvern_element.get_id()), stop_execution_on_failure=False)]
+
     # force to move focus back to the element
     await skyvern_element.get_locator().focus(timeout=timeout)
 

skyvern/webeye/scraper/domUtils.js

@@ -277,6 +277,7 @@ function hasASPClientControl() {
 }
 
 // from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L100-L119
+// NOTE: According this logic, some elements with aria-hidden won't be considered as invisible. And the result shows they are indeed interactable.
 function isElementVisible(element) {
   // TODO: This is a hack to not check visibility for option elements
   // because they are not visible by default. We check their parent instead for visibility.

skyvern/webeye/utils/dom.py

@@ -297,6 +297,15 @@ class SkyvernElement:
         skyvern_frame = await SkyvernFrame.create_instance(self.get_frame())
         return await skyvern_frame.is_sibling(await self.get_element_handler(), target)
 
+    async def has_hidden_attr(self) -> bool:
+        hidden: str | None = await self.get_attr("hidden", mode="dynamic")
+        aria_hidden: str | None = await self.get_attr("aria-hidden", mode="dynamic")
+        if hidden is not None and hidden.lower() != "false":
+            return True
+        if aria_hidden is not None and aria_hidden.lower() != "false":
+            return True
+        return False
+
     def get_element_dict(self) -> dict:
         return self.__static_element