add disable element id (#1052)

skyvern/exceptions.py

@@ -521,3 +521,10 @@ class InvalidWorkflowParameter(SkyvernHTTPException):
             message,
             status_code=status.HTTP_400_BAD_REQUEST,
         )
+
+
+class InteractWithDisabledElement(SkyvernException):
+    def __init__(self, element_id: str):
+        super().__init__(
+            f"The element(id={element_id}) now is disabled, try to interact with it later when it's enabled."
+        )

skyvern/webeye/actions/handler.py

@@ -25,6 +25,7 @@ from skyvern.exceptions import (
     FailToSelectByValue,
     IllegitComplete,
     ImaginaryFileUrl,
+    InteractWithDisabledElement,
     InvalidElementForTextInput,
     MissingElement,
     MissingFileUrl,
@@ -347,6 +348,18 @@ async def handle_click_action(
     dom = DomUtil(scraped_page=scraped_page, page=page)
     skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)
     await asyncio.sleep(0.3)
+
+    # dynamically validate the attr, since it could change into enabled after the previous actions
+    if await skyvern_element.is_disabled(dynamic=True):
+        LOG.warning(
+            "Try to click on a disabled element",
+            action_type=action.action_type,
+            task_id=task.task_id,
+            step_id=step.step_id,
+            element_id=skyvern_element.get_id(),
+        )
+        return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]
+
     if action.download:
         results = await handle_click_to_download_file_action(action, page, scraped_page, task)
     else:
@@ -417,6 +430,17 @@ async def handle_input_text_action(
     if text is None:
         return [ActionFailure(FailedToFetchSecret())]
 
+    # dynamically validate the attr, since it could change into enabled after the previous actions
+    if await skyvern_element.is_disabled(dynamic=True):
+        LOG.warning(
+            "Try to input text on a disabled element",
+            action_type=action.action_type,
+            task_id=task.task_id,
+            step_id=step.step_id,
+            element_id=skyvern_element.get_id(),
+        )
+        return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]
+
     incremental_element: list[dict] = []
     # check if it's selectable
     if skyvern_element.get_tag_name() == InteractiveElement.INPUT and not await skyvern_element.is_raw_input():
@@ -572,6 +596,18 @@ async def handle_upload_file_action(
 
     dom = DomUtil(scraped_page=scraped_page, page=page)
     skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)
+
+    # dynamically validate the attr, since it could change into enabled after the previous actions
+    if await skyvern_element.is_disabled(dynamic=True):
+        LOG.warning(
+            "Try to upload file on a disabled element",
+            action_type=action.action_type,
+            task_id=task.task_id,
+            step_id=step.step_id,
+            element_id=skyvern_element.get_id(),
+        )
+        return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]
+
     locator = skyvern_element.locator
 
     file_path = await download_file(file_url)
@@ -717,6 +753,17 @@ async def handle_select_option_action(
             )
             return await handle_select_option_action(select_action, page, scraped_page, task, step)
 
+    # dynamically validate the attr, since it could change into enabled after the previous actions
+    if await skyvern_element.is_disabled(dynamic=True):
+        LOG.warning(
+            "Try to select on a disabled element",
+            action_type=action.action_type,
+            task_id=task.task_id,
+            step_id=step.step_id,
+            element_id=skyvern_element.get_id(),
+        )
+        return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]
+
     if tag_name == InteractiveElement.SELECT:
         LOG.info(
             "SelectOptionAction is on <select>",

skyvern/webeye/scraper/domUtils.js

@@ -836,7 +836,8 @@ function buildElementObject(frame, element, interactable, purgeable = false) {
       attr.name === "aria-selected" ||
       attr.name === "readonly" ||
       attr.name === "aria-readonly" ||
-      attr.name === "disabled"
+      attr.name === "disabled" ||
+      attr.name === "aria-disabled"
     ) {
       if (attrValue && attrValue.toLowerCase() === "false") {
         attrValue = false;

skyvern/webeye/scraper/scraper.py

@@ -32,6 +32,7 @@ RESERVED_ATTRIBUTES = {
     "data-original-title",  # for bootstrap tooltip
     "data-ui",
     "disabled",  # for button
+    "aria-disabled",
     "for",
     "href",  # For a tags
     "maxlength",
@@ -550,6 +551,22 @@ class IncrementalScrapePage:
         return "".join([json_to_html(element) for element in (element_tree or self.element_tree_trimmed)])
 
 
+def _should_keep_unique_id(element: dict) -> bool:
+    # case where we shouldn't keep unique_id
+    # 1. not disable attr and no interactable
+    # 2. disable=false and intrecatable=false
+
+    attributes = element.get("attributes", {})
+    if "disabled" not in attributes and "aria-disabled" not in attributes:
+        return element.get("interactable", False)
+
+    disabled = attributes.get("disabled")
+    aria_disabled = attributes.get("aria-disabled")
+    if disabled or aria_disabled:
+        return True
+    return element.get("interactable", False)
+
+
 def trim_element(element: dict) -> dict:
     queue = [element]
     while queue:
@@ -557,7 +574,7 @@ def trim_element(element: dict) -> dict:
         if "frame" in queue_ele:
             del queue_ele["frame"]
 
-        if "id" in queue_ele and not queue_ele.get("interactable"):
+        if "id" in queue_ele and not _should_keep_unique_id(queue_ele):
             del queue_ele["id"]
 
         if "attributes" in queue_ele:

skyvern/webeye/utils/dom.py

@@ -207,6 +207,43 @@ class SkyvernElement:
     def is_interactable(self) -> bool:
         return self.__static_element.get("interactable", False)
 
+    async def is_disabled(self, dynamic: bool = False) -> bool:
+        # if attr not exist, return None
+        # if attr is like 'disabled', return empty string or True
+        # if attr is like `disabled=false`, return the value
+        disabled = False
+        aria_disabled = False
+
+        disabled_attr: bool | str | None = None
+        aria_disabled_attr: bool | str | None = None
+
+        try:
+            disabled_attr = await self.get_attr("disabled", dynamic=dynamic)
+            aria_disabled_attr = await self.get_attr("aria-disabled", dynamic=dynamic)
+        except Exception:
+            # FIXME: maybe it should be considered as "disabled" element if failed to get the attributes?
+            LOG.exception(
+                "Failed to get the disabled attribute",
+                element=self.__static_element,
+                element_id=self.get_id(),
+            )
+
+        if disabled_attr is not None:
+            # disabled_attr should be bool or str
+            if isinstance(disabled_attr, bool):
+                disabled = disabled_attr
+            if isinstance(disabled_attr, str):
+                disabled = disabled_attr.lower() != "false"
+
+        if aria_disabled_attr is not None:
+            # aria_disabled_attr should be bool or str
+            if isinstance(aria_disabled_attr, bool):
+                aria_disabled = aria_disabled_attr
+            if isinstance(aria_disabled_attr, str):
+                aria_disabled = aria_disabled_attr.lower() != "false"
+
+        return disabled or aria_disabled
+
     async def is_selectable(self) -> bool:
         return self.get_selectable() or self.get_tag_name() in SELECTABLE_ELEMENT
 
@@ -367,7 +404,8 @@ class SkyvernElement:
         timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS,
     ) -> typing.Any:
         if not dynamic:
-            if attr := self.get_attributes().get(attr_name):
+            attr = self.get_attributes().get(attr_name)
+            if attr is not None:
                 return attr
 
         return await self.locator.get_attribute(attr_name, timeout=timeout)