do not parse the dropped element twice (#2801)

skyvern/forge/agent_functions.py

@@ -106,7 +106,19 @@ def _remove_skyvern_attributes(element: Dict) -> Dict:
     return element_copied
 
 
-def _mark_element_as_dropped(element: dict) -> None:
+def _add_to_dropped_css_svg_element_map(hashed_key: str | None) -> None:
+    context = skyvern_context.ensure_context()
+    if hashed_key:
+        context.dropped_css_svg_element_map[hashed_key] = True
+
+
+def _is_element_already_dropped(hashed_key: str) -> bool:
+    context = skyvern_context.ensure_context()
+    return hashed_key in context.dropped_css_svg_element_map
+
+
+def _mark_element_as_dropped(element: dict, *, hashed_key: str | None) -> None:
+    _add_to_dropped_css_svg_element_map(hashed_key)
     if "children" in element:
         del element["children"]
     element["isDropped"] = True
@@ -127,7 +139,7 @@ async def _check_svg_eligibility(
         return False
 
     if always_drop:
-        _mark_element_as_dropped(element)
+        _mark_element_as_dropped(element, hashed_key=None)
         return False
 
     task_id = task.task_id if task else None
@@ -137,11 +149,11 @@ async def _check_svg_eligibility(
     try:
         locater = skyvern_frame.get_frame().locator(f'[{SKYVERN_ID_ATTR}="{element_id}"]')
         if await locater.count() == 0:
-            _mark_element_as_dropped(element)
+            _mark_element_as_dropped(element, hashed_key=None)
             return False
 
         if not await locater.is_visible(timeout=settings.BROWSER_ACTION_TIMEOUT_MS):
-            _mark_element_as_dropped(element)
+            _mark_element_as_dropped(element, hashed_key=None)
             return False
 
         skyvern_element = SkyvernElement(locator=locater, frame=skyvern_frame.get_frame(), static_element=element)
@@ -150,7 +162,7 @@ async def _check_svg_eligibility(
             await skyvern_element.get_element_handler(timeout=1000)
         )
         if not skyvern_element.is_interactable() and blocked:
-            _mark_element_as_dropped(element)
+            _mark_element_as_dropped(element, hashed_key=None)
             return False
     except Exception:
         LOG.warning(
@@ -195,6 +207,11 @@ async def _convert_svg_to_string(
     if svg_shape:
         LOG.debug("SVG loaded from cache", element_id=element_id, key=svg_key, shape=svg_shape)
     else:
+        if _is_element_already_dropped(svg_key):
+            LOG.debug("SVG is already dropped, going to abort conversion", element_id=element_id, key=svg_key)
+            _mark_element_as_dropped(element, hashed_key=svg_key)
+            return
+
         if len(svg_html) > settings.SVG_MAX_LENGTH:
             # TODO: implement a fallback solution for "too large" case, maybe convert by screenshot
             LOG.warning(
@@ -205,7 +222,7 @@ async def _convert_svg_to_string(
                 length=len(svg_html),
                 key=svg_key,
             )
-            _mark_element_as_dropped(element)
+            _mark_element_as_dropped(element, hashed_key=svg_key)
             return
 
         LOG.debug("call LLM to convert SVG to string shape", element_id=element_id)
@@ -244,7 +261,7 @@ async def _convert_svg_to_string(
                     element_id=element_id,
                     key=svg_key,
                 )
-                _mark_element_as_dropped(element)
+                _mark_element_as_dropped(element, hashed_key=svg_key)
                 return
             except Exception:
                 LOG.info(
@@ -268,7 +285,7 @@ async def _convert_svg_to_string(
                 key=svg_key,
                 length=len(svg_html),
             )
-            _mark_element_as_dropped(element)
+            _mark_element_as_dropped(element, hashed_key=svg_key)
             return
 
     element["attributes"] = dict()
@@ -313,6 +330,9 @@ async def _convert_css_shape_to_string(
     if css_shape:
         LOG.debug("CSS shape loaded from cache", element_id=element_id, key=shape_key, shape=css_shape)
     else:
+        if _is_element_already_dropped(shape_key):
+            LOG.debug("CSS shape is already dropped, going to abort conversion", element_id=element_id, key=shape_key)
+            return None
         try:
             locater = skyvern_frame.get_frame().locator(f'[{SKYVERN_ID_ATTR}="{element_id}"]')
             if await locater.count() == 0:
@@ -399,6 +419,7 @@ async def _convert_css_shape_to_string(
                         element_id=element_id,
                         key=shape_key,
                     )
+                    _add_to_dropped_css_svg_element_map(shape_key)
                     return None
                 except Exception:
                     LOG.info(
@@ -422,6 +443,7 @@ async def _convert_css_shape_to_string(
                     element_id=element_id,
                     key=shape_key,
                 )
+                _add_to_dropped_css_svg_element_map(shape_key)
                 return None
         except Exception:
             LOG.warning(
@@ -432,6 +454,7 @@ async def _convert_css_shape_to_string(
                 element_id=element_id,
                 exc_info=True,
             )
+            _add_to_dropped_css_svg_element_map(shape_key)
             return None
 
     if "attributes" not in element:

skyvern/forge/sdk/core/skyvern_context.py

@@ -25,6 +25,7 @@ class SkyvernContext:
     refresh_working_page: bool = False
     frame_index_map: dict[Frame, int] = field(default_factory=dict)
     max_screenshot_scrolling_times: int | None = None
+    dropped_css_svg_element_map: dict[str, bool] = field(default_factory=dict)
 
     def __repr__(self) -> str:
         return f"SkyvernContext(request_id={self.request_id}, organization_id={self.organization_id}, task_id={self.task_id}, workflow_id={self.workflow_id}, workflow_run_id={self.workflow_run_id}, task_v2_id={self.task_v2_id}, max_steps_override={self.max_steps_override}, run_id={self.run_id})"