laminar integration (#2887)

skyvern/webeye/actions/handler.py

@@ -71,6 +71,7 @@ from skyvern.forge.sdk.models import Step
 from skyvern.forge.sdk.schemas.tasks import Task
 from skyvern.forge.sdk.services.bitwarden import BitwardenConstants
 from skyvern.forge.sdk.services.credentials import OnePasswordConstants
+from skyvern.forge.sdk.trace import TraceManager
 from skyvern.services.task_v1_service import is_cua_task
 from skyvern.utils.prompt_engine import CheckPhoneNumberFormatResponse, load_prompt_with_elements
 from skyvern.webeye.actions import actions, handler_utils
@@ -340,6 +341,7 @@ class ActionHandler:
         cls._teardown_action_types[action_type] = handler
 
     @staticmethod
+    @TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
     async def handle_action(
         scraped_page: ScrapedPage,
         task: Task,
@@ -459,6 +461,7 @@ def check_for_invalid_web_action(
     return []
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_solve_captcha_action(
     action: actions.SolveCaptchaAction,
     page: Page,
@@ -474,6 +477,7 @@ async def handle_solve_captcha_action(
     return [ActionSuccess()]
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_click_action(
     action: actions.ClickAction,
     page: Page,
@@ -650,6 +654,7 @@ async def handle_click_action(
     return results
 
 
+@TraceManager.traced_async(ignore_inputs=["anchor_element", "scraped_page", "page", "incremental_scraped", "dom"])
 async def handle_sequential_click_for_dropdown(
     action: actions.ClickAction,
     anchor_element: SkyvernElement,
@@ -722,6 +727,7 @@ async def handle_sequential_click_for_dropdown(
     )
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_click_to_download_file_action(
     action: actions.ClickAction,
     page: Page,
@@ -814,6 +820,7 @@ async def handle_click_to_download_file_action(
     return [ActionSuccess(download_triggered=True)]
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_input_text_action(
     action: actions.InputTextAction,
     page: Page,
@@ -1135,6 +1142,7 @@ async def handle_input_text_action(
             await skyvern_element.press_key("Tab")
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_upload_file_action(
     action: actions.UploadFileAction,
     page: Page,
@@ -1212,6 +1220,7 @@ async def handle_upload_file_action(
 
 
 # This function is deprecated. Downloads are handled by the click action handler now.
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_download_file_action(
     action: actions.DownloadFileAction,
     page: Page,
@@ -1253,6 +1262,7 @@ async def handle_download_file_action(
     return [ActionSuccess(data={"file_path": full_file_path})]
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_null_action(
     action: actions.NullAction,
     page: Page,
@@ -1263,6 +1273,7 @@ async def handle_null_action(
     return [ActionSuccess()]
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_select_option_action(
     action: actions.SelectOptionAction,
     page: Page,
@@ -1600,6 +1611,7 @@ async def handle_select_option_action(
         await incremental_scraped.stop_listen_dom_increment()
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_checkbox_action(
     action: actions.CheckboxAction,
     page: Page,
@@ -1628,6 +1640,7 @@ async def handle_checkbox_action(
     return [ActionSuccess()]
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_wait_action(
     action: actions.WaitAction,
     page: Page,
@@ -1639,6 +1652,7 @@ async def handle_wait_action(
     return [ActionFailure(exception=Exception("Wait action is treated as a failure"))]
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_terminate_action(
     action: actions.TerminateAction,
     page: Page,
@@ -1649,6 +1663,7 @@ async def handle_terminate_action(
     return [ActionSuccess()]
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_complete_action(
     action: actions.CompleteAction,
     page: Page,
@@ -1694,6 +1709,7 @@ async def handle_complete_action(
     return [ActionSuccess()]
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_extract_action(
     action: actions.ExtractAction,
     page: Page,
@@ -1715,6 +1731,7 @@ async def handle_extract_action(
         return [ActionFailure(exception=Exception("No data extraction goal"))]
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_scroll_action(
     action: actions.ScrollAction,
     page: Page,
@@ -1728,6 +1745,7 @@ async def handle_scroll_action(
     return [ActionSuccess()]
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_keypress_action(
     action: actions.KeypressAction,
     page: Page,
@@ -1787,6 +1805,7 @@ async def handle_keypress_action(
     return [ActionSuccess()]
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_move_action(
     action: actions.MoveAction,
     page: Page,
@@ -1798,6 +1817,7 @@ async def handle_move_action(
     return [ActionSuccess()]
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_drag_action(
     action: actions.DragAction,
     page: Page,
@@ -1815,6 +1835,7 @@ async def handle_drag_action(
     return [ActionSuccess()]
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_verification_code_action(
     action: actions.VerificationCodeAction,
     page: Page,
@@ -1833,6 +1854,7 @@ async def handle_verification_code_action(
     return [ActionSuccess()]
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def handle_left_mouse_action(
     action: actions.LeftMouseAction,
     page: Page,
@@ -2092,6 +2114,7 @@ async def chain_click(
             return [ActionFailure(WrongElementToUploadFile(action.element_id))]
 
 
+@TraceManager.traced_async(ignore_inputs=["context", "page", "dom", "text", "skyvern_element", "preserved_elements"])
 async def choose_auto_completion_dropdown(
     context: InputOrSelectContext,
     page: Page,
@@ -2418,6 +2441,19 @@ async def input_or_auto_complete_input(
         return None
 
 
+@TraceManager.traced_async(
+    ignore_inputs=[
+        "input_or_select_context",
+        "page",
+        "dom",
+        "skyvern_element",
+        "skyvern_frame",
+        "incremental_scraped",
+        "dropdown_menu_element",
+        "target_value",
+        "continue_until_close",
+    ]
+)
 async def sequentially_select_from_dropdown(
     action: SelectOptionAction,
     input_or_select_context: InputOrSelectContext,
@@ -2470,6 +2506,7 @@ async def sequentially_select_from_dropdown(
             force_select=force_select,
             target_value=target_value,
         )
+        assert single_select_result is not None
         select_history.append(single_select_result)
         values.append(single_select_result.value)
         # wait 1s until DOM finished updating
@@ -2614,6 +2651,7 @@ class CustomSelectPromptOptions(BaseModel):
     target_value: str | None = None
 
 
+@TraceManager.traced_async(ignore_inputs=["scraped_page", "page"])
 async def select_from_emerging_elements(
     current_element_id: str,
     options: CustomSelectPromptOptions,
@@ -2702,6 +2740,19 @@ async def select_from_emerging_elements(
     return ActionSuccess()
 
 
+@TraceManager.traced_async(
+    ignore_inputs=[
+        "context",
+        "page",
+        "skyvern_element",
+        "skyvern_frame",
+        "incremental_scraped",
+        "check_filter_funcs",
+        "dropdown_menu_element",
+        "select_history",
+        "target_value",
+    ]
+)
 async def select_from_dropdown(
     context: InputOrSelectContext,
     page: Page,
@@ -2893,6 +2944,17 @@ async def select_from_dropdown(
         return single_select_result
 
 
+@TraceManager.traced_async(
+    ignore_inputs=[
+        "value",
+        "page",
+        "skyvern_element",
+        "skyvern_frame",
+        "dom",
+        "incremental_scraped",
+        "dropdown_menu_element",
+    ]
+)
 async def select_from_dropdown_by_value(
     value: str,
     page: Page,
@@ -3133,6 +3195,9 @@ async def try_to_find_potential_scrollable_element(
     return skyvern_element
 
 
+@TraceManager.traced_async(
+    ignore_inputs=["scrollable_element", "page", "skyvern_frame", "incremental_scraped", "is_continue"]
+)
 async def scroll_down_to_load_all_options(
     scrollable_element: SkyvernElement,
     page: Page,

skyvern/webeye/scraper/scraper.py

@@ -16,6 +16,7 @@ from skyvern.constants import BUILDING_ELEMENT_TREE_TIMEOUT_MS, DEFAULT_MAX_TOKE
 from skyvern.exceptions import FailedToTakeScreenshot, ScrapingFailed, UnknownElementTreeFormat
 from skyvern.forge.sdk.api.crypto import calculate_sha256
 from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.trace import TraceManager
 from skyvern.utils.image_resizer import Resolution
 from skyvern.utils.token_counter import count_tokens
 from skyvern.webeye.browser_factory import BrowserState
@@ -392,6 +393,7 @@ class ScrapedPage(BaseModel, ElementTreeBuilder):
         return await self.generate_scraped_page(take_screenshots=False)
 
 
+@TraceManager.traced_async(ignore_input=True)
 async def scrape_website(
     browser_state: BrowserState,
     url: str,
@@ -646,11 +648,9 @@ async def add_frame_interactable_elements(
         )
         return elements, element_tree
 
-    frame_js_script = f"async () => await buildTreeFromBody('{unique_id}', {frame_index})"
-
-    await SkyvernFrame.evaluate(frame=frame, expression=JS_FUNCTION_DEFS)
-    frame_elements, frame_element_tree = await SkyvernFrame.evaluate(
-        frame=frame, expression=frame_js_script, timeout_ms=BUILDING_ELEMENT_TREE_TIMEOUT_MS
+    skyvern_frame = await SkyvernFrame.create_instance(frame)
+    frame_elements, frame_element_tree = await skyvern_frame.build_tree_from_body(
+        frame_name=unique_id, frame_index=frame_index
     )
 
     for element in elements:
@@ -662,6 +662,7 @@ async def add_frame_interactable_elements(
     return elements, element_tree
 
 
+@TraceManager.traced_async(ignore_input=True)
 async def get_interactable_element_tree(
     page: Page,
     scrape_exclude: ScrapeExcludeFunc | None = None,
@@ -671,12 +672,9 @@ async def get_interactable_element_tree(
     :param page: Page instance to get the element tree from.
     :return: Tuple containing the element tree and a map of element IDs to elements.
     """
-    await SkyvernFrame.evaluate(frame=page, expression=JS_FUNCTION_DEFS)
     # main page index is 0
-    main_frame_js_script = "async () => await buildTreeFromBody('main.frame', 0)"
-    elements, element_tree = await SkyvernFrame.evaluate(
-        frame=page, expression=main_frame_js_script, timeout_ms=BUILDING_ELEMENT_TREE_TIMEOUT_MS
-    )
+    skyvern_page = await SkyvernFrame.create_instance(page)
+    elements, element_tree = await skyvern_page.build_tree_from_body(frame_name="main.frame", frame_index=0)
 
     context = skyvern_context.ensure_context()
     frames = await get_all_children_frames(page)
@@ -718,25 +716,23 @@ class IncrementalScrapePage(ElementTreeBuilder):
             return True
         return False
 
+    @TraceManager.traced_async(ignore_input=True)
     async def get_incremental_element_tree(
         self,
         cleanup_element_tree: CleanupElementTreeFunc,
     ) -> list[dict]:
         frame = self.skyvern_frame.get_frame()
 
-        js_script = "async () => await getIncrementElements()"
         try:
-            incremental_elements, incremental_tree = await SkyvernFrame.evaluate(
-                frame=frame, expression=js_script, timeout_ms=BUILDING_ELEMENT_TREE_TIMEOUT_MS
+            incremental_elements, incremental_tree = await self.skyvern_frame.get_incremental_element_tree(
+                wait_until_finished=True
             )
         except TimeoutError:
             LOG.warning(
                 "Timeout to get incremental elements with wait_until_finished, going to get incremental elements without waiting",
             )
-
-            js_script = "async () => await getIncrementElements(false)"
-            incremental_elements, incremental_tree = await SkyvernFrame.evaluate(
-                frame=frame, expression=js_script, timeout_ms=BUILDING_ELEMENT_TREE_TIMEOUT_MS
+            incremental_elements, incremental_tree = await self.skyvern_frame.get_incremental_element_tree(
+                wait_until_finished=False
             )
 
         # we listen the incremental elements seperated by frames, so all elements will be in the same SkyvernFrame

skyvern/webeye/utils/page.py

@@ -14,6 +14,7 @@ from playwright.async_api import ElementHandle, Frame, Page
 from skyvern.config import settings
 from skyvern.constants import BUILDING_ELEMENT_TREE_TIMEOUT_MS, PAGE_CONTENT_TIMEOUT, SKYVERN_DIR
 from skyvern.exceptions import FailedToTakeScreenshot
+from skyvern.forge.sdk.trace import TraceManager
 
 LOG = structlog.get_logger()
 
@@ -221,6 +222,7 @@ class SkyvernFrame:
         return await SkyvernFrame.evaluate(frame=frame, expression="() => document.location.href")
 
     @staticmethod
+    @TraceManager.traced_async(ignore_inputs=["file_path", "timeout"])
     async def take_scrolling_screenshot(
         page: Page,
         file_path: str | None = None,
@@ -286,6 +288,7 @@ class SkyvernFrame:
             await skyvern_frame.scroll_to_x_y(x, y)
 
     @staticmethod
+    @TraceManager.traced_async(ignore_inputs=["page"])
     async def take_split_screenshots(
         page: Page,
         url: str | None = None,
@@ -436,3 +439,21 @@ class SkyvernFrame:
     async def get_select_options(self, element: ElementHandle) -> tuple[list, str]:
         js_script = "([element]) => getSelectOptions(element)"
         return await self.evaluate(frame=self.frame, expression=js_script, arg=[element])
+
+    @TraceManager.traced_async()
+    async def build_tree_from_body(
+        self, frame_name: str | None, frame_index: int, timeout_ms: float = BUILDING_ELEMENT_TREE_TIMEOUT_MS
+    ) -> tuple[list[dict], list[dict]]:
+        js_script = "async ([frame_name, frame_index]) => await buildTreeFromBody(frame_name, frame_index)"
+        return await self.evaluate(
+            frame=self.frame, expression=js_script, timeout_ms=timeout_ms, arg=[frame_name, frame_index]
+        )
+
+    @TraceManager.traced_async()
+    async def get_incremental_element_tree(
+        self, wait_until_finished: bool = True, timeout_ms: float = BUILDING_ELEMENT_TREE_TIMEOUT_MS
+    ) -> tuple[list[dict], list[dict]]:
+        js_script = "async ([wait_until_finished]) => await getIncrementElements(wait_until_finished)"
+        return await self.evaluate(
+            frame=self.frame, expression=js_script, timeout_ms=timeout_ms, arg=[wait_until_finished]
+        )