introduce action.has_mini_agent (#4295)

skyvern/core/script_generations/script_skyvern_page.py

@@ -15,6 +15,7 @@ from skyvern.forge import app
 from skyvern.forge.prompts import prompt_engine
 from skyvern.forge.sdk.artifact.models import ArtifactType
 from skyvern.forge.sdk.core import skyvern_context
+from skyvern.services.otp_service import poll_otp_value
 from skyvern.utils.url_validators import prepend_scheme_and_validate_url
 from skyvern.webeye.actions.action_types import ActionType
 from skyvern.webeye.actions.actions import (
@@ -398,15 +399,36 @@ class ScriptSkyvernPage(SkyvernPage):
             # If screenshot creation fails, don't block execution
             pass
 
-    async def get_actual_value(self, value: str) -> str:
+    async def get_actual_value(
+        self,
+        value: str,
+        totp_identifier: str | None = None,
+        totp_url: str | None = None,
+    ) -> str:
         """Input text into an element identified by ``selector``."""
         context = skyvern_context.ensure_context()
         if context and context.workflow_run_id:
+            task_id = context.task_id
+            workflow_run_id = context.workflow_run_id
+            organization_id = context.organization_id
+            value = get_actual_value_of_parameter_if_secret(workflow_run_id, value)
+
             # support TOTP secret and internal it to TOTP code
             is_totp_value = value == "BW_TOTP" or value == "OP_TOTP" or value == "AZ_TOTP"
             if is_totp_value:
                 value = generate_totp_value(context.workflow_run_id, value)
-            value = get_actual_value_of_parameter_if_secret(context.workflow_run_id, value)
+            elif (totp_identifier or totp_url) and organization_id:
+                totp_value = await poll_otp_value(
+                    organization_id=organization_id,
+                    task_id=task_id,
+                    workflow_run_id=workflow_run_id,
+                    totp_verification_url=totp_url,
+                    totp_identifier=totp_identifier,
+                )
+                if totp_value:
+                    # use the totp verification code
+                    value = totp_value.value
+
         return value
 
     async def goto(self, url: str, **kwargs: Any) -> None:

skyvern/core/script_generations/skyvern_page.py

@@ -96,7 +96,12 @@ class SkyvernPage(Page):
         timeout = kwargs.pop("timeout", settings.BROWSER_LOADING_TIMEOUT_MS)
         await self.page.goto(url, timeout=timeout, **kwargs)
 
-    async def get_actual_value(self, value: str) -> str:
+    async def get_actual_value(
+        self,
+        value: str,
+        totp_identifier: str | None = None,
+        totp_url: str | None = None,
+    ) -> str:
         return value
 
     ######### Public Interfaces #########
@@ -390,7 +395,11 @@ class SkyvernPage(Page):
             error_to_raise = None
             if selector:
                 try:
-                    value = await self.get_actual_value(value)
+                    value = await self.get_actual_value(
+                        value,
+                        totp_identifier=totp_identifier,
+                        totp_url=totp_url,
+                    )
                     locator = self.page.locator(selector)
                     await handler_utils.input_sequentially(locator, value, timeout=timeout)
                     return value

skyvern/schemas/scripts.py

@@ -137,6 +137,7 @@ class ScriptBlock(BaseModel):
     run_signature: str | None = None  # The function call code to execute this block
     workflow_run_id: str | None = None
     workflow_run_block_id: str | None = None
+    input_fields: list[str] | None = None
     created_at: datetime
     modified_at: datetime
     deleted_at: datetime | None = None

skyvern/webeye/actions/actions.py

@@ -135,10 +135,19 @@ class Action(BaseModel):
     # TOTP timing information for multi-field TOTP sequences
     totp_timing_info: dict[str, Any] | None = None
 
+    # flag indicating whether the action requires mini-agent mode
+    has_mini_agent: bool | None = None
+
     created_at: datetime | None = None
     modified_at: datetime | None = None
     created_by: str | None = None
 
+    def set_has_mini_agent(self) -> None:
+        """
+        Set the has_mini_agent flag to True if any mini-agent is involved when handling the action.
+        """
+        self.has_mini_agent = True
+
     @classmethod
     def validate(cls: Type[T], value: Any) -> T:
         if isinstance(value, dict):

skyvern/webeye/actions/handler.py

@@ -1124,6 +1124,7 @@ async def handle_input_text_action(
             element_id=skyvern_element.get_id(),
             action=action,
         )
+        action.set_has_mini_agent()
         return await handle_select_option_action(select_action, page, scraped_page, task, step)
 
     incremental_element: list[dict] = []
@@ -1219,6 +1220,7 @@ async def handle_input_text_action(
             try_to_quit_dropdown = True
             try:
                 # TODO: we don't select by value for the auto completion detect case
+                action.set_has_mini_agent()
 
                 select_result = await sequentially_select_from_dropdown(
                     action=select_action,
@@ -1303,6 +1305,7 @@ async def handle_input_text_action(
     # check the phone number format when type=tel and the text is not a secret value
     if not is_secret_value and await skyvern_element.get_attr("type") == "tel":
         try:
+            action.set_has_mini_agent()
             text = await check_phone_number_format(
                 value=text,
                 action=action,
@@ -1373,6 +1376,7 @@ async def handle_input_text_action(
     if action.totp_timing_info:
         timing_info = action.totp_timing_info
         if timing_info.get("is_totp_sequence"):
+            action.set_has_mini_agent()
             result = await _handle_multi_field_totp_sequence(timing_info, task)
             if result is not None:
                 return result  # Return ActionFailure if TOTP handling failed
@@ -1405,6 +1409,7 @@ async def handle_input_text_action(
 
         if tag_name == InteractiveElement.INPUT and await skyvern_element.get_attr("type") == "date":
             try:
+                action.set_has_mini_agent()
                 text = await check_date_format(
                     value=text,
                     action=action,
@@ -1426,6 +1431,7 @@ async def handle_input_text_action(
         if not await skyvern_element.is_raw_input():
             is_location_input = input_or_select_context.is_location_input if input_or_select_context else False
             if input_or_select_context and (await skyvern_element.is_auto_completion_input() or is_location_input):
+                action.set_has_mini_agent()
                 if result := await input_or_auto_complete_input(
                     input_or_select_context=input_or_select_context,
                     scraped_page=scraped_page,
@@ -1667,6 +1673,7 @@ async def handle_select_option_action(
     # Confirm if the select action is on the custom option element
     if await skyvern_element.is_custom_option():
         click_action = ClickAction(element_id=action.element_id)
+        action.set_has_mini_agent()
         return await chain_click(task, scraped_page, page, click_action, skyvern_element)
 
     if not await skyvern_element.is_selectable():
@@ -1775,6 +1782,7 @@ async def handle_select_option_action(
             action=action,
         )
         check_action = CheckboxAction(element_id=action.element_id, is_checked=True)
+        action.set_has_mini_agent()
         return await handle_checkbox_action(check_action, page, scraped_page, task, step)
 
     if await skyvern_element.is_radio():
@@ -1783,6 +1791,7 @@ async def handle_select_option_action(
             action=action,
         )
         click_action = ClickAction(element_id=action.element_id)
+        action.set_has_mini_agent()
         return await chain_click(task, scraped_page, page, click_action, skyvern_element)
 
     # FIXME: maybe there's a case where <input type="button"> could trigger dropdown menu?
@@ -1792,6 +1801,7 @@ async def handle_select_option_action(
             action=action,
         )
         click_action = ClickAction(element_id=action.element_id)
+        action.set_has_mini_agent()
         return await chain_click(task, scraped_page, page, click_action, skyvern_element)
 
     LOG.info(
@@ -3634,6 +3644,7 @@ async def normal_select(
     step: Step,
     builder: ElementTreeBuilder,
 ) -> List[ActionResult]:
+    action.set_has_mini_agent()
     try:
         current_text = await skyvern_element.get_attr("selected")
         if current_text and (current_text == action.option.label or current_text == action.option.value):