diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.py index e685e3a9..9172b866 100644 --- a/skyvern/core/script_generations/script_skyvern_page.py +++ b/skyvern/core/script_generations/script_skyvern_page.py @@ -15,6 +15,7 @@ from skyvern.forge import app from skyvern.forge.prompts import prompt_engine from skyvern.forge.sdk.artifact.models import ArtifactType from skyvern.forge.sdk.core import skyvern_context +from skyvern.services.otp_service import poll_otp_value from skyvern.utils.url_validators import prepend_scheme_and_validate_url from skyvern.webeye.actions.action_types import ActionType from skyvern.webeye.actions.actions import ( @@ -398,15 +399,36 @@ class ScriptSkyvernPage(SkyvernPage): # If screenshot creation fails, don't block execution pass - async def get_actual_value(self, value: str) -> str: + async def get_actual_value( + self, + value: str, + totp_identifier: str | None = None, + totp_url: str | None = None, + ) -> str: """Input text into an element identified by ``selector``.""" context = skyvern_context.ensure_context() if context and context.workflow_run_id: + task_id = context.task_id + workflow_run_id = context.workflow_run_id + organization_id = context.organization_id + value = get_actual_value_of_parameter_if_secret(workflow_run_id, value) + # support TOTP secret and internal it to TOTP code is_totp_value = value == "BW_TOTP" or value == "OP_TOTP" or value == "AZ_TOTP" if is_totp_value: value = generate_totp_value(context.workflow_run_id, value) - value = get_actual_value_of_parameter_if_secret(context.workflow_run_id, value) + elif (totp_identifier or totp_url) and organization_id: + totp_value = await poll_otp_value( + organization_id=organization_id, + task_id=task_id, + workflow_run_id=workflow_run_id, + totp_verification_url=totp_url, + totp_identifier=totp_identifier, + ) + if totp_value: + # use the totp verification code + value = totp_value.value + return value async def goto(self, url: str, **kwargs: Any) -> None: diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.py index c17d53fd..06e502ee 100644 --- a/skyvern/core/script_generations/skyvern_page.py +++ b/skyvern/core/script_generations/skyvern_page.py @@ -96,7 +96,12 @@ class SkyvernPage(Page): timeout = kwargs.pop("timeout", settings.BROWSER_LOADING_TIMEOUT_MS) await self.page.goto(url, timeout=timeout, **kwargs) - async def get_actual_value(self, value: str) -> str: + async def get_actual_value( + self, + value: str, + totp_identifier: str | None = None, + totp_url: str | None = None, + ) -> str: return value ######### Public Interfaces ######### @@ -390,7 +395,11 @@ class SkyvernPage(Page): error_to_raise = None if selector: try: - value = await self.get_actual_value(value) + value = await self.get_actual_value( + value, + totp_identifier=totp_identifier, + totp_url=totp_url, + ) locator = self.page.locator(selector) await handler_utils.input_sequentially(locator, value, timeout=timeout) return value diff --git a/skyvern/schemas/scripts.py b/skyvern/schemas/scripts.py index 2481e8a4..f110a5f0 100644 --- a/skyvern/schemas/scripts.py +++ b/skyvern/schemas/scripts.py @@ -137,6 +137,7 @@ class ScriptBlock(BaseModel): run_signature: str | None = None # The function call code to execute this block workflow_run_id: str | None = None workflow_run_block_id: str | None = None + input_fields: list[str] | None = None created_at: datetime modified_at: datetime deleted_at: datetime | None = None diff --git a/skyvern/webeye/actions/actions.py b/skyvern/webeye/actions/actions.py index 743f0df6..b9e066bc 100644 --- a/skyvern/webeye/actions/actions.py +++ b/skyvern/webeye/actions/actions.py @@ -135,10 +135,19 @@ class Action(BaseModel): # TOTP timing information for multi-field TOTP sequences totp_timing_info: dict[str, Any] | None = None + # flag indicating whether the action requires mini-agent mode + has_mini_agent: bool | None = None + created_at: datetime | None = None modified_at: datetime | None = None created_by: str | None = None + def set_has_mini_agent(self) -> None: + """ + Set the has_mini_agent flag to True if any mini-agent is involved when handling the action. + """ + self.has_mini_agent = True + @classmethod def validate(cls: Type[T], value: Any) -> T: if isinstance(value, dict): diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py index 2ea6c140..d4172046 100644 --- a/skyvern/webeye/actions/handler.py +++ b/skyvern/webeye/actions/handler.py @@ -1124,6 +1124,7 @@ async def handle_input_text_action( element_id=skyvern_element.get_id(), action=action, ) + action.set_has_mini_agent() return await handle_select_option_action(select_action, page, scraped_page, task, step) incremental_element: list[dict] = [] @@ -1219,6 +1220,7 @@ async def handle_input_text_action( try_to_quit_dropdown = True try: # TODO: we don't select by value for the auto completion detect case + action.set_has_mini_agent() select_result = await sequentially_select_from_dropdown( action=select_action, @@ -1303,6 +1305,7 @@ async def handle_input_text_action( # check the phone number format when type=tel and the text is not a secret value if not is_secret_value and await skyvern_element.get_attr("type") == "tel": try: + action.set_has_mini_agent() text = await check_phone_number_format( value=text, action=action, @@ -1373,6 +1376,7 @@ async def handle_input_text_action( if action.totp_timing_info: timing_info = action.totp_timing_info if timing_info.get("is_totp_sequence"): + action.set_has_mini_agent() result = await _handle_multi_field_totp_sequence(timing_info, task) if result is not None: return result # Return ActionFailure if TOTP handling failed @@ -1405,6 +1409,7 @@ async def handle_input_text_action( if tag_name == InteractiveElement.INPUT and await skyvern_element.get_attr("type") == "date": try: + action.set_has_mini_agent() text = await check_date_format( value=text, action=action, @@ -1426,6 +1431,7 @@ async def handle_input_text_action( if not await skyvern_element.is_raw_input(): is_location_input = input_or_select_context.is_location_input if input_or_select_context else False if input_or_select_context and (await skyvern_element.is_auto_completion_input() or is_location_input): + action.set_has_mini_agent() if result := await input_or_auto_complete_input( input_or_select_context=input_or_select_context, scraped_page=scraped_page, @@ -1667,6 +1673,7 @@ async def handle_select_option_action( # Confirm if the select action is on the custom option element if await skyvern_element.is_custom_option(): click_action = ClickAction(element_id=action.element_id) + action.set_has_mini_agent() return await chain_click(task, scraped_page, page, click_action, skyvern_element) if not await skyvern_element.is_selectable(): @@ -1775,6 +1782,7 @@ async def handle_select_option_action( action=action, ) check_action = CheckboxAction(element_id=action.element_id, is_checked=True) + action.set_has_mini_agent() return await handle_checkbox_action(check_action, page, scraped_page, task, step) if await skyvern_element.is_radio(): @@ -1783,6 +1791,7 @@ async def handle_select_option_action( action=action, ) click_action = ClickAction(element_id=action.element_id) + action.set_has_mini_agent() return await chain_click(task, scraped_page, page, click_action, skyvern_element) # FIXME: maybe there's a case where could trigger dropdown menu? @@ -1792,6 +1801,7 @@ async def handle_select_option_action( action=action, ) click_action = ClickAction(element_id=action.element_id) + action.set_has_mini_agent() return await chain_click(task, scraped_page, page, click_action, skyvern_element) LOG.info( @@ -3634,6 +3644,7 @@ async def normal_select( step: Step, builder: ElementTreeBuilder, ) -> List[ActionResult]: + action.set_has_mini_agent() try: current_text = await skyvern_element.get_attr("selected") if current_text and (current_text == action.option.label or current_text == action.option.value):