From 32ad324af7ccb03148ff5323d09ff80fcb4b57e9 Mon Sep 17 00:00:00 2001 From: Shuchang Zheng Date: Wed, 19 Mar 2025 18:16:55 -0700 Subject: [PATCH] Do not run complete verification for extract blocks or tasks without nav goal + disable complete verification for nav blocks in task v2 run (#1973) --- skyvern/forge/agent.py | 19 +++++++++++++++++-- skyvern/forge/sdk/services/task_v2_service.py | 2 ++ skyvern/forge/sdk/workflow/models/block.py | 2 ++ skyvern/forge/sdk/workflow/models/yaml.py | 3 +++ skyvern/forge/sdk/workflow/service.py | 8 ++++++++ 5 files changed, 32 insertions(+), 2 deletions(-) diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py index 54202092..d960c756 100644 --- a/skyvern/forge/agent.py +++ b/skyvern/forge/agent.py @@ -257,6 +257,7 @@ class ForgeAgent: close_browser_on_completion: bool = True, task_block: BaseTaskBlock | None = None, browser_session_id: str | None = None, + complete_verification: bool = True, ) -> Tuple[Step, DetailedAgentStepOutput | None, Step | None]: workflow_run: WorkflowRun | None = None if task.workflow_run_id: @@ -383,7 +384,12 @@ class ForgeAgent: await self.register_async_operations(organization, task, page) step, detailed_output = await self.agent_step( - task, step, browser_state, organization=organization, task_block=task_block + task, + step, + browser_state, + organization=organization, + task_block=task_block, + complete_verification=complete_verification, ) await app.AGENT_FUNCTION.post_step_execution(task, step) task = await self.update_task_errors_from_detailed_output(task, detailed_output) @@ -519,6 +525,7 @@ class ForgeAgent: close_browser_on_completion=close_browser_on_completion, browser_session_id=browser_session_id, task_block=task_block, + complete_verification=complete_verification, ) elif settings.execute_all_steps() and next_step: return await self.execute_step( @@ -529,6 +536,7 @@ class ForgeAgent: close_browser_on_completion=close_browser_on_completion, browser_session_id=browser_session_id, task_block=task_block, + complete_verification=complete_verification, ) else: LOG.info( @@ -743,6 +751,7 @@ class ForgeAgent: browser_state: BrowserState, organization: Organization | None = None, task_block: BaseTaskBlock | None = None, + complete_verification: bool = True, ) -> tuple[Step, DetailedAgentStepOutput]: detailed_agent_step_output = DetailedAgentStepOutput( scraped_page=None, @@ -1072,7 +1081,13 @@ class ForgeAgent: break task_completes_on_download = task_block and task_block.complete_on_download and task.workflow_run_id - if not has_decisive_action and not task_completes_on_download and not isinstance(task_block, ActionBlock): + if ( + not has_decisive_action + and not task_completes_on_download + and not isinstance(task_block, ActionBlock) + and complete_verification + and (task.navigation_goal or task.complete_criterion) + ): disable_user_goal_check = app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached( "DISABLE_USER_GOAL_CHECK", task.task_id, diff --git a/skyvern/forge/sdk/services/task_v2_service.py b/skyvern/forge/sdk/services/task_v2_service.py index 27f9a05b..91709696 100644 --- a/skyvern/forge/sdk/services/task_v2_service.py +++ b/skyvern/forge/sdk/services/task_v2_service.py @@ -1146,6 +1146,7 @@ async def _generate_navigation_task( navigation_goal=navigation_goal, totp_verification_url=totp_verification_url, totp_identifier=totp_identifier, + complete_verification=False, ) output_parameter = await app.WORKFLOW_SERVICE.create_output_parameter_for_block( workflow_id=workflow_id, @@ -1159,6 +1160,7 @@ async def _generate_navigation_task( totp_verification_url=totp_verification_url, totp_identifier=totp_identifier, output_parameter=output_parameter, + complete_verification=False, ), [navigation_block_yaml], [], diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py index 75fff2c3..b6a123cc 100644 --- a/skyvern/forge/sdk/workflow/models/block.py +++ b/skyvern/forge/sdk/workflow/models/block.py @@ -346,6 +346,7 @@ class BaseTaskBlock(Block): totp_verification_url: str | None = None totp_identifier: str | None = None cache_actions: bool = False + complete_verification: bool = True def get_all_parameters( self, @@ -601,6 +602,7 @@ class BaseTaskBlock(Block): task_block=self, browser_session_id=browser_session_id, close_browser_on_completion=browser_session_id is None, + complete_verification=self.complete_verification, ) except Exception as e: # Make sure the task is marked as failed in the database before raising the exception diff --git a/skyvern/forge/sdk/workflow/models/yaml.py b/skyvern/forge/sdk/workflow/models/yaml.py index 14d127ca..e9af4265 100644 --- a/skyvern/forge/sdk/workflow/models/yaml.py +++ b/skyvern/forge/sdk/workflow/models/yaml.py @@ -141,6 +141,7 @@ class TaskBlockYAML(BlockYAML): cache_actions: bool = False complete_criterion: str | None = None terminate_criterion: str | None = None + complete_verification: bool = True class ForLoopBlockYAML(BlockYAML): @@ -273,6 +274,7 @@ class NavigationBlockYAML(BlockYAML): cache_actions: bool = False complete_criterion: str | None = None terminate_criterion: str | None = None + complete_verification: bool = True class ExtractionBlockYAML(BlockYAML): @@ -303,6 +305,7 @@ class LoginBlockYAML(BlockYAML): cache_actions: bool = False complete_criterion: str | None = None terminate_criterion: str | None = None + complete_verification: bool = True class WaitBlockYAML(BlockYAML): diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py index abe519ac..ebfe4957 100644 --- a/skyvern/forge/sdk/workflow/service.py +++ b/skyvern/forge/sdk/workflow/service.py @@ -1595,6 +1595,7 @@ class WorkflowService: cache_actions=block_yaml.cache_actions, complete_criterion=block_yaml.complete_criterion, terminate_criterion=block_yaml.terminate_criterion, + complete_verification=block_yaml.complete_verification, ) elif block_yaml.block_type == BlockType.FOR_LOOP: loop_blocks = [ @@ -1747,6 +1748,8 @@ class WorkflowService: totp_verification_url=block_yaml.totp_verification_url, totp_identifier=block_yaml.totp_identifier, cache_actions=block_yaml.cache_actions, + # DO NOT run complete verification for action block + complete_verification=False, max_steps_per_run=1, ) @@ -1774,6 +1777,7 @@ class WorkflowService: cache_actions=block_yaml.cache_actions, complete_criterion=block_yaml.complete_criterion, terminate_criterion=block_yaml.terminate_criterion, + complete_verification=block_yaml.complete_verification, ) elif block_yaml.block_type == BlockType.EXTRACTION: @@ -1794,6 +1798,7 @@ class WorkflowService: max_retries=block_yaml.max_retries, continue_on_failure=block_yaml.continue_on_failure, cache_actions=block_yaml.cache_actions, + complete_verification=False, ) elif block_yaml.block_type == BlockType.LOGIN: @@ -1818,6 +1823,7 @@ class WorkflowService: cache_actions=block_yaml.cache_actions, complete_criterion=block_yaml.complete_criterion, terminate_criterion=block_yaml.terminate_criterion, + complete_verification=block_yaml.complete_verification, ) elif block_yaml.block_type == BlockType.WAIT: @@ -1853,6 +1859,7 @@ class WorkflowService: totp_identifier=block_yaml.totp_identifier, cache_actions=block_yaml.cache_actions, complete_on_download=True, + complete_verification=False, ) elif block_yaml.block_type == BlockType.TaskV2: return TaskV2Block( @@ -1870,6 +1877,7 @@ class WorkflowService: label=block_yaml.label, url=block_yaml.url, output_parameter=output_parameter, + complete_verification=False, ) raise ValueError(f"Invalid block type {block_yaml.block_type}")