From 26f0e614c4aca808f439363d3cef7cc531e7d335 Mon Sep 17 00:00:00 2001 From: Shuchang Zheng Date: Thu, 13 Mar 2025 10:01:29 -0700 Subject: [PATCH] update action history prompt (#1932) --- skyvern/forge/agent.py | 43 +++++++++---------- .../forge/prompts/skyvern/check-user-goal.j2 | 10 ++--- .../forge/prompts/skyvern/extract-action.j2 | 23 +++++----- .../prompts/skyvern/single-click-action.j2 | 14 +++--- .../prompts/skyvern/single-input-action.j2 | 14 +++--- .../prompts/skyvern/single-select-action.j2 | 14 +++--- .../prompts/skyvern/single-upload-action.j2 | 14 +++--- 7 files changed, 67 insertions(+), 65 deletions(-) diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py index e41f93b0..1f048f5d 100644 --- a/skyvern/forge/agent.py +++ b/skyvern/forge/agent.py @@ -1542,29 +1542,28 @@ class ForgeAgent: actions_and_results.extend(window_step.output.actions_and_results) # exclude successful action from history - return json.dumps( - [ - { - "action": action.model_dump( + action_history = [ + { + "action": action.model_dump( + exclude_none=True, + include={"action_type", "element_id", "status", "reasoning", "option", "download"}, + ), + "results": [ + result.model_dump( exclude_none=True, - include={"action_type", "element_id", "status", "reasoning", "option", "download"}, - ), - "results": [ - result.model_dump( - exclude_none=True, - include={ - "success", - "exception_type", - "exception_message", - }, - ) - for result in results - ], - } - for action, results in actions_and_results - if len(results) > 0 - ] - ) + include={ + "success", + "exception_type", + "exception_message", + }, + ) + for result in results + ], + } + for action, results in actions_and_results + if len(results) > 0 + ] + return json.dumps(action_history) if action_history else "" async def get_extracted_information_for_task(self, task: Task) -> dict[str, Any] | list | str | None: """ diff --git a/skyvern/forge/prompts/skyvern/check-user-goal.j2 b/skyvern/forge/prompts/skyvern/check-user-goal.j2 index beb82577..a4eef6b4 100644 --- a/skyvern/forge/prompts/skyvern/check-user-goal.j2 +++ b/skyvern/forge/prompts/skyvern/check-user-goal.j2 @@ -9,11 +9,6 @@ Make sure to ONLY return the JSON object in this format with no additional text } ``` -Elements on the page: -``` -{{ elements }} -``` - User Goal: ``` {{ navigation_goal }} @@ -28,3 +23,8 @@ Complete Criterion: ``` {{ complete_criterion }} ```{% endif %} + +Elements on the page: +``` +{{ elements }} +``` \ No newline at end of file diff --git a/skyvern/forge/prompts/skyvern/extract-action.j2 b/skyvern/forge/prompts/skyvern/extract-action.j2 index 4fe0d8e2..4ed00a8c 100644 --- a/skyvern/forge/prompts/skyvern/extract-action.j2 +++ b/skyvern/forge/prompts/skyvern/extract-action.j2 @@ -42,13 +42,12 @@ Reply in JSON format with the following keys: } {% if action_history %} Consider the action history from the last step and the screenshot together, if actions from the last step don't yield positive impact, try other actions or other action combinations. -{% endif %} -Clickable elements from `{{ current_url }}`: -``` -{{ elements }} -``` -The URL of the page you're on right now is `{{ current_url }}`. +Action history from previous steps: (note: even if the action history suggests goal is achieved, check the screenshot and the DOM elements to make sure the goal is achieved) +``` +{{ action_history }} +``` +{% endif %} {% if complete_criterion %} Complete criterion: ``` @@ -74,10 +73,14 @@ User details: ``` {{ navigation_payload_str }} ``` -{% if action_history %} -Action results from previous steps: (note: even if the action history suggests goal is achieved, check the screenshot and the DOM elements to make sure the goal is achieved) -{{ action_history }} -{% endif %} + +Clickable elements from `{{ current_url }}`: +``` +{{ elements }} +``` + +The URL of the page you're on right now is `{{ current_url }}`. + Current datetime, ISO format: ``` {{ local_datetime }} diff --git a/skyvern/forge/prompts/skyvern/single-click-action.j2 b/skyvern/forge/prompts/skyvern/single-click-action.j2 index c886df82..0dcb0287 100644 --- a/skyvern/forge/prompts/skyvern/single-click-action.j2 +++ b/skyvern/forge/prompts/skyvern/single-click-action.j2 @@ -19,13 +19,6 @@ Reply in JSON format with the following keys: }] } -The URL of the page you're on right now is `{{ current_url }}`. - -HTML elements from `{{ current_url }}`: -``` -{{ elements }} -``` - User instruction (user's intention or self questioning to help figure out what to click): ``` {{ navigation_goal }} @@ -41,6 +34,13 @@ Context of the big goal user wants to achieve: {{ user_context }} ```{% endif %} +The URL of the page you're on right now is `{{ current_url }}`. + +HTML elements from `{{ current_url }}`: +``` +{{ elements }} +``` + Current datetime, ISO format: ``` {{ local_datetime }} diff --git a/skyvern/forge/prompts/skyvern/single-input-action.j2 b/skyvern/forge/prompts/skyvern/single-input-action.j2 index bcf6cec7..dd057609 100644 --- a/skyvern/forge/prompts/skyvern/single-input-action.j2 +++ b/skyvern/forge/prompts/skyvern/single-input-action.j2 @@ -20,13 +20,6 @@ Reply in JSON format with the following keys: "need_verification_code": bool, // Whether a verification code is needed to proceed. True only if the code is available to user. If the code is not sent, return false {% endif %} } -The URL of the page you're on right now is `{{ current_url }}`. - -HTML elements from `{{ current_url }}`: -``` -{{ elements }} -``` - User instruction: ``` {{ navigation_goal }} @@ -37,6 +30,13 @@ User details: {{ navigation_payload_str }} ``` +The URL of the page you're on right now is `{{ current_url }}`. + +HTML elements from `{{ current_url }}`: +``` +{{ elements }} +``` + Current datetime, ISO format: ``` {{ local_datetime }} diff --git a/skyvern/forge/prompts/skyvern/single-select-action.j2 b/skyvern/forge/prompts/skyvern/single-select-action.j2 index 89ae123b..eea267ad 100644 --- a/skyvern/forge/prompts/skyvern/single-select-action.j2 +++ b/skyvern/forge/prompts/skyvern/single-select-action.j2 @@ -22,13 +22,6 @@ Reply in JSON format with the following keys: }] } -The URL of the page you're on right now is `{{ current_url }}`. - -HTML elements from `{{ current_url }}`: -``` -{{ elements }} -``` - User instruction: ``` {{ navigation_goal }} @@ -39,6 +32,13 @@ User details: {{ navigation_payload_str }} ``` +The URL of the page you're on right now is `{{ current_url }}`. + +HTML elements from `{{ current_url }}`: +``` +{{ elements }} +``` + Current datetime, ISO format: ``` {{ local_datetime }} diff --git a/skyvern/forge/prompts/skyvern/single-upload-action.j2 b/skyvern/forge/prompts/skyvern/single-upload-action.j2 index af6c4cb5..edd4ca53 100644 --- a/skyvern/forge/prompts/skyvern/single-upload-action.j2 +++ b/skyvern/forge/prompts/skyvern/single-upload-action.j2 @@ -18,13 +18,6 @@ Reply in JSON format with the following keys: }] } -The URL of the page you're on right now is `{{ current_url }}`. - -HTML elements from `{{ current_url }}`: -``` -{{ elements }} -``` - User instruction: ``` {{ navigation_goal }} @@ -35,6 +28,13 @@ User details: {{ navigation_payload_str }} ``` +The URL of the page you're on right now is `{{ current_url }}`. + +HTML elements from `{{ current_url }}`: +``` +{{ elements }} +``` + Current datetime, ISO format: ``` {{ local_datetime }}