add history to surface errorcode prompt (#3428)

skyvern/forge/agent.py

@@ -81,6 +81,7 @@ from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, Wo
 from skyvern.schemas.runs import CUA_ENGINES, RunEngine
 from skyvern.schemas.steps import AgentStepOutput
 from skyvern.services import run_service
+from skyvern.services.action_service import get_action_history
 from skyvern.services.task_v1_service import is_cua_task
 from skyvern.utils.image_resizer import Resolution
 from skyvern.utils.prompt_engine import MaxStepsReasonResponse, load_prompt_with_elements
@@ -2100,47 +2101,7 @@ class ForgeAgent:
         return final_navigation_payload
 
     async def _get_action_results(self, task: Task, current_step: Step | None = None) -> str:
-        """
-        Get the action results from the last app.SETTINGS.PROMPT_ACTION_HISTORY_WINDOW steps.
-        If current_step is provided, the current executing step will be included in the action history.
-        Default is excluding the current executing step from the action history.
-        """
-
-        # Get action results from the last app.SETTINGS.PROMPT_ACTION_HISTORY_WINDOW steps
-        steps = await app.DATABASE.get_task_steps(task_id=task.task_id, organization_id=task.organization_id)
-        # the last step is always the newly created one and it should be excluded from the history window
-        window_steps = steps[-1 - settings.PROMPT_ACTION_HISTORY_WINDOW : -1]
-        if current_step:
-            window_steps.append(current_step)
-
-        actions_and_results: list[tuple[Action, list[ActionResult]]] = []
-        for window_step in window_steps:
-            if window_step.output and window_step.output.actions_and_results:
-                actions_and_results.extend(window_step.output.actions_and_results)
-
-        # exclude successful action from history
-        action_history = [
-            {
-                "action": action.model_dump(
-                    exclude_none=True,
-                    include={"action_type", "element_id", "status", "reasoning", "option", "download"},
-                ),
-                "results": [
-                    result.model_dump(
-                        exclude_none=True,
-                        include={
-                            "success",
-                            "exception_type",
-                            "exception_message",
-                        },
-                    )
-                    for result in results
-                ],
-            }
-            for action, results in actions_and_results
-            if len(results) > 0
-        ]
-        return json.dumps(action_history)
+        return json.dumps(await get_action_history(task=task, current_step=current_step))
 
     async def get_extracted_information_for_task(self, task: Task) -> dict[str, Any] | list | str | None:
         """

skyvern/forge/prompts/skyvern/surface-user-defined-errors.j2

@@ -1,4 +1,4 @@
-You are here to help the user use the error codes and their descriptions to surface user-defined errors based on the screenshots, user goal, user details and the HTML elements.
+You are here to help the user use the error codes and their descriptions to surface user-defined errors based on the screenshots, user goal, user details, action history and the HTML elements.
 Do not return any error that's not defined by the user. 
 
 Reply in JSON format with the following keys:
@@ -27,6 +27,12 @@ User details:
 {{ navigation_payload_str }}
 ```
 
+Consider the action history and the screenshot together.
+Action history from previous steps: (note: even if the action history suggests goal is achieved, check the screenshot and the DOM elements to make sure the goal is achieved)
+```
+{{action_history}}
+```
+
 Clickable elements from `{{ current_url }}`:
 ```
 {{ elements }}

skyvern/services/action_service.py

@@ -0,0 +1,54 @@
+from typing import Any
+
+from skyvern.config import settings
+from skyvern.forge import app
+from skyvern.forge.sdk.models import Step
+from skyvern.forge.sdk.schemas.tasks import Task
+from skyvern.webeye.actions.actions import Action
+from skyvern.webeye.actions.responses import ActionResult
+
+
+async def get_action_history(
+    task: Task, current_step: Step | None = None, history_window: int = settings.PROMPT_ACTION_HISTORY_WINDOW
+) -> list[dict[str, Any]]:
+    """
+    Get the action results from the last history_window steps.
+    If current_step is provided, the current executing step will be included in the action history.
+    Default is excluding the current executing step from the action history.
+    """
+
+    # Get action results from the last history_window steps
+    steps = await app.DATABASE.get_task_steps(task_id=task.task_id, organization_id=task.organization_id)
+    # the last step is always the newly created one and it should be excluded from the history window
+    window_steps = steps[-1 - history_window : -1]
+    if current_step:
+        window_steps.append(current_step)
+
+    actions_and_results: list[tuple[Action, list[ActionResult]]] = []
+    for window_step in window_steps:
+        if window_step.output and window_step.output.actions_and_results:
+            actions_and_results.extend(window_step.output.actions_and_results)
+
+    # exclude successful action from history
+    action_history = [
+        {
+            "action": action.model_dump(
+                exclude_none=True,
+                include={"action_type", "element_id", "status", "reasoning", "option", "download"},
+            ),
+            "results": [
+                result.model_dump(
+                    exclude_none=True,
+                    include={
+                        "success",
+                        "exception_type",
+                        "exception_message",
+                    },
+                )
+                for result in results
+            ],
+        }
+        for action, results in actions_and_results
+        if len(results) > 0
+    ]
+    return action_history

skyvern/webeye/actions/handler.py

@@ -69,6 +69,7 @@ from skyvern.forge.sdk.schemas.tasks import Task
 from skyvern.forge.sdk.services.bitwarden import BitwardenConstants
 from skyvern.forge.sdk.services.credentials import AzureVaultConstants, OnePasswordConstants
 from skyvern.forge.sdk.trace import TraceManager
+from skyvern.services.action_service import get_action_history
 from skyvern.services.task_v1_service import is_cua_task
 from skyvern.utils.prompt_engine import (
     CheckDateFormatResponse,
@@ -3732,6 +3733,7 @@ async def _get_input_or_select_context(
 
 
 async def extract_user_defined_errors(task: Task, step: Step, scraped_page: ScrapedPage) -> list[UserDefinedError]:
+    action_history = await get_action_history(task=task, current_step=step)
     scraped_page_refreshed = await scraped_page.refresh(draw_boxes=False)
     prompt = prompt_engine.load_prompt(
         "surface-user-defined-errors",
@@ -3739,6 +3741,7 @@ async def extract_user_defined_errors(task: Task, step: Step, scraped_page: Scra
         navigation_payload_str=json.dumps(task.navigation_payload),
         elements=scraped_page_refreshed.build_element_tree(fmt=ElementTreeFormat.HTML),
         current_url=scraped_page_refreshed.url,
+        action_history=json.dumps(action_history),
         error_code_mapping_str=json.dumps(task.error_code_mapping) if task.error_code_mapping else "{}",
         local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),
     )