Decorate bounding boxes with element_ids to improve Skyvern accuracy (+ a few more changes) (#536)

skyvern/forge/agent.py

@@ -930,7 +930,7 @@ class ForgeAgent:
             LOG.info("Using Claude3 Sonnet prompt template for action extraction")
             prompt_template = "extract-action-claude3-sonnet"
 
-        # TODO: we only use HTML element for now, introduce a way to swith in the future
+        # TODO: we only use HTML element for now, introduce a way to switch in the future
         element_tree_format = ElementTreeFormat.HTML
         LOG.info(
             "Building element tree",

skyvern/forge/prompts/skyvern/extract-action-claude3-sonnet.j2

@@ -9,6 +9,7 @@ If you see a popup in the page screenshot, prioritize actions on the popup.
 
 Reply in JSON format with the following keys:
 {
+    "action_plan": str, // A string that describes the plan of actions you're going to take. Be specific and to the point. Use this as a quick summary of the actions you're going to take, and what order you're going to take them in
     "actions": array // An array of actions. Here's the format of each action:
     [{
         "reasoning": str, // The reasoning behind the action. Be specific, referencing any user information and their fields and element ids in your reasoning. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point.

skyvern/forge/prompts/skyvern/extract-action.j2

@@ -9,6 +9,7 @@ If you see a popup in the page screenshot, prioritize actions on the popup.
 
 Reply in JSON format with the following keys:
 {
+    "action_plan": str, // A string that describes the plan of actions you're going to take. Be specific and to the point. Use this as a quick summary of the actions you're going to take, and what order you're going to take them in
     "actions": array // An array of actions. Here's the format of each action:
     [{
         "reasoning": str, // The reasoning behind the action. Be specific, referencing any user information and their fields and element ids in your reasoning. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point.