From 9cd1f15763c7b14ff56ef6088ae47a426bff011b Mon Sep 17 00:00:00 2001 From: LawyZheng Date: Thu, 21 Nov 2024 16:12:00 +0800 Subject: [PATCH] add user detail query and answer to single action prompt (#1230) --- skyvern/forge/prompts/skyvern/single-click-action.j2 | 2 ++ skyvern/forge/prompts/skyvern/single-input-action.j2 | 2 ++ skyvern/forge/prompts/skyvern/single-select-action.j2 | 2 ++ skyvern/forge/prompts/skyvern/single-upload-action.j2 | 2 ++ 4 files changed, 8 insertions(+) diff --git a/skyvern/forge/prompts/skyvern/single-click-action.j2 b/skyvern/forge/prompts/skyvern/single-click-action.j2 index 6ab52874..d53d5ebf 100644 --- a/skyvern/forge/prompts/skyvern/single-click-action.j2 +++ b/skyvern/forge/prompts/skyvern/single-click-action.j2 @@ -9,6 +9,8 @@ Reply in JSON format with the following keys: "actions": array // You are supposed to give only one action("CLICK") in the action list. Here's the format of the action: [{ "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point. + "user_detail_query": str, // Think of this value as a Jeopardy question. Ask the user for the details you need for executing this action. Ask the question even if the details are disclosed in user instruction or user details. If you are clicking on something specific, ask about what to click on. If you're downloading a file and you have multiple options, ask the user which one to download. Otherwise, use null. Examples are: "What is the previous insurance provider of the user?", "Which invoice should I download?", "Does the user have any pets?". If the action doesn't require any user details, use null. + "user_detail_answer": str, // The answer to the `user_detail_query`. The source of this answer can be user instruction or user details. "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence "action_type": str, // It's a string enum: "CLICK". "CLICK" is an element you'd like to click. "id": str, // The id of the element to take action on. The id has to be one from the elements list. diff --git a/skyvern/forge/prompts/skyvern/single-input-action.j2 b/skyvern/forge/prompts/skyvern/single-input-action.j2 index bb220b3a..7bff5867 100644 --- a/skyvern/forge/prompts/skyvern/single-input-action.j2 +++ b/skyvern/forge/prompts/skyvern/single-input-action.j2 @@ -9,6 +9,8 @@ Reply in JSON format with the following keys: "actions": array // You are supposed to give only one action("INPUT_TEXT") in the action list. Here's the format of the action: [{ "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point. + "user_detail_query": str, // Think of this value as a Jeopardy question. Ask the user for the details you need for executing this action. Ask the question even if the details are disclosed in user instruction or user details. If it's a text field, ask for the text. Otherwise, use null. Examples are: "What product ID should I input into the search bar?", "What is the previous insurance provider of the user?", "Does the user have any pets?". If the action doesn't require any user details, use null. + "user_detail_answer": str, // The answer to the `user_detail_query`. The source of this answer can be user instruction or user details. "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence "action_type": str, // It's a string enum: "INPUT_TEXT". "INPUT_TEXT" is an element you'd like to input text into. "id": str, // The id of the element to take action on. The id has to be one from the elements list. diff --git a/skyvern/forge/prompts/skyvern/single-select-action.j2 b/skyvern/forge/prompts/skyvern/single-select-action.j2 index d99ae548..34de0ff7 100644 --- a/skyvern/forge/prompts/skyvern/single-select-action.j2 +++ b/skyvern/forge/prompts/skyvern/single-select-action.j2 @@ -9,6 +9,8 @@ Reply in JSON format with the following keys: "actions": array // You are supposed to give only one action("SELECT_OPTION") in the action list. Here's the format of the action: [{ "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point. + "user_detail_query": str, // Think of this value as a Jeopardy question. Ask the user for the details you need for executing this action. Ask the question even if the details are disclosed in user instruction or user details. If it's a dropdown, ask for the relevant information. Otherwise, use null. Examples are: "Which invoice should I download?", "Does the user have any pets?". If the action doesn't require any user details, use null. + "user_detail_answer": str, // The answer to the `user_detail_query`. The source of this answer can be user instruction or user details. "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence "action_type": str, // It's a string enum: "SELECT_OPTION". "SELECT_OPTION" is an element you'd like to select an option from. "id": str, // The id of the element to take action on. The id has to be one from the elements list. diff --git a/skyvern/forge/prompts/skyvern/single-upload-action.j2 b/skyvern/forge/prompts/skyvern/single-upload-action.j2 index df2ef32b..9bbcf034 100644 --- a/skyvern/forge/prompts/skyvern/single-upload-action.j2 +++ b/skyvern/forge/prompts/skyvern/single-upload-action.j2 @@ -9,6 +9,8 @@ Reply in JSON format with the following keys: "actions": array // You are supposed to give only one action("UPLOAD_FILE") in the action list. Here's the format of the action: [{ "reasoning": str, // The reasoning behind the action. This reasoning must be user information agnostic. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point. + "user_detail_query": str, // Think of this value as a Jeopardy question. Ask the user for the details you need for executing this action. Ask the question even if the details are disclosed in user instruction or user details. If it's a file upload, ask for the file. Otherwise, use null. Examples are: "What file should I upload?". If the action doesn't require any user details, use null. + "user_detail_answer": str, // The answer to the `user_detail_query`. The source of this answer can be user instruction or user details. "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence "action_type": str, // It's a string enum: "UPLOAD_FILE". "UPLOAD_FILE" is an element you'd like to upload a file into. "id": str, // The id of the element to take action on. The id has to be one from the elements list.