Magic Wand Button to improve prompts in all blocks (#4060)

skyvern-frontend/src/routes/workflows/editor/constants.ts

@@ -20,3 +20,95 @@ export const BITWARDEN_CLIENT_SECRET_AWS_SECRET_KEY =
   "SKYVERN_BITWARDEN_CLIENT_SECRET";
 export const BITWARDEN_MASTER_PASSWORD_AWS_SECRET_KEY =
   "SKYVERN_BITWARDEN_MASTER_PASSWORD";
+
+type AiImproveConfig = {
+  useCase: string;
+  context: Record<string, unknown>;
+};
+
+const createAiImproveConfig = (
+  block: string,
+  field: string,
+  extraContext: Record<string, unknown> = {},
+): AiImproveConfig => ({
+  useCase: `workflow_editor.${block}.${field}`,
+  context: {
+    block_type: block,
+    field,
+    ...extraContext,
+  },
+});
+
+export const AI_IMPROVE_CONFIGS = {
+  task: {
+    navigationGoal: createAiImproveConfig("task", "navigation_goal"),
+    dataExtractionGoal: createAiImproveConfig("task", "data_extraction_goal"),
+    completeCriterion: createAiImproveConfig("task", "complete_criterion"),
+  },
+  action: {
+    navigationGoal: createAiImproveConfig("action", "navigation_goal"),
+    errorCodeMapping: createAiImproveConfig("action", "error_code_mapping"),
+  },
+  navigation: {
+    navigationGoal: createAiImproveConfig("navigation", "navigation_goal"),
+    completeCriterion: createAiImproveConfig(
+      "navigation",
+      "complete_criterion",
+    ),
+  },
+  extraction: {
+    dataExtractionGoal: createAiImproveConfig(
+      "extraction",
+      "data_extraction_goal",
+    ),
+    dataSchema: createAiImproveConfig("extraction", "data_schema"),
+  },
+  validation: {
+    completeCriterion: createAiImproveConfig(
+      "validation",
+      "complete_criterion",
+    ),
+    terminateCriterion: createAiImproveConfig(
+      "validation",
+      "terminate_criterion",
+    ),
+  },
+  login: {
+    navigationGoal: createAiImproveConfig("login", "navigation_goal"),
+    completeCriterion: createAiImproveConfig("login", "complete_criterion"),
+    terminateCriterion: createAiImproveConfig("login", "terminate_criterion"),
+  },
+  fileDownload: {
+    navigationGoal: createAiImproveConfig("file_download", "navigation_goal"),
+    completeCriterion: createAiImproveConfig(
+      "file_download",
+      "complete_criterion",
+    ),
+  },
+  taskV2: {
+    prompt: createAiImproveConfig("task_v2", "prompt"),
+  },
+  textPrompt: {
+    prompt: createAiImproveConfig("text_prompt", "prompt"),
+    jsonSchema: createAiImproveConfig("text_prompt", "json_schema"),
+  },
+  humanInteraction: {
+    instructions: createAiImproveConfig("human_interaction", "instructions"),
+    positiveDescriptor: createAiImproveConfig(
+      "human_interaction",
+      "positive_descriptor",
+    ),
+    negativeDescriptor: createAiImproveConfig(
+      "human_interaction",
+      "negative_descriptor",
+    ),
+    body: createAiImproveConfig("human_interaction", "body"),
+  },
+  sendEmail: {
+    subject: createAiImproveConfig("send_email", "subject"),
+    body: createAiImproveConfig("send_email", "body"),
+  },
+  httpRequest: {
+    body: createAiImproveConfig("http_request", "body"),
+  },
+} as const;

skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/ActionNode.tsx

@@ -17,6 +17,7 @@ import { errorMappingExampleValue } from "../types";
 import { CodeEditor } from "@/routes/workflows/components/CodeEditor";
 import { Switch } from "@/components/ui/switch";
 import { placeholders, helpTooltips } from "../../helpContent";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
 import { WorkflowBlockInputTextarea } from "@/components/WorkflowBlockInputTextarea";
 import { useRerender } from "@/hooks/useRerender";
 import { BlockCodeEditor } from "@/routes/workflows/components/BlockCodeEditor";
@@ -139,10 +140,7 @@ function ActionNode({ id, data, type }: NodeProps<ActionNode>) {
                 <HelpTooltip content={navigationGoalTooltip} />
               </div>
               <WorkflowBlockInputTextarea
-                aiImprove={{
+                aiImprove={AI_IMPROVE_CONFIGS.action.navigationGoal}
-                  context: { block_type: "Action Block" },
-                  useCase: "task_v1_prompt",
-                }}
                 nodeId={id}
                 onChange={(value) => {
                   update({ navigationGoal: value });

skyvern-frontend/src/routes/workflows/editor/nodes/ExtractionNode/ExtractionNode.tsx

@@ -37,6 +37,7 @@ import { useUpdate } from "@/routes/workflows/editor/useUpdate";
 import { useRerender } from "@/hooks/useRerender";
 
 import { DisableCache } from "../DisableCache";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
 
 function ExtractionNode({ id, data, type }: NodeProps<ExtractionNode>) {
   const [facing, setFacing] = useState<"front" | "back">("front");
@@ -113,6 +114,7 @@ function ExtractionNode({ id, data, type }: NodeProps<ExtractionNode>) {
             </div>
 
             <WorkflowBlockInputTextarea
+              aiImprove={AI_IMPROVE_CONFIGS.extraction.dataExtractionGoal}
               nodeId={id}
               onChange={(value) => {
                 if (!editable) {

skyvern-frontend/src/routes/workflows/editor/nodes/FileDownloadNode/FileDownloadNode.tsx

@@ -37,6 +37,7 @@ import { useRerender } from "@/hooks/useRerender";
 import { BROWSER_DOWNLOAD_TIMEOUT_SECONDS } from "@/api/types";
 
 import { DisableCache } from "../DisableCache";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
 
 const urlTooltip =
   "The URL Skyvern is navigating to. Leave this field blank to pick up from where the last block left off.";
@@ -132,6 +133,7 @@ function FileDownloadNode({ id, data }: NodeProps<FileDownloadNode>) {
                 <HelpTooltip content={navigationGoalTooltip} />
               </div>
               <WorkflowBlockInputTextarea
+                aiImprove={AI_IMPROVE_CONFIGS.fileDownload.navigationGoal}
                 nodeId={id}
                 onChange={(value) => {
                   update({ navigationGoal: value });

skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx

@@ -19,6 +19,7 @@ import {
   AccordionTrigger,
 } from "@/components/ui/accordion";
 import { useRerender } from "@/hooks/useRerender";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
 
 const instructionsTooltip =
   "Instructions shown to the user for review. Explain what needs to be reviewed and what action should be taken.";
@@ -169,6 +170,7 @@ function HumanInteractionNode({
             <div className="space-y-2">
               <Label className="text-xs text-slate-300">Body</Label>
               <WorkflowBlockInputTextarea
+                aiImprove={AI_IMPROVE_CONFIGS.humanInteraction.body}
                 nodeId={id}
                 onChange={(value) => {
                   update({ body: value });

skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginNode.tsx

@@ -36,6 +36,7 @@ import { useUpdate } from "@/routes/workflows/editor/useUpdate";
 import { useRerender } from "@/hooks/useRerender";
 
 import { DisableCache } from "../DisableCache";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
 
 function LoginNode({ id, data, type }: NodeProps<LoginNode>) {
   const blockScriptStore = useBlockScriptStore();
@@ -127,6 +128,7 @@ function LoginNode({ id, data, type }: NodeProps<LoginNode>) {
                 />
               </div>
               <WorkflowBlockInputTextarea
+                aiImprove={AI_IMPROVE_CONFIGS.login.navigationGoal}
                 nodeId={id}
                 onChange={(value) => {
                   update({ navigationGoal: value });
@@ -187,6 +189,7 @@ function LoginNode({ id, data, type }: NodeProps<LoginNode>) {
                       Complete if...
                     </Label>
                     <WorkflowBlockInputTextarea
+                      aiImprove={AI_IMPROVE_CONFIGS.login.completeCriterion}
                       nodeId={id}
                       onChange={(value) => {
                         update({ completeCriterion: value });

skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/NavigationNode.tsx

@@ -37,6 +37,7 @@ import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuer
 import { useUpdate } from "@/routes/workflows/editor/useUpdate";
 
 import { DisableCache } from "../DisableCache";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
 
 function NavigationNode({ id, data, type }: NodeProps<NavigationNode>) {
   const { blockLabel: urlBlockLabel } = useParams();
@@ -136,6 +137,7 @@ function NavigationNode({ id, data, type }: NodeProps<NavigationNode>) {
                 />
               </div>
               <WorkflowBlockInputTextarea
+                aiImprove={AI_IMPROVE_CONFIGS.navigation.navigationGoal}
                 nodeId={id}
                 onChange={(value) => {
                   update({ navigationGoal: value });
@@ -184,6 +186,9 @@ function NavigationNode({ id, data, type }: NodeProps<NavigationNode>) {
                       Complete if...
                     </Label>
                     <WorkflowBlockInputTextarea
+                      aiImprove={
+                        AI_IMPROVE_CONFIGS.navigation.completeCriterion
+                      }
                       nodeId={id}
                       onChange={(value) => {
                         update({ completeCriterion: value });

skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx

@@ -13,6 +13,7 @@ import { useParams } from "react-router-dom";
 import { statusIsRunningOrQueued } from "@/routes/tasks/types";
 import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
 import { useUpdate } from "@/routes/workflows/editor/useUpdate";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
 
 function SendEmailNode({ id, data }: NodeProps<SendEmailNode>) {
   const { editable, label } = data;
@@ -94,6 +95,7 @@ function SendEmailNode({ id, data }: NodeProps<SendEmailNode>) {
         <div className="space-y-2">
           <Label className="text-xs text-slate-300">Body</Label>
           <WorkflowBlockInputTextarea
+            aiImprove={AI_IMPROVE_CONFIGS.sendEmail.body}
             nodeId={id}
             onChange={(value) => {
               update({ body: value });

skyvern-frontend/src/routes/workflows/editor/nodes/TaskNode/TaskNode.tsx

@@ -21,6 +21,7 @@ import { Handle, NodeProps, Position, useEdges, useNodes } from "@xyflow/react";
 import { useState } from "react";
 import { AppNode } from "..";
 import { helpTooltips, placeholders } from "../../helpContent";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
 import { getAvailableOutputParameterKeys } from "../../workflowEditorUtils";
 import { dataSchemaExampleValue, errorMappingExampleValue } from "../types";
 import { ParametersMultiSelect } from "./ParametersMultiSelect";
@@ -137,10 +138,7 @@ function TaskNode({ id, data, type }: NodeProps<TaskNode>) {
                       />
                     </div>
                     <WorkflowBlockInputTextarea
-                      aiImprove={{
+                      aiImprove={AI_IMPROVE_CONFIGS.task.navigationGoal}
-                        context: { block_type: "Task Block" },
-                        useCase: "task_v1_prompt",
-                      }}
                       nodeId={id}
                       onChange={(value) => {
                         update({ navigationGoal: value });
@@ -176,6 +174,7 @@ function TaskNode({ id, data, type }: NodeProps<TaskNode>) {
                       />
                     </div>
                     <WorkflowBlockInputTextarea
+                      aiImprove={AI_IMPROVE_CONFIGS.task.dataExtractionGoal}
                       nodeId={id}
                       onChange={(value) => {
                         update({ dataExtractionGoal: value });
@@ -209,6 +208,7 @@ function TaskNode({ id, data, type }: NodeProps<TaskNode>) {
                       Complete if...
                     </Label>
                     <WorkflowBlockInputTextarea
+                      aiImprove={AI_IMPROVE_CONFIGS.task.completeCriterion}
                       nodeId={id}
                       onChange={(value) => {
                         update({ completeCriterion: value });

skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/Taskv2Node.tsx

@@ -19,6 +19,7 @@ import { ModelSelector } from "@/components/ModelSelector";
 import { cn } from "@/util/utils";
 import { NodeHeader } from "../components/NodeHeader";
 import { NodeTabs } from "../components/NodeTabs";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
 import { useParams } from "react-router-dom";
 import { statusIsRunningOrQueued } from "@/routes/tasks/types";
 import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
@@ -107,7 +108,7 @@ function Taskv2Node({ id, data, type }: NodeProps<Taskv2Node>) {
                 ) : null}
               </div>
               <WorkflowBlockInputTextarea
-                aiImprove={{ useCase: "task_v2_prompt" }}
+                aiImprove={AI_IMPROVE_CONFIGS.taskV2.prompt}
                 nodeId={id}
                 onChange={(value) => {
                   update({ prompt: value });

skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx

@@ -15,6 +15,7 @@ import { useParams } from "react-router-dom";
 import { statusIsRunningOrQueued } from "@/routes/tasks/types";
 import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
 import { useUpdate } from "@/routes/workflows/editor/useUpdate";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
 
 function TextPromptNode({ id, data }: NodeProps<TextPromptNode>) {
   const { editable, label } = data;
@@ -75,6 +76,7 @@ function TextPromptNode({ id, data }: NodeProps<TextPromptNode>) {
           </div>
 
           <WorkflowBlockInputTextarea
+            aiImprove={AI_IMPROVE_CONFIGS.textPrompt.prompt}
             nodeId={id}
             onChange={(value) => {
               update({ prompt: value });

skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx

@@ -34,6 +34,7 @@ import { useUpdate } from "@/routes/workflows/editor/useUpdate";
 import { useRerender } from "@/hooks/useRerender";
 
 import { DisableCache } from "../DisableCache";
+import { AI_IMPROVE_CONFIGS } from "../../constants";
 
 function ValidationNode({ id, data, type }: NodeProps<ValidationNode>) {
   const [facing, setFacing] = useState<"front" | "back">("front");
@@ -114,6 +115,7 @@ function ValidationNode({ id, data, type }: NodeProps<ValidationNode>) {
               ) : null}
             </div>
             <WorkflowBlockInputTextarea
+              aiImprove={AI_IMPROVE_CONFIGS.validation.completeCriterion}
               nodeId={id}
               onChange={(value) => {
                 update({ completeCriterion: value });
@@ -125,6 +127,7 @@ function ValidationNode({ id, data, type }: NodeProps<ValidationNode>) {
           <div className="space-y-2">
             <Label className="text-xs text-slate-300">Terminate if...</Label>
             <WorkflowBlockInputTextarea
+              aiImprove={AI_IMPROVE_CONFIGS.validation.terminateCriterion}
               nodeId={id}
               onChange={(value) => {
                 update({ terminateCriterion: value });