From 0cbab39a2724d3002099a2cdf01ec9d8ab01e202 Mon Sep 17 00:00:00 2001 From: Celal Zamanoglu <95054566+celalzamanoglu@users.noreply.github.com> Date: Fri, 21 Nov 2025 18:24:23 +0300 Subject: [PATCH] Magic Wand Button to improve prompts in all blocks (#4060) --- .../src/routes/workflows/editor/constants.ts | 92 +++++++++++++++++++ .../editor/nodes/ActionNode/ActionNode.tsx | 6 +- .../nodes/ExtractionNode/ExtractionNode.tsx | 2 + .../FileDownloadNode/FileDownloadNode.tsx | 2 + .../HumanInteractionNode.tsx | 2 + .../editor/nodes/LoginNode/LoginNode.tsx | 3 + .../nodes/NavigationNode/NavigationNode.tsx | 5 + .../nodes/SendEmailNode/SendEmailNode.tsx | 2 + .../editor/nodes/TaskNode/TaskNode.tsx | 8 +- .../editor/nodes/Taskv2Node/Taskv2Node.tsx | 3 +- .../nodes/TextPromptNode/TextPromptNode.tsx | 2 + .../nodes/ValidationNode/ValidationNode.tsx | 3 + 12 files changed, 121 insertions(+), 9 deletions(-) diff --git a/skyvern-frontend/src/routes/workflows/editor/constants.ts b/skyvern-frontend/src/routes/workflows/editor/constants.ts index 7da2dfd3..4793dd21 100644 --- a/skyvern-frontend/src/routes/workflows/editor/constants.ts +++ b/skyvern-frontend/src/routes/workflows/editor/constants.ts @@ -20,3 +20,95 @@ export const BITWARDEN_CLIENT_SECRET_AWS_SECRET_KEY = "SKYVERN_BITWARDEN_CLIENT_SECRET"; export const BITWARDEN_MASTER_PASSWORD_AWS_SECRET_KEY = "SKYVERN_BITWARDEN_MASTER_PASSWORD"; + +type AiImproveConfig = { + useCase: string; + context: Record; +}; + +const createAiImproveConfig = ( + block: string, + field: string, + extraContext: Record = {}, +): AiImproveConfig => ({ + useCase: `workflow_editor.${block}.${field}`, + context: { + block_type: block, + field, + ...extraContext, + }, +}); + +export const AI_IMPROVE_CONFIGS = { + task: { + navigationGoal: createAiImproveConfig("task", "navigation_goal"), + dataExtractionGoal: createAiImproveConfig("task", "data_extraction_goal"), + completeCriterion: createAiImproveConfig("task", "complete_criterion"), + }, + action: { + navigationGoal: createAiImproveConfig("action", "navigation_goal"), + errorCodeMapping: createAiImproveConfig("action", "error_code_mapping"), + }, + navigation: { + navigationGoal: createAiImproveConfig("navigation", "navigation_goal"), + completeCriterion: createAiImproveConfig( + "navigation", + "complete_criterion", + ), + }, + extraction: { + dataExtractionGoal: createAiImproveConfig( + "extraction", + "data_extraction_goal", + ), + dataSchema: createAiImproveConfig("extraction", "data_schema"), + }, + validation: { + completeCriterion: createAiImproveConfig( + "validation", + "complete_criterion", + ), + terminateCriterion: createAiImproveConfig( + "validation", + "terminate_criterion", + ), + }, + login: { + navigationGoal: createAiImproveConfig("login", "navigation_goal"), + completeCriterion: createAiImproveConfig("login", "complete_criterion"), + terminateCriterion: createAiImproveConfig("login", "terminate_criterion"), + }, + fileDownload: { + navigationGoal: createAiImproveConfig("file_download", "navigation_goal"), + completeCriterion: createAiImproveConfig( + "file_download", + "complete_criterion", + ), + }, + taskV2: { + prompt: createAiImproveConfig("task_v2", "prompt"), + }, + textPrompt: { + prompt: createAiImproveConfig("text_prompt", "prompt"), + jsonSchema: createAiImproveConfig("text_prompt", "json_schema"), + }, + humanInteraction: { + instructions: createAiImproveConfig("human_interaction", "instructions"), + positiveDescriptor: createAiImproveConfig( + "human_interaction", + "positive_descriptor", + ), + negativeDescriptor: createAiImproveConfig( + "human_interaction", + "negative_descriptor", + ), + body: createAiImproveConfig("human_interaction", "body"), + }, + sendEmail: { + subject: createAiImproveConfig("send_email", "subject"), + body: createAiImproveConfig("send_email", "body"), + }, + httpRequest: { + body: createAiImproveConfig("http_request", "body"), + }, +} as const; diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/ActionNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/ActionNode.tsx index 64f83f5f..7a8c0a8b 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/ActionNode.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/ActionNode/ActionNode.tsx @@ -17,6 +17,7 @@ import { errorMappingExampleValue } from "../types"; import { CodeEditor } from "@/routes/workflows/components/CodeEditor"; import { Switch } from "@/components/ui/switch"; import { placeholders, helpTooltips } from "../../helpContent"; +import { AI_IMPROVE_CONFIGS } from "../../constants"; import { WorkflowBlockInputTextarea } from "@/components/WorkflowBlockInputTextarea"; import { useRerender } from "@/hooks/useRerender"; import { BlockCodeEditor } from "@/routes/workflows/components/BlockCodeEditor"; @@ -139,10 +140,7 @@ function ActionNode({ id, data, type }: NodeProps) { { update({ navigationGoal: value }); diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/ExtractionNode/ExtractionNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/ExtractionNode/ExtractionNode.tsx index 2e72e209..ff7ffb1b 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/ExtractionNode/ExtractionNode.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/ExtractionNode/ExtractionNode.tsx @@ -37,6 +37,7 @@ import { useUpdate } from "@/routes/workflows/editor/useUpdate"; import { useRerender } from "@/hooks/useRerender"; import { DisableCache } from "../DisableCache"; +import { AI_IMPROVE_CONFIGS } from "../../constants"; function ExtractionNode({ id, data, type }: NodeProps) { const [facing, setFacing] = useState<"front" | "back">("front"); @@ -113,6 +114,7 @@ function ExtractionNode({ id, data, type }: NodeProps) { { if (!editable) { diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/FileDownloadNode/FileDownloadNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/FileDownloadNode/FileDownloadNode.tsx index e2cca065..13c7d686 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/FileDownloadNode/FileDownloadNode.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/FileDownloadNode/FileDownloadNode.tsx @@ -37,6 +37,7 @@ import { useRerender } from "@/hooks/useRerender"; import { BROWSER_DOWNLOAD_TIMEOUT_SECONDS } from "@/api/types"; import { DisableCache } from "../DisableCache"; +import { AI_IMPROVE_CONFIGS } from "../../constants"; const urlTooltip = "The URL Skyvern is navigating to. Leave this field blank to pick up from where the last block left off."; @@ -132,6 +133,7 @@ function FileDownloadNode({ id, data }: NodeProps) { { update({ navigationGoal: value }); diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx index c3c36af8..515133d3 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx @@ -19,6 +19,7 @@ import { AccordionTrigger, } from "@/components/ui/accordion"; import { useRerender } from "@/hooks/useRerender"; +import { AI_IMPROVE_CONFIGS } from "../../constants"; const instructionsTooltip = "Instructions shown to the user for review. Explain what needs to be reviewed and what action should be taken."; @@ -169,6 +170,7 @@ function HumanInteractionNode({
{ update({ body: value }); diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginNode.tsx index 3f35a739..c5a9aef6 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginNode.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginNode.tsx @@ -36,6 +36,7 @@ import { useUpdate } from "@/routes/workflows/editor/useUpdate"; import { useRerender } from "@/hooks/useRerender"; import { DisableCache } from "../DisableCache"; +import { AI_IMPROVE_CONFIGS } from "../../constants"; function LoginNode({ id, data, type }: NodeProps) { const blockScriptStore = useBlockScriptStore(); @@ -127,6 +128,7 @@ function LoginNode({ id, data, type }: NodeProps) { />
{ update({ navigationGoal: value }); @@ -187,6 +189,7 @@ function LoginNode({ id, data, type }: NodeProps) { Complete if... { update({ completeCriterion: value }); diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/NavigationNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/NavigationNode.tsx index 8f148e98..339a5579 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/NavigationNode.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/NavigationNode/NavigationNode.tsx @@ -37,6 +37,7 @@ import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuer import { useUpdate } from "@/routes/workflows/editor/useUpdate"; import { DisableCache } from "../DisableCache"; +import { AI_IMPROVE_CONFIGS } from "../../constants"; function NavigationNode({ id, data, type }: NodeProps) { const { blockLabel: urlBlockLabel } = useParams(); @@ -136,6 +137,7 @@ function NavigationNode({ id, data, type }: NodeProps) { /> { update({ navigationGoal: value }); @@ -184,6 +186,9 @@ function NavigationNode({ id, data, type }: NodeProps) { Complete if... { update({ completeCriterion: value }); diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx index 644f4116..30105a99 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx @@ -13,6 +13,7 @@ import { useParams } from "react-router-dom"; import { statusIsRunningOrQueued } from "@/routes/tasks/types"; import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery"; import { useUpdate } from "@/routes/workflows/editor/useUpdate"; +import { AI_IMPROVE_CONFIGS } from "../../constants"; function SendEmailNode({ id, data }: NodeProps) { const { editable, label } = data; @@ -94,6 +95,7 @@ function SendEmailNode({ id, data }: NodeProps) {
{ update({ body: value }); diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/TaskNode/TaskNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/TaskNode/TaskNode.tsx index b4855a9c..2850f985 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/TaskNode/TaskNode.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/TaskNode/TaskNode.tsx @@ -21,6 +21,7 @@ import { Handle, NodeProps, Position, useEdges, useNodes } from "@xyflow/react"; import { useState } from "react"; import { AppNode } from ".."; import { helpTooltips, placeholders } from "../../helpContent"; +import { AI_IMPROVE_CONFIGS } from "../../constants"; import { getAvailableOutputParameterKeys } from "../../workflowEditorUtils"; import { dataSchemaExampleValue, errorMappingExampleValue } from "../types"; import { ParametersMultiSelect } from "./ParametersMultiSelect"; @@ -137,10 +138,7 @@ function TaskNode({ id, data, type }: NodeProps) { />
{ update({ navigationGoal: value }); @@ -176,6 +174,7 @@ function TaskNode({ id, data, type }: NodeProps) { /> { update({ dataExtractionGoal: value }); @@ -209,6 +208,7 @@ function TaskNode({ id, data, type }: NodeProps) { Complete if... { update({ completeCriterion: value }); diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/Taskv2Node.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/Taskv2Node.tsx index 55d6570c..4ceafb49 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/Taskv2Node.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/Taskv2Node/Taskv2Node.tsx @@ -19,6 +19,7 @@ import { ModelSelector } from "@/components/ModelSelector"; import { cn } from "@/util/utils"; import { NodeHeader } from "../components/NodeHeader"; import { NodeTabs } from "../components/NodeTabs"; +import { AI_IMPROVE_CONFIGS } from "../../constants"; import { useParams } from "react-router-dom"; import { statusIsRunningOrQueued } from "@/routes/tasks/types"; import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery"; @@ -107,7 +108,7 @@ function Taskv2Node({ id, data, type }: NodeProps) { ) : null} { update({ prompt: value }); diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx index 9e15ba09..545969d3 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx @@ -15,6 +15,7 @@ import { useParams } from "react-router-dom"; import { statusIsRunningOrQueued } from "@/routes/tasks/types"; import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery"; import { useUpdate } from "@/routes/workflows/editor/useUpdate"; +import { AI_IMPROVE_CONFIGS } from "../../constants"; function TextPromptNode({ id, data }: NodeProps) { const { editable, label } = data; @@ -75,6 +76,7 @@ function TextPromptNode({ id, data }: NodeProps) { { update({ prompt: value }); diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx index dcd97687..59897866 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/ValidationNode/ValidationNode.tsx @@ -34,6 +34,7 @@ import { useUpdate } from "@/routes/workflows/editor/useUpdate"; import { useRerender } from "@/hooks/useRerender"; import { DisableCache } from "../DisableCache"; +import { AI_IMPROVE_CONFIGS } from "../../constants"; function ValidationNode({ id, data, type }: NodeProps) { const [facing, setFacing] = useState<"front" | "back">("front"); @@ -114,6 +115,7 @@ function ValidationNode({ id, data, type }: NodeProps) { ) : null} { update({ completeCriterion: value }); @@ -125,6 +127,7 @@ function ValidationNode({ id, data, type }: NodeProps) {
{ update({ terminateCriterion: value });