From 26a137418bc27d80aff8c1d4e842bb6dc87528c2 Mon Sep 17 00:00:00 2001 From: Jonathan Dobson Date: Wed, 3 Dec 2025 13:08:23 -0500 Subject: [PATCH] Enable browser recording (#4182) --- .../src/components/BrowserStream.tsx | 11 +- skyvern-frontend/src/components/Flippable.tsx | 10 +- .../hooks/useProcessRecordingMutation.ts | 141 +++++++----------- .../routes/workflows/editor/FlowRenderer.tsx | 30 +++- .../workflows/editor/WorkflowAdderBusy.tsx | 7 +- .../workflows/editor/WorkflowHeader.tsx | 21 ++- .../src/routes/workflows/editor/Workspace.tsx | 12 +- .../editor/edges/EdgeWithAddButton.tsx | 40 +++-- .../nodes/CodeBlockNode/CodeBlockNode.tsx | 8 +- .../nodes/DownloadNode/DownloadNode.tsx | 8 +- .../nodes/FileParserNode/FileParserNode.tsx | 8 +- .../nodes/FileUploadNode/FileUploadNode.tsx | 8 +- .../nodes/HttpRequestNode/HttpRequestNode.tsx | 9 +- .../HumanInteractionNode.tsx | 8 +- .../editor/nodes/LoopNode/LoopNode.tsx | 8 +- .../workflows/editor/nodes/NodeActionMenu.tsx | 8 +- .../nodes/NodeAdderNode/NodeAdderNode.tsx | 35 +++-- .../nodes/PDFParserNode/PDFParserNode.tsx | 8 +- .../nodes/SendEmailNode/SendEmailNode.tsx | 8 +- .../editor/nodes/StartNode/StartNode.tsx | 9 +- .../nodes/TextPromptNode/TextPromptNode.tsx | 8 +- .../editor/nodes/WaitNode/WaitNode.tsx | 8 +- .../editor/nodes/components/NodeHeader.tsx | 7 +- .../src/store/RecordedBlocksStore.ts | 12 +- .../src/store/useRecordingStore.ts | 29 ++++ 25 files changed, 310 insertions(+), 151 deletions(-) diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx index 1263bf80..b80ccde9 100644 --- a/skyvern-frontend/src/components/BrowserStream.tsx +++ b/skyvern-frontend/src/components/BrowserStream.tsx @@ -627,11 +627,14 @@ function BrowserStream({ "source" in data && typeof data.source === "string" ) { + const event = data as MessageInExfiltratedEvent; + return { kind: "exfiltrated-event", - event_name: data.event_name, - params: data.params, - source: data.source, + event_name: event.event_name, + params: event.params, + source: event.source, + timestamp: event.timestamp, } as MessageInExfiltratedEvent; } break; @@ -803,6 +806,7 @@ function BrowserStream({ if (!hasEvents) { e.preventDefault(); recordingStore.setIsRecording(false); + recordingStore.reset(); } }} > @@ -826,6 +830,7 @@ function BrowserStream({ variant="destructive" onClick={() => { recordingStore.setIsRecording(false); + recordingStore.reset(); }} > Cancel recording diff --git a/skyvern-frontend/src/components/Flippable.tsx b/skyvern-frontend/src/components/Flippable.tsx index d6bb2c14..8f43e63d 100644 --- a/skyvern-frontend/src/components/Flippable.tsx +++ b/skyvern-frontend/src/components/Flippable.tsx @@ -1,4 +1,6 @@ import { ReactNode, Children, useRef, useEffect } from "react"; + +import { useRecordingStore } from "@/store/useRecordingStore"; import { cn } from "@/util/utils"; interface FlippableProps { @@ -22,6 +24,7 @@ export function Flippable({ className, preserveFrontsideHeight = false, }: FlippableProps) { + const recordingStore = useRecordingStore(); const childrenArray = Children.toArray(children); const front = childrenArray[0]; const back = childrenArray[1]; @@ -48,7 +51,12 @@ export function Flippable({ }, [facing, preserveFrontsideHeight]); return ( -
+
new Promise((resolve) => setTimeout(resolve, ms)); - -const FAIL_QUITE_NO_EVENTS = "FAIL-QUIET:NO-EVENTS" as const; +const FAIL_QUIET_NO_EVENTS = "FAIL-QUIET:NO-EVENTS" as const; const useProcessRecordingMutation = ({ browserSessionId, onSuccess, }: { browserSessionId: string | null; - onSuccess?: (workflowBlocks: Array) => void; + onSuccess?: (args: { + blocks: Array; + parameters: Array; + }) => void; }) => { - // const credentialGetter = useCredentialGetter(); + const credentialGetter = useCredentialGetter(); const recordingStore = useRecordingStore(); + const { workflowPermanentId } = useParams(); const processRecordingMutation = useMutation({ mutationFn: async () => { @@ -32,97 +32,64 @@ const useProcessRecordingMutation = ({ ); } + if (!workflowPermanentId) { + throw new Error( + "Cannot process recording without a valid workflow permanent ID.", + ); + } + const eventCount = recordingStore.getEventCount(); if (eventCount === 0) { - throw new Error(FAIL_QUITE_NO_EVENTS); + throw new Error(FAIL_QUIET_NO_EVENTS); } // (this flushes any pending events) const compressedChunks = await recordingStore.getCompressedChunks(); // TODO: Replace this mock with actual API call when endpoint is ready - // const client = await getClient(credentialGetter, "sans-api-v1"); - // return client - // .post< - // { compressed_chunks: string[] }, - // { data: Array } - // >(`/browser_sessions/${browserSessionId}/process_recording`, { - // compressed_chunks: compressedChunks, - // }) - // .then((response) => response.data); - - // Mock response with 2-second delay - console.log( - `Processing ${eventCount} events in ${compressedChunks.length} compressed chunks`, - ); - await sleep(2000); - - // Return mock workflow blocks with two ActionBlocks - const mockWorkflowBlocks: Array = [ - { - block_type: "action", - label: "action_1", - title: "Enter search term", - navigation_goal: "Enter 'foo' in the search field", - url: null, - error_code_mapping: null, - parameters: [], - engine: null, - continue_on_failure: false, - output_parameter: { - parameter_type: "output", - key: "action_1_output", - description: null, - output_parameter_id: "mock-output-1", - workflow_id: browserSessionId || "mock-workflow-id", - created_at: new Date().toISOString(), - modified_at: new Date().toISOString(), - deleted_at: null, - }, - model: null, - } satisfies ActionBlock, - { - block_type: "action", - label: "action_2", - title: "Click search", - navigation_goal: "Click the search button", - url: null, - error_code_mapping: null, - parameters: [], - engine: null, - continue_on_failure: false, - output_parameter: { - parameter_type: "output", - key: "action_2_output", - description: null, - output_parameter_id: "mock-output-2", - workflow_id: browserSessionId || "mock-workflow-id", - created_at: new Date().toISOString(), - modified_at: new Date().toISOString(), - deleted_at: null, - }, - model: null, - } satisfies ActionBlock, - ]; - return mockWorkflowBlocks; + const client = await getClient(credentialGetter, "sans-api-v1"); + return client + .post< + { compressed_chunks: string[] }, + { + data: { + blocks: Array; + parameters: Array; + }; + } + >(`/browser_sessions/${browserSessionId}/process_recording`, { + compressed_chunks: compressedChunks, + workflow_permanent_id: workflowPermanentId, + }) + .then((response) => ({ + blocks: response.data.blocks, + parameters: response.data.parameters, + })); }, - onSuccess: (workflowBlocks) => { - // Clear events after successful flush + onSuccess: ({ blocks, parameters }) => { recordingStore.clear(); - toast({ - variant: "success", - title: "Recording Processed", - description: "The recording has been successfully processed.", - }); + if (blocks && blocks.length > 0) { + toast({ + variant: "success", + title: "Recording Processed", + description: "The recording has been successfully processed.", + }); - if (workflowBlocks) { - onSuccess?.(workflowBlocks); + onSuccess?.({ blocks, parameters: parameters }); + + return; } + + toast({ + variant: "warning", + title: "Recording Processed (No Blocks)", + description: "No blocks could be created from the recording.", + }); }, onError: (error) => { - if (error instanceof Error && error.message === FAIL_QUITE_NO_EVENTS) { + if (error instanceof Error && error.message === FAIL_QUIET_NO_EVENTS) { return; } diff --git a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx index 385ee523..74dd5461 100644 --- a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx @@ -324,6 +324,9 @@ function FlowRenderer({ setGetSaveDataRef.current = workflowChangesStore.setGetSaveData; const saveWorkflow = useWorkflowSave({ status: "published" }); const recordedBlocks = useRecordedBlocksStore((state) => state.blocks); + const recordedParameters = useRecordedBlocksStore( + (state) => state.parameters, + ); const recordedInsertionPoint = useRecordedBlocksStore( (state) => state.insertionPoint, ); @@ -583,7 +586,8 @@ function FlowRenderer({ doLayout(nodes, edges); } - // effect to add new blocks that were generated from a browser recording + // effect to add new blocks that were generated from a browser recording, + // along with any new parameters useEffect(() => { if (!recordedBlocks || !recordedInsertionPoint) { return; @@ -658,6 +662,30 @@ function FlowRenderer({ workflowChangesStore.setHasChanges(true); doLayout(newNodesAfter, [...editedEdges, ...newEdges]); + const newParameters = Array(); + + for (const newParameter of recordedParameters ?? []) { + const exists = parameters.some((param) => param.key === newParameter.key); + + if (!exists) { + newParameters.push({ + key: newParameter.key, + parameterType: "workflow", + dataType: newParameter.workflow_parameter_type, + description: newParameter.description ?? null, + defaultValue: newParameter.default_value ?? "", + }); + } + } + + if (newParameters.length > 0) { + const workflowParametersStore = useWorkflowParametersStore.getState(); + workflowParametersStore.setParameters([ + ...workflowParametersStore.parameters, + ...newParameters, + ]); + } + clearRecordedBlocks(); // eslint-disable-next-line react-hooks/exhaustive-deps }, [recordedBlocks, recordedInsertionPoint]); diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx index 54f341e6..f925eca5 100644 --- a/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx @@ -6,7 +6,7 @@ import { TooltipProvider, TooltipTrigger, } from "@/components/ui/tooltip"; -import { useRecordingStore, CHUNK_SIZE } from "@/store/useRecordingStore"; +import { useRecordingStore } from "@/store/useRecordingStore"; import { cn } from "@/util/utils"; import "./WorkflowAdderBusy.css"; @@ -45,10 +45,7 @@ function WorkflowAdderBusy({ const [shouldBump, setShouldBump] = useState(false); const bumpTimeoutRef = useRef(null); const prevCountRef = useRef(0); - - const eventCount = - recordingStore.pendingEvents.length + - recordingStore.compressedChunks.length * CHUNK_SIZE; + const eventCount = recordingStore.exposedEventCount; // effect for bump animation when count changes useEffect(() => { diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx index 3c827597..3b61fff5 100644 --- a/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx @@ -26,6 +26,7 @@ import { useCreateWorkflowMutation } from "../hooks/useCreateWorkflowMutation"; import { convert } from "./workflowEditorUtils"; import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery"; import { useDebugStore } from "@/store/useDebugStore"; +import { useRecordingStore } from "@/store/useRecordingStore"; import { useWorkflowTitleStore } from "@/store/WorkflowTitleStore"; import { useWorkflowHasChangesStore } from "@/store/WorkflowHasChangesStore"; import { cn } from "@/util/utils"; @@ -82,6 +83,7 @@ function WorkflowHeader({ const createWorkflowMutation = useCreateWorkflowMutation(); const { data: workflowRun } = useWorkflowRunQuery(); const debugStore = useDebugStore(); + const recordingStore = useRecordingStore(); const workflowRunIsRunningOrQueued = workflowRun && statusIsRunningOrQueued(workflowRun); const [chosenCacheKeyValue, setChosenCacheKeyValue] = useState( @@ -105,8 +107,10 @@ function WorkflowHeader({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [cacheKeyValue]); + const isRecording = recordingStore.isRecording; + const shouldShowCacheControls = - !isGeneratingCode && (cacheKeyValues?.total_count ?? 0) > 0; + !isRecording && !isGeneratingCode && (cacheKeyValues?.total_count ?? 0) > 0; if (!globalWorkflows) { return null; // this should be loaded already by some other components @@ -124,7 +128,7 @@ function WorkflowHeader({ >
{ setTitle(newTitle); workflowChangesStore.setHasChanges(true); @@ -247,7 +251,7 @@ function WorkflowHeader({ size="icon" variant={debugStore.isDebugMode ? "default" : "tertiary"} className="size-10 min-w-[2.5rem]" - disabled={workflowRunIsRunningOrQueued} + disabled={workflowRunIsRunningOrQueued || isRecording} onClick={() => { if (debugStore.isDebugMode) { navigate(`/workflows/${workflowPermanentId}/edit`); @@ -277,7 +281,7 @@ function WorkflowHeader({ size="icon" variant="tertiary" className="size-10 min-w-[2.5rem]" - disabled={isGlobalWorkflow} + disabled={isGlobalWorkflow || isRecording} onClick={() => { onSave(); }} @@ -297,6 +301,7 @@ function WorkflowHeader({
diff --git a/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx b/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx index b9b7bbe7..f5467017 100644 --- a/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx @@ -14,6 +14,7 @@ import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore"; import { useRecordingStore } from "@/store/useRecordingStore"; import { useSettingsStore } from "@/store/SettingsStore"; import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore"; +import { cn } from "@/util/utils"; import { REACT_FLOW_EDGE_Z_INDEX } from "../constants"; import { WorkflowAddMenu } from "../WorkflowAddMenu"; @@ -52,8 +53,8 @@ function EdgeWithAddButton({ ); const processRecordingMutation = useProcessRecordingMutation({ browserSessionId: settingsStore.browserSessionId, - onSuccess: (blocks) => { - setRecordedBlocks(blocks, { + onSuccess: (result) => { + setRecordedBlocks(result, { previous: source, next: target, parent: sourceNode?.parentId, @@ -66,6 +67,17 @@ function EdgeWithAddButton({ const sourceNode = nodes.find((node) => node.id === source); + const isBusy = + (isProcessing || recordingStore.isRecording) && + debugStore.isDebugMode && + settingsStore.isUsingABrowser && + workflowStatePanel.workflowPanelState.data?.previous === source && + workflowStatePanel.workflowPanelState.data?.next === target && + workflowStatePanel.workflowPanelState.data?.parent === + (sourceNode?.parentId || undefined); + + const isDisabled = !isBusy && recordingStore.isRecording; + const updateWorkflowPanelState = (active: boolean) => { setWorkflowPanelState({ active, @@ -78,7 +90,13 @@ function EdgeWithAddButton({ }); }; - const onAdd = () => updateWorkflowPanelState(true); + const onAdd = () => { + if (isDisabled) { + return; + } + + updateWorkflowPanelState(true); + }; const onRecord = () => { if (recordingStore.isRecording) { @@ -100,7 +118,10 @@ function EdgeWithAddButton({ const adder = (
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockNode.tsx index ea64be6d..0fbafd1a 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockNode.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockNode.tsx @@ -7,6 +7,7 @@ import { CodeEditor } from "@/routes/workflows/components/CodeEditor"; import { statusIsRunningOrQueued } from "@/routes/tasks/types"; import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery"; import { useUpdate } from "@/routes/workflows/editor/useUpdate"; +import { useRecordingStore } from "@/store/useRecordingStore"; import { deepEqualStringArrays } from "@/util/equality"; import { cn } from "@/util/utils"; @@ -17,6 +18,7 @@ function CodeBlockNode({ id, data }: NodeProps) { const { editable, label } = data; const { blockLabel: urlBlockLabel } = useParams(); const { data: workflowRun } = useWorkflowRunQuery(); + const recordingStore = useRecordingStore(); const workflowRunIsRunningOrQueued = workflowRun && statusIsRunningOrQueued(workflowRun); const thisBlockIsTargetted = @@ -26,7 +28,11 @@ function CodeBlockNode({ id, data }: NodeProps) { const update = useUpdate({ id, editable }); return ( -
+
) { const { editable, label } = data; const { blockLabel: urlBlockLabel } = useParams(); const { data: workflowRun } = useWorkflowRunQuery(); + const recordingStore = useRecordingStore(); const workflowRunIsRunningOrQueued = workflowRun && statusIsRunningOrQueued(workflowRun); const thisBlockIsTargetted = @@ -22,7 +24,11 @@ function DownloadNode({ id, data }: NodeProps) { workflowRunIsRunningOrQueued && thisBlockIsTargetted; return ( -
+
) { const { editable, label } = data; @@ -27,9 +28,14 @@ function FileParserNode({ id, data }: NodeProps) { workflowRunIsRunningOrQueued && thisBlockIsTargetted; const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id }); const update = useUpdate({ id, editable }); + const recordingStore = useRecordingStore(); return ( -
+
) { const { editable, label } = data; @@ -30,9 +31,14 @@ function FileUploadNode({ id, data }: NodeProps) { const thisBlockIsPlaying = workflowRunIsRunningOrQueued && thisBlockIsTargetted; const update = useUpdate({ id, editable }); + const recordingStore = useRecordingStore(); return ( -
+
) { ); const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id }); + const recordingStore = useRecordingStore(); const showBodyEditor = data.method !== "GET" && data.method !== "HEAD" && data.method !== "DELETE"; return ( -
+
+
) { const nodes = useNodes(); @@ -34,6 +35,7 @@ function LoopNode({ id, data }: NodeProps) { const update = useUpdate({ id, editable }); const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id }); const children = nodes.filter((node) => node.parentId === id); + const recordingStore = useRecordingStore(); const furthestDownChild: Node | null = children.reduce( (acc, child) => { @@ -56,7 +58,11 @@ function LoopNode({ id, data }: NodeProps) { const loopNodeWidth = getLoopNodeWidth(node, nodes); return ( -
+
{isDeletable && ( { onDelete?.(); }} diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx index c8d46b30..8b76ed83 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx @@ -7,6 +7,7 @@ import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore"; import { useRecordingStore } from "@/store/useRecordingStore"; import { useSettingsStore } from "@/store/SettingsStore"; import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore"; +import { cn } from "@/util/utils"; import type { NodeAdderNode } from "./types"; import { WorkflowAddMenu } from "../../WorkflowAddMenu"; @@ -29,8 +30,8 @@ function NodeAdderNode({ id, parentId }: NodeProps) { const processRecordingMutation = useProcessRecordingMutation({ browserSessionId: settingsStore.browserSessionId, - onSuccess: (blocks) => { - setRecordedBlocks(blocks, { + onSuccess: (result) => { + setRecordedBlocks(result, { previous, next: id, parent: parentId, @@ -41,6 +42,17 @@ function NodeAdderNode({ id, parentId }: NodeProps) { const isProcessing = processRecordingMutation.isPending; + const isBusy = + (isProcessing || recordingStore.isRecording) && + debugStore.isDebugMode && + settingsStore.isUsingABrowser && + workflowStatePanel.workflowPanelState.data?.previous === previous && + workflowStatePanel.workflowPanelState.data?.next === id && + workflowStatePanel.workflowPanelState.data?.parent === + (parentId || undefined); + + const isDisabled = !isBusy && recordingStore.isRecording; + const updateWorkflowPanelState = (active: boolean) => { const previous = edges.find((edge) => edge.target === id)?.source; @@ -57,6 +69,10 @@ function NodeAdderNode({ id, parentId }: NodeProps) { }; const onAdd = () => { + if (isDisabled) { + return; + } + updateWorkflowPanelState(true); }; @@ -79,7 +95,9 @@ function NodeAdderNode({ id, parentId }: NodeProps) { const adder = (
{ onAdd(); }} @@ -106,15 +124,6 @@ function NodeAdderNode({ id, parentId }: NodeProps) { ); - const isBusy = - (isProcessing || recordingStore.isRecording) && - debugStore.isDebugMode && - settingsStore.isUsingABrowser && - workflowStatePanel.workflowPanelState.data?.previous === previous && - workflowStatePanel.workflowPanelState.data?.next === id && - workflowStatePanel.workflowPanelState.data?.parent === - (parentId || undefined); - return (
) { id="b" className="opacity-0" /> - {isBusy ? busy : menu} + {isBusy ? busy : isDisabled ? adder : menu}
); } diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserNode.tsx index de171b02..de64993f 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserNode.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserNode.tsx @@ -14,6 +14,7 @@ import { statusIsRunningOrQueued } from "@/routes/tasks/types"; import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery"; import { useUpdate } from "@/routes/workflows/editor/useUpdate"; import { ModelSelector } from "@/components/ModelSelector"; +import { useRecordingStore } from "@/store/useRecordingStore"; function PDFParserNode({ id, data }: NodeProps) { const { editable, label } = data; @@ -27,9 +28,14 @@ function PDFParserNode({ id, data }: NodeProps) { workflowRunIsRunningOrQueued && thisBlockIsTargetted; const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id }); const update = useUpdate({ id, editable }); + const recordingStore = useRecordingStore(); return ( -
+
) { const { editable, label } = data; @@ -27,9 +28,14 @@ function SendEmailNode({ id, data }: NodeProps) { workflowRunIsRunningOrQueued && thisBlockIsTargetted; const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id }); const update = useUpdate({ id, editable }); + const recordingStore = useRecordingStore(); return ( -
+
) { const reactFlowInstance = useReactFlow(); const [facing, setFacing] = useState<"front" | "back">("front"); const blockScriptStore = useBlockScriptStore(); + const recordingStore = useRecordingStore(); const script = blockScriptStore.scripts.__start_block__; const rerender = useRerender({ prefix: "accordion" }); const toggleScriptForNodeCallback = useToggleScriptForNodeCallback(); + const isRecording = recordingStore.isRecording; const makeStartSettings = (data: StartNode["data"]): StartSettings => { return { @@ -396,7 +399,11 @@ function StartNode({ id, data }: NodeProps) { } return ( -
+
) { const { editable, label } = data; @@ -29,9 +30,14 @@ function TextPromptNode({ id, data }: NodeProps) { workflowRunIsRunningOrQueued && thisBlockIsTargetted; const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id }); const update = useUpdate({ id, editable }); + const recordingStore = useRecordingStore(); return ( -
+
) { const { editable, label } = data; @@ -25,9 +26,14 @@ function WaitNode({ id, data, type }: NodeProps) { const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id }); const update = useUpdate({ id, editable }); + const recordingStore = useRecordingStore(); return ( -
+
{ handleOnPlay(); diff --git a/skyvern-frontend/src/store/RecordedBlocksStore.ts b/skyvern-frontend/src/store/RecordedBlocksStore.ts index 4daa6891..336fb667 100644 --- a/skyvern-frontend/src/store/RecordedBlocksStore.ts +++ b/skyvern-frontend/src/store/RecordedBlocksStore.ts @@ -1,5 +1,6 @@ import { create } from "zustand"; import type { WorkflowBlock } from "@/routes/workflows/types/workflowTypes"; +import type { WorkflowParameter } from "@/routes/workflows/types/workflowTypes"; type InsertionPoint = { previous: string | null; @@ -10,12 +11,16 @@ type InsertionPoint = { type RecordedBlocksState = { blocks: Array | null; + parameters: Array | null; insertionPoint: InsertionPoint | null; }; type RecordedBlocksStore = RecordedBlocksState & { setRecordedBlocks: ( - blocks: Array, + data: { + blocks: Array; + parameters: Array; + }, insertionPoint: InsertionPoint, ) => void; clearRecordedBlocks: () => void; @@ -23,9 +28,10 @@ type RecordedBlocksStore = RecordedBlocksState & { const useRecordedBlocksStore = create((set) => ({ blocks: null, + parameters: null, insertionPoint: null, - setRecordedBlocks: (blocks, insertionPoint) => { - set({ blocks, insertionPoint }); + setRecordedBlocks: ({ blocks, parameters }, insertionPoint) => { + set({ blocks, parameters, insertionPoint }); }, clearRecordedBlocks: () => { set({ blocks: null, insertionPoint: null }); diff --git a/skyvern-frontend/src/store/useRecordingStore.ts b/skyvern-frontend/src/store/useRecordingStore.ts index 5369d006..76802d82 100644 --- a/skyvern-frontend/src/store/useRecordingStore.ts +++ b/skyvern-frontend/src/store/useRecordingStore.ts @@ -81,6 +81,7 @@ export interface MessageInExfiltratedCdpEvent { event_name: string; params: ExfiltratedEventCdpParams; source: "cdp"; + timestamp: number; } export interface MessageInExfiltratedConsoleEvent { @@ -88,6 +89,7 @@ export interface MessageInExfiltratedConsoleEvent { event_name: string; params: ExfiltratedEventConsoleParams; source: "console"; + timestamp: number; } export type MessageInExfiltratedEvent = @@ -105,6 +107,11 @@ interface RecordingStore { * Each chunk contains up to CHUNK_SIZE events. */ compressedChunks: string[]; + /** + * The number of events to show the user. This elides noisy events, like + * `mousemove`. + */ + exposedEventCount: number; /** * Buffer of events not yet compressed into a chunk. */ @@ -194,8 +201,25 @@ async function compressEventsToB64(jsonString: string): Promise { return btoa(binary); } +const isExposedEvent = (event: MessageInExfiltratedEvent): boolean => { + const exposedConsoleEventTypes = new Set(["focus", "click", "keypress"]); + + if (event.source === "console") { + if (exposedConsoleEventTypes.has(event.params.type)) { + return true; + } + } + + if (event.source === "cdp") { + return true; + } + + return false; +}; + export const useRecordingStore = create((set, get) => ({ compressedChunks: [], + exposedEventCount: 0, pendingEvents: [], isCompressing: false, isRecording: false, @@ -204,6 +228,10 @@ export const useRecordingStore = create((set, get) => ({ const state = get(); const newPendingEvents = [...state.pendingEvents, event]; + if (isExposedEvent(event)) { + set({ exposedEventCount: state.exposedEventCount + 1 }); + } + if (newPendingEvents.length >= CHUNK_SIZE && !state.isCompressing) { const eventsToCompress = newPendingEvents.slice(0, CHUNK_SIZE); const remainingEvents = newPendingEvents.slice(CHUNK_SIZE); @@ -241,6 +269,7 @@ export const useRecordingStore = create((set, get) => ({ reset: () => set({ compressedChunks: [], + exposedEventCount: 0, pendingEvents: [], isCompressing: false, isRecording: false,