From ef3d88c1b9edadf74d51a2cb269f960229b495d3 Mon Sep 17 00:00:00 2001 From: Jonathan Dobson Date: Fri, 28 Nov 2025 11:23:06 -0500 Subject: [PATCH] Browser recording action (#4130) --- .../src/components/BrowserStream.tsx | 329 ++++++++++++++---- .../src/components/RadialMenu.tsx | 7 +- .../src/components/browser-stream.css | 1 + .../hooks/useProcessRecordingMutation.ts | 140 ++++++++ .../routes/workflows/editor/FlowRenderer.tsx | 90 +++++ .../workflows/editor/WorkflowAddMenu.tsx | 82 +++++ .../workflows/editor/WorkflowAdderBusy.css | 29 ++ .../workflows/editor/WorkflowAdderBusy.tsx | 196 +++++++++++ .../src/routes/workflows/editor/Workspace.tsx | 3 + .../editor/edges/EdgeWithAddButton.tsx | 126 ++++--- .../nodes/NodeAdderNode/NodeAdderNode.tsx | 121 +++++-- .../workflows/editor/workflowEditorUtils.ts | 1 + .../src/store/RecordedBlocksStore.ts | 36 ++ skyvern-frontend/src/store/SettingsStore.ts | 17 + .../src/store/useRecordingStore.ts | 298 ++++++++++++++++ 15 files changed, 1320 insertions(+), 156 deletions(-) create mode 100644 skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts create mode 100644 skyvern-frontend/src/routes/workflows/editor/WorkflowAddMenu.tsx create mode 100644 skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.css create mode 100644 skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx create mode 100644 skyvern-frontend/src/store/RecordedBlocksStore.ts create mode 100644 skyvern-frontend/src/store/useRecordingStore.ts diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx index 3b4d5a2f..1263bf80 100644 --- a/skyvern-frontend/src/components/BrowserStream.tsx +++ b/skyvern-frontend/src/components/BrowserStream.tsx @@ -1,5 +1,5 @@ import RFB from "@novnc/novnc/lib/rfb.js"; -import { ExitIcon, HandIcon } from "@radix-ui/react-icons"; +import { ExitIcon, HandIcon, InfoCircledIcon } from "@radix-ui/react-icons"; import { useEffect, useState, useRef, useCallback } from "react"; import { useQuery } from "@tanstack/react-query"; @@ -9,12 +9,28 @@ import type { TaskApiResponse, WorkflowRunStatusApiResponse, } from "@/api/types"; +import { Tip } from "@/components/Tip"; import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogClose, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; import { AnimatedWave } from "@/components/AnimatedWave"; import { toast } from "@/components/ui/use-toast"; import { useCredentialGetter } from "@/hooks/useCredentialGetter"; import { statusIsNotFinalized } from "@/routes/tasks/types"; import { useClientIdStore } from "@/store/useClientIdStore"; +import { + useRecordingStore, + type MessageInExfiltratedEvent, +} from "@/store/useRecordingStore"; +import { useSettingsStore } from "@/store/SettingsStore"; import { environment, wssBaseUrl, @@ -31,18 +47,34 @@ interface BrowserSession { completed_at?: string; } -interface CommandTakeControl { - kind: "take-control"; +interface CommandBeginExfiltration { + kind: "begin-exfiltration"; } interface CommandCedeControl { kind: "cede-control"; } -// a "Command" is an fire-n-forget out-message - it does not require a response -type Command = CommandTakeControl | CommandCedeControl; +interface CommandEndExfiltration { + kind: "end-exfiltration"; +} -const messageInKinds = ["ask-for-clipboard", "copied-text"] as const; +interface CommandTakeControl { + kind: "take-control"; +} + +// a "Command" is an fire-n-forget out-message - it does not require a response +type Command = + | CommandBeginExfiltration + | CommandCedeControl + | CommandEndExfiltration + | CommandTakeControl; + +const messageInKinds = [ + "ask-for-clipboard", + "copied-text", + "exfiltrated-event", +] as const; type MessageInKind = (typeof messageInKinds)[number]; @@ -55,7 +87,10 @@ interface MessageInCopiedText { text: string; } -type MessageIn = MessageInCopiedText | MessageInAskForClipboard; +type MessageIn = + | MessageInCopiedText + | MessageInAskForClipboard + | MessageInExfiltratedEvent; interface MessageOutAskForClipboardResponse { kind: "ask-for-clipboard-response"; @@ -66,6 +101,7 @@ type MessageOut = MessageOutAskForClipboardResponse; type Props = { browserSessionId?: string; + exfiltrate?: boolean; interactive?: boolean; showControlButtons?: boolean; task?: { @@ -82,6 +118,7 @@ type Props = { function BrowserStream({ browserSessionId = undefined, + exfiltrate = false, interactive = true, showControlButtons = undefined, task = undefined, @@ -175,6 +212,8 @@ function BrowserStream({ const rfbRef = useRef(null); const observerRef = useRef(null); const clientId = useClientIdStore((state) => state.clientId); + const recordingStore = useRecordingStore(); + const settingsStore = useSettingsStore(); const credentialGetter = useCredentialGetter(); const getWebSocketParams = useCallback(async () => { @@ -198,6 +237,15 @@ function BrowserStream({ setIsReady(isVncConnected && isCanvasReady && hasBrowserSession); }, [hasBrowserSession, isCanvasReady, isVncConnected]); + // update global settings store about browser usage + useEffect(() => { + settingsStore.setIsUsingABrowser(isReady); + settingsStore.setBrowserSessionId( + isReady ? browserSessionId ?? null : null, + ); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [isReady, browserSessionId]); + // effect for vnc disconnects only useEffect(() => { if (prevVncConnectedRef.current && !isVncConnected) { @@ -417,8 +465,6 @@ function BrowserStream({ const sendCommand = (command: Command) => { if (!messageSocket) { - console.warn("Cannot send command, as command socket is closed."); - console.warn(command); return; } @@ -478,11 +524,58 @@ function BrowserStream({ } }, [task, workflow]); + // effect for exfiltration + useEffect(() => { + const sendCommand = (command: Command) => { + if (!messageSocket) { + return; + } + + messageSocket.send(JSON.stringify(command)); + }; + + sendCommand({ + kind: exfiltrate ? "begin-exfiltration" : "end-exfiltration", + }); + }, [exfiltrate, messageSocket]); + useEffect(() => { if (!interactive) { setUserIsControlling(false); } }, [interactive]); + + // effect to ensure the recordingStore is reset when the component unmounts + useEffect(() => { + return () => { + recordingStore.reset(); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + // effect to ensure 'take-control' is sent on the rising edge of + // recordingStore.isRecording + useEffect(() => { + if (!recordingStore.isRecording) { + return; + } + + if (!isMessageConnected) { + return; + } + + const sendCommand = (command: Command) => { + if (!messageSocket) { + return; + } + + messageSocket.send(JSON.stringify(command)); + }; + + sendCommand({ kind: "take-control" }); + setUserIsControlling(true); + }, [recordingStore.isRecording, isMessageConnected, messageSocket]); + /** * TODO(jdo): could use zod or smth similar */ @@ -524,6 +617,25 @@ function BrowserStream({ } break; } + case "exfiltrated-event": { + if ( + "event_name" in data && + typeof data.event_name === "string" && + "params" in data && + typeof data.params === "object" && + data.params !== null && + "source" in data && + typeof data.source === "string" + ) { + return { + kind: "exfiltrated-event", + event_name: data.event_name, + params: data.params, + source: data.source, + } as MessageInExfiltratedEvent; + } + break; + } default: { const _exhaustive: never = kind; return _exhaustive; @@ -604,6 +716,10 @@ function BrowserStream({ break; } + case "exfiltrated-event": { + recordingStore.add(message); + break; + } default: { const _exhaustive: never = kind; return _exhaustive; @@ -615,72 +731,137 @@ function BrowserStream({ userIsControlling || (interactive && !showControlButtons); return ( -
- {isReady && isVisible && ( -
- {showControlButtons && ( -
- - + <> +
+ {isReady && isVisible && ( +
+ {showControlButtons && ( +
+ + +
+ )} +
+ )} + {recordingStore.isRecording && ( + <> +
+
+
+
Browser is recording
+ +
+ +
+ + + + + + + + Cancel recording? + + You have recorded events that will be lost if you + cancel. Are you sure you want to cancel the recording? + + + + + + + + + + + + +
+
- )} -
- )} - {!isReady && ( -
- {browserSessionId && !hasBrowserSession ? ( -
This live browser session is no longer streaming.
- ) : ( - <> - - Hm, working on the connection... - Hang tight, we're almost there... - Just a moment... - Backpropagating... - Attention is all I need... - Consulting the manual... - Looking for the bat phone... - Where's Shu?... - - - - )} -
- )} -
+ + )} + {!isReady && ( +
+ {browserSessionId && !hasBrowserSession ? ( +
This live browser session is no longer streaming.
+ ) : ( + <> + + Hm, working on the connection... + Hang tight, we're almost there... + Just a moment... + Backpropagating... + Attention is all I need... + Consulting the manual... + Looking for the bat phone... + Where's Shu?... + + + + )} +
+ )} +
+ ); } diff --git a/skyvern-frontend/src/components/RadialMenu.tsx b/skyvern-frontend/src/components/RadialMenu.tsx index 77e8b49b..fe64fc59 100644 --- a/skyvern-frontend/src/components/RadialMenu.tsx +++ b/skyvern-frontend/src/components/RadialMenu.tsx @@ -1,4 +1,4 @@ -import { useState, useRef, useEffect, ReactNode } from "react"; +import { useState, useRef, useEffect, ReactNode, Fragment } from "react"; export interface RadialMenuItem { id: string; @@ -175,9 +175,8 @@ export function RadialMenu({ : "scaleY(1)"; return ( - <> +
)} - + ); })} diff --git a/skyvern-frontend/src/components/browser-stream.css b/skyvern-frontend/src/components/browser-stream.css index 85999982..4c463277 100644 --- a/skyvern-frontend/src/components/browser-stream.css +++ b/skyvern-frontend/src/components/browser-stream.css @@ -70,6 +70,7 @@ .browser-stream > div { background: transparent !important; + aspect-ratio: 16/9 !important; } .browser-stream .control-button { diff --git a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts new file mode 100644 index 00000000..79142d5a --- /dev/null +++ b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts @@ -0,0 +1,140 @@ +import { useMutation } from "@tanstack/react-query"; + +// import { getClient } from "@/api/AxiosClient"; +import { toast } from "@/components/ui/use-toast"; +// import { useCredentialGetter } from "@/hooks/useCredentialGetter"; +// import { type MessageInExfiltratedEvent } from "@/store/useRecordingStore"; +import { useRecordingStore } from "@/store/useRecordingStore"; +import { + type ActionBlock, + type WorkflowBlock, +} from "@/routes/workflows/types/workflowTypes"; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const FAIL_QUITE_NO_EVENTS = "FAIL-QUIET:NO-EVENTS" as const; + +const useProcessRecordingMutation = ({ + browserSessionId, + onSuccess, +}: { + browserSessionId: string | null; + onSuccess?: (workflowBlocks: Array) => void; +}) => { + // const credentialGetter = useCredentialGetter(); + const recordingStore = useRecordingStore(); + + const processRecordingMutation = useMutation({ + mutationFn: async () => { + if (!browserSessionId) { + throw new Error( + "Cannot process recording without a valid browser session ID.", + ); + } + + const eventCount = recordingStore.getEventCount(); + + if (eventCount === 0) { + throw new Error(FAIL_QUITE_NO_EVENTS); + } + + // (this flushes any pending events) + const compressedChunks = await recordingStore.getCompressedChunks(); + + // TODO: Replace this mock with actual API call when endpoint is ready + // const client = await getClient(credentialGetter, "sans-api-v1"); + // return client + // .post< + // { compressed_chunks: string[] }, + // { data: Array } + // >(`/browser_sessions/${browserSessionId}/process_recording`, { + // compressed_chunks: compressedChunks, + // }) + // .then((response) => response.data); + + // Mock response with 2-second delay + console.log( + `Processing ${eventCount} events in ${compressedChunks.length} compressed chunks`, + ); + await sleep(2000); + + // Return mock workflow blocks with two ActionBlocks + const mockWorkflowBlocks: Array = [ + { + block_type: "action", + label: "action_1", + title: "Enter search term", + navigation_goal: "Enter 'foo' in the search field", + url: null, + error_code_mapping: null, + parameters: [], + engine: null, + continue_on_failure: false, + output_parameter: { + parameter_type: "output", + key: "action_1_output", + description: null, + output_parameter_id: "mock-output-1", + workflow_id: browserSessionId || "mock-workflow-id", + created_at: new Date().toISOString(), + modified_at: new Date().toISOString(), + deleted_at: null, + }, + model: null, + } satisfies ActionBlock, + { + block_type: "action", + label: "action_2", + title: "Click search", + navigation_goal: "Click the search button", + url: null, + error_code_mapping: null, + parameters: [], + engine: null, + continue_on_failure: false, + output_parameter: { + parameter_type: "output", + key: "action_2_output", + description: null, + output_parameter_id: "mock-output-2", + workflow_id: browserSessionId || "mock-workflow-id", + created_at: new Date().toISOString(), + modified_at: new Date().toISOString(), + deleted_at: null, + }, + model: null, + } satisfies ActionBlock, + ]; + return mockWorkflowBlocks; + }, + onSuccess: (workflowBlocks) => { + // Clear events after successful flush + recordingStore.clear(); + + toast({ + variant: "success", + title: "Recording Processed", + description: "The recording has been successfully processed.", + }); + + if (workflowBlocks) { + onSuccess?.(workflowBlocks); + } + }, + onError: (error) => { + if (error instanceof Error && error.message === FAIL_QUITE_NO_EVENTS) { + return; + } + + toast({ + variant: "destructive", + title: "Error Processing Recording", + description: error.message, + }); + }, + }); + + return processRecordingMutation; +}; + +export { useProcessRecordingMutation }; diff --git a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx index 1f172feb..385ee523 100644 --- a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx @@ -11,6 +11,7 @@ import { useOnChange } from "@/hooks/useOnChange"; import { useShouldNotifyWhenClosingTab } from "@/hooks/useShouldNotifyWhenClosingTab"; import { BlockActionContext } from "@/store/BlockActionContext"; import { useDebugStore } from "@/store/useDebugStore"; +import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore"; import { useWorkflowHasChangesStore, useWorkflowSave, @@ -33,6 +34,7 @@ import { EdgeChange, } from "@xyflow/react"; import "@xyflow/react/dist/style.css"; +import { nanoid } from "nanoid"; import { useCallback, useEffect, useRef, useState } from "react"; import { useBlocker } from "react-router-dom"; import { @@ -76,8 +78,10 @@ import { import "./reactFlowOverrideStyles.css"; import { convertEchoParameters, + convertToNode, createNode, descendants, + generateNodeLabel, getAdditionalParametersForEmailBlock, getOrderedChildrenBlocks, getOutputParameterKey, @@ -319,6 +323,13 @@ function FlowRenderer({ const setGetSaveDataRef = useRef(workflowChangesStore.setGetSaveData); setGetSaveDataRef.current = workflowChangesStore.setGetSaveData; const saveWorkflow = useWorkflowSave({ status: "published" }); + const recordedBlocks = useRecordedBlocksStore((state) => state.blocks); + const recordedInsertionPoint = useRecordedBlocksStore( + (state) => state.insertionPoint, + ); + const clearRecordedBlocks = useRecordedBlocksStore( + (state) => state.clearRecordedBlocks, + ); useShouldNotifyWhenClosingTab(workflowChangesStore.hasChanges); const blocker = useBlocker(({ currentLocation, nextLocation }) => { return ( @@ -572,6 +583,85 @@ function FlowRenderer({ doLayout(nodes, edges); } + // effect to add new blocks that were generated from a browser recording + useEffect(() => { + if (!recordedBlocks || !recordedInsertionPoint) { + return; + } + + const { previous, next, parent, connectingEdgeType } = + recordedInsertionPoint; + + const newNodes: Array = []; + const newEdges: Array = []; + + let existingLabels = nodes + .filter(isWorkflowBlockNode) + .map((node) => node.data.label); + + let prevNodeId = previous; + + // convert each WorkflowBlock to an AppNode + recordedBlocks.forEach((block, index) => { + const id = nanoid(); + const label = generateNodeLabel(existingLabels); + existingLabels = [...existingLabels, label]; + const blockWithLabel = { ...block, label: block.label || label }; + + const node = convertToNode( + { id, parentId: parent }, + blockWithLabel, + true, + ); + newNodes.push(node); + + // create edge from previous node to this one + if (prevNodeId) { + newEdges.push({ + id: nanoid(), + type: "edgeWithAddButton", + source: prevNodeId, + target: id, + style: { strokeWidth: 2 }, + }); + } + + // if this is the last block, connect to next + if (index === recordedBlocks.length - 1 && next) { + newEdges.push({ + id: nanoid(), + type: connectingEdgeType, + source: id, + target: next, + style: { strokeWidth: 2 }, + }); + } + + prevNodeId = id; + }); + + const editedEdges = previous + ? edges.filter((edge) => edge.source !== previous) + : edges; + + const previousNode = nodes.find((node) => node.id === previous); + const previousNodeIndex = previousNode + ? nodes.indexOf(previousNode) + : nodes.length - 1; + + const newNodesAfter = [ + ...nodes.slice(0, previousNodeIndex + 1), + ...newNodes, + ...nodes.slice(previousNodeIndex + 1), + ]; + + workflowChangesStore.setHasChanges(true); + doLayout(newNodesAfter, [...editedEdges, ...newEdges]); + + clearRecordedBlocks(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [recordedBlocks, recordedInsertionPoint]); + const editorElementRef = useRef(null); useAutoPan(editorElementRef, nodes); diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowAddMenu.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowAddMenu.tsx new file mode 100644 index 00000000..60c838dc --- /dev/null +++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowAddMenu.tsx @@ -0,0 +1,82 @@ +import { SquareIcon, PlusIcon } from "@radix-ui/react-icons"; +import { ReactNode } from "react"; + +import { RadialMenu } from "@/components/RadialMenu"; +import { useIsSkyvernUser } from "@/hooks/useIsSkyvernUser"; +import { useDebugStore } from "@/store/useDebugStore"; +import { useRecordingStore } from "@/store/useRecordingStore"; +import { useSettingsStore } from "@/store/SettingsStore"; + +type WorkflowAddMenuProps = { + buttonSize?: string; + children: ReactNode; + gap?: number; + radius?: string; + rotateText?: boolean; + startAt?: number; + // -- + onAdd: () => void; + onRecord: () => void; +}; + +function WorkflowAddMenu({ + buttonSize, + children, + gap, + radius = "80px", + rotateText = true, + startAt = 90, + // -- + onAdd, + onRecord, +}: WorkflowAddMenuProps) { + const debugStore = useDebugStore(); + const recordingStore = useRecordingStore(); + const settingsStore = useSettingsStore(); + const isSkyvernUser = useIsSkyvernUser(); + + if ( + !isSkyvernUser || + !debugStore.isDebugMode || + !settingsStore.isUsingABrowser + ) { + return <>{children}; + } + + return ( + , + text: "Add Block", + onClick: () => { + onAdd(); + }, + }, + { + id: "2", + icon: , + enabled: !recordingStore.isRecording && settingsStore.isUsingABrowser, + text: "Record Browser", + onClick: () => { + if (!settingsStore.isUsingABrowser) { + return; + } + + onRecord(); + }, + }, + ]} + buttonSize={buttonSize} + radius={radius} + startAt={startAt} + gap={gap} + rotateText={rotateText} + > + {children} + + ); +} + +export { WorkflowAddMenu }; diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.css b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.css new file mode 100644 index 00000000..f83cc025 --- /dev/null +++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.css @@ -0,0 +1,29 @@ +@keyframes pulse-dash { + 0% { + stroke-dasharray: 141.4 141.4; + stroke-width: 6; + } + 50% { + stroke-dasharray: 10 11.4; + stroke-width: 8; + } + 100% { + stroke-dasharray: 141.4 141.4; + stroke-width: 6; + } +} + +@keyframes pulse-dash-small { + 0% { + stroke-dasharray: 100.4 100.4; + stroke-width: 3; + } + 50% { + stroke-dasharray: 10 11.4; + stroke-width: 5; + } + 100% { + stroke-dasharray: 100.4 100.4; + stroke-width: 3; + } +} diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx new file mode 100644 index 00000000..54f341e6 --- /dev/null +++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx @@ -0,0 +1,196 @@ +import { ReactNode, useEffect, useRef, useState } from "react"; + +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import { useRecordingStore, CHUNK_SIZE } from "@/store/useRecordingStore"; +import { cn } from "@/util/utils"; + +import "./WorkflowAdderBusy.css"; + +type Operation = "recording" | "processing"; + +type Size = "small" | "large"; + +type Props = { + children: ReactNode; + /** + * The operation being performed (e.g., recording or processing). + */ + operation: Operation; + /** + * An explicit sizing; otherwise the size will be determined by the child content. + */ + size?: Size; + /** + * Color for the cover and ellipses. Defaults to "red". + */ + color?: string; + // -- + onComplete: () => void; +}; + +function WorkflowAdderBusy({ + children, + operation, + size, + color = "red", + onComplete, +}: Props) { + const recordingStore = useRecordingStore(); + const [isHovered, setIsHovered] = useState(false); + const [shouldBump, setShouldBump] = useState(false); + const bumpTimeoutRef = useRef(null); + const prevCountRef = useRef(0); + + const eventCount = + recordingStore.pendingEvents.length + + recordingStore.compressedChunks.length * CHUNK_SIZE; + + // effect for bump animation when count changes + useEffect(() => { + if (eventCount > prevCountRef.current && prevCountRef.current > 0) { + if (bumpTimeoutRef.current) { + clearTimeout(bumpTimeoutRef.current); + } + + setShouldBump(true); + + bumpTimeoutRef.current = setTimeout(() => { + setShouldBump(false); + }, 300); + } + + prevCountRef.current = eventCount; + + return () => { + if (bumpTimeoutRef.current) { + clearTimeout(bumpTimeoutRef.current); + } + }; + }, [eventCount]); + + const handleClick = (e: React.MouseEvent) => { + e.stopPropagation(); + e.preventDefault(); + onComplete(); + + return false; + }; + + return ( + +
+ + +
setIsHovered(true)} + onMouseLeave={() => setIsHovered(false)} + > + {/* cover */} +
+
+ {children} +
+
+ + + +
+ {isHovered && ( +
+ + + +
+ )} +
+ + +

+ {operation === "recording" ? "Finish Recording" : "Processing..."} +

+ + + {recordingStore.isRecording && eventCount > 0 && ( + + +
+ {eventCount} +
+ + +

Event Count

+ + + )} +
+ + ); +} + +export { WorkflowAdderBusy }; diff --git a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx index f249c08d..86d0076d 100644 --- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx @@ -29,6 +29,7 @@ import { useBlockScriptsQuery } from "@/routes/workflows/hooks/useBlockScriptsQu import { WorkflowRunStream } from "@/routes/workflows/workflowRun/WorkflowRunStream"; import { useCacheKeyValuesQuery } from "../hooks/useCacheKeyValuesQuery"; import { useBlockScriptStore } from "@/store/BlockScriptStore"; +import { useRecordingStore } from "@/store/useRecordingStore"; import { useSidebarStore } from "@/store/SidebarStore"; import { AnimatedWave } from "@/components/AnimatedWave"; @@ -238,6 +239,7 @@ function Workspace({ const queryClient = useQueryClient(); const [shouldFetchDebugSession, setShouldFetchDebugSession] = useState(false); const blockScriptStore = useBlockScriptStore(); + const recordingStore = useRecordingStore(); const cacheKey = workflow?.cache_key ?? ""; const [cacheKeyValue, setCacheKeyValue] = useState( @@ -1339,6 +1341,7 @@ function Workspace({
state.setRecordedBlocks, + ); const setWorkflowPanelState = useWorkflowPanelStore( (state) => state.setWorkflowPanelState, ); + const processRecordingMutation = useProcessRecordingMutation({ + browserSessionId: settingsStore.browserSessionId, + onSuccess: (blocks) => { + setRecordedBlocks(blocks, { + previous: source, + next: target, + parent: sourceNode?.parentId, + connectingEdgeType: "edgeWithAddButton", + }); + }, + }); + + const isProcessing = processRecordingMutation.isPending; + const sourceNode = nodes.find((node) => node.id === source); - const onAdd = () => { + const updateWorkflowPanelState = (active: boolean) => { setWorkflowPanelState({ - active: true, + active, content: "nodeLibrary", data: { previous: source, @@ -55,6 +78,25 @@ function EdgeWithAddButton({ }); }; + const onAdd = () => updateWorkflowPanelState(true); + + const onRecord = () => { + if (recordingStore.isRecording) { + recordingStore.setIsRecording(false); + } else { + recordingStore.setIsRecording(true); + updateWorkflowPanelState(false); + } + }; + + const onEndRecord = () => { + if (recordingStore.isRecording) { + recordingStore.setIsRecording(false); + } + + processRecordingMutation.mutate(); + }; + const adder = (
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx index a2d0e3cb..c8d46b30 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx @@ -1,25 +1,51 @@ -import { SquareIcon, PlusIcon } from "@radix-ui/react-icons"; +import { PlusIcon } from "@radix-ui/react-icons"; import { Handle, NodeProps, Position, useEdges } from "@xyflow/react"; -import { RadialMenu } from "@/components/RadialMenu"; -import { useIsSkyvernUser } from "@/hooks/useIsSkyvernUser"; +import { useProcessRecordingMutation } from "@/routes/browserSessions/hooks/useProcessRecordingMutation"; import { useDebugStore } from "@/store/useDebugStore"; +import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore"; +import { useRecordingStore } from "@/store/useRecordingStore"; +import { useSettingsStore } from "@/store/SettingsStore"; import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore"; import type { NodeAdderNode } from "./types"; +import { WorkflowAddMenu } from "../../WorkflowAddMenu"; +import { WorkflowAdderBusy } from "../../WorkflowAdderBusy"; function NodeAdderNode({ id, parentId }: NodeProps) { - const debugStore = useDebugStore(); - const isSkyvernUser = useIsSkyvernUser(); const edges = useEdges(); + const debugStore = useDebugStore(); + const recordingStore = useRecordingStore(); + const settingsStore = useSettingsStore(); const setWorkflowPanelState = useWorkflowPanelStore( (state) => state.setWorkflowPanelState, ); + const workflowStatePanel = useWorkflowPanelStore(); + const setRecordedBlocks = useRecordedBlocksStore( + (state) => state.setRecordedBlocks, + ); - const onAdd = () => { + const previous = edges.find((edge) => edge.target === id)?.source ?? null; + + const processRecordingMutation = useProcessRecordingMutation({ + browserSessionId: settingsStore.browserSessionId, + onSuccess: (blocks) => { + setRecordedBlocks(blocks, { + previous, + next: id, + parent: parentId, + connectingEdgeType: "default", + }); + }, + }); + + const isProcessing = processRecordingMutation.isPending; + + const updateWorkflowPanelState = (active: boolean) => { const previous = edges.find((edge) => edge.target === id)?.source; + setWorkflowPanelState({ - active: true, + active, content: "nodeLibrary", data: { previous: previous ?? null, @@ -30,9 +56,30 @@ function NodeAdderNode({ id, parentId }: NodeProps) { }); }; + const onAdd = () => { + updateWorkflowPanelState(true); + }; + + const onRecord = () => { + if (recordingStore.isRecording) { + recordingStore.setIsRecording(false); + } else { + recordingStore.setIsRecording(true); + updateWorkflowPanelState(false); + } + }; + + const onEndRecord = () => { + if (recordingStore.isRecording) { + recordingStore.setIsRecording(false); + } + + processRecordingMutation.mutate(); + }; + const adder = (
{ onAdd(); }} @@ -41,6 +88,33 @@ function NodeAdderNode({ id, parentId }: NodeProps) {
); + const busy = ( + { + onEndRecord(); + }} + > + {adder} + + ); + + const menu = ( + + {adder} + + ); + + const isBusy = + (isProcessing || recordingStore.isRecording) && + debugStore.isDebugMode && + settingsStore.isUsingABrowser && + workflowStatePanel.workflowPanelState.data?.previous === previous && + workflowStatePanel.workflowPanelState.data?.next === id && + workflowStatePanel.workflowPanelState.data?.parent === + (parentId || undefined); + return (
) { id="b" className="opacity-0" /> - {isSkyvernUser && debugStore.isDebugMode ? ( - , - text: "Add Block", - onClick: () => { - onAdd(); - }, - }, - { - id: "2", - icon: , - enabled: false, - text: "Record Browser", - onClick: () => { - console.log("Record"); - }, - }, - ]} - radius="80px" - startAt={90} - rotateText={true} - > - {adder} - - ) : ( - adder - )} + {isBusy ? busy : menu}
); } diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts index 1bb9912e..833cbb96 100644 --- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts +++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts @@ -2458,6 +2458,7 @@ function getLabelForWorkflowParameterType(type: WorkflowParameterValueType) { export { convert, convertEchoParameters, + convertToNode, createNode, generateNodeData, generateNodeLabel, diff --git a/skyvern-frontend/src/store/RecordedBlocksStore.ts b/skyvern-frontend/src/store/RecordedBlocksStore.ts new file mode 100644 index 00000000..4daa6891 --- /dev/null +++ b/skyvern-frontend/src/store/RecordedBlocksStore.ts @@ -0,0 +1,36 @@ +import { create } from "zustand"; +import type { WorkflowBlock } from "@/routes/workflows/types/workflowTypes"; + +type InsertionPoint = { + previous: string | null; + next: string | null; + parent?: string; + connectingEdgeType: string; +}; + +type RecordedBlocksState = { + blocks: Array | null; + insertionPoint: InsertionPoint | null; +}; + +type RecordedBlocksStore = RecordedBlocksState & { + setRecordedBlocks: ( + blocks: Array, + insertionPoint: InsertionPoint, + ) => void; + clearRecordedBlocks: () => void; +}; + +const useRecordedBlocksStore = create((set) => ({ + blocks: null, + insertionPoint: null, + setRecordedBlocks: (blocks, insertionPoint) => { + set({ blocks, insertionPoint }); + }, + clearRecordedBlocks: () => { + set({ blocks: null, insertionPoint: null }); + }, +})); + +export { useRecordedBlocksStore }; +export type { InsertionPoint }; diff --git a/skyvern-frontend/src/store/SettingsStore.ts b/skyvern-frontend/src/store/SettingsStore.ts index b8050b59..57ed62cd 100644 --- a/skyvern-frontend/src/store/SettingsStore.ts +++ b/skyvern-frontend/src/store/SettingsStore.ts @@ -2,16 +2,33 @@ import { create } from "zustand"; type SettingsStore = { environment: string; + /** + * The user is currently operating or viewing a live, remote browser. NOTE: if + * the browser is still connecting, or otherwise not ready, then this should + * be false. + */ + isUsingABrowser: boolean; + /** + * The current browser session ID when a browser is active. + */ + browserSessionId: string | null; organization: string; setEnvironment: (environment: string) => void; + setIsUsingABrowser: (isUsing: boolean) => void; + setBrowserSessionId: (browserSessionId: string | null) => void; setOrganization: (organization: string) => void; }; const useSettingsStore = create((set) => { return { environment: "local", + isUsingABrowser: false, + browserSessionId: null, organization: "skyvern", setEnvironment: (environment: string) => set({ environment }), + setIsUsingABrowser: (isUsing: boolean) => set({ isUsingABrowser: isUsing }), + setBrowserSessionId: (browserSessionId: string | null) => + set({ browserSessionId }), setOrganization: (organization: string) => set({ organization }), }; }); diff --git a/skyvern-frontend/src/store/useRecordingStore.ts b/skyvern-frontend/src/store/useRecordingStore.ts new file mode 100644 index 00000000..5369d006 --- /dev/null +++ b/skyvern-frontend/src/store/useRecordingStore.ts @@ -0,0 +1,298 @@ +import { create } from "zustand"; + +/** + * example: { + * 'targetInfo': { + * 'targetId': '8B698E27F1F32372718DA73DCA0C5944', + * 'type': 'page', + * 'title': 'New Tab', + * 'url': 'chrome://newtab/', + * 'attached': True, + * 'canAccessOpener': False, + * 'browserContextId': 'FD13D5C556E681BB49AEED0AB2CA1972', + * } + */ +export interface ExfiltratedEventCdpParams { + targetInfo: { + attached?: boolean; + browserContextId?: string; + canAccessOpener?: boolean; + targetId?: string; + title?: string; + type?: string; + url?: string; + }; +} + +export interface ExfiltratedEventConsoleParams { + type: string; + url: string; + timestamp: number; + target: { + className?: string; + id?: string; + innerText?: string; + tagName?: string; + text: string[]; + value?: string; + }; + inputValue?: string; + mousePosition: { + xa: number | null; + ya: number | null; + xp: number | null; + yp: number | null; + }; + key?: string; + code?: string; + activeElement: { + tagName?: string; + id?: string; + className?: string; + boundingRect?: { + x: number; + y: number; + width: number; + height: number; + top: number; + right: number; + bottom: number; + left: number; + } | null; + scroll?: { + scrollTop: number; + scrollLeft: number; + scrollHeight: number; + scrollWidth: number; + clientHeight: number; + clientWidth: number; + } | null; + }; + window: { + width: number; + height: number; + scrollX: number; + scrollY: number; + }; +} + +export interface MessageInExfiltratedCdpEvent { + kind: "exfiltrated-event"; + event_name: string; + params: ExfiltratedEventCdpParams; + source: "cdp"; +} + +export interface MessageInExfiltratedConsoleEvent { + kind: "exfiltrated-event"; + event_name: string; + params: ExfiltratedEventConsoleParams; + source: "console"; +} + +export type MessageInExfiltratedEvent = + | MessageInExfiltratedCdpEvent + | MessageInExfiltratedConsoleEvent; + +/** + * Number of events per compressed chunk. + */ +export const CHUNK_SIZE = 1000 as const; + +interface RecordingStore { + /** + * Compressed chunks of recorded events (base64 gzip). + * Each chunk contains up to CHUNK_SIZE events. + */ + compressedChunks: string[]; + /** + * Buffer of events not yet compressed into a chunk. + */ + pendingEvents: MessageInExfiltratedEvent[]; + /** + * Whether a compression operation is currently in progress. + */ + isCompressing: boolean; + /** + * Whether the user is currently in browser recording mode. + */ + isRecording: boolean; + /** + * Add a new recorded event. Triggers async compression when buffer is full. + */ + add: (event: MessageInExfiltratedEvent) => void; + /** + * Clear all recorded events and compressed chunks. + */ + clear: () => void; + /** + * Reset the recording store (clear events and set isRecording to false). + */ + reset: () => void; + /** + * Set whether the user is in browser recording mode. + */ + setIsRecording: (isRecording: boolean) => void; + /** + * Flush any pending events into a compressed chunk. + * Call this before consuming the data. + */ + flush: () => Promise; + /** + * Get all compressed chunks (after flushing pending events). + */ + getCompressedChunks: () => Promise; + /** + * Get the total number of events (compressed + pending). + */ + getEventCount: () => number; +} + +/** + * compresses a JSON string using the Gzip algorithm and returns the result + * as a Base64 encoded string + */ +async function compressEventsToB64(jsonString: string): Promise { + // 1. Convert the string to a Uint8Array (a byte array). + const encoder = new TextEncoder(); + const uint8Array = encoder.encode(jsonString); + + // 2. Create a ReadableStream from the byte array. + const readableStream = new ReadableStream({ + start(controller) { + controller.enqueue(uint8Array); + controller.close(); + }, + }); + + // 3. Pipe the data through the Gzip compression stream. + const compressedStream = readableStream.pipeThrough( + new CompressionStream("gzip"), // Use 'gzip' for standard network transport + ); + + // 4. Read the entire compressed stream back into a single ArrayBuffer. + // The Response object provides an easy way to convert streams into a single buffer. + const compressedBuffer = await new Response(compressedStream).arrayBuffer(); + + // 5. Convert the ArrayBuffer (binary data) to a Base64 string for transport. + // Base64 is used to safely transmit binary data over text-based protocols (like JSON). + const bytes = new Uint8Array(compressedBuffer); + let binary = ""; + + // Convert Uint8Array to a raw binary string (this is needed for btoa) + for (let i = 0; i < bytes.length; i++) { + const nextByte = bytes[i]; + + if (nextByte === undefined) { + continue; + } + + binary += String.fromCharCode(nextByte); + } + + // Convert the raw binary string to Base64 + return btoa(binary); +} + +export const useRecordingStore = create((set, get) => ({ + compressedChunks: [], + pendingEvents: [], + isCompressing: false, + isRecording: false, + + add: (event) => { + const state = get(); + const newPendingEvents = [...state.pendingEvents, event]; + + if (newPendingEvents.length >= CHUNK_SIZE && !state.isCompressing) { + const eventsToCompress = newPendingEvents.slice(0, CHUNK_SIZE); + const remainingEvents = newPendingEvents.slice(CHUNK_SIZE); + + set({ pendingEvents: remainingEvents, isCompressing: true }); + + // compress asynchronously + queueMicrotask(async () => { + try { + const jsonString = JSON.stringify(eventsToCompress); + const compressed = await compressEventsToB64(jsonString); + const currentState = get(); + set({ + compressedChunks: [...currentState.compressedChunks, compressed], + isCompressing: false, + }); + } catch (error) { + console.error("Failed to compress events chunk:", error); + + // on error, put events back into pending + const currentState = get(); + set({ + pendingEvents: [...eventsToCompress, ...currentState.pendingEvents], + isCompressing: false, + }); + } + }); + } else { + set({ pendingEvents: newPendingEvents }); + } + }, + + clear: () => set({ compressedChunks: [], pendingEvents: [] }), + + reset: () => + set({ + compressedChunks: [], + pendingEvents: [], + isCompressing: false, + isRecording: false, + }), + + setIsRecording: (isRecording) => { + const state = get(); + // clear events on rising edge + if (!state.isRecording && isRecording) { + get().clear(); + } + set({ isRecording }); + }, + + flush: async () => { + // Wait for any in-progress compression to complete + while (get().isCompressing) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + + const pending = get().pendingEvents; + if (pending.length === 0) { + return; + } + + set({ isCompressing: true }); + + try { + const jsonString = JSON.stringify(pending); + const compressed = await compressEventsToB64(jsonString); + const currentState = get(); + set({ + compressedChunks: [...currentState.compressedChunks, compressed], + pendingEvents: [], + isCompressing: false, + }); + } catch (error) { + console.error("Failed to flush pending events:", error); + set({ isCompressing: false }); + throw error; + } + }, + + getCompressedChunks: async () => { + await get().flush(); + return get().compressedChunks; + }, + + getEventCount: () => { + const state = get(); + return ( + state.compressedChunks.length * CHUNK_SIZE + state.pendingEvents.length + ); + }, +}));