Browser recording action (#4130)
This commit is contained in:
Jonathan Dobson
@@ -1,5 +1,5 @@
|
||||
import RFB from "@novnc/novnc/lib/rfb.js";
|
||||
import { ExitIcon, HandIcon } from "@radix-ui/react-icons";
|
||||
import { ExitIcon, HandIcon, InfoCircledIcon } from "@radix-ui/react-icons";
|
||||
import { useEffect, useState, useRef, useCallback } from "react";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
|
||||
@@ -9,12 +9,28 @@ import type {
|
||||
TaskApiResponse,
|
||||
WorkflowRunStatusApiResponse,
|
||||
} from "@/api/types";
|
||||
import { Tip } from "@/components/Tip";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
Dialog,
|
||||
DialogClose,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
DialogTrigger,
|
||||
} from "@/components/ui/dialog";
|
||||
import { AnimatedWave } from "@/components/AnimatedWave";
|
||||
import { toast } from "@/components/ui/use-toast";
|
||||
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
|
||||
import { statusIsNotFinalized } from "@/routes/tasks/types";
|
||||
import { useClientIdStore } from "@/store/useClientIdStore";
|
||||
import {
|
||||
useRecordingStore,
|
||||
type MessageInExfiltratedEvent,
|
||||
} from "@/store/useRecordingStore";
|
||||
import { useSettingsStore } from "@/store/SettingsStore";
|
||||
import {
|
||||
environment,
|
||||
wssBaseUrl,
|
||||
@@ -31,18 +47,34 @@ interface BrowserSession {
|
||||
completed_at?: string;
|
||||
}
|
||||
|
||||
interface CommandTakeControl {
|
||||
kind: "take-control";
|
||||
interface CommandBeginExfiltration {
|
||||
kind: "begin-exfiltration";
|
||||
}
|
||||
|
||||
interface CommandCedeControl {
|
||||
kind: "cede-control";
|
||||
}
|
||||
|
||||
// a "Command" is an fire-n-forget out-message - it does not require a response
|
||||
type Command = CommandTakeControl | CommandCedeControl;
|
||||
interface CommandEndExfiltration {
|
||||
kind: "end-exfiltration";
|
||||
}
|
||||
|
||||
const messageInKinds = ["ask-for-clipboard", "copied-text"] as const;
|
||||
interface CommandTakeControl {
|
||||
kind: "take-control";
|
||||
}
|
||||
|
||||
// a "Command" is an fire-n-forget out-message - it does not require a response
|
||||
type Command =
|
||||
| CommandBeginExfiltration
|
||||
| CommandCedeControl
|
||||
| CommandEndExfiltration
|
||||
| CommandTakeControl;
|
||||
|
||||
const messageInKinds = [
|
||||
"ask-for-clipboard",
|
||||
"copied-text",
|
||||
"exfiltrated-event",
|
||||
] as const;
|
||||
|
||||
type MessageInKind = (typeof messageInKinds)[number];
|
||||
|
||||
@@ -55,7 +87,10 @@ interface MessageInCopiedText {
|
||||
text: string;
|
||||
}
|
||||
|
||||
type MessageIn = MessageInCopiedText | MessageInAskForClipboard;
|
||||
type MessageIn =
|
||||
| MessageInCopiedText
|
||||
| MessageInAskForClipboard
|
||||
| MessageInExfiltratedEvent;
|
||||
|
||||
interface MessageOutAskForClipboardResponse {
|
||||
kind: "ask-for-clipboard-response";
|
||||
@@ -66,6 +101,7 @@ type MessageOut = MessageOutAskForClipboardResponse;
|
||||
|
||||
type Props = {
|
||||
browserSessionId?: string;
|
||||
exfiltrate?: boolean;
|
||||
interactive?: boolean;
|
||||
showControlButtons?: boolean;
|
||||
task?: {
|
||||
@@ -82,6 +118,7 @@ type Props = {
|
||||
|
||||
function BrowserStream({
|
||||
browserSessionId = undefined,
|
||||
exfiltrate = false,
|
||||
interactive = true,
|
||||
showControlButtons = undefined,
|
||||
task = undefined,
|
||||
@@ -175,6 +212,8 @@ function BrowserStream({
|
||||
const rfbRef = useRef<RFB | null>(null);
|
||||
const observerRef = useRef<MutationObserver | null>(null);
|
||||
const clientId = useClientIdStore((state) => state.clientId);
|
||||
const recordingStore = useRecordingStore();
|
||||
const settingsStore = useSettingsStore();
|
||||
const credentialGetter = useCredentialGetter();
|
||||
|
||||
const getWebSocketParams = useCallback(async () => {
|
||||
@@ -198,6 +237,15 @@ function BrowserStream({
|
||||
setIsReady(isVncConnected && isCanvasReady && hasBrowserSession);
|
||||
}, [hasBrowserSession, isCanvasReady, isVncConnected]);
|
||||
|
||||
// update global settings store about browser usage
|
||||
useEffect(() => {
|
||||
settingsStore.setIsUsingABrowser(isReady);
|
||||
settingsStore.setBrowserSessionId(
|
||||
isReady ? browserSessionId ?? null : null,
|
||||
);
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [isReady, browserSessionId]);
|
||||
|
||||
// effect for vnc disconnects only
|
||||
useEffect(() => {
|
||||
if (prevVncConnectedRef.current && !isVncConnected) {
|
||||
@@ -417,8 +465,6 @@ function BrowserStream({
|
||||
|
||||
const sendCommand = (command: Command) => {
|
||||
if (!messageSocket) {
|
||||
console.warn("Cannot send command, as command socket is closed.");
|
||||
console.warn(command);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -478,11 +524,58 @@ function BrowserStream({
|
||||
}
|
||||
}, [task, workflow]);
|
||||
|
||||
// effect for exfiltration
|
||||
useEffect(() => {
|
||||
const sendCommand = (command: Command) => {
|
||||
if (!messageSocket) {
|
||||
return;
|
||||
}
|
||||
|
||||
messageSocket.send(JSON.stringify(command));
|
||||
};
|
||||
|
||||
sendCommand({
|
||||
kind: exfiltrate ? "begin-exfiltration" : "end-exfiltration",
|
||||
});
|
||||
}, [exfiltrate, messageSocket]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!interactive) {
|
||||
setUserIsControlling(false);
|
||||
}
|
||||
}, [interactive]);
|
||||
|
||||
// effect to ensure the recordingStore is reset when the component unmounts
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
recordingStore.reset();
|
||||
};
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, []);
|
||||
|
||||
// effect to ensure 'take-control' is sent on the rising edge of
|
||||
// recordingStore.isRecording
|
||||
useEffect(() => {
|
||||
if (!recordingStore.isRecording) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isMessageConnected) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sendCommand = (command: Command) => {
|
||||
if (!messageSocket) {
|
||||
return;
|
||||
}
|
||||
|
||||
messageSocket.send(JSON.stringify(command));
|
||||
};
|
||||
|
||||
sendCommand({ kind: "take-control" });
|
||||
setUserIsControlling(true);
|
||||
}, [recordingStore.isRecording, isMessageConnected, messageSocket]);
|
||||
|
||||
/**
|
||||
* TODO(jdo): could use zod or smth similar
|
||||
*/
|
||||
@@ -524,6 +617,25 @@ function BrowserStream({
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "exfiltrated-event": {
|
||||
if (
|
||||
"event_name" in data &&
|
||||
typeof data.event_name === "string" &&
|
||||
"params" in data &&
|
||||
typeof data.params === "object" &&
|
||||
data.params !== null &&
|
||||
"source" in data &&
|
||||
typeof data.source === "string"
|
||||
) {
|
||||
return {
|
||||
kind: "exfiltrated-event",
|
||||
event_name: data.event_name,
|
||||
params: data.params,
|
||||
source: data.source,
|
||||
} as MessageInExfiltratedEvent;
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
const _exhaustive: never = kind;
|
||||
return _exhaustive;
|
||||
@@ -604,6 +716,10 @@ function BrowserStream({
|
||||
|
||||
break;
|
||||
}
|
||||
case "exfiltrated-event": {
|
||||
recordingStore.add(message);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
const _exhaustive: never = kind;
|
||||
return _exhaustive;
|
||||
@@ -615,72 +731,137 @@ function BrowserStream({
|
||||
userIsControlling || (interactive && !showControlButtons);
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
"browser-stream relative flex items-center justify-center",
|
||||
{
|
||||
"user-is-controlling": theUserIsControlling,
|
||||
},
|
||||
)}
|
||||
ref={setCanvasContainerRef}
|
||||
>
|
||||
{isReady && isVisible && (
|
||||
<div className="overlay z-10 flex items-center justify-center overflow-hidden">
|
||||
{showControlButtons && (
|
||||
<div className="control-buttons pointer-events-none relative flex h-full w-full items-center justify-center">
|
||||
<Button
|
||||
onClick={() => {
|
||||
setUserIsControlling(true);
|
||||
}}
|
||||
className={cn("control-button pointer-events-auto border", {
|
||||
hide: userIsControlling,
|
||||
})}
|
||||
size="sm"
|
||||
>
|
||||
<HandIcon className="mr-2 h-4 w-4" />
|
||||
take control
|
||||
</Button>
|
||||
<Button
|
||||
onClick={() => {
|
||||
setUserIsControlling(false);
|
||||
}}
|
||||
className={cn(
|
||||
"control-button pointer-events-auto absolute bottom-0 border",
|
||||
{
|
||||
hide: !userIsControlling,
|
||||
},
|
||||
)}
|
||||
size="sm"
|
||||
>
|
||||
<ExitIcon className="mr-2 h-4 w-4" />
|
||||
stop controlling
|
||||
</Button>
|
||||
<>
|
||||
<div
|
||||
className={cn(
|
||||
"browser-stream relative flex flex-col items-center justify-center",
|
||||
{
|
||||
"user-is-controlling": theUserIsControlling,
|
||||
},
|
||||
)}
|
||||
ref={setCanvasContainerRef}
|
||||
>
|
||||
{isReady && isVisible && (
|
||||
<div className="overlay z-10 flex items-center justify-center overflow-hidden">
|
||||
{showControlButtons && (
|
||||
<div className="control-buttons pointer-events-none relative flex h-full w-full items-center justify-center">
|
||||
<Button
|
||||
onClick={() => {
|
||||
setUserIsControlling(true);
|
||||
}}
|
||||
className={cn("control-button pointer-events-auto border", {
|
||||
hide: userIsControlling,
|
||||
})}
|
||||
size="sm"
|
||||
>
|
||||
<HandIcon className="mr-2 h-4 w-4" />
|
||||
take control
|
||||
</Button>
|
||||
<Button
|
||||
onClick={() => {
|
||||
setUserIsControlling(false);
|
||||
}}
|
||||
className={cn(
|
||||
"control-button pointer-events-auto absolute bottom-0 border",
|
||||
{
|
||||
hide: !userIsControlling,
|
||||
},
|
||||
)}
|
||||
size="sm"
|
||||
>
|
||||
<ExitIcon className="mr-2 h-4 w-4" />
|
||||
stop controlling
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{recordingStore.isRecording && (
|
||||
<>
|
||||
<div className="pointer-events-none absolute flex aspect-video w-full items-center justify-center rounded-xl p-2 outline outline-8 outline-offset-[-2px] outline-red-500 animate-in fade-in">
|
||||
<div className="relative h-full w-full">
|
||||
<div className="pointer-events-auto absolute top-[-3rem] flex w-full items-center justify-start gap-2 text-red-500">
|
||||
<div className="truncate">Browser is recording</div>
|
||||
<Tip content="To finish the recording, press stop on the animated recording button in the workflow.">
|
||||
<div className="cursor-pointer">
|
||||
<InfoCircledIcon />
|
||||
</div>
|
||||
</Tip>
|
||||
<Dialog>
|
||||
<DialogTrigger asChild>
|
||||
<Button
|
||||
className="ml-auto cursor-pointer"
|
||||
size="sm"
|
||||
variant="destructive"
|
||||
style={{
|
||||
marginTop: "-0.5rem",
|
||||
}}
|
||||
onClick={(e) => {
|
||||
const hasEvents =
|
||||
recordingStore.pendingEvents.length > 0 ||
|
||||
recordingStore.compressedChunks.length > 0;
|
||||
if (!hasEvents) {
|
||||
e.preventDefault();
|
||||
recordingStore.setIsRecording(false);
|
||||
}
|
||||
}}
|
||||
>
|
||||
cancel
|
||||
</Button>
|
||||
</DialogTrigger>
|
||||
<DialogContent>
|
||||
<DialogHeader>
|
||||
<DialogTitle>Cancel recording?</DialogTitle>
|
||||
<DialogDescription>
|
||||
You have recorded events that will be lost if you
|
||||
cancel. Are you sure you want to cancel the recording?
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<DialogFooter>
|
||||
<DialogClose asChild>
|
||||
<Button variant="outline">Keep recording</Button>
|
||||
</DialogClose>
|
||||
<DialogClose asChild>
|
||||
<Button
|
||||
variant="destructive"
|
||||
onClick={() => {
|
||||
recordingStore.setIsRecording(false);
|
||||
}}
|
||||
>
|
||||
Cancel recording
|
||||
</Button>
|
||||
</DialogClose>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{!isReady && (
|
||||
<div className="absolute left-0 top-1/2 flex aspect-video max-h-full w-full -translate-y-1/2 flex-col items-center justify-center gap-2 rounded-md border border-slate-800 text-sm text-slate-400">
|
||||
{browserSessionId && !hasBrowserSession ? (
|
||||
<div>This live browser session is no longer streaming.</div>
|
||||
) : (
|
||||
<>
|
||||
<RotateThrough interval={7 * 1000}>
|
||||
<span>Hm, working on the connection...</span>
|
||||
<span>Hang tight, we're almost there...</span>
|
||||
<span>Just a moment...</span>
|
||||
<span>Backpropagating...</span>
|
||||
<span>Attention is all I need...</span>
|
||||
<span>Consulting the manual...</span>
|
||||
<span>Looking for the bat phone...</span>
|
||||
<span>Where's Shu?...</span>
|
||||
</RotateThrough>
|
||||
<AnimatedWave text=".‧₊˚ ⋅ ? ✨ ?★ ‧₊˚ ⋅" />
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
{!isReady && (
|
||||
<div className="absolute left-0 top-1/2 flex aspect-video max-h-full w-full -translate-y-1/2 flex-col items-center justify-center gap-2 rounded-md border border-slate-800 text-sm text-slate-400">
|
||||
{browserSessionId && !hasBrowserSession ? (
|
||||
<div>This live browser session is no longer streaming.</div>
|
||||
) : (
|
||||
<>
|
||||
<RotateThrough interval={7 * 1000}>
|
||||
<span>Hm, working on the connection...</span>
|
||||
<span>Hang tight, we're almost there...</span>
|
||||
<span>Just a moment...</span>
|
||||
<span>Backpropagating...</span>
|
||||
<span>Attention is all I need...</span>
|
||||
<span>Consulting the manual...</span>
|
||||
<span>Looking for the bat phone...</span>
|
||||
<span>Where's Shu?...</span>
|
||||
</RotateThrough>
|
||||
<AnimatedWave text=".‧₊˚ ⋅ ? ✨ ?★ ‧₊˚ ⋅" />
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useState, useRef, useEffect, ReactNode } from "react";
|
||||
import { useState, useRef, useEffect, ReactNode, Fragment } from "react";
|
||||
|
||||
export interface RadialMenuItem {
|
||||
id: string;
|
||||
@@ -175,9 +175,8 @@ export function RadialMenu({
|
||||
: "scaleY(1)";
|
||||
|
||||
return (
|
||||
<>
|
||||
<Fragment key={item.id}>
|
||||
<button
|
||||
key={item.id}
|
||||
onClick={() => {
|
||||
item.onClick();
|
||||
setIsOpen(false);
|
||||
@@ -225,7 +224,7 @@ export function RadialMenu({
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
</Fragment>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
@@ -70,6 +70,7 @@
|
||||
|
||||
.browser-stream > div {
|
||||
background: transparent !important;
|
||||
aspect-ratio: 16/9 !important;
|
||||
}
|
||||
|
||||
.browser-stream .control-button {
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
import { useMutation } from "@tanstack/react-query";
|
||||
|
||||
// import { getClient } from "@/api/AxiosClient";
|
||||
import { toast } from "@/components/ui/use-toast";
|
||||
// import { useCredentialGetter } from "@/hooks/useCredentialGetter";
|
||||
// import { type MessageInExfiltratedEvent } from "@/store/useRecordingStore";
|
||||
import { useRecordingStore } from "@/store/useRecordingStore";
|
||||
import {
|
||||
type ActionBlock,
|
||||
type WorkflowBlock,
|
||||
} from "@/routes/workflows/types/workflowTypes";
|
||||
|
||||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
const FAIL_QUITE_NO_EVENTS = "FAIL-QUIET:NO-EVENTS" as const;
|
||||
|
||||
const useProcessRecordingMutation = ({
|
||||
browserSessionId,
|
||||
onSuccess,
|
||||
}: {
|
||||
browserSessionId: string | null;
|
||||
onSuccess?: (workflowBlocks: Array<WorkflowBlock>) => void;
|
||||
}) => {
|
||||
// const credentialGetter = useCredentialGetter();
|
||||
const recordingStore = useRecordingStore();
|
||||
|
||||
const processRecordingMutation = useMutation({
|
||||
mutationFn: async () => {
|
||||
if (!browserSessionId) {
|
||||
throw new Error(
|
||||
"Cannot process recording without a valid browser session ID.",
|
||||
);
|
||||
}
|
||||
|
||||
const eventCount = recordingStore.getEventCount();
|
||||
|
||||
if (eventCount === 0) {
|
||||
throw new Error(FAIL_QUITE_NO_EVENTS);
|
||||
}
|
||||
|
||||
// (this flushes any pending events)
|
||||
const compressedChunks = await recordingStore.getCompressedChunks();
|
||||
|
||||
// TODO: Replace this mock with actual API call when endpoint is ready
|
||||
// const client = await getClient(credentialGetter, "sans-api-v1");
|
||||
// return client
|
||||
// .post<
|
||||
// { compressed_chunks: string[] },
|
||||
// { data: Array<WorkflowBlock> }
|
||||
// >(`/browser_sessions/${browserSessionId}/process_recording`, {
|
||||
// compressed_chunks: compressedChunks,
|
||||
// })
|
||||
// .then((response) => response.data);
|
||||
|
||||
// Mock response with 2-second delay
|
||||
console.log(
|
||||
`Processing ${eventCount} events in ${compressedChunks.length} compressed chunks`,
|
||||
);
|
||||
await sleep(2000);
|
||||
|
||||
// Return mock workflow blocks with two ActionBlocks
|
||||
const mockWorkflowBlocks: Array<WorkflowBlock> = [
|
||||
{
|
||||
block_type: "action",
|
||||
label: "action_1",
|
||||
title: "Enter search term",
|
||||
navigation_goal: "Enter 'foo' in the search field",
|
||||
url: null,
|
||||
error_code_mapping: null,
|
||||
parameters: [],
|
||||
engine: null,
|
||||
continue_on_failure: false,
|
||||
output_parameter: {
|
||||
parameter_type: "output",
|
||||
key: "action_1_output",
|
||||
description: null,
|
||||
output_parameter_id: "mock-output-1",
|
||||
workflow_id: browserSessionId || "mock-workflow-id",
|
||||
created_at: new Date().toISOString(),
|
||||
modified_at: new Date().toISOString(),
|
||||
deleted_at: null,
|
||||
},
|
||||
model: null,
|
||||
} satisfies ActionBlock,
|
||||
{
|
||||
block_type: "action",
|
||||
label: "action_2",
|
||||
title: "Click search",
|
||||
navigation_goal: "Click the search button",
|
||||
url: null,
|
||||
error_code_mapping: null,
|
||||
parameters: [],
|
||||
engine: null,
|
||||
continue_on_failure: false,
|
||||
output_parameter: {
|
||||
parameter_type: "output",
|
||||
key: "action_2_output",
|
||||
description: null,
|
||||
output_parameter_id: "mock-output-2",
|
||||
workflow_id: browserSessionId || "mock-workflow-id",
|
||||
created_at: new Date().toISOString(),
|
||||
modified_at: new Date().toISOString(),
|
||||
deleted_at: null,
|
||||
},
|
||||
model: null,
|
||||
} satisfies ActionBlock,
|
||||
];
|
||||
return mockWorkflowBlocks;
|
||||
},
|
||||
onSuccess: (workflowBlocks) => {
|
||||
// Clear events after successful flush
|
||||
recordingStore.clear();
|
||||
|
||||
toast({
|
||||
variant: "success",
|
||||
title: "Recording Processed",
|
||||
description: "The recording has been successfully processed.",
|
||||
});
|
||||
|
||||
if (workflowBlocks) {
|
||||
onSuccess?.(workflowBlocks);
|
||||
}
|
||||
},
|
||||
onError: (error) => {
|
||||
if (error instanceof Error && error.message === FAIL_QUITE_NO_EVENTS) {
|
||||
return;
|
||||
}
|
||||
|
||||
toast({
|
||||
variant: "destructive",
|
||||
title: "Error Processing Recording",
|
||||
description: error.message,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
return processRecordingMutation;
|
||||
};
|
||||
|
||||
export { useProcessRecordingMutation };
|
||||
@@ -11,6 +11,7 @@ import { useOnChange } from "@/hooks/useOnChange";
|
||||
import { useShouldNotifyWhenClosingTab } from "@/hooks/useShouldNotifyWhenClosingTab";
|
||||
import { BlockActionContext } from "@/store/BlockActionContext";
|
||||
import { useDebugStore } from "@/store/useDebugStore";
|
||||
import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
|
||||
import {
|
||||
useWorkflowHasChangesStore,
|
||||
useWorkflowSave,
|
||||
@@ -33,6 +34,7 @@ import {
|
||||
EdgeChange,
|
||||
} from "@xyflow/react";
|
||||
import "@xyflow/react/dist/style.css";
|
||||
import { nanoid } from "nanoid";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { useBlocker } from "react-router-dom";
|
||||
import {
|
||||
@@ -76,8 +78,10 @@ import {
|
||||
import "./reactFlowOverrideStyles.css";
|
||||
import {
|
||||
convertEchoParameters,
|
||||
convertToNode,
|
||||
createNode,
|
||||
descendants,
|
||||
generateNodeLabel,
|
||||
getAdditionalParametersForEmailBlock,
|
||||
getOrderedChildrenBlocks,
|
||||
getOutputParameterKey,
|
||||
@@ -319,6 +323,13 @@ function FlowRenderer({
|
||||
const setGetSaveDataRef = useRef(workflowChangesStore.setGetSaveData);
|
||||
setGetSaveDataRef.current = workflowChangesStore.setGetSaveData;
|
||||
const saveWorkflow = useWorkflowSave({ status: "published" });
|
||||
const recordedBlocks = useRecordedBlocksStore((state) => state.blocks);
|
||||
const recordedInsertionPoint = useRecordedBlocksStore(
|
||||
(state) => state.insertionPoint,
|
||||
);
|
||||
const clearRecordedBlocks = useRecordedBlocksStore(
|
||||
(state) => state.clearRecordedBlocks,
|
||||
);
|
||||
useShouldNotifyWhenClosingTab(workflowChangesStore.hasChanges);
|
||||
const blocker = useBlocker(({ currentLocation, nextLocation }) => {
|
||||
return (
|
||||
@@ -572,6 +583,85 @@ function FlowRenderer({
|
||||
doLayout(nodes, edges);
|
||||
}
|
||||
|
||||
// effect to add new blocks that were generated from a browser recording
|
||||
useEffect(() => {
|
||||
if (!recordedBlocks || !recordedInsertionPoint) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { previous, next, parent, connectingEdgeType } =
|
||||
recordedInsertionPoint;
|
||||
|
||||
const newNodes: Array<AppNode> = [];
|
||||
const newEdges: Array<Edge> = [];
|
||||
|
||||
let existingLabels = nodes
|
||||
.filter(isWorkflowBlockNode)
|
||||
.map((node) => node.data.label);
|
||||
|
||||
let prevNodeId = previous;
|
||||
|
||||
// convert each WorkflowBlock to an AppNode
|
||||
recordedBlocks.forEach((block, index) => {
|
||||
const id = nanoid();
|
||||
const label = generateNodeLabel(existingLabels);
|
||||
existingLabels = [...existingLabels, label];
|
||||
const blockWithLabel = { ...block, label: block.label || label };
|
||||
|
||||
const node = convertToNode(
|
||||
{ id, parentId: parent },
|
||||
blockWithLabel,
|
||||
true,
|
||||
);
|
||||
newNodes.push(node);
|
||||
|
||||
// create edge from previous node to this one
|
||||
if (prevNodeId) {
|
||||
newEdges.push({
|
||||
id: nanoid(),
|
||||
type: "edgeWithAddButton",
|
||||
source: prevNodeId,
|
||||
target: id,
|
||||
style: { strokeWidth: 2 },
|
||||
});
|
||||
}
|
||||
|
||||
// if this is the last block, connect to next
|
||||
if (index === recordedBlocks.length - 1 && next) {
|
||||
newEdges.push({
|
||||
id: nanoid(),
|
||||
type: connectingEdgeType,
|
||||
source: id,
|
||||
target: next,
|
||||
style: { strokeWidth: 2 },
|
||||
});
|
||||
}
|
||||
|
||||
prevNodeId = id;
|
||||
});
|
||||
|
||||
const editedEdges = previous
|
||||
? edges.filter((edge) => edge.source !== previous)
|
||||
: edges;
|
||||
|
||||
const previousNode = nodes.find((node) => node.id === previous);
|
||||
const previousNodeIndex = previousNode
|
||||
? nodes.indexOf(previousNode)
|
||||
: nodes.length - 1;
|
||||
|
||||
const newNodesAfter = [
|
||||
...nodes.slice(0, previousNodeIndex + 1),
|
||||
...newNodes,
|
||||
...nodes.slice(previousNodeIndex + 1),
|
||||
];
|
||||
|
||||
workflowChangesStore.setHasChanges(true);
|
||||
doLayout(newNodesAfter, [...editedEdges, ...newEdges]);
|
||||
|
||||
clearRecordedBlocks();
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [recordedBlocks, recordedInsertionPoint]);
|
||||
|
||||
const editorElementRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
useAutoPan(editorElementRef, nodes);
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import { SquareIcon, PlusIcon } from "@radix-ui/react-icons";
|
||||
import { ReactNode } from "react";
|
||||
|
||||
import { RadialMenu } from "@/components/RadialMenu";
|
||||
import { useIsSkyvernUser } from "@/hooks/useIsSkyvernUser";
|
||||
import { useDebugStore } from "@/store/useDebugStore";
|
||||
import { useRecordingStore } from "@/store/useRecordingStore";
|
||||
import { useSettingsStore } from "@/store/SettingsStore";
|
||||
|
||||
type WorkflowAddMenuProps = {
|
||||
buttonSize?: string;
|
||||
children: ReactNode;
|
||||
gap?: number;
|
||||
radius?: string;
|
||||
rotateText?: boolean;
|
||||
startAt?: number;
|
||||
// --
|
||||
onAdd: () => void;
|
||||
onRecord: () => void;
|
||||
};
|
||||
|
||||
function WorkflowAddMenu({
|
||||
buttonSize,
|
||||
children,
|
||||
gap,
|
||||
radius = "80px",
|
||||
rotateText = true,
|
||||
startAt = 90,
|
||||
// --
|
||||
onAdd,
|
||||
onRecord,
|
||||
}: WorkflowAddMenuProps) {
|
||||
const debugStore = useDebugStore();
|
||||
const recordingStore = useRecordingStore();
|
||||
const settingsStore = useSettingsStore();
|
||||
const isSkyvernUser = useIsSkyvernUser();
|
||||
|
||||
if (
|
||||
!isSkyvernUser ||
|
||||
!debugStore.isDebugMode ||
|
||||
!settingsStore.isUsingABrowser
|
||||
) {
|
||||
return <>{children}</>;
|
||||
}
|
||||
|
||||
return (
|
||||
<RadialMenu
|
||||
items={[
|
||||
{
|
||||
id: "1",
|
||||
icon: <PlusIcon className={buttonSize ? "h-3 w-3" : undefined} />,
|
||||
text: "Add Block",
|
||||
onClick: () => {
|
||||
onAdd();
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "2",
|
||||
icon: <SquareIcon className={buttonSize ? "h-3 w-3" : undefined} />,
|
||||
enabled: !recordingStore.isRecording && settingsStore.isUsingABrowser,
|
||||
text: "Record Browser",
|
||||
onClick: () => {
|
||||
if (!settingsStore.isUsingABrowser) {
|
||||
return;
|
||||
}
|
||||
|
||||
onRecord();
|
||||
},
|
||||
},
|
||||
]}
|
||||
buttonSize={buttonSize}
|
||||
radius={radius}
|
||||
startAt={startAt}
|
||||
gap={gap}
|
||||
rotateText={rotateText}
|
||||
>
|
||||
{children}
|
||||
</RadialMenu>
|
||||
);
|
||||
}
|
||||
|
||||
export { WorkflowAddMenu };
|
||||
@@ -0,0 +1,29 @@
|
||||
@keyframes pulse-dash {
|
||||
0% {
|
||||
stroke-dasharray: 141.4 141.4;
|
||||
stroke-width: 6;
|
||||
}
|
||||
50% {
|
||||
stroke-dasharray: 10 11.4;
|
||||
stroke-width: 8;
|
||||
}
|
||||
100% {
|
||||
stroke-dasharray: 141.4 141.4;
|
||||
stroke-width: 6;
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes pulse-dash-small {
|
||||
0% {
|
||||
stroke-dasharray: 100.4 100.4;
|
||||
stroke-width: 3;
|
||||
}
|
||||
50% {
|
||||
stroke-dasharray: 10 11.4;
|
||||
stroke-width: 5;
|
||||
}
|
||||
100% {
|
||||
stroke-dasharray: 100.4 100.4;
|
||||
stroke-width: 3;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
import { ReactNode, useEffect, useRef, useState } from "react";
|
||||
|
||||
import {
|
||||
Tooltip,
|
||||
TooltipContent,
|
||||
TooltipProvider,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui/tooltip";
|
||||
import { useRecordingStore, CHUNK_SIZE } from "@/store/useRecordingStore";
|
||||
import { cn } from "@/util/utils";
|
||||
|
||||
import "./WorkflowAdderBusy.css";
|
||||
|
||||
type Operation = "recording" | "processing";
|
||||
|
||||
type Size = "small" | "large";
|
||||
|
||||
type Props = {
|
||||
children: ReactNode;
|
||||
/**
|
||||
* The operation being performed (e.g., recording or processing).
|
||||
*/
|
||||
operation: Operation;
|
||||
/**
|
||||
* An explicit sizing; otherwise the size will be determined by the child content.
|
||||
*/
|
||||
size?: Size;
|
||||
/**
|
||||
* Color for the cover and ellipses. Defaults to "red".
|
||||
*/
|
||||
color?: string;
|
||||
// --
|
||||
onComplete: () => void;
|
||||
};
|
||||
|
||||
function WorkflowAdderBusy({
|
||||
children,
|
||||
operation,
|
||||
size,
|
||||
color = "red",
|
||||
onComplete,
|
||||
}: Props) {
|
||||
const recordingStore = useRecordingStore();
|
||||
const [isHovered, setIsHovered] = useState(false);
|
||||
const [shouldBump, setShouldBump] = useState(false);
|
||||
const bumpTimeoutRef = useRef<NodeJS.Timeout | null>(null);
|
||||
const prevCountRef = useRef(0);
|
||||
|
||||
const eventCount =
|
||||
recordingStore.pendingEvents.length +
|
||||
recordingStore.compressedChunks.length * CHUNK_SIZE;
|
||||
|
||||
// effect for bump animation when count changes
|
||||
useEffect(() => {
|
||||
if (eventCount > prevCountRef.current && prevCountRef.current > 0) {
|
||||
if (bumpTimeoutRef.current) {
|
||||
clearTimeout(bumpTimeoutRef.current);
|
||||
}
|
||||
|
||||
setShouldBump(true);
|
||||
|
||||
bumpTimeoutRef.current = setTimeout(() => {
|
||||
setShouldBump(false);
|
||||
}, 300);
|
||||
}
|
||||
|
||||
prevCountRef.current = eventCount;
|
||||
|
||||
return () => {
|
||||
if (bumpTimeoutRef.current) {
|
||||
clearTimeout(bumpTimeoutRef.current);
|
||||
}
|
||||
};
|
||||
}, [eventCount]);
|
||||
|
||||
const handleClick = (e: React.MouseEvent) => {
|
||||
e.stopPropagation();
|
||||
e.preventDefault();
|
||||
onComplete();
|
||||
|
||||
return false;
|
||||
};
|
||||
|
||||
return (
|
||||
<TooltipProvider>
|
||||
<div className="relative inline-block">
|
||||
<Tooltip open={isHovered}>
|
||||
<TooltipTrigger asChild>
|
||||
<div
|
||||
className={cn("relative inline-block", {
|
||||
"flex items-center justify-center": size !== undefined,
|
||||
"min-h-[40px] min-w-[40px]": size === "small",
|
||||
"min-h-[80px] min-w-[80px]": size === "large",
|
||||
})}
|
||||
onMouseEnter={() => setIsHovered(true)}
|
||||
onMouseLeave={() => setIsHovered(false)}
|
||||
>
|
||||
{/* cover */}
|
||||
<div
|
||||
className={cn("absolute inset-0 rounded-full opacity-40", {
|
||||
"opacity-30": isHovered,
|
||||
})}
|
||||
style={{ backgroundColor: color }}
|
||||
onClick={handleClick}
|
||||
/>
|
||||
<div className="pointer-events-none flex items-center justify-center">
|
||||
{children}
|
||||
</div>
|
||||
<div className="pointer-events-none absolute inset-0">
|
||||
<svg
|
||||
className="h-full w-full animate-spin"
|
||||
viewBox="0 0 100 100"
|
||||
preserveAspectRatio="none"
|
||||
style={{ transformOrigin: "center" }}
|
||||
>
|
||||
<ellipse
|
||||
cx="50"
|
||||
cy="50"
|
||||
rx="45"
|
||||
ry="45"
|
||||
fill="none"
|
||||
stroke={color}
|
||||
strokeWidth={size === "small" ? "3" : "6"}
|
||||
strokeDasharray="141.4 141.4"
|
||||
strokeLinecap="round"
|
||||
vectorEffect="non-scaling-stroke"
|
||||
style={{
|
||||
animation: `${size === "small" ? "pulse-dash-small" : "pulse-dash"} 10s ease-in-out infinite`,
|
||||
}}
|
||||
/>
|
||||
</svg>
|
||||
</div>
|
||||
{isHovered && (
|
||||
<div className="pointer-events-none absolute inset-0">
|
||||
<svg
|
||||
className="h-full w-full"
|
||||
viewBox="0 0 100 100"
|
||||
preserveAspectRatio="none"
|
||||
>
|
||||
<rect
|
||||
x="30"
|
||||
y="30"
|
||||
width="40"
|
||||
height="40"
|
||||
fill={color}
|
||||
vectorEffect="non-scaling-stroke"
|
||||
className="animate-in zoom-in-0"
|
||||
style={{
|
||||
transformOrigin: "center",
|
||||
transformBox: "fill-box",
|
||||
animationDuration: "200ms",
|
||||
animationTimingFunction:
|
||||
"cubic-bezier(0.34, 1.56, 0.64, 1)",
|
||||
}}
|
||||
/>
|
||||
</svg>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
<p>
|
||||
{operation === "recording" ? "Finish Recording" : "Processing..."}
|
||||
</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
{recordingStore.isRecording && eventCount > 0 && (
|
||||
<Tooltip delayDuration={0}>
|
||||
<TooltipTrigger asChild>
|
||||
<div
|
||||
className={cn(
|
||||
"absolute -right-2 -top-2 flex h-6 min-w-6 items-center justify-center rounded-full px-1.5 text-xs font-semibold text-white shadow-lg transition-transform",
|
||||
{
|
||||
"scale-125": shouldBump,
|
||||
"scale-100": !shouldBump,
|
||||
},
|
||||
)}
|
||||
style={{
|
||||
backgroundColor: color,
|
||||
transition: "transform 0.6s",
|
||||
}}
|
||||
>
|
||||
{eventCount}
|
||||
</div>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent>
|
||||
<p>Event Count</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
)}
|
||||
</div>
|
||||
</TooltipProvider>
|
||||
);
|
||||
}
|
||||
|
||||
export { WorkflowAdderBusy };
|
||||
@@ -29,6 +29,7 @@ import { useBlockScriptsQuery } from "@/routes/workflows/hooks/useBlockScriptsQu
|
||||
import { WorkflowRunStream } from "@/routes/workflows/workflowRun/WorkflowRunStream";
|
||||
import { useCacheKeyValuesQuery } from "../hooks/useCacheKeyValuesQuery";
|
||||
import { useBlockScriptStore } from "@/store/BlockScriptStore";
|
||||
import { useRecordingStore } from "@/store/useRecordingStore";
|
||||
import { useSidebarStore } from "@/store/SidebarStore";
|
||||
|
||||
import { AnimatedWave } from "@/components/AnimatedWave";
|
||||
@@ -238,6 +239,7 @@ function Workspace({
|
||||
const queryClient = useQueryClient();
|
||||
const [shouldFetchDebugSession, setShouldFetchDebugSession] = useState(false);
|
||||
const blockScriptStore = useBlockScriptStore();
|
||||
const recordingStore = useRecordingStore();
|
||||
const cacheKey = workflow?.cache_key ?? "";
|
||||
|
||||
const [cacheKeyValue, setCacheKeyValue] = useState(
|
||||
@@ -1339,6 +1341,7 @@ function Workspace({
|
||||
<div className="skyvern-vnc-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">
|
||||
<div key={reloadKey} className="w-full flex-1">
|
||||
<BrowserStream
|
||||
exfiltrate={recordingStore.isRecording}
|
||||
interactive={true}
|
||||
browserSessionId={
|
||||
activeDebugSession?.browser_session_id
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { SquareIcon, PlusIcon } from "@radix-ui/react-icons";
|
||||
import { PlusIcon } from "@radix-ui/react-icons";
|
||||
import {
|
||||
BaseEdge,
|
||||
EdgeLabelRenderer,
|
||||
@@ -8,12 +8,16 @@ import {
|
||||
} from "@xyflow/react";
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { RadialMenu } from "@/components/RadialMenu";
|
||||
import { useIsSkyvernUser } from "@/hooks/useIsSkyvernUser";
|
||||
import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
|
||||
import { useProcessRecordingMutation } from "@/routes/browserSessions/hooks/useProcessRecordingMutation";
|
||||
import { useDebugStore } from "@/store/useDebugStore";
|
||||
import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
|
||||
import { useRecordingStore } from "@/store/useRecordingStore";
|
||||
import { useSettingsStore } from "@/store/SettingsStore";
|
||||
import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
|
||||
|
||||
import { REACT_FLOW_EDGE_Z_INDEX } from "../constants";
|
||||
import { WorkflowAddMenu } from "../WorkflowAddMenu";
|
||||
import { WorkflowAdderBusy } from "../WorkflowAdderBusy";
|
||||
|
||||
function EdgeWithAddButton({
|
||||
source,
|
||||
@@ -27,8 +31,6 @@ function EdgeWithAddButton({
|
||||
style = {},
|
||||
markerEnd,
|
||||
}: EdgeProps) {
|
||||
const debugStore = useDebugStore();
|
||||
const isSkyvernUser = useIsSkyvernUser();
|
||||
const nodes = useNodes();
|
||||
const [edgePath, labelX, labelY] = getBezierPath({
|
||||
sourceX,
|
||||
@@ -38,14 +40,35 @@ function EdgeWithAddButton({
|
||||
targetY,
|
||||
targetPosition,
|
||||
});
|
||||
const debugStore = useDebugStore();
|
||||
const recordingStore = useRecordingStore();
|
||||
const settingsStore = useSettingsStore();
|
||||
const workflowStatePanel = useWorkflowPanelStore();
|
||||
const setRecordedBlocks = useRecordedBlocksStore(
|
||||
(state) => state.setRecordedBlocks,
|
||||
);
|
||||
const setWorkflowPanelState = useWorkflowPanelStore(
|
||||
(state) => state.setWorkflowPanelState,
|
||||
);
|
||||
const processRecordingMutation = useProcessRecordingMutation({
|
||||
browserSessionId: settingsStore.browserSessionId,
|
||||
onSuccess: (blocks) => {
|
||||
setRecordedBlocks(blocks, {
|
||||
previous: source,
|
||||
next: target,
|
||||
parent: sourceNode?.parentId,
|
||||
connectingEdgeType: "edgeWithAddButton",
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
const isProcessing = processRecordingMutation.isPending;
|
||||
|
||||
const sourceNode = nodes.find((node) => node.id === source);
|
||||
|
||||
const onAdd = () => {
|
||||
const updateWorkflowPanelState = (active: boolean) => {
|
||||
setWorkflowPanelState({
|
||||
active: true,
|
||||
active,
|
||||
content: "nodeLibrary",
|
||||
data: {
|
||||
previous: source,
|
||||
@@ -55,6 +78,25 @@ function EdgeWithAddButton({
|
||||
});
|
||||
};
|
||||
|
||||
const onAdd = () => updateWorkflowPanelState(true);
|
||||
|
||||
const onRecord = () => {
|
||||
if (recordingStore.isRecording) {
|
||||
recordingStore.setIsRecording(false);
|
||||
} else {
|
||||
recordingStore.setIsRecording(true);
|
||||
updateWorkflowPanelState(false);
|
||||
}
|
||||
};
|
||||
|
||||
const onEndRecord = () => {
|
||||
if (recordingStore.isRecording) {
|
||||
recordingStore.setIsRecording(false);
|
||||
}
|
||||
|
||||
processRecordingMutation.mutate();
|
||||
};
|
||||
|
||||
const adder = (
|
||||
<Button
|
||||
size="icon"
|
||||
@@ -65,6 +107,41 @@ function EdgeWithAddButton({
|
||||
</Button>
|
||||
);
|
||||
|
||||
const menu = (
|
||||
<WorkflowAddMenu
|
||||
buttonSize="25px"
|
||||
gap={35}
|
||||
radius="50px"
|
||||
startAt={72.5}
|
||||
onAdd={onAdd}
|
||||
onRecord={onRecord}
|
||||
>
|
||||
{adder}
|
||||
</WorkflowAddMenu>
|
||||
);
|
||||
|
||||
const busy = (
|
||||
<WorkflowAdderBusy
|
||||
color={isProcessing ? "white" : "red"}
|
||||
operation={isProcessing ? "processing" : "recording"}
|
||||
size="small"
|
||||
onComplete={() => {
|
||||
onEndRecord();
|
||||
}}
|
||||
>
|
||||
{adder}
|
||||
</WorkflowAdderBusy>
|
||||
);
|
||||
|
||||
const isBusy =
|
||||
(isProcessing || recordingStore.isRecording) &&
|
||||
debugStore.isDebugMode &&
|
||||
settingsStore.isUsingABrowser &&
|
||||
workflowStatePanel.workflowPanelState.data?.previous === source &&
|
||||
workflowStatePanel.workflowPanelState.data?.next === target &&
|
||||
workflowStatePanel.workflowPanelState.data?.parent ===
|
||||
(sourceNode?.parentId || undefined);
|
||||
|
||||
return (
|
||||
<>
|
||||
<BaseEdge path={edgePath} markerEnd={markerEnd} style={style} />
|
||||
@@ -81,38 +158,7 @@ function EdgeWithAddButton({
|
||||
}}
|
||||
className="nodrag nopan"
|
||||
>
|
||||
{isSkyvernUser && debugStore.isDebugMode ? (
|
||||
<RadialMenu
|
||||
items={[
|
||||
{
|
||||
id: "1",
|
||||
icon: <PlusIcon className="h-3 w-3" />,
|
||||
text: "Add Block",
|
||||
onClick: () => {
|
||||
onAdd();
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "2",
|
||||
icon: <SquareIcon className="h-3 w-3" />,
|
||||
enabled: false,
|
||||
text: "Record Browser",
|
||||
onClick: () => {
|
||||
console.log("Record");
|
||||
},
|
||||
},
|
||||
]}
|
||||
buttonSize="25px"
|
||||
radius="50px"
|
||||
startAt={72.5}
|
||||
gap={35}
|
||||
rotateText={true}
|
||||
>
|
||||
{adder}
|
||||
</RadialMenu>
|
||||
) : (
|
||||
adder
|
||||
)}
|
||||
{isBusy ? busy : menu}
|
||||
</div>
|
||||
</EdgeLabelRenderer>
|
||||
</>
|
||||
|
||||
@@ -1,25 +1,51 @@
|
||||
import { SquareIcon, PlusIcon } from "@radix-ui/react-icons";
|
||||
import { PlusIcon } from "@radix-ui/react-icons";
|
||||
import { Handle, NodeProps, Position, useEdges } from "@xyflow/react";
|
||||
|
||||
import { RadialMenu } from "@/components/RadialMenu";
|
||||
import { useIsSkyvernUser } from "@/hooks/useIsSkyvernUser";
|
||||
import { useProcessRecordingMutation } from "@/routes/browserSessions/hooks/useProcessRecordingMutation";
|
||||
import { useDebugStore } from "@/store/useDebugStore";
|
||||
import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
|
||||
import { useRecordingStore } from "@/store/useRecordingStore";
|
||||
import { useSettingsStore } from "@/store/SettingsStore";
|
||||
import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
|
||||
|
||||
import type { NodeAdderNode } from "./types";
|
||||
import { WorkflowAddMenu } from "../../WorkflowAddMenu";
|
||||
import { WorkflowAdderBusy } from "../../WorkflowAdderBusy";
|
||||
|
||||
function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
|
||||
const debugStore = useDebugStore();
|
||||
const isSkyvernUser = useIsSkyvernUser();
|
||||
const edges = useEdges();
|
||||
const debugStore = useDebugStore();
|
||||
const recordingStore = useRecordingStore();
|
||||
const settingsStore = useSettingsStore();
|
||||
const setWorkflowPanelState = useWorkflowPanelStore(
|
||||
(state) => state.setWorkflowPanelState,
|
||||
);
|
||||
const workflowStatePanel = useWorkflowPanelStore();
|
||||
const setRecordedBlocks = useRecordedBlocksStore(
|
||||
(state) => state.setRecordedBlocks,
|
||||
);
|
||||
|
||||
const onAdd = () => {
|
||||
const previous = edges.find((edge) => edge.target === id)?.source ?? null;
|
||||
|
||||
const processRecordingMutation = useProcessRecordingMutation({
|
||||
browserSessionId: settingsStore.browserSessionId,
|
||||
onSuccess: (blocks) => {
|
||||
setRecordedBlocks(blocks, {
|
||||
previous,
|
||||
next: id,
|
||||
parent: parentId,
|
||||
connectingEdgeType: "default",
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
const isProcessing = processRecordingMutation.isPending;
|
||||
|
||||
const updateWorkflowPanelState = (active: boolean) => {
|
||||
const previous = edges.find((edge) => edge.target === id)?.source;
|
||||
|
||||
setWorkflowPanelState({
|
||||
active: true,
|
||||
active,
|
||||
content: "nodeLibrary",
|
||||
data: {
|
||||
previous: previous ?? null,
|
||||
@@ -30,9 +56,30 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
|
||||
});
|
||||
};
|
||||
|
||||
const onAdd = () => {
|
||||
updateWorkflowPanelState(true);
|
||||
};
|
||||
|
||||
const onRecord = () => {
|
||||
if (recordingStore.isRecording) {
|
||||
recordingStore.setIsRecording(false);
|
||||
} else {
|
||||
recordingStore.setIsRecording(true);
|
||||
updateWorkflowPanelState(false);
|
||||
}
|
||||
};
|
||||
|
||||
const onEndRecord = () => {
|
||||
if (recordingStore.isRecording) {
|
||||
recordingStore.setIsRecording(false);
|
||||
}
|
||||
|
||||
processRecordingMutation.mutate();
|
||||
};
|
||||
|
||||
const adder = (
|
||||
<div
|
||||
className="rounded-full bg-slate-50 p-2"
|
||||
className={"rounded-full bg-slate-50 p-2"}
|
||||
onClick={() => {
|
||||
onAdd();
|
||||
}}
|
||||
@@ -41,6 +88,33 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
|
||||
</div>
|
||||
);
|
||||
|
||||
const busy = (
|
||||
<WorkflowAdderBusy
|
||||
color={isProcessing ? "gray" : "red"}
|
||||
operation={isProcessing ? "processing" : "recording"}
|
||||
onComplete={() => {
|
||||
onEndRecord();
|
||||
}}
|
||||
>
|
||||
{adder}
|
||||
</WorkflowAdderBusy>
|
||||
);
|
||||
|
||||
const menu = (
|
||||
<WorkflowAddMenu onAdd={onAdd} onRecord={onRecord}>
|
||||
{adder}
|
||||
</WorkflowAddMenu>
|
||||
);
|
||||
|
||||
const isBusy =
|
||||
(isProcessing || recordingStore.isRecording) &&
|
||||
debugStore.isDebugMode &&
|
||||
settingsStore.isUsingABrowser &&
|
||||
workflowStatePanel.workflowPanelState.data?.previous === previous &&
|
||||
workflowStatePanel.workflowPanelState.data?.next === id &&
|
||||
workflowStatePanel.workflowPanelState.data?.parent ===
|
||||
(parentId || undefined);
|
||||
|
||||
return (
|
||||
<div>
|
||||
<Handle
|
||||
@@ -55,36 +129,7 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
|
||||
id="b"
|
||||
className="opacity-0"
|
||||
/>
|
||||
{isSkyvernUser && debugStore.isDebugMode ? (
|
||||
<RadialMenu
|
||||
items={[
|
||||
{
|
||||
id: "1",
|
||||
icon: <PlusIcon />,
|
||||
text: "Add Block",
|
||||
onClick: () => {
|
||||
onAdd();
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "2",
|
||||
icon: <SquareIcon />,
|
||||
enabled: false,
|
||||
text: "Record Browser",
|
||||
onClick: () => {
|
||||
console.log("Record");
|
||||
},
|
||||
},
|
||||
]}
|
||||
radius="80px"
|
||||
startAt={90}
|
||||
rotateText={true}
|
||||
>
|
||||
{adder}
|
||||
</RadialMenu>
|
||||
) : (
|
||||
adder
|
||||
)}
|
||||
{isBusy ? busy : menu}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2458,6 +2458,7 @@ function getLabelForWorkflowParameterType(type: WorkflowParameterValueType) {
|
||||
export {
|
||||
convert,
|
||||
convertEchoParameters,
|
||||
convertToNode,
|
||||
createNode,
|
||||
generateNodeData,
|
||||
generateNodeLabel,
|
||||
|
||||
36
skyvern-frontend/src/store/RecordedBlocksStore.ts
Normal file
36
skyvern-frontend/src/store/RecordedBlocksStore.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import { create } from "zustand";
|
||||
import type { WorkflowBlock } from "@/routes/workflows/types/workflowTypes";
|
||||
|
||||
type InsertionPoint = {
|
||||
previous: string | null;
|
||||
next: string | null;
|
||||
parent?: string;
|
||||
connectingEdgeType: string;
|
||||
};
|
||||
|
||||
type RecordedBlocksState = {
|
||||
blocks: Array<WorkflowBlock> | null;
|
||||
insertionPoint: InsertionPoint | null;
|
||||
};
|
||||
|
||||
type RecordedBlocksStore = RecordedBlocksState & {
|
||||
setRecordedBlocks: (
|
||||
blocks: Array<WorkflowBlock>,
|
||||
insertionPoint: InsertionPoint,
|
||||
) => void;
|
||||
clearRecordedBlocks: () => void;
|
||||
};
|
||||
|
||||
const useRecordedBlocksStore = create<RecordedBlocksStore>((set) => ({
|
||||
blocks: null,
|
||||
insertionPoint: null,
|
||||
setRecordedBlocks: (blocks, insertionPoint) => {
|
||||
set({ blocks, insertionPoint });
|
||||
},
|
||||
clearRecordedBlocks: () => {
|
||||
set({ blocks: null, insertionPoint: null });
|
||||
},
|
||||
}));
|
||||
|
||||
export { useRecordedBlocksStore };
|
||||
export type { InsertionPoint };
|
||||
@@ -2,16 +2,33 @@ import { create } from "zustand";
|
||||
|
||||
type SettingsStore = {
|
||||
environment: string;
|
||||
/**
|
||||
* The user is currently operating or viewing a live, remote browser. NOTE: if
|
||||
* the browser is still connecting, or otherwise not ready, then this should
|
||||
* be false.
|
||||
*/
|
||||
isUsingABrowser: boolean;
|
||||
/**
|
||||
* The current browser session ID when a browser is active.
|
||||
*/
|
||||
browserSessionId: string | null;
|
||||
organization: string;
|
||||
setEnvironment: (environment: string) => void;
|
||||
setIsUsingABrowser: (isUsing: boolean) => void;
|
||||
setBrowserSessionId: (browserSessionId: string | null) => void;
|
||||
setOrganization: (organization: string) => void;
|
||||
};
|
||||
|
||||
const useSettingsStore = create<SettingsStore>((set) => {
|
||||
return {
|
||||
environment: "local",
|
||||
isUsingABrowser: false,
|
||||
browserSessionId: null,
|
||||
organization: "skyvern",
|
||||
setEnvironment: (environment: string) => set({ environment }),
|
||||
setIsUsingABrowser: (isUsing: boolean) => set({ isUsingABrowser: isUsing }),
|
||||
setBrowserSessionId: (browserSessionId: string | null) =>
|
||||
set({ browserSessionId }),
|
||||
setOrganization: (organization: string) => set({ organization }),
|
||||
};
|
||||
});
|
||||
|
||||
298
skyvern-frontend/src/store/useRecordingStore.ts
Normal file
298
skyvern-frontend/src/store/useRecordingStore.ts
Normal file
@@ -0,0 +1,298 @@
|
||||
import { create } from "zustand";
|
||||
|
||||
/**
|
||||
* example: {
|
||||
* 'targetInfo': {
|
||||
* 'targetId': '8B698E27F1F32372718DA73DCA0C5944',
|
||||
* 'type': 'page',
|
||||
* 'title': 'New Tab',
|
||||
* 'url': 'chrome://newtab/',
|
||||
* 'attached': True,
|
||||
* 'canAccessOpener': False,
|
||||
* 'browserContextId': 'FD13D5C556E681BB49AEED0AB2CA1972',
|
||||
* }
|
||||
*/
|
||||
export interface ExfiltratedEventCdpParams {
|
||||
targetInfo: {
|
||||
attached?: boolean;
|
||||
browserContextId?: string;
|
||||
canAccessOpener?: boolean;
|
||||
targetId?: string;
|
||||
title?: string;
|
||||
type?: string;
|
||||
url?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ExfiltratedEventConsoleParams {
|
||||
type: string;
|
||||
url: string;
|
||||
timestamp: number;
|
||||
target: {
|
||||
className?: string;
|
||||
id?: string;
|
||||
innerText?: string;
|
||||
tagName?: string;
|
||||
text: string[];
|
||||
value?: string;
|
||||
};
|
||||
inputValue?: string;
|
||||
mousePosition: {
|
||||
xa: number | null;
|
||||
ya: number | null;
|
||||
xp: number | null;
|
||||
yp: number | null;
|
||||
};
|
||||
key?: string;
|
||||
code?: string;
|
||||
activeElement: {
|
||||
tagName?: string;
|
||||
id?: string;
|
||||
className?: string;
|
||||
boundingRect?: {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
top: number;
|
||||
right: number;
|
||||
bottom: number;
|
||||
left: number;
|
||||
} | null;
|
||||
scroll?: {
|
||||
scrollTop: number;
|
||||
scrollLeft: number;
|
||||
scrollHeight: number;
|
||||
scrollWidth: number;
|
||||
clientHeight: number;
|
||||
clientWidth: number;
|
||||
} | null;
|
||||
};
|
||||
window: {
|
||||
width: number;
|
||||
height: number;
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface MessageInExfiltratedCdpEvent {
|
||||
kind: "exfiltrated-event";
|
||||
event_name: string;
|
||||
params: ExfiltratedEventCdpParams;
|
||||
source: "cdp";
|
||||
}
|
||||
|
||||
export interface MessageInExfiltratedConsoleEvent {
|
||||
kind: "exfiltrated-event";
|
||||
event_name: string;
|
||||
params: ExfiltratedEventConsoleParams;
|
||||
source: "console";
|
||||
}
|
||||
|
||||
export type MessageInExfiltratedEvent =
|
||||
| MessageInExfiltratedCdpEvent
|
||||
| MessageInExfiltratedConsoleEvent;
|
||||
|
||||
/**
|
||||
* Number of events per compressed chunk.
|
||||
*/
|
||||
export const CHUNK_SIZE = 1000 as const;
|
||||
|
||||
interface RecordingStore {
|
||||
/**
|
||||
* Compressed chunks of recorded events (base64 gzip).
|
||||
* Each chunk contains up to CHUNK_SIZE events.
|
||||
*/
|
||||
compressedChunks: string[];
|
||||
/**
|
||||
* Buffer of events not yet compressed into a chunk.
|
||||
*/
|
||||
pendingEvents: MessageInExfiltratedEvent[];
|
||||
/**
|
||||
* Whether a compression operation is currently in progress.
|
||||
*/
|
||||
isCompressing: boolean;
|
||||
/**
|
||||
* Whether the user is currently in browser recording mode.
|
||||
*/
|
||||
isRecording: boolean;
|
||||
/**
|
||||
* Add a new recorded event. Triggers async compression when buffer is full.
|
||||
*/
|
||||
add: (event: MessageInExfiltratedEvent) => void;
|
||||
/**
|
||||
* Clear all recorded events and compressed chunks.
|
||||
*/
|
||||
clear: () => void;
|
||||
/**
|
||||
* Reset the recording store (clear events and set isRecording to false).
|
||||
*/
|
||||
reset: () => void;
|
||||
/**
|
||||
* Set whether the user is in browser recording mode.
|
||||
*/
|
||||
setIsRecording: (isRecording: boolean) => void;
|
||||
/**
|
||||
* Flush any pending events into a compressed chunk.
|
||||
* Call this before consuming the data.
|
||||
*/
|
||||
flush: () => Promise<void>;
|
||||
/**
|
||||
* Get all compressed chunks (after flushing pending events).
|
||||
*/
|
||||
getCompressedChunks: () => Promise<string[]>;
|
||||
/**
|
||||
* Get the total number of events (compressed + pending).
|
||||
*/
|
||||
getEventCount: () => number;
|
||||
}
|
||||
|
||||
/**
|
||||
* compresses a JSON string using the Gzip algorithm and returns the result
|
||||
* as a Base64 encoded string
|
||||
*/
|
||||
async function compressEventsToB64(jsonString: string): Promise<string> {
|
||||
// 1. Convert the string to a Uint8Array (a byte array).
|
||||
const encoder = new TextEncoder();
|
||||
const uint8Array = encoder.encode(jsonString);
|
||||
|
||||
// 2. Create a ReadableStream from the byte array.
|
||||
const readableStream = new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(uint8Array);
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
|
||||
// 3. Pipe the data through the Gzip compression stream.
|
||||
const compressedStream = readableStream.pipeThrough(
|
||||
new CompressionStream("gzip"), // Use 'gzip' for standard network transport
|
||||
);
|
||||
|
||||
// 4. Read the entire compressed stream back into a single ArrayBuffer.
|
||||
// The Response object provides an easy way to convert streams into a single buffer.
|
||||
const compressedBuffer = await new Response(compressedStream).arrayBuffer();
|
||||
|
||||
// 5. Convert the ArrayBuffer (binary data) to a Base64 string for transport.
|
||||
// Base64 is used to safely transmit binary data over text-based protocols (like JSON).
|
||||
const bytes = new Uint8Array(compressedBuffer);
|
||||
let binary = "";
|
||||
|
||||
// Convert Uint8Array to a raw binary string (this is needed for btoa)
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
const nextByte = bytes[i];
|
||||
|
||||
if (nextByte === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
||||
binary += String.fromCharCode(nextByte);
|
||||
}
|
||||
|
||||
// Convert the raw binary string to Base64
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
export const useRecordingStore = create<RecordingStore>((set, get) => ({
|
||||
compressedChunks: [],
|
||||
pendingEvents: [],
|
||||
isCompressing: false,
|
||||
isRecording: false,
|
||||
|
||||
add: (event) => {
|
||||
const state = get();
|
||||
const newPendingEvents = [...state.pendingEvents, event];
|
||||
|
||||
if (newPendingEvents.length >= CHUNK_SIZE && !state.isCompressing) {
|
||||
const eventsToCompress = newPendingEvents.slice(0, CHUNK_SIZE);
|
||||
const remainingEvents = newPendingEvents.slice(CHUNK_SIZE);
|
||||
|
||||
set({ pendingEvents: remainingEvents, isCompressing: true });
|
||||
|
||||
// compress asynchronously
|
||||
queueMicrotask(async () => {
|
||||
try {
|
||||
const jsonString = JSON.stringify(eventsToCompress);
|
||||
const compressed = await compressEventsToB64(jsonString);
|
||||
const currentState = get();
|
||||
set({
|
||||
compressedChunks: [...currentState.compressedChunks, compressed],
|
||||
isCompressing: false,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Failed to compress events chunk:", error);
|
||||
|
||||
// on error, put events back into pending
|
||||
const currentState = get();
|
||||
set({
|
||||
pendingEvents: [...eventsToCompress, ...currentState.pendingEvents],
|
||||
isCompressing: false,
|
||||
});
|
||||
}
|
||||
});
|
||||
} else {
|
||||
set({ pendingEvents: newPendingEvents });
|
||||
}
|
||||
},
|
||||
|
||||
clear: () => set({ compressedChunks: [], pendingEvents: [] }),
|
||||
|
||||
reset: () =>
|
||||
set({
|
||||
compressedChunks: [],
|
||||
pendingEvents: [],
|
||||
isCompressing: false,
|
||||
isRecording: false,
|
||||
}),
|
||||
|
||||
setIsRecording: (isRecording) => {
|
||||
const state = get();
|
||||
// clear events on rising edge
|
||||
if (!state.isRecording && isRecording) {
|
||||
get().clear();
|
||||
}
|
||||
set({ isRecording });
|
||||
},
|
||||
|
||||
flush: async () => {
|
||||
// Wait for any in-progress compression to complete
|
||||
while (get().isCompressing) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
|
||||
const pending = get().pendingEvents;
|
||||
if (pending.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
set({ isCompressing: true });
|
||||
|
||||
try {
|
||||
const jsonString = JSON.stringify(pending);
|
||||
const compressed = await compressEventsToB64(jsonString);
|
||||
const currentState = get();
|
||||
set({
|
||||
compressedChunks: [...currentState.compressedChunks, compressed],
|
||||
pendingEvents: [],
|
||||
isCompressing: false,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Failed to flush pending events:", error);
|
||||
set({ isCompressing: false });
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
|
||||
getCompressedChunks: async () => {
|
||||
await get().flush();
|
||||
return get().compressedChunks;
|
||||
},
|
||||
|
||||
getEventCount: () => {
|
||||
const state = get();
|
||||
return (
|
||||
state.compressedChunks.length * CHUNK_SIZE + state.pendingEvents.length
|
||||
);
|
||||
},
|
||||
}));
|
||||
Reference in New Issue
Block a user