vodorod/Dorod-Sky
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Enable browser recording (#4182)

Browse Source
This commit is contained in:
Jonathan Dobson
2025-12-03 13:08:23 -05:00
committed by GitHub
parent 3d94f415a4
commit 26a137418b
25 changed files with 310 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
skyvern-frontend/src/components/BrowserStream.tsx
View File

@@ -627,11 +627,14 @@ function BrowserStream({
"source" in data &&
typeof data.source === "string"
) {
const event = data as MessageInExfiltratedEvent;
return {
kind: "exfiltrated-event",
event_name: data.event_name,
params: data.params,
source: data.source,
event_name: event.event_name,
params: event.params,
source: event.source,
timestamp: event.timestamp,
} as MessageInExfiltratedEvent;
}
break;
@@ -803,6 +806,7 @@ function BrowserStream({
if (!hasEvents) {
e.preventDefault();
recordingStore.setIsRecording(false);
recordingStore.reset();
}
}}
>
@@ -826,6 +830,7 @@ function BrowserStream({
variant="destructive"
onClick={() => {
recordingStore.setIsRecording(false);
recordingStore.reset();
}}
>
Cancel recording

10
skyvern-frontend/src/components/Flippable.tsx
View File

@@ -1,4 +1,6 @@
import { ReactNode, Children, useRef, useEffect } from "react";
import { useRecordingStore } from "@/store/useRecordingStore";
import { cn } from "@/util/utils";
interface FlippableProps {
@@ -22,6 +24,7 @@ export function Flippable({
className,
preserveFrontsideHeight = false,
}: FlippableProps) {
const recordingStore = useRecordingStore();
const childrenArray = Children.toArray(children);
const front = childrenArray[0];
const back = childrenArray[1];
@@ -48,7 +51,12 @@ export function Flippable({
}, [facing, preserveFrontsideHeight]);
return (
<div className={className} style={{ perspective: "1000px" }}>
<div
className={cn(className, {
"pointer-events-none opacity-50": recordingStore.isRecording,
})}
style={{ perspective: "1000px" }}
>
<div
className={cn(
"transition-transform duration-700",

141
skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
View File

@@ -1,28 +1,28 @@
import { useParams } from "react-router-dom";
import { useMutation } from "@tanstack/react-query";
// import { getClient } from "@/api/AxiosClient";
import { getClient } from "@/api/AxiosClient";
import { toast } from "@/components/ui/use-toast";
// import { useCredentialGetter } from "@/hooks/useCredentialGetter";
// import { type MessageInExfiltratedEvent } from "@/store/useRecordingStore";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { useRecordingStore } from "@/store/useRecordingStore";
import {
type ActionBlock,
type WorkflowBlock,
} from "@/routes/workflows/types/workflowTypes";
import { type WorkflowBlock } from "@/routes/workflows/types/workflowTypes";
import { type WorkflowParameter } from "@/routes/workflows/types/workflowTypes";
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
const FAIL_QUITE_NO_EVENTS = "FAIL-QUIET:NO-EVENTS" as const;
const FAIL_QUIET_NO_EVENTS = "FAIL-QUIET:NO-EVENTS" as const;
const useProcessRecordingMutation = ({
browserSessionId,
onSuccess,
}: {
browserSessionId: string | null;
onSuccess?: (workflowBlocks: Array<WorkflowBlock>) => void;
onSuccess?: (args: {
blocks: Array<WorkflowBlock>;
parameters: Array<WorkflowParameter>;
}) => void;
}) => {
// const credentialGetter = useCredentialGetter();
const credentialGetter = useCredentialGetter();
const recordingStore = useRecordingStore();
const { workflowPermanentId } = useParams();
const processRecordingMutation = useMutation({
mutationFn: async () => {
@@ -32,97 +32,64 @@ const useProcessRecordingMutation = ({
);
}
if (!workflowPermanentId) {
throw new Error(
"Cannot process recording without a valid workflow permanent ID.",
);
}
const eventCount = recordingStore.getEventCount();
if (eventCount === 0) {
throw new Error(FAIL_QUITE_NO_EVENTS);
throw new Error(FAIL_QUIET_NO_EVENTS);
}
// (this flushes any pending events)
const compressedChunks = await recordingStore.getCompressedChunks();
// TODO: Replace this mock with actual API call when endpoint is ready
// const client = await getClient(credentialGetter, "sans-api-v1");
// return client
// .post<
// { compressed_chunks: string[] },
// { data: Array<WorkflowBlock> }
// >(`/browser_sessions/${browserSessionId}/process_recording`, {
// compressed_chunks: compressedChunks,
// })
// .then((response) => response.data);
// Mock response with 2-second delay
console.log(
`Processing ${eventCount} events in ${compressedChunks.length} compressed chunks`,
);
await sleep(2000);
// Return mock workflow blocks with two ActionBlocks
const mockWorkflowBlocks: Array<WorkflowBlock> = [
{
block_type: "action",
label: "action_1",
title: "Enter search term",
navigation_goal: "Enter 'foo' in the search field",
url: null,
error_code_mapping: null,
parameters: [],
engine: null,
continue_on_failure: false,
output_parameter: {
parameter_type: "output",
key: "action_1_output",
description: null,
output_parameter_id: "mock-output-1",
workflow_id: browserSessionId || "mock-workflow-id",
created_at: new Date().toISOString(),
modified_at: new Date().toISOString(),
deleted_at: null,
},
model: null,
} satisfies ActionBlock,
{
block_type: "action",
label: "action_2",
title: "Click search",
navigation_goal: "Click the search button",
url: null,
error_code_mapping: null,
parameters: [],
engine: null,
continue_on_failure: false,
output_parameter: {
parameter_type: "output",
key: "action_2_output",
description: null,
output_parameter_id: "mock-output-2",
workflow_id: browserSessionId || "mock-workflow-id",
created_at: new Date().toISOString(),
modified_at: new Date().toISOString(),
deleted_at: null,
},
model: null,
} satisfies ActionBlock,
];
return mockWorkflowBlocks;
const client = await getClient(credentialGetter, "sans-api-v1");
return client
.post<
{ compressed_chunks: string[] },
{
data: {
blocks: Array<WorkflowBlock>;
parameters: Array<WorkflowParameter>;
};
}
>(`/browser_sessions/${browserSessionId}/process_recording`, {
compressed_chunks: compressedChunks,
workflow_permanent_id: workflowPermanentId,
})
.then((response) => ({
blocks: response.data.blocks,
parameters: response.data.parameters,
}));
},
onSuccess: (workflowBlocks) => {
// Clear events after successful flush
onSuccess: ({ blocks, parameters }) => {
recordingStore.clear();
toast({
variant: "success",
title: "Recording Processed",
description: "The recording has been successfully processed.",
});
if (blocks && blocks.length > 0) {
toast({
variant: "success",
title: "Recording Processed",
description: "The recording has been successfully processed.",
});
if (workflowBlocks) {
onSuccess?.(workflowBlocks);
onSuccess?.({ blocks, parameters: parameters });
return;
}
toast({
variant: "warning",
title: "Recording Processed (No Blocks)",
description: "No blocks could be created from the recording.",
});
},
onError: (error) => {
if (error instanceof Error && error.message === FAIL_QUITE_NO_EVENTS) {
if (error instanceof Error && error.message === FAIL_QUIET_NO_EVENTS) {
return;
}

30
skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx
View File

@@ -324,6 +324,9 @@ function FlowRenderer({
setGetSaveDataRef.current = workflowChangesStore.setGetSaveData;
const saveWorkflow = useWorkflowSave({ status: "published" });
const recordedBlocks = useRecordedBlocksStore((state) => state.blocks);
const recordedParameters = useRecordedBlocksStore(
(state) => state.parameters,
);
const recordedInsertionPoint = useRecordedBlocksStore(
(state) => state.insertionPoint,
);
@@ -583,7 +586,8 @@ function FlowRenderer({
doLayout(nodes, edges);
}
// effect to add new blocks that were generated from a browser recording
// effect to add new blocks that were generated from a browser recording,
// along with any new parameters
useEffect(() => {
if (!recordedBlocks || !recordedInsertionPoint) {
return;
@@ -658,6 +662,30 @@ function FlowRenderer({
workflowChangesStore.setHasChanges(true);
doLayout(newNodesAfter, [...editedEdges, ...newEdges]);
const newParameters = Array<ParametersState[number]>();
for (const newParameter of recordedParameters ?? []) {
const exists = parameters.some((param) => param.key === newParameter.key);
if (!exists) {
newParameters.push({
key: newParameter.key,
parameterType: "workflow",
dataType: newParameter.workflow_parameter_type,
description: newParameter.description ?? null,
defaultValue: newParameter.default_value ?? "",
});
}
}
if (newParameters.length > 0) {
const workflowParametersStore = useWorkflowParametersStore.getState();
workflowParametersStore.setParameters([
...workflowParametersStore.parameters,
...newParameters,
]);
}
clearRecordedBlocks();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [recordedBlocks, recordedInsertionPoint]);

7
skyvern-frontend/src/routes/workflows/editor/WorkflowAdderBusy.tsx
View File

@@ -6,7 +6,7 @@ import {
TooltipProvider,
TooltipTrigger,
} from "@/components/ui/tooltip";
import { useRecordingStore, CHUNK_SIZE } from "@/store/useRecordingStore";
import { useRecordingStore } from "@/store/useRecordingStore";
import { cn } from "@/util/utils";
import "./WorkflowAdderBusy.css";
@@ -45,10 +45,7 @@ function WorkflowAdderBusy({
const [shouldBump, setShouldBump] = useState(false);
const bumpTimeoutRef = useRef<NodeJS.Timeout | null>(null);
const prevCountRef = useRef(0);
const eventCount =
recordingStore.pendingEvents.length +
recordingStore.compressedChunks.length * CHUNK_SIZE;
const eventCount = recordingStore.exposedEventCount;
// effect for bump animation when count changes
useEffect(() => {

21
skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx
View File

@@ -26,6 +26,7 @@ import { useCreateWorkflowMutation } from "../hooks/useCreateWorkflowMutation";
import { convert } from "./workflowEditorUtils";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useDebugStore } from "@/store/useDebugStore";
import { useRecordingStore } from "@/store/useRecordingStore";
import { useWorkflowTitleStore } from "@/store/WorkflowTitleStore";
import { useWorkflowHasChangesStore } from "@/store/WorkflowHasChangesStore";
import { cn } from "@/util/utils";
@@ -82,6 +83,7 @@ function WorkflowHeader({
const createWorkflowMutation = useCreateWorkflowMutation();
const { data: workflowRun } = useWorkflowRunQuery();
const debugStore = useDebugStore();
const recordingStore = useRecordingStore();
const workflowRunIsRunningOrQueued =
workflowRun && statusIsRunningOrQueued(workflowRun);
const [chosenCacheKeyValue, setChosenCacheKeyValue] = useState<string | null>(
@@ -105,8 +107,10 @@ function WorkflowHeader({
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [cacheKeyValue]);
const isRecording = recordingStore.isRecording;
const shouldShowCacheControls =
!isGeneratingCode && (cacheKeyValues?.total_count ?? 0) > 0;
!isRecording && !isGeneratingCode && (cacheKeyValues?.total_count ?? 0) > 0;
if (!globalWorkflows) {
return null; // this should be loaded already by some other components
@@ -124,7 +128,7 @@ function WorkflowHeader({
>
<div className="flex h-full items-center">
<EditableNodeTitle
editable={true}
editable={!isRecording}
onChange={(newTitle) => {
setTitle(newTitle);
workflowChangesStore.setHasChanges(true);
@@ -247,7 +251,7 @@ function WorkflowHeader({
size="icon"
variant={debugStore.isDebugMode ? "default" : "tertiary"}
className="size-10 min-w-[2.5rem]"
disabled={workflowRunIsRunningOrQueued}
disabled={workflowRunIsRunningOrQueued || isRecording}
onClick={() => {
if (debugStore.isDebugMode) {
navigate(`/workflows/${workflowPermanentId}/edit`);
@@ -277,7 +281,7 @@ function WorkflowHeader({
size="icon"
variant="tertiary"
className="size-10 min-w-[2.5rem]"
disabled={isGlobalWorkflow}
disabled={isGlobalWorkflow || isRecording}
onClick={() => {
onSave();
}}
@@ -297,6 +301,7 @@ function WorkflowHeader({
<Tooltip>
<TooltipTrigger asChild>
<Button
disabled={isRecording}
size="icon"
variant="tertiary"
className="size-10 min-w-[2.5rem]"
@@ -311,7 +316,12 @@ function WorkflowHeader({
</Tooltip>
</TooltipProvider>
)}
<Button variant="tertiary" size="lg" onClick={onParametersClick}>
<Button
disabled={isRecording}
variant="tertiary"
size="lg"
onClick={onParametersClick}
>
<span className="mr-2">Parameters</span>
{parametersPanelOpen ? (
<ChevronUpIcon className="h-6 w-6" />
@@ -320,6 +330,7 @@ function WorkflowHeader({
)}
</Button>
<Button
disabled={isRecording}
size="lg"
onClick={() => {
onRun?.();

12
skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
View File

@@ -1362,13 +1362,15 @@ function Workspace({
"mr-16": !blockLabel,
})}
>
{showPowerButton && (
{!recordingStore.isRecording && showPowerButton && (
<PowerButton onClick={() => cycle()} />
)}
<ReloadButton
isReloading={isReloading}
onClick={() => reload()}
/>
{!recordingStore.isRecording && (
<ReloadButton
isReloading={isReloading}
onClick={() => reload()}
/>
)}
</div>
</footer>
</div>

40
skyvern-frontend/src/routes/workflows/editor/edges/EdgeWithAddButton.tsx
View File

@@ -14,6 +14,7 @@ import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
import { useRecordingStore } from "@/store/useRecordingStore";
import { useSettingsStore } from "@/store/SettingsStore";
import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
import { cn } from "@/util/utils";
import { REACT_FLOW_EDGE_Z_INDEX } from "../constants";
import { WorkflowAddMenu } from "../WorkflowAddMenu";
@@ -52,8 +53,8 @@ function EdgeWithAddButton({
);
const processRecordingMutation = useProcessRecordingMutation({
browserSessionId: settingsStore.browserSessionId,
onSuccess: (blocks) => {
setRecordedBlocks(blocks, {
onSuccess: (result) => {
setRecordedBlocks(result, {
previous: source,
next: target,
parent: sourceNode?.parentId,
@@ -66,6 +67,17 @@ function EdgeWithAddButton({
const sourceNode = nodes.find((node) => node.id === source);
const isBusy =
(isProcessing || recordingStore.isRecording) &&
debugStore.isDebugMode &&
settingsStore.isUsingABrowser &&
workflowStatePanel.workflowPanelState.data?.previous === source &&
workflowStatePanel.workflowPanelState.data?.next === target &&
workflowStatePanel.workflowPanelState.data?.parent ===
(sourceNode?.parentId || undefined);
const isDisabled = !isBusy && recordingStore.isRecording;
const updateWorkflowPanelState = (active: boolean) => {
setWorkflowPanelState({
active,
@@ -78,7 +90,13 @@ function EdgeWithAddButton({
});
};
const onAdd = () => updateWorkflowPanelState(true);
const onAdd = () => {
if (isDisabled) {
return;
}
updateWorkflowPanelState(true);
};
const onRecord = () => {
if (recordingStore.isRecording) {
@@ -100,7 +118,10 @@ function EdgeWithAddButton({
const adder = (
<Button
size="icon"
className="h-4 w-4 rounded-full transition-all hover:scale-150"
className={cn("h-4 w-4 rounded-full transition-all hover:scale-150", {
"cursor-not-allowed bg-[grey] hover:scale-100 hover:bg-[grey] active:bg-[grey]":
isDisabled,
})}
onClick={() => onAdd()}
>
<PlusIcon />
@@ -133,15 +154,6 @@ function EdgeWithAddButton({
</WorkflowAdderBusy>
);
const isBusy =
(isProcessing || recordingStore.isRecording) &&
debugStore.isDebugMode &&
settingsStore.isUsingABrowser &&
workflowStatePanel.workflowPanelState.data?.previous === source &&
workflowStatePanel.workflowPanelState.data?.next === target &&
workflowStatePanel.workflowPanelState.data?.parent ===
(sourceNode?.parentId || undefined);
return (
<>
<BaseEdge path={edgePath} markerEnd={markerEnd} style={style} />
@@ -158,7 +170,7 @@ function EdgeWithAddButton({
}}
className="nodrag nopan"
>
{isBusy ? busy : menu}
{isBusy ? busy : isDisabled ? adder : menu}
</div>
</EdgeLabelRenderer>
</>

8
skyvern-frontend/src/routes/workflows/editor/nodes/CodeBlockNode/CodeBlockNode.tsx
View File

@@ -7,6 +7,7 @@ import { CodeEditor } from "@/routes/workflows/components/CodeEditor";
import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { useRecordingStore } from "@/store/useRecordingStore";
import { deepEqualStringArrays } from "@/util/equality";
import { cn } from "@/util/utils";
@@ -17,6 +18,7 @@ function CodeBlockNode({ id, data }: NodeProps<CodeBlockNode>) {
const { editable, label } = data;
const { blockLabel: urlBlockLabel } = useParams();
const { data: workflowRun } = useWorkflowRunQuery();
const recordingStore = useRecordingStore();
const workflowRunIsRunningOrQueued =
workflowRun && statusIsRunningOrQueued(workflowRun);
const thisBlockIsTargetted =
@@ -26,7 +28,11 @@ function CodeBlockNode({ id, data }: NodeProps<CodeBlockNode>) {
const update = useUpdate<CodeBlockNode["data"]>({ id, editable });
return (
<div>
<div
className={cn({
"pointer-events-none opacity-50": recordingStore.isRecording,
})}
>
<Handle
type="source"
position={Position.Bottom}

8
skyvern-frontend/src/routes/workflows/editor/nodes/DownloadNode/DownloadNode.tsx
View File

@@ -9,11 +9,13 @@ import { NodeHeader } from "../components/NodeHeader";
import { useParams } from "react-router-dom";
import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useRecordingStore } from "@/store/useRecordingStore";
function DownloadNode({ id, data }: NodeProps<DownloadNode>) {
const { editable, label } = data;
const { blockLabel: urlBlockLabel } = useParams();
const { data: workflowRun } = useWorkflowRunQuery();
const recordingStore = useRecordingStore();
const workflowRunIsRunningOrQueued =
workflowRun && statusIsRunningOrQueued(workflowRun);
const thisBlockIsTargetted =
@@ -22,7 +24,11 @@ function DownloadNode({ id, data }: NodeProps<DownloadNode>) {
workflowRunIsRunningOrQueued && thisBlockIsTargetted;
return (
<div>
<div
className={cn({
"pointer-events-none opacity-50": recordingStore.isRecording,
})}
>
<Handle
type="source"
position={Position.Bottom}

8
skyvern-frontend/src/routes/workflows/editor/nodes/FileParserNode/FileParserNode.tsx
View File

@@ -14,6 +14,7 @@ import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { ModelSelector } from "@/components/ModelSelector";
import { useRecordingStore } from "@/store/useRecordingStore";
function FileParserNode({ id, data }: NodeProps<FileParserNode>) {
const { editable, label } = data;
@@ -27,9 +28,14 @@ function FileParserNode({ id, data }: NodeProps<FileParserNode>) {
workflowRunIsRunningOrQueued && thisBlockIsTargetted;
const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });
const update = useUpdate<FileParserNode["data"]>({ id, editable });
const recordingStore = useRecordingStore();
return (
<div>
<div
className={cn({
"pointer-events-none opacity-50": recordingStore.isRecording,
})}
>
<Handle
type="source"
position={Position.Bottom}

8
skyvern-frontend/src/routes/workflows/editor/nodes/FileUploadNode/FileUploadNode.tsx
View File

@@ -18,6 +18,7 @@ import {
import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { useRecordingStore } from "@/store/useRecordingStore";
function FileUploadNode({ id, data }: NodeProps<FileUploadNode>) {
const { editable, label } = data;
@@ -30,9 +31,14 @@ function FileUploadNode({ id, data }: NodeProps<FileUploadNode>) {
const thisBlockIsPlaying =
workflowRunIsRunningOrQueued && thisBlockIsTargetted;
const update = useUpdate<FileUploadNode["data"]>({ id, editable });
const recordingStore = useRecordingStore();
return (
<div>
<div
className={cn({
"pointer-events-none opacity-50": recordingStore.isRecording,
})}
>
<Handle
type="source"
position={Position.Bottom}

9
skyvern-frontend/src/routes/workflows/editor/nodes/HttpRequestNode/HttpRequestNode.tsx
View File

@@ -38,6 +38,8 @@ import { CurlImportDialog } from "./CurlImportDialog";
import { QuickHeadersDialog } from "./QuickHeadersDialog";
import { MethodBadge, UrlValidator, RequestPreview } from "./HttpUtils";
import { useRerender } from "@/hooks/useRerender";
import { useRecordingStore } from "@/store/useRecordingStore";
import { cn } from "@/util/utils";
const httpMethods = [
"GET",
@@ -111,12 +113,17 @@ function HttpRequestNode({ id, data }: NodeProps<HttpRequestNodeType>) {
);
const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });
const recordingStore = useRecordingStore();
const showBodyEditor =
data.method !== "GET" && data.method !== "HEAD" && data.method !== "DELETE";
return (
<div>
<div
className={cn({
"pointer-events-none opacity-50": recordingStore.isRecording,
})}
>
<Handle
type="source"
position={Position.Bottom}

8
skyvern-frontend/src/routes/workflows/editor/nodes/HumanInteractionNode/HumanInteractionNode.tsx
View File

@@ -19,6 +19,7 @@ import {
AccordionTrigger,
} from "@/components/ui/accordion";
import { useRerender } from "@/hooks/useRerender";
import { useRecordingStore } from "@/store/useRecordingStore";
import { AI_IMPROVE_CONFIGS } from "../../constants";
const instructionsTooltip =
@@ -38,6 +39,7 @@ function HumanInteractionNode({
const { editable, label } = data;
const { blockLabel: urlBlockLabel } = useParams();
const { data: workflowRun } = useWorkflowRunQuery();
const recordingStore = useRecordingStore();
const workflowRunIsRunningOrQueued =
workflowRun && statusIsRunningOrQueued(workflowRun);
const thisBlockIsTargetted =
@@ -48,7 +50,11 @@ function HumanInteractionNode({
const rerender = useRerender({ prefix: "accordian" });
return (
<div>
<div
className={cn({
"pointer-events-none opacity-50": recordingStore.isRecording,
})}
>
<Handle
type="source"
position={Position.Bottom}

8
skyvern-frontend/src/routes/workflows/editor/nodes/LoopNode/LoopNode.tsx
View File

@@ -15,6 +15,7 @@ import { useParams } from "react-router-dom";
import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { useRecordingStore } from "@/store/useRecordingStore";
function LoopNode({ id, data }: NodeProps<LoopNode>) {
const nodes = useNodes<AppNode>();
@@ -34,6 +35,7 @@ function LoopNode({ id, data }: NodeProps<LoopNode>) {
const update = useUpdate<LoopNode["data"]>({ id, editable });
const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });
const children = nodes.filter((node) => node.parentId === id);
const recordingStore = useRecordingStore();
const furthestDownChild: Node | null = children.reduce(
(acc, child) => {
@@ -56,7 +58,11 @@ function LoopNode({ id, data }: NodeProps<LoopNode>) {
const loopNodeWidth = getLoopNodeWidth(node, nodes);
return (
<div>
<div
className={cn({
"pointer-events-none opacity-50": recordingStore.isRecording,
})}
>
<Handle
type="source"
position={Position.Bottom}

8
skyvern-frontend/src/routes/workflows/editor/nodes/NodeActionMenu.tsx
View File

@@ -1,3 +1,5 @@
import { DotsHorizontalIcon } from "@radix-ui/react-icons";
import {
DropdownMenu,
DropdownMenuContent,
@@ -6,7 +8,7 @@ import {
DropdownMenuSeparator,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import { DotsHorizontalIcon } from "@radix-ui/react-icons";
import { useRecordingStore } from "@/store/useRecordingStore";
type Props = {
isDeletable?: boolean;
@@ -23,6 +25,9 @@ function NodeActionMenu({
onDelete,
onShowScript,
}: Props) {
const recordingStore = useRecordingStore();
const isRecording = recordingStore.isRecording;
if (!isDeletable && !isScriptable) {
return null;
}
@@ -37,6 +42,7 @@ function NodeActionMenu({
<DropdownMenuSeparator />
{isDeletable && (
<DropdownMenuItem
disabled={isRecording}
onSelect={() => {
onDelete?.();
}}

35
skyvern-frontend/src/routes/workflows/editor/nodes/NodeAdderNode/NodeAdderNode.tsx
View File

@@ -7,6 +7,7 @@ import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
import { useRecordingStore } from "@/store/useRecordingStore";
import { useSettingsStore } from "@/store/SettingsStore";
import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
import { cn } from "@/util/utils";
import type { NodeAdderNode } from "./types";
import { WorkflowAddMenu } from "../../WorkflowAddMenu";
@@ -29,8 +30,8 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
const processRecordingMutation = useProcessRecordingMutation({
browserSessionId: settingsStore.browserSessionId,
onSuccess: (blocks) => {
setRecordedBlocks(blocks, {
onSuccess: (result) => {
setRecordedBlocks(result, {
previous,
next: id,
parent: parentId,
@@ -41,6 +42,17 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
const isProcessing = processRecordingMutation.isPending;
const isBusy =
(isProcessing || recordingStore.isRecording) &&
debugStore.isDebugMode &&
settingsStore.isUsingABrowser &&
workflowStatePanel.workflowPanelState.data?.previous === previous &&
workflowStatePanel.workflowPanelState.data?.next === id &&
workflowStatePanel.workflowPanelState.data?.parent ===
(parentId || undefined);
const isDisabled = !isBusy && recordingStore.isRecording;
const updateWorkflowPanelState = (active: boolean) => {
const previous = edges.find((edge) => edge.target === id)?.source;
@@ -57,6 +69,10 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
};
const onAdd = () => {
if (isDisabled) {
return;
}
updateWorkflowPanelState(true);
};
@@ -79,7 +95,9 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
const adder = (
<div
className={"rounded-full bg-slate-50 p-2"}
className={cn("rounded-full bg-slate-50 p-2", {
"cursor-not-allowed bg-[grey]": isDisabled,
})}
onClick={() => {
onAdd();
}}
@@ -106,15 +124,6 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
</WorkflowAddMenu>
);
const isBusy =
(isProcessing || recordingStore.isRecording) &&
debugStore.isDebugMode &&
settingsStore.isUsingABrowser &&
workflowStatePanel.workflowPanelState.data?.previous === previous &&
workflowStatePanel.workflowPanelState.data?.next === id &&
workflowStatePanel.workflowPanelState.data?.parent ===
(parentId || undefined);
return (
<div>
<Handle
@@ -129,7 +138,7 @@ function NodeAdderNode({ id, parentId }: NodeProps<NodeAdderNode>) {
id="b"
className="opacity-0"
/>
{isBusy ? busy : menu}
{isBusy ? busy : isDisabled ? adder : menu}
</div>
);
}

8
skyvern-frontend/src/routes/workflows/editor/nodes/PDFParserNode/PDFParserNode.tsx
View File

@@ -14,6 +14,7 @@ import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { ModelSelector } from "@/components/ModelSelector";
import { useRecordingStore } from "@/store/useRecordingStore";
function PDFParserNode({ id, data }: NodeProps<PDFParserNode>) {
const { editable, label } = data;
@@ -27,9 +28,14 @@ function PDFParserNode({ id, data }: NodeProps<PDFParserNode>) {
workflowRunIsRunningOrQueued && thisBlockIsTargetted;
const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });
const update = useUpdate<PDFParserNode["data"]>({ id, editable });
const recordingStore = useRecordingStore();
return (
<div>
<div
className={cn({
"pointer-events-none opacity-50": recordingStore.isRecording,
})}
>
<Handle
type="source"
position={Position.Bottom}

8
skyvern-frontend/src/routes/workflows/editor/nodes/SendEmailNode/SendEmailNode.tsx
View File

@@ -14,6 +14,7 @@ import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { AI_IMPROVE_CONFIGS } from "../../constants";
import { useRecordingStore } from "@/store/useRecordingStore";
function SendEmailNode({ id, data }: NodeProps<SendEmailNode>) {
const { editable, label } = data;
@@ -27,9 +28,14 @@ function SendEmailNode({ id, data }: NodeProps<SendEmailNode>) {
workflowRunIsRunningOrQueued && thisBlockIsTargetted;
const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });
const update = useUpdate<SendEmailNode["data"]>({ id, editable });
const recordingStore = useRecordingStore();
return (
<div>
<div
className={cn({
"pointer-events-none opacity-50": recordingStore.isRecording,
})}
>
<Handle
type="source"
position={Position.Bottom}

9
skyvern-frontend/src/routes/workflows/editor/nodes/StartNode/StartNode.tsx
View File

@@ -36,6 +36,7 @@ import {
import { Flippable } from "@/components/Flippable";
import { useRerender } from "@/hooks/useRerender";
import { useBlockScriptStore } from "@/store/BlockScriptStore";
import { useRecordingStore } from "@/store/useRecordingStore";
import { BlockCodeEditor } from "@/routes/workflows/components/BlockCodeEditor";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { cn } from "@/util/utils";
@@ -56,9 +57,11 @@ function StartNode({ id, data }: NodeProps<StartNode>) {
const reactFlowInstance = useReactFlow();
const [facing, setFacing] = useState<"front" | "back">("front");
const blockScriptStore = useBlockScriptStore();
const recordingStore = useRecordingStore();
const script = blockScriptStore.scripts.__start_block__;
const rerender = useRerender({ prefix: "accordion" });
const toggleScriptForNodeCallback = useToggleScriptForNodeCallback();
const isRecording = recordingStore.isRecording;
const makeStartSettings = (data: StartNode["data"]): StartSettings => {
return {
@@ -396,7 +399,11 @@ function StartNode({ id, data }: NodeProps<StartNode>) {
}
return (
<div>
<div
className={cn({
"pointer-events-none opacity-50": isRecording,
})}
>
<Handle
type="source"
position={Position.Bottom}

8
skyvern-frontend/src/routes/workflows/editor/nodes/TextPromptNode/TextPromptNode.tsx
View File

@@ -16,6 +16,7 @@ import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { AI_IMPROVE_CONFIGS } from "../../constants";
import { useRecordingStore } from "@/store/useRecordingStore";
function TextPromptNode({ id, data }: NodeProps<TextPromptNode>) {
const { editable, label } = data;
@@ -29,9 +30,14 @@ function TextPromptNode({ id, data }: NodeProps<TextPromptNode>) {
workflowRunIsRunningOrQueued && thisBlockIsTargetted;
const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });
const update = useUpdate<TextPromptNode["data"]>({ id, editable });
const recordingStore = useRecordingStore();
return (
<div>
<div
className={cn({
"pointer-events-none opacity-50": recordingStore.isRecording,
})}
>
<Handle
type="source"
position={Position.Bottom}

8
skyvern-frontend/src/routes/workflows/editor/nodes/WaitNode/WaitNode.tsx
View File

@@ -11,6 +11,7 @@ import { useParams } from "react-router-dom";
import { statusIsRunningOrQueued } from "@/routes/tasks/types";
import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery";
import { useUpdate } from "@/routes/workflows/editor/useUpdate";
import { useRecordingStore } from "@/store/useRecordingStore";
function WaitNode({ id, data, type }: NodeProps<WaitNode>) {
const { editable, label } = data;
@@ -25,9 +26,14 @@ function WaitNode({ id, data, type }: NodeProps<WaitNode>) {
const isFirstWorkflowBlock = useIsFirstBlockInWorkflow({ id });
const update = useUpdate<WaitNode["data"]>({ id, editable });
const recordingStore = useRecordingStore();
return (
<div>
<div
className={cn({
"pointer-events-none opacity-50": recordingStore.isRecording,
})}
>
<Handle
type="source"
position={Position.Bottom}

7
skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx
View File

@@ -30,6 +30,7 @@ import {
import { getInitialValues } from "@/routes/workflows/utils";
import { useBlockOutputStore } from "@/store/BlockOutputStore";
import { useDebugStore } from "@/store/useDebugStore";
import { useRecordingStore } from "@/store/useRecordingStore";
import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
import { useWorkflowSave } from "@/store/WorkflowHasChangesStore";
import {
@@ -169,6 +170,7 @@ function NodeHeader({
} = useParams();
const blockOutputsStore = useBlockOutputStore();
const debugStore = useDebugStore();
const recordingStore = useRecordingStore();
const { closeWorkflowPanel } = useWorkflowPanelStore();
const workflowSettingsStore = useWorkflowSettingsStore();
const [label, setLabel] = useNodeLabelChangeHandler({
@@ -204,6 +206,8 @@ function NodeHeader({
const thisBlockIsTargetted =
urlBlockLabel !== undefined && urlBlockLabel === blockLabel;
const isRecording = recordingStore.isRecording;
const [workflowRunStatus, setWorkflowRunStatus] = useState(
workflowRun?.status,
);
@@ -590,7 +594,8 @@ function NodeHeader({
"pointer-events-none fill-gray-500 text-gray-500":
workflowRunIsRunningOrQueued ||
!workflowPermanentId ||
debugSession === undefined,
debugSession === undefined ||
isRecording,
})}
onClick={() => {
handleOnPlay();

12
skyvern-frontend/src/store/RecordedBlocksStore.ts
View File

@@ -1,5 +1,6 @@
import { create } from "zustand";
import type { WorkflowBlock } from "@/routes/workflows/types/workflowTypes";
import type { WorkflowParameter } from "@/routes/workflows/types/workflowTypes";
type InsertionPoint = {
previous: string | null;
@@ -10,12 +11,16 @@ type InsertionPoint = {
type RecordedBlocksState = {
blocks: Array<WorkflowBlock> | null;
parameters: Array<WorkflowParameter> | null;
insertionPoint: InsertionPoint | null;
};
type RecordedBlocksStore = RecordedBlocksState & {
setRecordedBlocks: (
blocks: Array<WorkflowBlock>,
data: {
blocks: Array<WorkflowBlock>;
parameters: Array<WorkflowParameter>;
},
insertionPoint: InsertionPoint,
) => void;
clearRecordedBlocks: () => void;
@@ -23,9 +28,10 @@ type RecordedBlocksStore = RecordedBlocksState & {
const useRecordedBlocksStore = create<RecordedBlocksStore>((set) => ({
blocks: null,
parameters: null,
insertionPoint: null,
setRecordedBlocks: (blocks, insertionPoint) => {
set({ blocks, insertionPoint });
setRecordedBlocks: ({ blocks, parameters }, insertionPoint) => {
set({ blocks, parameters, insertionPoint });
},
clearRecordedBlocks: () => {
set({ blocks: null, insertionPoint: null });

29
skyvern-frontend/src/store/useRecordingStore.ts
View File

@@ -81,6 +81,7 @@ export interface MessageInExfiltratedCdpEvent {
event_name: string;
params: ExfiltratedEventCdpParams;
source: "cdp";
timestamp: number;
}
export interface MessageInExfiltratedConsoleEvent {
@@ -88,6 +89,7 @@ export interface MessageInExfiltratedConsoleEvent {
event_name: string;
params: ExfiltratedEventConsoleParams;
source: "console";
timestamp: number;
}
export type MessageInExfiltratedEvent =
@@ -105,6 +107,11 @@ interface RecordingStore {
* Each chunk contains up to CHUNK_SIZE events.
*/
compressedChunks: string[];
/**
* The number of events to show the user. This elides noisy events, like
* `mousemove`.
*/
exposedEventCount: number;
/**
* Buffer of events not yet compressed into a chunk.
*/
@@ -194,8 +201,25 @@ async function compressEventsToB64(jsonString: string): Promise<string> {
return btoa(binary);
}
const isExposedEvent = (event: MessageInExfiltratedEvent): boolean => {
const exposedConsoleEventTypes = new Set(["focus", "click", "keypress"]);
if (event.source === "console") {
if (exposedConsoleEventTypes.has(event.params.type)) {
return true;
}
}
if (event.source === "cdp") {
return true;
}
return false;
};
export const useRecordingStore = create<RecordingStore>((set, get) => ({
compressedChunks: [],
exposedEventCount: 0,
pendingEvents: [],
isCompressing: false,
isRecording: false,
@@ -204,6 +228,10 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({
const state = get();
const newPendingEvents = [...state.pendingEvents, event];
if (isExposedEvent(event)) {
set({ exposedEventCount: state.exposedEventCount + 1 });
}
if (newPendingEvents.length >= CHUNK_SIZE && !state.isCompressing) {
const eventsToCompress = newPendingEvents.slice(0, CHUNK_SIZE);
const remainingEvents = newPendingEvents.slice(CHUNK_SIZE);
@@ -241,6 +269,7 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({
reset: () =>
set({
compressedChunks: [],
exposedEventCount: 0,
pendingEvents: [],
isCompressing: false,
isRecording: false,