From 71540849bfb09a49851e7a479ccc70f91a02ceab Mon Sep 17 00:00:00 2001 From: Jonathan Dobson Date: Wed, 16 Jul 2025 17:39:51 -0400 Subject: [PATCH] Debugger Lite (#2970) --- .../src/components/BrowserStream.tsx | 41 +----- .../src/components/FloatingWindow.tsx | 41 +++++- .../src/components/browser-stream.css | 23 +-- skyvern-frontend/src/router.tsx | 5 +- .../routes/workflows/editor/FlowRenderer.tsx | 50 +++++-- .../workflows/editor/WorkflowDebugger.tsx | 100 +++++++++++++ .../workflows/editor/WorkflowDebuggerRun.tsx | 37 +++++ .../editor/WorkflowDebuggerRunTimeline.tsx | 129 ++++++++++++++++ .../workflows/editor/WorkflowEditor.tsx | 138 ++---------------- .../workflows/editor/WorkflowHeader.tsx | 2 +- .../editor/nodes/components/NodeHeader.tsx | 1 - .../panels/WorkflowDebugOverviewWindow.tsx | 72 --------- .../src/routes/workflows/editor/utils.ts | 100 +++++++++++++ 13 files changed, 470 insertions(+), 269 deletions(-) create mode 100644 skyvern-frontend/src/routes/workflows/editor/WorkflowDebugger.tsx create mode 100644 skyvern-frontend/src/routes/workflows/editor/WorkflowDebuggerRun.tsx create mode 100644 skyvern-frontend/src/routes/workflows/editor/WorkflowDebuggerRunTimeline.tsx delete mode 100644 skyvern-frontend/src/routes/workflows/editor/panels/WorkflowDebugOverviewWindow.tsx create mode 100644 skyvern-frontend/src/routes/workflows/editor/utils.ts diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx index 1e58b3fc..3733257d 100644 --- a/skyvern-frontend/src/components/BrowserStream.tsx +++ b/skyvern-frontend/src/components/BrowserStream.tsx @@ -1,7 +1,5 @@ import { Status } from "@/api/types"; import { useEffect, useState, useRef, useCallback } from "react"; -import { HandIcon, StopIcon } from "@radix-ui/react-icons"; -import { Button } from "@/components/ui/button"; import { Skeleton } from "@/components/ui/skeleton"; import { statusIsNotFinalized } from "@/routes/tasks/types"; import { useCredentialGetter } from "@/hooks/useCredentialGetter"; @@ -29,6 +27,7 @@ type Command = CommandTakeControl | CommandCedeControl; type Props = { browserSessionId?: string; + interactive?: boolean; task?: { run: TaskApiResponse; }; @@ -41,6 +40,7 @@ type Props = { function BrowserStream({ browserSessionId = undefined, + interactive = true, task = undefined, workflow = undefined, // -- @@ -67,7 +67,6 @@ function BrowserStream({ } const [commandSocket, setCommandSocket] = useState(null); - const [userIsControlling, setUserIsControlling] = useState(false); const [vncDisconnectedTrigger, setVncDisconnectedTrigger] = useState(0); const prevVncConnectedRef = useRef(false); const [isVncConnected, setIsVncConnected] = useState(false); @@ -273,13 +272,13 @@ function BrowserStream({ commandSocket.send(JSON.stringify(command)); }; - if (userIsControlling) { + if (interactive) { sendCommand({ kind: "take-control" }); } else { sendCommand({ kind: "cede-control" }); } // eslint-disable-next-line react-hooks/exhaustive-deps - }, [userIsControlling, isCommandConnected]); + }, [interactive, isCommandConnected]); // Effect to show toast when task or workflow reaches a final state based on hook updates useEffect(() => { @@ -316,39 +315,11 @@ function BrowserStream({ return (
- {isVncConnected && ( -
-
- -
- -
-
-
- )} + {isVncConnected &&
} {!isVncConnected && (
diff --git a/skyvern-frontend/src/components/FloatingWindow.tsx b/skyvern-frontend/src/components/FloatingWindow.tsx index 3add89f4..42235ca9 100644 --- a/skyvern-frontend/src/components/FloatingWindow.tsx +++ b/skyvern-frontend/src/components/FloatingWindow.tsx @@ -6,6 +6,7 @@ * and `re-resizable`; but I don't want to do that until it's worth the effort.) */ +import { ReloadIcon } from "@radix-ui/react-icons"; import { Resizable } from "re-resizable"; import { useCallback, @@ -69,6 +70,14 @@ function WindowsButton(props: { ); } +function ReloadButton(props: { isReloading: boolean; onClick: () => void }) { + return ( + + ); +} + function getOs(): OS { if (typeof navigator === "undefined") { return "Unknown"; // For non-browser environments @@ -105,6 +114,7 @@ function FloatingWindow({ showCloseButton, showMaximizeButton, showMinimizeButton, + showReloadButton = false, title, zIndex, // -- @@ -118,11 +128,14 @@ function FloatingWindow({ showCloseButton?: boolean; showMaximizeButton?: boolean; showMinimizeButton?: boolean; + showReloadButton?: boolean; title: string; zIndex?: string; // -- onInteract?: () => void; }) { + const [reloadKey, setReloadKey] = useState(0); + const [isReloading, setIsReloading] = useState(false); const [position, setPosition] = useState({ x: 0, y: 0 }); const [size, setSize] = useState({ left: 0, @@ -394,6 +407,19 @@ function FloatingWindow({ setIsMinimized(false); }; + const reload = () => { + if (isReloading) { + return; + } + + setReloadKey((prev) => prev + 1); + setIsReloading(true); + + setTimeout(() => { + setIsReloading(false); + }, 1000); + }; + /** * If maximized, need to retain max size during parent resizing. */ @@ -507,6 +533,7 @@ function FloatingWindow({ >
{os === "macOS" ? ( @@ -557,9 +584,21 @@ function FloatingWindow({ )}
{title}
+ {showReloadButton && ( + reload()} + /> + )} ) : ( <> + {showReloadButton && ( + reload()} + /> + )}
{title}
{showMinimizeButton && ( diff --git a/skyvern-frontend/src/components/browser-stream.css b/skyvern-frontend/src/components/browser-stream.css index edefc92d..fe3b9237 100644 --- a/skyvern-frontend/src/components/browser-stream.css +++ b/skyvern-frontend/src/components/browser-stream.css @@ -13,35 +13,18 @@ padding: 0rem; } -.browser-stream .overlay-container { +.browser-stream .overlay { position: absolute; top: 0; left: 0; width: 100%; height: 100%; - display: flex; - align-items: center; - justify-content: center; -} - -.browser-stream .overlay { - position: relative; - height: auto; - width: 100%; - max-height: 100%; - aspect-ratio: 16 / 9; - object-fit: cover; - display: flex; - align-items: center; - justify-content: center; + cursor: not-allowed; } .browser-stream.user-is-controlling .overlay { pointer-events: none; -} - -.browser-stream.user-is-controlling .overlay-container { - pointer-events: none; + cursor: unset; } .browser-stream .take-control { diff --git a/skyvern-frontend/src/router.tsx b/skyvern-frontend/src/router.tsx index 72edd503..36e46301 100644 --- a/skyvern-frontend/src/router.tsx +++ b/skyvern-frontend/src/router.tsx @@ -18,6 +18,7 @@ import { WorkflowRun } from "./routes/workflows/WorkflowRun"; import { WorkflowRunParameters } from "./routes/workflows/WorkflowRunParameters"; import { Workflows } from "./routes/workflows/Workflows"; import { WorkflowsPageLayout } from "./routes/workflows/WorkflowsPageLayout"; +import { WorkflowDebugger } from "./routes/workflows/editor/WorkflowDebugger"; import { WorkflowEditor } from "./routes/workflows/editor/WorkflowEditor"; import { WorkflowPostRunParameters } from "./routes/workflows/workflowRun/WorkflowPostRunParameters"; import { WorkflowRunOutput } from "./routes/workflows/workflowRun/WorkflowRunOutput"; @@ -110,11 +111,11 @@ const router = createBrowserRouter([ }, { path: "debug", - element: , + element: , }, { path: ":workflowRunId/:blockLabel/debug", - element: , + element: , }, { path: "edit", diff --git a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx index 481c2367..1f3a1f9f 100644 --- a/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/FlowRenderer.tsx @@ -26,6 +26,7 @@ import { Panel, PanOnScrollMode, ReactFlow, + Viewport, useEdgesState, useNodesInitialized, useNodesState, @@ -97,6 +98,7 @@ import { startNode, } from "./workflowEditorUtils"; import { cn } from "@/util/utils"; +import { WorkflowDebuggerRun } from "@/routes/workflows/editor/WorkflowDebuggerRun"; import { useAutoPan } from "./useAutoPan"; function convertToParametersYAML( @@ -273,6 +275,13 @@ function FlowRenderer({ const [title, setTitle] = useState(initialTitle); const [debuggableBlockCount, setDebuggableBlockCount] = useState(0); const nodesInitialized = useNodesInitialized(); + const [shouldConstrainPan, setShouldConstrainPan] = useState(false); + + useEffect(() => { + if (nodesInitialized) { + setShouldConstrainPan(true); + } + }, [nodesInitialized]); const { hasChanges, setHasChanges } = useWorkflowHasChangesStore(); useShouldNotifyWhenClosingTab(hasChanges); const blocker = useBlocker(({ currentLocation, nextLocation }) => { @@ -625,7 +634,8 @@ function FlowRenderer({ } }); - const constrainPan = (y: number) => { + const constrainPan = (viewport: Viewport) => { + const y = viewport.y; const yLockMin = nodes.reduce( (acc, node) => { const nodeBottom = node.position.y + (node.height ?? 0); @@ -636,15 +646,22 @@ function FlowRenderer({ }, { value: -Infinity }, ); - const yLockMinValue = yLockMin.value; const xLock = getXLock(); const newY = Math.max(-yLockMinValue + yLockMax, Math.min(yLockMax, y)); - reactFlowInstance.setViewport({ - x: xLock, - y: newY, - zoom: zoomLock, - }); + + // avoid infinite recursion with onMove + if ( + viewport.x !== xLock || + viewport.y !== newY || + viewport.zoom !== zoomLock + ) { + reactFlowInstance.setViewport({ + x: xLock, + y: newY, + zoom: zoomLock, + }); + } }; return ( @@ -734,14 +751,15 @@ function FlowRenderer({ nodeTypes={nodeTypes} edgeTypes={edgeTypes} colorMode="dark" - fitView={!debugStore.isDebugMode} + fitView={true} fitViewOptions={{ maxZoom: 1, }} deleteKeyCode={null} onMove={(_, viewport) => { - const y = viewport.y; - debugStore.isDebugMode && constrainPan(y); + if (debugStore.isDebugMode && shouldConstrainPan) { + constrainPan(viewport); + } }} maxZoom={debugStore.isDebugMode ? 1 : 2} minZoom={debugStore.isDebugMode ? 1 : 0.5} @@ -758,6 +776,18 @@ function FlowRenderer({ > + {debugStore.isDebugMode && ( + +
+
+ +
+
+ + )} { + return state.setCollapsed; + }); + + const setHasChanges = useWorkflowHasChangesStore( + (state) => state.setHasChanges, + ); + + useMountEffect(() => { + setCollapsed(true); + setHasChanges(false); + }); + + if (!workflow) { + return null; + } + + const settings: WorkflowSettings = { + persistBrowserSession: workflow.persist_browser_session, + proxyLocation: workflow.proxy_location, + webhookCallbackUrl: workflow.webhook_callback_url, + model: workflow.model, + maxScreenshotScrolls: workflow.max_screenshot_scrolls, + extraHttpHeaders: workflow.extra_http_headers + ? JSON.stringify(workflow.extra_http_headers) + : null, + }; + + const elements = getElements( + workflow.workflow_definition.blocks, + settings, + true, + ); + + const isFinalized = workflowRun ? statusIsFinalized(workflowRun) : null; + const interactor = workflowRun && isFinalized === false ? "agent" : "human"; + const browserTitle = interactor === "agent" ? `Browser [🤖]` : `Browser [👤]`; + + return ( +
+ + + + + {workflowRun && ( + + {workflowRun && workflowRun.browser_session_id ? ( + + ) : ( + + )} + + )} +
+ ); +} + +export { WorkflowDebugger }; diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowDebuggerRun.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowDebuggerRun.tsx new file mode 100644 index 00000000..3d3af1ca --- /dev/null +++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowDebuggerRun.tsx @@ -0,0 +1,37 @@ +import { useWorkflowRunQuery } from "@/routes/workflows/hooks/useWorkflowRunQuery"; +import { WorkflowDebuggerRunTimeline } from "./WorkflowDebuggerRunTimeline"; + +function WorkflowDebuggerRun() { + const { data: workflowRun } = useWorkflowRunQuery(); + + const workflowFailureReason = workflowRun?.failure_reason ? ( +
+
Workflow Failure Reason
+
{workflowRun.failure_reason}
+
+ ) : null; + + return ( +
+
+ {workflowFailureReason} +
+ {}} + onBlockItemSelected={() => {}} + onObserverThoughtCardSelected={() => {}} + /> +
+
+
+ ); +} + +export { WorkflowDebuggerRun }; diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowDebuggerRunTimeline.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowDebuggerRunTimeline.tsx new file mode 100644 index 00000000..f385877e --- /dev/null +++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowDebuggerRunTimeline.tsx @@ -0,0 +1,129 @@ +import { ScrollArea, ScrollAreaViewport } from "@/components/ui/scroll-area"; +import { Skeleton } from "@/components/ui/skeleton"; +import { statusIsFinalized } from "@/routes/tasks/types"; +import { useWorkflowRunQuery } from "../hooks/useWorkflowRunQuery"; +import { useWorkflowRunTimelineQuery } from "../hooks/useWorkflowRunTimelineQuery"; +import { + isBlockItem, + isObserverThought, + isTaskVariantBlockItem, + isThoughtItem, + ObserverThought, + WorkflowRunBlock, +} from "../types/workflowRunTypes"; +import { ThoughtCard } from "@/routes/workflows/workflowRun/ThoughtCard"; +import { + ActionItem, + WorkflowRunOverviewActiveElement, +} from "@/routes/workflows/workflowRun/WorkflowRunOverview"; +import { WorkflowRunTimelineBlockItem } from "@/routes/workflows/workflowRun/WorkflowRunTimelineBlockItem"; + +type Props = { + activeItem: WorkflowRunOverviewActiveElement; + onObserverThoughtCardSelected: (item: ObserverThought) => void; + onActionItemSelected: (item: ActionItem) => void; + onBlockItemSelected: (item: WorkflowRunBlock) => void; +}; + +function WorkflowDebuggerRunTimeline({ + activeItem, + onObserverThoughtCardSelected, + onActionItemSelected, + onBlockItemSelected, +}: Props) { + const { data: workflowRun, isLoading: workflowRunIsLoading } = + useWorkflowRunQuery(); + + const { data: workflowRunTimeline, isLoading: workflowRunTimelineIsLoading } = + useWorkflowRunTimelineQuery(); + + if (workflowRunIsLoading || workflowRunTimelineIsLoading) { + return ; + } + + if (!workflowRun || !workflowRunTimeline) { + return ( +
+
+
+ Hi! 👋 We're experimenting with a new feature called debugger. +
+
+ This debugger allows you to see the state of your workflow in a live + browser. +
+
+ You can run individual blocks, instead of the whole workflow. +
+
+ To get started, press the play button on a block in your workflow. +
+
+
+ ); + } + + const workflowRunIsFinalized = statusIsFinalized(workflowRun); + + const numberOfActions = workflowRunTimeline.reduce((total, current) => { + if (isTaskVariantBlockItem(current)) { + return total + current.block!.actions!.length; + } + return total + 0; + }, 0); + + return ( +
+
+
+ Actions: {numberOfActions} +
+
+ Steps: {workflowRun.total_steps ?? 0} +
+
+ {!workflowRunIsFinalized && workflowRunTimeline.length === 0 && ( + + )} + + +
+ {workflowRunIsFinalized && workflowRunTimeline.length === 0 && ( +
Workflow timeline is empty
+ )} + {workflowRunTimeline?.map((timelineItem) => { + if (isBlockItem(timelineItem)) { + return ( + + ); + } + if (isThoughtItem(timelineItem)) { + return ( + + ); + } + })} +
+ + +
+ ); +} + +export { WorkflowDebuggerRunTimeline }; diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx index 02378bf4..bcd0af0a 100644 --- a/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowEditor.tsx @@ -7,21 +7,11 @@ import { useWorkflowQuery } from "../hooks/useWorkflowQuery"; import { FlowRenderer } from "./FlowRenderer"; import { getElements } from "./workflowEditorUtils"; import { LogoMinimized } from "@/components/LogoMinimized"; -import { useDebugStore } from "@/store/useDebugStore"; -import { - isDisplayedInWorkflowEditor, - WorkflowEditorParameterTypes, - WorkflowParameterTypes, - WorkflowParameterValueType, - WorkflowSettings, -} from "../types/workflowTypes"; -import { ParametersState } from "./types"; +import { WorkflowSettings } from "../types/workflowTypes"; import { useGlobalWorkflowsQuery } from "../hooks/useGlobalWorkflowsQuery"; -import { WorkflowDebugOverviewWindow } from "./panels/WorkflowDebugOverviewWindow"; -import { cn } from "@/util/utils"; +import { getInitialParameters } from "./utils"; function WorkflowEditor() { - const debugStore = useDebugStore(); const { workflowPermanentId } = useParams(); const setCollapsed = useSidebarStore((state) => { return state.setCollapsed; @@ -76,123 +66,17 @@ function WorkflowEditor() { !isGlobalWorkflow, ); - const getInitialParameters = () => { - return workflow.workflow_definition.parameters - .filter((parameter) => isDisplayedInWorkflowEditor(parameter)) - .map((parameter) => { - if (parameter.parameter_type === WorkflowParameterTypes.Workflow) { - if ( - parameter.workflow_parameter_type === - WorkflowParameterValueType.CredentialId - ) { - return { - key: parameter.key, - parameterType: WorkflowEditorParameterTypes.Credential, - credentialId: parameter.default_value as string, - description: parameter.description, - }; - } - return { - key: parameter.key, - parameterType: WorkflowEditorParameterTypes.Workflow, - dataType: parameter.workflow_parameter_type, - defaultValue: parameter.default_value, - description: parameter.description, - }; - } else if ( - parameter.parameter_type === WorkflowParameterTypes.Context - ) { - return { - key: parameter.key, - parameterType: WorkflowEditorParameterTypes.Context, - sourceParameterKey: parameter.source.key, - description: parameter.description, - }; - } else if ( - parameter.parameter_type === - WorkflowParameterTypes.Bitwarden_Sensitive_Information - ) { - return { - key: parameter.key, - parameterType: WorkflowEditorParameterTypes.Secret, - collectionId: parameter.bitwarden_collection_id, - identityKey: parameter.bitwarden_identity_key, - identityFields: parameter.bitwarden_identity_fields, - description: parameter.description, - }; - } else if ( - parameter.parameter_type === - WorkflowParameterTypes.Bitwarden_Credit_Card_Data - ) { - return { - key: parameter.key, - parameterType: WorkflowEditorParameterTypes.CreditCardData, - collectionId: parameter.bitwarden_collection_id, - itemId: parameter.bitwarden_item_id, - description: parameter.description, - }; - } else if ( - parameter.parameter_type === WorkflowParameterTypes.Credential - ) { - return { - key: parameter.key, - parameterType: WorkflowEditorParameterTypes.Credential, - credentialId: parameter.credential_id, - description: parameter.description, - }; - } else if ( - parameter.parameter_type === WorkflowParameterTypes.OnePassword - ) { - return { - key: parameter.key, - parameterType: WorkflowEditorParameterTypes.OnePassword, - vaultId: parameter.vault_id, - itemId: parameter.item_id, - description: parameter.description, - }; - } else if ( - parameter.parameter_type === - WorkflowParameterTypes.Bitwarden_Login_Credential - ) { - return { - key: parameter.key, - parameterType: WorkflowEditorParameterTypes.Credential, - collectionId: parameter.bitwarden_collection_id, - itemId: parameter.bitwarden_item_id, - urlParameterKey: parameter.url_parameter_key, - description: parameter.description, - }; - } - return undefined; - }) - .filter(Boolean) as ParametersState; - }; - return (
-
- - - -
- {debugStore.isDebugMode && ( -
- -
- )} + + +
); } diff --git a/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx b/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx index 64e0eeeb..3a0db6c7 100644 --- a/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/WorkflowHeader.tsx @@ -110,7 +110,7 @@ function WorkflowHeader({ }} > - {debugStore.isDebugMode ? "End" : "Start Debugging"} + {debugStore.isDebugMode ? "End Debugging" : "Start Debugging"} diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx index 5bae1307..4c2a3302 100644 --- a/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx +++ b/skyvern-frontend/src/routes/workflows/editor/nodes/components/NodeHeader.tsx @@ -306,7 +306,6 @@ function NodeHeader({ title: "Workflow Canceled", description: "The workflow has been successfully canceled.", }); - navigate(`/workflows/${workflowPermanentId}/debug`); }, onError: (error) => { toast({ diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowDebugOverviewWindow.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowDebugOverviewWindow.tsx deleted file mode 100644 index c37c975b..00000000 --- a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowDebugOverviewWindow.tsx +++ /dev/null @@ -1,72 +0,0 @@ -/** - * NOTE(jdo): this is a hack: we are iframe-ing the overview page, but we really - * need dedicated UI component for this. - */ - -import { FloatingWindow } from "@/components/FloatingWindow"; -import { useMemo, useRef } from "react"; -import { useParams } from "react-router-dom"; - -function WorkflowDebugOverviewWindow() { - return ( - - - - ); -} - -function WorkflowDebugOverviewWindowIframe() { - const { workflowPermanentId: wpid, workflowRunId: wrid } = useParams(); - const lastCompletePair = useRef<{ wpid: string; wrid: string } | null>(null); - - if (wpid !== undefined && wrid !== undefined) { - lastCompletePair.current = { - wpid, - wrid, - }; - } - - const paramsToUse = useMemo(() => { - if (wpid && wrid) { - return { wpid, wrid }; - } - return lastCompletePair.current; - }, [wpid, wrid]); - - const origin = location.origin; - const dest = paramsToUse - ? `${origin}/workflows/${paramsToUse.wpid}/${paramsToUse.wrid}/overview?embed=true` - : null; - - return dest ? ( -
-