From 8c02c3c4e038d09882be35d3db8a581d44f32ae9 Mon Sep 17 00:00:00 2001 From: Shuchang Zheng Date: Wed, 18 Jun 2025 10:04:54 -0700 Subject: [PATCH] add a vnc streaming component for workflows (#2746) --- skyvern-frontend/src/api/types.ts | 1 + skyvern-frontend/src/novnc.d.ts | 39 ++++ .../workflowRun/WorkflowRunOverview.tsx | 11 +- .../workflowRun/WorkflowRunStreamVnc.tsx | 208 ++++++++++++++++++ .../workflowRun/workflow-run-stream-vnc.css | 96 ++++++++ 5 files changed, 353 insertions(+), 2 deletions(-) create mode 100644 skyvern-frontend/src/novnc.d.ts create mode 100644 skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStreamVnc.tsx create mode 100644 skyvern-frontend/src/routes/workflows/workflowRun/workflow-run-stream-vnc.css diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.ts index e63ba76d..ed4a72df 100644 --- a/skyvern-frontend/src/api/types.ts +++ b/skyvern-frontend/src/api/types.ts @@ -294,6 +294,7 @@ export type WorkflowRunStatusApiResponse = { total_cost: number | null; task_v2: TaskV2 | null; workflow_title: string | null; + browser_session_id: string | null; max_screenshot_scrolling_times: number | null; }; diff --git a/skyvern-frontend/src/novnc.d.ts b/skyvern-frontend/src/novnc.d.ts new file mode 100644 index 00000000..68706d6e --- /dev/null +++ b/skyvern-frontend/src/novnc.d.ts @@ -0,0 +1,39 @@ +declare module "@novnc/novnc/lib/rfb.js" { + export interface RfbEvent { + detail: { + clean: boolean; + reason: string; + error: { + message: string; + }; + message: string; + }; + } + + export interface RfbDisplay { + autoscale(): void; + _scale: number; + } + + export interface RFBOptions { + credentials?: { username?: string; password?: string }; + clipViewport?: boolean; + scaleViewport?: boolean; + shared?: boolean; + resizeSession?: boolean; + viewOnly?: boolean; + [key: string]: unknown; + } + + export default class RFB { + _display: RfbDisplay; + resizeSession: boolean; + scaleViewport: boolean; + constructor(target: HTMLElement, url: string, options?: RFBOptions); + + addEventListener(event: string, listener: (e: RfbEvent) => void): void; + removeEventListener(event: string, listener: (e: RfbEvent) => void): void; + disconnect(): void; + viewportChange(): void; + } +} diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx index 9bbba88b..e0a7ef79 100644 --- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx +++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx @@ -14,6 +14,7 @@ import { import { ObserverThoughtScreenshot } from "./ObserverThoughtScreenshot"; import { WorkflowRunBlockScreenshot } from "./WorkflowRunBlockScreenshot"; import { WorkflowRunStream } from "./WorkflowRunStream"; +import { WorkflowRunStreamVnc } from "./WorkflowRunStreamVnc"; import { useSearchParams } from "react-router-dom"; import { findActiveItem } from "./workflowTimelineUtils"; import { Skeleton } from "@/components/ui/skeleton"; @@ -62,9 +63,15 @@ function WorkflowRunOverview() { workflowRunIsFinalized, ); + const streamingComponent = workflowRun.browser_session_id ? ( + + ) : ( + + ); + return ( - - {selection === "stream" && } + + {selection === "stream" && streamingComponent} {selection !== "stream" && isAction(selection) && ( (); + + const [userIsControlling, setUserIsControlling] = useState(false); + const [vncDisconnectedTrigger, setVncDisconnectedTrigger] = useState(0); + const prevVncConnectedRef = useRef(false); + const [isVncConnected, setIsVncConnected] = useState(false); + const showStream = workflowRun && statusIsNotFinalized(workflowRun); + const credentialGetter = useCredentialGetter(); + const queryClient = useQueryClient(); + const [canvasContainer, setCanvasContainer] = useState( + null, + ); + const setCanvasContainerRef = useCallback((node: HTMLDivElement | null) => { + setCanvasContainer(node); + }, []); + + const rfbRef = useRef(null); + + // effect for disconnects only + useEffect(() => { + if (prevVncConnectedRef.current && !isVncConnected) { + setVncDisconnectedTrigger((x) => x + 1); + } + prevVncConnectedRef.current = isVncConnected; + }, [isVncConnected]); + + useEffect( + () => { + if (!showStream || !canvasContainer || !workflowRunId) { + if (rfbRef.current) { + rfbRef.current.disconnect(); + rfbRef.current = null; + setIsVncConnected(false); + } + return; + } + + async function setupVnc() { + let credentialQueryParam = ""; + + if (environment === "local") { + credentialQueryParam = `?apikey=${envCredential}`; + } else { + if (credentialGetter) { + const token = await credentialGetter(); + credentialQueryParam = `?token=Bearer ${token}`; + } else { + credentialQueryParam = `?apikey=${envCredential}`; + } + } + + if (rfbRef.current && isVncConnected) { + return; + } + + const vncUrl = `${wssBaseUrl}/stream/vnc/workflow_run/${workflowRunId}${credentialQueryParam}`; + + if (rfbRef.current) { + rfbRef.current.disconnect(); + } + + const canvas = canvasContainer; + + if (!canvas) { + throw new Error("Canvas element not found"); + } + + const rfb = new RFB(canvas, vncUrl); + + rfb.scaleViewport = true; + + rfbRef.current = rfb; + + rfb.addEventListener("connect", () => { + setIsVncConnected(true); + }); + + rfb.addEventListener("disconnect", async (/* e: RfbEvent */) => { + setIsVncConnected(false); + + queryClient.invalidateQueries({ + queryKey: ["workflowRun", workflowPermanentId, workflowRunId], + }); + queryClient.invalidateQueries({ queryKey: ["workflowRuns"] }); + queryClient.invalidateQueries({ + queryKey: ["workflowTasks", workflowRunId], + }); + queryClient.invalidateQueries({ queryKey: ["runs"] }); + }); + } + + setupVnc(); + + return () => { + if (rfbRef.current) { + rfbRef.current.disconnect(); + rfbRef.current = null; + } + setIsVncConnected(false); + }; + }, + // cannot include isVncConnected in deps as it will cause infinite loop + // eslint-disable-next-line react-hooks/exhaustive-deps + [ + credentialGetter, + workflowRunId, + workflowPermanentId, + showStream, + queryClient, + canvasContainer, + vncDisconnectedTrigger, // will re-run on disconnects + ], + ); + + // Effect to show toast when workflow reaches a final state based on hook updates + useEffect(() => { + if (workflowRun) { + if ( + workflowRun.status === Status.Failed || + workflowRun.status === Status.Terminated + ) { + // Only show toast if VNC is not connected or was never connected, + // to avoid double toasting if disconnect handler also triggers similar logic. + // However, the disconnect handler now primarily invalidates queries. + toast({ + title: "Run Ended", + description: `The workflow run has ${workflowRun.status}.`, + variant: "destructive", + }); + } else if (workflowRun.status === Status.Completed) { + toast({ + title: "Run Completed", + description: "The workflow run has been completed.", + variant: "success", + }); + } + } + }, [workflowRun, workflowRun?.status]); + + return ( +
+ {isVncConnected && ( +
+
+ +
+ +
+
+
+ )} + {!isVncConnected && ( +
+ +
+ )} +
+ ); +} + +export { WorkflowRunStreamVnc }; diff --git a/skyvern-frontend/src/routes/workflows/workflowRun/workflow-run-stream-vnc.css b/skyvern-frontend/src/routes/workflows/workflowRun/workflow-run-stream-vnc.css new file mode 100644 index 00000000..fe0185ad --- /dev/null +++ b/skyvern-frontend/src/routes/workflows/workflowRun/workflow-run-stream-vnc.css @@ -0,0 +1,96 @@ +.workflow-run-stream-vnc { + position: relative; + width: 100%; + height: 100%; + min-height: 0; + padding: 0.5rem; + overflow: visible; + + transition: padding 0.2s ease-in-out; +} + +.workflow-run-stream-vnc.user-is-controlling { + padding: 0rem; +} + +.workflow-run-stream-vnc .overlay-container { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + display: flex; + align-items: center; + justify-content: center; +} + +.workflow-run-stream-vnc .overlay { + position: relative; + height: auto; + width: 100%; + max-height: 100%; + aspect-ratio: 16 / 9; + object-fit: cover; + display: flex; + align-items: center; + justify-content: center; +} + +.workflow-run-stream-vnc.user-is-controlling .overlay { + pointer-events: none; +} + +.workflow-run-stream-vnc.user-is-controlling .overlay-container { + pointer-events: none; +} + +.workflow-run-stream-vnc .take-control { + transform: translateY(0); + transition: + transform 0.2s ease-in-out, + opacity 0.2s ease-in-out; + opacity: 0.3; +} + +.workflow-run-stream-vnc .take-control:not(.hide):hover { + opacity: 1; +} + +.workflow-run-stream-vnc .take-control.hide { + transform: translateY(100%); + opacity: 0; + pointer-events: none; +} + +.workflow-run-stream-vnc .relinquish-control { + transform: translateY(0); + transition: + transform 0.2s ease-in-out, + opacity 0.2s ease-in-out; + opacity: 0.3; + pointer-events: all; +} + +.workflow-run-stream-vnc .relinquish-control:not(.hide):hover { + opacity: 1; +} + +.workflow-run-stream-vnc .relinquish-control.hide { + transform: translateY(100%); + opacity: 0; + pointer-events: none; +} + +.workflow-run-stream-vnc > div > canvas { + opacity: 0; + animation: skyvern-anim-fadeIn 1s ease-in forwards; +} + +@keyframes skyvern-anim-fadeIn { + from { + opacity: 0; + } + to { + opacity: 1; + } +}