add a vnc streaming component for workflows (#2746)

skyvern-frontend/src/api/types.ts

@@ -294,6 +294,7 @@ export type WorkflowRunStatusApiResponse = {
   total_cost: number | null;
   task_v2: TaskV2 | null;
   workflow_title: string | null;
+  browser_session_id: string | null;
   max_screenshot_scrolling_times: number | null;
 };
 

skyvern-frontend/src/novnc.d.ts

@@ -0,0 +1,39 @@
+declare module "@novnc/novnc/lib/rfb.js" {
+  export interface RfbEvent {
+    detail: {
+      clean: boolean;
+      reason: string;
+      error: {
+        message: string;
+      };
+      message: string;
+    };
+  }
+
+  export interface RfbDisplay {
+    autoscale(): void;
+    _scale: number;
+  }
+
+  export interface RFBOptions {
+    credentials?: { username?: string; password?: string };
+    clipViewport?: boolean;
+    scaleViewport?: boolean;
+    shared?: boolean;
+    resizeSession?: boolean;
+    viewOnly?: boolean;
+    [key: string]: unknown;
+  }
+
+  export default class RFB {
+    _display: RfbDisplay;
+    resizeSession: boolean;
+    scaleViewport: boolean;
+    constructor(target: HTMLElement, url: string, options?: RFBOptions);
+
+    addEventListener(event: string, listener: (e: RfbEvent) => void): void;
+    removeEventListener(event: string, listener: (e: RfbEvent) => void): void;
+    disconnect(): void;
+    viewportChange(): void;
+  }
+}

skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunOverview.tsx

@@ -14,6 +14,7 @@ import {
 import { ObserverThoughtScreenshot } from "./ObserverThoughtScreenshot";
 import { WorkflowRunBlockScreenshot } from "./WorkflowRunBlockScreenshot";
 import { WorkflowRunStream } from "./WorkflowRunStream";
+import { WorkflowRunStreamVnc } from "./WorkflowRunStreamVnc";
 import { useSearchParams } from "react-router-dom";
 import { findActiveItem } from "./workflowTimelineUtils";
 import { Skeleton } from "@/components/ui/skeleton";
@@ -62,9 +63,15 @@ function WorkflowRunOverview() {
     workflowRunIsFinalized,
   );
 
+  const streamingComponent = workflowRun.browser_session_id ? (
+    <WorkflowRunStreamVnc />
+  ) : (
+    <WorkflowRunStream />
+  );
+
   return (
-    <AspectRatio ratio={16 / 9} className="overflow-y-hidden">
-      {selection === "stream" && <WorkflowRunStream />}
+    <AspectRatio ratio={16 / 9}>
+      {selection === "stream" && streamingComponent}
       {selection !== "stream" && isAction(selection) && (
         <ActionScreenshot
           index={selection.action_order ?? 0}

skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStreamVnc.tsx

@@ -0,0 +1,208 @@
+import { Status } from "@/api/types";
+import { useWorkflowRunQuery } from "../hooks/useWorkflowRunQuery";
+import { useEffect, useState, useRef, useCallback } from "react";
+import { HandIcon, PlayIcon } from "@radix-ui/react-icons";
+import { Button } from "@/components/ui/button";
+import { Skeleton } from "@/components/ui/skeleton";
+import { statusIsNotFinalized } from "@/routes/tasks/types";
+import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { useParams } from "react-router-dom";
+import { envCredential } from "@/util/env";
+import { toast } from "@/components/ui/use-toast";
+import { useQueryClient } from "@tanstack/react-query";
+import RFB from "@novnc/novnc/lib/rfb.js";
+import { environment } from "@/util/env";
+import { cn } from "@/util/utils";
+
+import "./workflow-run-stream-vnc.css";
+
+const wssBaseUrl = import.meta.env.VITE_WSS_BASE_URL;
+
+function WorkflowRunStreamVnc() {
+  const { data: workflowRun } = useWorkflowRunQuery();
+  const { workflowRunId, workflowPermanentId } = useParams<{
+    workflowRunId: string;
+    workflowPermanentId: string;
+  }>();
+
+  const [userIsControlling, setUserIsControlling] = useState<boolean>(false);
+  const [vncDisconnectedTrigger, setVncDisconnectedTrigger] = useState(0);
+  const prevVncConnectedRef = useRef<boolean>(false);
+  const [isVncConnected, setIsVncConnected] = useState<boolean>(false);
+  const showStream = workflowRun && statusIsNotFinalized(workflowRun);
+  const credentialGetter = useCredentialGetter();
+  const queryClient = useQueryClient();
+  const [canvasContainer, setCanvasContainer] = useState<HTMLDivElement | null>(
+    null,
+  );
+  const setCanvasContainerRef = useCallback((node: HTMLDivElement | null) => {
+    setCanvasContainer(node);
+  }, []);
+
+  const rfbRef = useRef<RFB | null>(null);
+
+  // effect for disconnects only
+  useEffect(() => {
+    if (prevVncConnectedRef.current && !isVncConnected) {
+      setVncDisconnectedTrigger((x) => x + 1);
+    }
+    prevVncConnectedRef.current = isVncConnected;
+  }, [isVncConnected]);
+
+  useEffect(
+    () => {
+      if (!showStream || !canvasContainer || !workflowRunId) {
+        if (rfbRef.current) {
+          rfbRef.current.disconnect();
+          rfbRef.current = null;
+          setIsVncConnected(false);
+        }
+        return;
+      }
+
+      async function setupVnc() {
+        let credentialQueryParam = "";
+
+        if (environment === "local") {
+          credentialQueryParam = `?apikey=${envCredential}`;
+        } else {
+          if (credentialGetter) {
+            const token = await credentialGetter();
+            credentialQueryParam = `?token=Bearer ${token}`;
+          } else {
+            credentialQueryParam = `?apikey=${envCredential}`;
+          }
+        }
+
+        if (rfbRef.current && isVncConnected) {
+          return;
+        }
+
+        const vncUrl = `${wssBaseUrl}/stream/vnc/workflow_run/${workflowRunId}${credentialQueryParam}`;
+
+        if (rfbRef.current) {
+          rfbRef.current.disconnect();
+        }
+
+        const canvas = canvasContainer;
+
+        if (!canvas) {
+          throw new Error("Canvas element not found");
+        }
+
+        const rfb = new RFB(canvas, vncUrl);
+
+        rfb.scaleViewport = true;
+
+        rfbRef.current = rfb;
+
+        rfb.addEventListener("connect", () => {
+          setIsVncConnected(true);
+        });
+
+        rfb.addEventListener("disconnect", async (/* e: RfbEvent */) => {
+          setIsVncConnected(false);
+
+          queryClient.invalidateQueries({
+            queryKey: ["workflowRun", workflowPermanentId, workflowRunId],
+          });
+          queryClient.invalidateQueries({ queryKey: ["workflowRuns"] });
+          queryClient.invalidateQueries({
+            queryKey: ["workflowTasks", workflowRunId],
+          });
+          queryClient.invalidateQueries({ queryKey: ["runs"] });
+        });
+      }
+
+      setupVnc();
+
+      return () => {
+        if (rfbRef.current) {
+          rfbRef.current.disconnect();
+          rfbRef.current = null;
+        }
+        setIsVncConnected(false);
+      };
+    },
+    // cannot include isVncConnected in deps as it will cause infinite loop
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [
+      credentialGetter,
+      workflowRunId,
+      workflowPermanentId,
+      showStream,
+      queryClient,
+      canvasContainer,
+      vncDisconnectedTrigger, // will re-run on disconnects
+    ],
+  );
+
+  // Effect to show toast when workflow reaches a final state based on hook updates
+  useEffect(() => {
+    if (workflowRun) {
+      if (
+        workflowRun.status === Status.Failed ||
+        workflowRun.status === Status.Terminated
+      ) {
+        // Only show toast if VNC is not connected or was never connected,
+        // to avoid double toasting if disconnect handler also triggers similar logic.
+        // However, the disconnect handler now primarily invalidates queries.
+        toast({
+          title: "Run Ended",
+          description: `The workflow run has ${workflowRun.status}.`,
+          variant: "destructive",
+        });
+      } else if (workflowRun.status === Status.Completed) {
+        toast({
+          title: "Run Completed",
+          description: "The workflow run has been completed.",
+          variant: "success",
+        });
+      }
+    }
+  }, [workflowRun, workflowRun?.status]);
+
+  return (
+    <div
+      className={cn("workflow-run-stream-vnc", {
+        "user-is-controlling": userIsControlling,
+      })}
+      ref={setCanvasContainerRef}
+    >
+      {isVncConnected && (
+        <div className="overlay-container">
+          <div className="overlay">
+            <Button
+              // className="take-control"
+              className={cn("take-control", { hide: userIsControlling })}
+              type="button"
+              onClick={() => setUserIsControlling(true)}
+            >
+              <HandIcon className="mr-2 h-4 w-4" />
+              take control
+            </Button>
+            <div className="absolute bottom-[-1rem] right-[1rem]">
+              <Button
+                className={cn("relinquish-control", {
+                  hide: !userIsControlling,
+                })}
+                type="button"
+                onClick={() => setUserIsControlling(false)}
+              >
+                <PlayIcon className="mr-2 h-4 w-4" />
+                run agent
+              </Button>
+            </div>
+          </div>
+        </div>
+      )}
+      {!isVncConnected && (
+        <div className="absolute left-0 top-0 flex h-full w-full items-center justify-center bg-black">
+          <Skeleton className="aspect-[16/9] h-auto max-h-full w-full max-w-full rounded-lg object-cover" />
+        </div>
+      )}
+    </div>
+  );
+}
+
+export { WorkflowRunStreamVnc };

skyvern-frontend/src/routes/workflows/workflowRun/workflow-run-stream-vnc.css

@@ -0,0 +1,96 @@
+.workflow-run-stream-vnc {
+  position: relative;
+  width: 100%;
+  height: 100%;
+  min-height: 0;
+  padding: 0.5rem;
+  overflow: visible;
+
+  transition: padding 0.2s ease-in-out;
+}
+
+.workflow-run-stream-vnc.user-is-controlling {
+  padding: 0rem;
+}
+
+.workflow-run-stream-vnc .overlay-container {
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+.workflow-run-stream-vnc .overlay {
+  position: relative;
+  height: auto;
+  width: 100%;
+  max-height: 100%;
+  aspect-ratio: 16 / 9;
+  object-fit: cover;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+}
+
+.workflow-run-stream-vnc.user-is-controlling .overlay {
+  pointer-events: none;
+}
+
+.workflow-run-stream-vnc.user-is-controlling .overlay-container {
+  pointer-events: none;
+}
+
+.workflow-run-stream-vnc .take-control {
+  transform: translateY(0);
+  transition:
+    transform 0.2s ease-in-out,
+    opacity 0.2s ease-in-out;
+  opacity: 0.3;
+}
+
+.workflow-run-stream-vnc .take-control:not(.hide):hover {
+  opacity: 1;
+}
+
+.workflow-run-stream-vnc .take-control.hide {
+  transform: translateY(100%);
+  opacity: 0;
+  pointer-events: none;
+}
+
+.workflow-run-stream-vnc .relinquish-control {
+  transform: translateY(0);
+  transition:
+    transform 0.2s ease-in-out,
+    opacity 0.2s ease-in-out;
+  opacity: 0.3;
+  pointer-events: all;
+}
+
+.workflow-run-stream-vnc .relinquish-control:not(.hide):hover {
+  opacity: 1;
+}
+
+.workflow-run-stream-vnc .relinquish-control.hide {
+  transform: translateY(100%);
+  opacity: 0;
+  pointer-events: none;
+}
+
+.workflow-run-stream-vnc > div > canvas {
+  opacity: 0;
+  animation: skyvern-anim-fadeIn 1s ease-in forwards;
+}
+
+@keyframes skyvern-anim-fadeIn {
+  from {
+    opacity: 0;
+  }
+  to {
+    opacity: 1;
+  }
+}