Jon/browser stream component (#2808)
This commit is contained in:
Shuchang Zheng
352
skyvern-frontend/src/components/BrowserStream.tsx
Normal file
352
skyvern-frontend/src/components/BrowserStream.tsx
Normal file
@@ -0,0 +1,352 @@
|
||||
import { Status } from "@/api/types";
|
||||
import { useEffect, useState, useRef, useCallback } from "react";
|
||||
import { HandIcon, PlayIcon } from "@radix-ui/react-icons";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { statusIsNotFinalized } from "@/routes/tasks/types";
|
||||
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
|
||||
import { envCredential } from "@/util/env";
|
||||
import { toast } from "@/components/ui/use-toast";
|
||||
import RFB from "@novnc/novnc/lib/rfb.js";
|
||||
import { environment } from "@/util/env";
|
||||
import { cn } from "@/util/utils";
|
||||
import { useClientIdStore } from "@/store/useClientIdStore";
|
||||
import type {
|
||||
TaskApiResponse,
|
||||
WorkflowRunStatusApiResponse,
|
||||
} from "@/api/types";
|
||||
|
||||
import "./browser-stream.css";
|
||||
|
||||
const wssBaseUrl = import.meta.env.VITE_WSS_BASE_URL;
|
||||
|
||||
interface CommandTakeControl {
|
||||
kind: "take-control";
|
||||
}
|
||||
|
||||
interface CommandCedeControl {
|
||||
kind: "cede-control";
|
||||
}
|
||||
|
||||
type Command = CommandTakeControl | CommandCedeControl;
|
||||
|
||||
type Props = {
|
||||
task?: {
|
||||
run: TaskApiResponse;
|
||||
};
|
||||
workflow?: {
|
||||
run: WorkflowRunStatusApiResponse;
|
||||
};
|
||||
// --
|
||||
onClose?: () => void;
|
||||
};
|
||||
|
||||
function BrowserStream({
|
||||
task = undefined,
|
||||
workflow = undefined,
|
||||
// --
|
||||
onClose,
|
||||
}: Props) {
|
||||
let showStream: boolean = false;
|
||||
let runId: string;
|
||||
let entity: "task" | "workflow";
|
||||
|
||||
if (task) {
|
||||
runId = task.run.task_id;
|
||||
showStream = statusIsNotFinalized(task.run);
|
||||
entity = "task";
|
||||
} else if (workflow) {
|
||||
runId = workflow.run.workflow_run_id;
|
||||
showStream = statusIsNotFinalized(workflow.run);
|
||||
entity = "workflow";
|
||||
} else {
|
||||
throw new Error("No task or workflow provided");
|
||||
}
|
||||
|
||||
const [commandSocket, setCommandSocket] = useState<WebSocket | null>(null);
|
||||
const [userIsControlling, setUserIsControlling] = useState<boolean>(false);
|
||||
const [vncDisconnectedTrigger, setVncDisconnectedTrigger] = useState(0);
|
||||
const prevVncConnectedRef = useRef<boolean>(false);
|
||||
const [isVncConnected, setIsVncConnected] = useState<boolean>(false);
|
||||
const [commandDisconnectedTrigger, setCommandDisconnectedTrigger] =
|
||||
useState(0);
|
||||
const prevCommandConnectedRef = useRef<boolean>(false);
|
||||
const [isCommandConnected, setIsCommandConnected] = useState<boolean>(false);
|
||||
// goes up a level
|
||||
// const queryClient = useQueryClient();
|
||||
const [canvasContainer, setCanvasContainer] = useState<HTMLDivElement | null>(
|
||||
null,
|
||||
);
|
||||
const setCanvasContainerRef = useCallback((node: HTMLDivElement | null) => {
|
||||
setCanvasContainer(node);
|
||||
}, []);
|
||||
const rfbRef = useRef<RFB | null>(null);
|
||||
const clientId = useClientIdStore((state) => state.clientId);
|
||||
const credentialGetter = useCredentialGetter();
|
||||
|
||||
const getWebSocketParams = useCallback(async () => {
|
||||
const clientIdQueryParam = `client_id=${clientId}`;
|
||||
let credentialQueryParam = "";
|
||||
|
||||
if (environment === "local") {
|
||||
credentialQueryParam = `apikey=${envCredential}`;
|
||||
} else {
|
||||
if (credentialGetter) {
|
||||
const token = await credentialGetter();
|
||||
credentialQueryParam = `token=Bearer ${token}`;
|
||||
} else {
|
||||
credentialQueryParam = `apikey=${envCredential}`;
|
||||
}
|
||||
}
|
||||
|
||||
const params = [credentialQueryParam, clientIdQueryParam].join("&");
|
||||
|
||||
return `${params}`;
|
||||
}, [clientId, credentialGetter]);
|
||||
|
||||
// effect for vnc disconnects only
|
||||
useEffect(() => {
|
||||
if (prevVncConnectedRef.current && !isVncConnected) {
|
||||
setVncDisconnectedTrigger((x) => x + 1);
|
||||
onClose?.();
|
||||
}
|
||||
prevVncConnectedRef.current = isVncConnected;
|
||||
}, [isVncConnected, onClose]);
|
||||
|
||||
// effect for command disconnects only
|
||||
useEffect(() => {
|
||||
if (prevCommandConnectedRef.current && !isCommandConnected) {
|
||||
setCommandDisconnectedTrigger((x) => x + 1);
|
||||
onClose?.();
|
||||
}
|
||||
prevCommandConnectedRef.current = isCommandConnected;
|
||||
}, [isCommandConnected, onClose]);
|
||||
|
||||
// vnc socket
|
||||
useEffect(
|
||||
() => {
|
||||
if (!showStream || !canvasContainer || !runId) {
|
||||
if (rfbRef.current) {
|
||||
rfbRef.current.disconnect();
|
||||
rfbRef.current = null;
|
||||
setIsVncConnected(false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
async function setupVnc() {
|
||||
if (rfbRef.current && isVncConnected) {
|
||||
return;
|
||||
}
|
||||
|
||||
const wsParams = await getWebSocketParams();
|
||||
const vncUrl =
|
||||
entity === "task"
|
||||
? `${wssBaseUrl}/stream/vnc/task/${runId}?${wsParams}`
|
||||
: entity === "workflow"
|
||||
? `${wssBaseUrl}/stream/vnc/workflow_run/${runId}?${wsParams}`
|
||||
: null;
|
||||
|
||||
if (!vncUrl) {
|
||||
throw new Error("No vnc url");
|
||||
}
|
||||
|
||||
if (rfbRef.current) {
|
||||
rfbRef.current.disconnect();
|
||||
}
|
||||
|
||||
const canvas = canvasContainer;
|
||||
|
||||
if (!canvas) {
|
||||
throw new Error("Canvas element not found");
|
||||
}
|
||||
|
||||
const rfb = new RFB(canvas, vncUrl);
|
||||
|
||||
rfb.scaleViewport = true;
|
||||
|
||||
rfbRef.current = rfb;
|
||||
|
||||
rfb.addEventListener("connect", () => {
|
||||
setIsVncConnected(true);
|
||||
});
|
||||
|
||||
rfb.addEventListener("disconnect", async (/* e: RfbEvent */) => {
|
||||
setIsVncConnected(false);
|
||||
});
|
||||
}
|
||||
|
||||
setupVnc();
|
||||
|
||||
return () => {
|
||||
if (rfbRef.current) {
|
||||
rfbRef.current.disconnect();
|
||||
rfbRef.current = null;
|
||||
}
|
||||
setIsVncConnected(false);
|
||||
};
|
||||
},
|
||||
// cannot include isVncConnected in deps as it will cause infinite loop
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
[
|
||||
canvasContainer,
|
||||
showStream,
|
||||
vncDisconnectedTrigger, // will re-run on disconnects
|
||||
runId,
|
||||
entity,
|
||||
],
|
||||
);
|
||||
|
||||
// command socket
|
||||
useEffect(() => {
|
||||
if (!showStream || !canvasContainer || !runId) {
|
||||
return;
|
||||
}
|
||||
|
||||
let ws: WebSocket | null = null;
|
||||
|
||||
const connect = async () => {
|
||||
const wsParams = await getWebSocketParams();
|
||||
|
||||
const commandUrl =
|
||||
entity === "task"
|
||||
? `${wssBaseUrl}/stream/commands/task/${runId}?${wsParams}`
|
||||
: entity === "workflow"
|
||||
? `${wssBaseUrl}/stream/commands/workflow_run/${runId}?${wsParams}`
|
||||
: null;
|
||||
|
||||
if (!commandUrl) {
|
||||
throw new Error("No command url");
|
||||
}
|
||||
|
||||
ws = new WebSocket(commandUrl);
|
||||
|
||||
ws.onopen = () => {
|
||||
setIsCommandConnected(true);
|
||||
setCommandSocket(ws);
|
||||
};
|
||||
|
||||
ws.onclose = () => {
|
||||
setIsCommandConnected(false);
|
||||
setCommandSocket(null);
|
||||
};
|
||||
};
|
||||
|
||||
connect();
|
||||
|
||||
return () => {
|
||||
try {
|
||||
ws && ws.close();
|
||||
} catch (e) {
|
||||
// pass
|
||||
}
|
||||
};
|
||||
}, [
|
||||
canvasContainer,
|
||||
commandDisconnectedTrigger,
|
||||
entity,
|
||||
getWebSocketParams,
|
||||
runId,
|
||||
showStream,
|
||||
]);
|
||||
|
||||
// effect to send a command when the user is controlling, vs not controlling
|
||||
useEffect(() => {
|
||||
if (!isCommandConnected) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sendCommand = (command: Command) => {
|
||||
if (!commandSocket) {
|
||||
console.warn("Cannot send command, as command socket is closed.");
|
||||
console.warn(command);
|
||||
return;
|
||||
}
|
||||
|
||||
commandSocket.send(JSON.stringify(command));
|
||||
};
|
||||
|
||||
if (userIsControlling) {
|
||||
sendCommand({ kind: "take-control" });
|
||||
} else {
|
||||
sendCommand({ kind: "cede-control" });
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [userIsControlling, isCommandConnected]);
|
||||
|
||||
// Effect to show toast when task or workflow reaches a final state based on hook updates
|
||||
useEffect(() => {
|
||||
const run = task ? task.run : workflow ? workflow.run : null;
|
||||
|
||||
if (!run) {
|
||||
return;
|
||||
}
|
||||
|
||||
const name = task ? "task" : workflow ? "workflow" : null;
|
||||
|
||||
if (!name) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (run.status === Status.Failed || run.status === Status.Terminated) {
|
||||
// Only show toast if VNC is not connected or was never connected,
|
||||
// to avoid double toasting if disconnect handler also triggers similar logic.
|
||||
// However, the disconnect handler now primarily invalidates queries.
|
||||
toast({
|
||||
title: "Run Ended",
|
||||
description: `The ${name} run has ${run.status}.`,
|
||||
variant: "destructive",
|
||||
});
|
||||
} else if (run.status === Status.Completed) {
|
||||
toast({
|
||||
title: "Run Completed",
|
||||
description: `The ${name} run has been completed.`,
|
||||
variant: "success",
|
||||
});
|
||||
}
|
||||
}, [task, workflow]);
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn("browser-stream", {
|
||||
"user-is-controlling": userIsControlling,
|
||||
})}
|
||||
ref={setCanvasContainerRef}
|
||||
>
|
||||
{isVncConnected && (
|
||||
<div className="overlay-container">
|
||||
<div className="overlay">
|
||||
<Button
|
||||
// className="take-control"
|
||||
className={cn("take-control", { hide: userIsControlling })}
|
||||
type="button"
|
||||
onClick={() => setUserIsControlling(true)}
|
||||
>
|
||||
<HandIcon className="mr-2 h-4 w-4" />
|
||||
take control
|
||||
</Button>
|
||||
<div className="absolute bottom-[-1rem] right-[1rem]">
|
||||
<Button
|
||||
className={cn("relinquish-control", {
|
||||
hide: !userIsControlling,
|
||||
})}
|
||||
type="button"
|
||||
onClick={() => setUserIsControlling(false)}
|
||||
>
|
||||
<PlayIcon className="mr-2 h-4 w-4" />
|
||||
run agent
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{!isVncConnected && (
|
||||
<div className="absolute left-0 top-0 flex h-full w-full items-center justify-center bg-black">
|
||||
<Skeleton className="aspect-[16/9] h-auto max-h-full w-full max-w-full rounded-lg object-cover" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export { BrowserStream };
|
||||
96
skyvern-frontend/src/components/browser-stream.css
Normal file
96
skyvern-frontend/src/components/browser-stream.css
Normal file
@@ -0,0 +1,96 @@
|
||||
.browser-stream {
|
||||
position: relative;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
min-height: 0;
|
||||
padding: 0.5rem;
|
||||
overflow: visible;
|
||||
|
||||
transition: padding 0.2s ease-in-out;
|
||||
}
|
||||
|
||||
.browser-stream.user-is-controlling {
|
||||
padding: 0rem;
|
||||
}
|
||||
|
||||
.browser-stream .overlay-container {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.browser-stream .overlay {
|
||||
position: relative;
|
||||
height: auto;
|
||||
width: 100%;
|
||||
max-height: 100%;
|
||||
aspect-ratio: 16 / 9;
|
||||
object-fit: cover;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.browser-stream.user-is-controlling .overlay {
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.browser-stream.user-is-controlling .overlay-container {
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.browser-stream .take-control {
|
||||
transform: translateY(0);
|
||||
transition:
|
||||
transform 0.2s ease-in-out,
|
||||
opacity 0.2s ease-in-out;
|
||||
opacity: 0.3;
|
||||
}
|
||||
|
||||
.browser-stream .take-control:not(.hide):hover {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.browser-stream .take-control.hide {
|
||||
transform: translateY(100%);
|
||||
opacity: 0;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.browser-stream .relinquish-control {
|
||||
transform: translateY(0);
|
||||
transition:
|
||||
transform 0.2s ease-in-out,
|
||||
opacity 0.2s ease-in-out;
|
||||
opacity: 0.3;
|
||||
pointer-events: all;
|
||||
}
|
||||
|
||||
.browser-stream .relinquish-control:not(.hide):hover {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.browser-stream .relinquish-control.hide {
|
||||
transform: translateY(100%);
|
||||
opacity: 0;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.browser-stream > div > canvas {
|
||||
opacity: 0;
|
||||
animation: skyvern-anim-fadeIn 1s ease-in forwards;
|
||||
}
|
||||
|
||||
@keyframes skyvern-anim-fadeIn {
|
||||
from {
|
||||
opacity: 0;
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user