diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx index 34cda4a3..b3a5473f 100644 --- a/skyvern-frontend/src/components/BrowserStream.tsx +++ b/skyvern-frontend/src/components/BrowserStream.tsx @@ -39,8 +39,31 @@ interface CommandCedeControl { kind: "cede-control"; } +// a "Command" is an fire-n-forget out-message - it does not require a response type Command = CommandTakeControl | CommandCedeControl; +const messageInKinds = ["ask-for-clipboard", "copied-text"] as const; + +type MessageInKind = (typeof messageInKinds)[number]; + +interface MessageInAskForClipboard { + kind: "ask-for-clipboard"; +} + +interface MessageInCopiedText { + kind: "copied-text"; + text: string; +} + +type MessageIn = MessageInCopiedText | MessageInAskForClipboard; + +interface MessageOutAskForClipboardResponse { + kind: "ask-for-clipboard-response"; + text: string; +} + +type MessageOut = MessageOutAskForClipboardResponse; + type Props = { browserSessionId?: string; interactive?: boolean; @@ -102,6 +125,20 @@ function BrowserStream({ if (!browserSession || browserSession.completed_at) { setHasBrowserSession(false); return false; + + // NOTE(jdo:streaming-local-dev): remove above and use this instead + // if (browserSession && browserSession.completed_at) { + // console.warn( + // "Completed at:", + // browserSession.completed_at, + // "continuing anyway!", + // ); + // setHasBrowserSession(true); + // return true; + // } else { + // setHasBrowserSession(false); + // return false; + // } } setHasBrowserSession(true); @@ -117,16 +154,16 @@ function BrowserStream({ const [hasBrowserSession, setHasBrowserSession] = useState(true); // be optimistic const [userIsControlling, setUserIsControlling] = useState(false); - const [commandSocket, setCommandSocket] = useState(null); + const [messageSocket, setMessageSocket] = useState(null); const [vncDisconnectedTrigger, setVncDisconnectedTrigger] = useState(0); const prevVncConnectedRef = useRef(false); const [isVncConnected, setIsVncConnected] = useState(false); const [isCanvasReady, setIsCanvasReady] = useState(false); const [isReady, setIsReady] = useState(false); - const [commandDisconnectedTrigger, setCommandDisconnectedTrigger] = + const [messagesDisconnectedTrigger, setMessagesDisconnectedTrigger] = useState(0); - const prevCommandConnectedRef = useRef(false); - const [isCommandConnected, setIsCommandConnected] = useState(false); + const prevMessageConnectedRef = useRef(false); + const [isMessageConnected, setIsMessageConnected] = useState(false); const [canvasContainer, setCanvasContainer] = useState( null, ); @@ -168,14 +205,14 @@ function BrowserStream({ prevVncConnectedRef.current = isVncConnected; }, [isVncConnected, onClose]); - // effect for command disconnects only + // effect for message disconnects only useEffect(() => { - if (prevCommandConnectedRef.current && !isCommandConnected) { - setCommandDisconnectedTrigger((x) => x + 1); + if (prevMessageConnectedRef.current && !isMessageConnected) { + setMessagesDisconnectedTrigger((x) => x + 1); onClose?.(); } - prevCommandConnectedRef.current = isCommandConnected; - }, [isCommandConnected, onClose]); + prevMessageConnectedRef.current = isMessageConnected; + }, [isMessageConnected, onClose]); // vnc socket useEffect( @@ -289,7 +326,6 @@ function BrowserStream({ ], ); - // command socket useEffect(() => { if (!showStream || !canvasContainer || !runId) { return; @@ -300,34 +336,52 @@ function BrowserStream({ const connect = async () => { const wsParams = await getWebSocketParams(); - const commandUrl = + const messageUrl = entity === "browserSession" - ? `${newWssBaseUrl}/stream/commands/browser_session/${runId}?${wsParams}` + ? `${newWssBaseUrl}/stream/messages/browser_session/${runId}?${wsParams}` : entity === "task" - ? `${wssBaseUrl}/stream/commands/task/${runId}?${wsParams}` + ? `${wssBaseUrl}/stream/messages/task/${runId}?${wsParams}` : entity === "workflow" - ? `${wssBaseUrl}/stream/commands/workflow_run/${runId}?${wsParams}` + ? `${wssBaseUrl}/stream/messages/workflow_run/${runId}?${wsParams}` : null; - if (!commandUrl) { - throw new Error("No command url"); + if (!messageUrl) { + throw new Error("No message url"); } if (!hasBrowserSession) { - setIsCommandConnected(false); + setIsMessageConnected(false); return; } - ws = new WebSocket(commandUrl); + ws = new WebSocket(messageUrl); ws.onopen = () => { - setIsCommandConnected(true); - setCommandSocket(ws); + setIsMessageConnected(true); + setMessageSocket(ws); + }; + + ws.onmessage = (event) => { + const data = event.data; + + try { + const message = JSON.parse(data); + + handleMessage(message, ws); + + // handle incoming messages if needed + } catch (e) { + console.error( + "Error parsing message from message channel:", + e, + event, + ); + } }; ws.onclose = () => { - setIsCommandConnected(false); - setCommandSocket(null); + setIsMessageConnected(false); + setMessageSocket(null); }; }; @@ -340,31 +394,33 @@ function BrowserStream({ // pass } }; + // NOTE: adding getWebSocketParams causes constant reconnects of message channel when, + // for instance, take-control or cede-control is clicked + // eslint-disable-next-line react-hooks/exhaustive-deps }, [ browserSessionId, canvasContainer, - commandDisconnectedTrigger, + messagesDisconnectedTrigger, entity, - getWebSocketParams, hasBrowserSession, runId, showStream, ]); - // effect to send a command when the user is controlling, vs not controlling + // effect to send a message when the user is controlling, vs not controlling useEffect(() => { - if (!isCommandConnected) { + if (!isMessageConnected) { return; } const sendCommand = (command: Command) => { - if (!commandSocket) { + if (!messageSocket) { console.warn("Cannot send command, as command socket is closed."); console.warn(command); return; } - commandSocket.send(JSON.stringify(command)); + messageSocket.send(JSON.stringify(command)); }; if (interactive || userIsControlling) { @@ -373,7 +429,7 @@ function BrowserStream({ sendCommand({ kind: "cede-control" }); } // eslint-disable-next-line react-hooks/exhaustive-deps - }, [interactive, isCommandConnected, userIsControlling]); + }, [interactive, isMessageConnected, userIsControlling]); // Effect to handle window resize trigger for NoVNC canvas useEffect(() => { @@ -420,6 +476,134 @@ function BrowserStream({ } }, [task, workflow]); + /** + * TODO(jdo): could use zod or smth similar + */ + const getMessage = (data: unknown): MessageIn | undefined => { + if (!data) { + return; + } + + if (typeof data !== "object") { + return; + } + + if (!("kind" in data)) { + return; + } + + const k = data.kind; + + if (typeof k !== "string") { + return; + } + + if (!messageInKinds.includes(k as MessageInKind)) { + return; + } + + const kind = k as MessageInKind; + + switch (kind) { + case "ask-for-clipboard": { + return data as MessageInAskForClipboard; + } + case "copied-text": { + if ("text" in data && typeof data.text === "string") { + return { + kind: "copied-text", + text: data.text, + }; + } + break; + } + default: { + const _exhaustive: never = kind; + return _exhaustive; + } + } + }; + + const handleMessage = (data: unknown, ws: WebSocket | null) => { + const message = getMessage(data); + + if (!message) { + console.warn("Unknown message received on message channel:", data); + return; + } + + const kind = message.kind; + + const respond = (message: MessageOut) => { + if (!ws) { + console.warn("Cannot send message, as message socket is null."); + console.warn(message); + return; + } + + ws.send(JSON.stringify(message)); + }; + + switch (kind) { + case "ask-for-clipboard": { + if (!navigator.clipboard) { + console.warn("Clipboard API not available."); + return; + } + + navigator.clipboard + .readText() + .then((text) => { + toast({ + title: "Pasting Into Browser", + description: + "Pasting your current clipboard text into the web page. NOTE: copy-paste only works in the web page - not in the browser (like the address bar).", + }); + + const response: MessageOutAskForClipboardResponse = { + kind: "ask-for-clipboard-response", + text, + }; + + respond(response); + }) + .catch((err) => { + console.error("Failed to read clipboard contents: ", err); + }); + + break; + } + case "copied-text": { + const text = message.text; + + navigator.clipboard + .writeText(text) + .then(() => { + toast({ + title: "Copied to Clipboard", + description: + "The text has been copied to your clipboard. NOTE: copy-paste only works in the web page - not in the browser (like the address bar).", + }); + }) + .catch((err) => { + console.error("Failed to write to clipboard:", err); + + toast({ + variant: "destructive", + title: "Failed to write to Clipboard", + description: "The text could not be copied to your clipboard.", + }); + }); + + break; + } + default: { + const _exhaustive: never = kind; + return _exhaustive; + } + } + }; + const theUserIsControlling = userIsControlling || (interactive && !showControlButtons);