vodorod/Dorod-Sky
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

FE portion of seamless clipboard transfer in browser stream (#3799)

Browse Source
This commit is contained in:
Jonathan Dobson
2025-10-23 09:32:10 -04:00
committed by GitHub
parent 2852e1c7cf
commit f66f50fc93
1 changed files with 213 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

242
skyvern-frontend/src/components/BrowserStream.tsx
View File

@@ -39,8 +39,31 @@ interface CommandCedeControl {
kind: "cede-control";
}
// a "Command" is an fire-n-forget out-message - it does not require a response
type Command = CommandTakeControl | CommandCedeControl;
const messageInKinds = ["ask-for-clipboard", "copied-text"] as const;
type MessageInKind = (typeof messageInKinds)[number];
interface MessageInAskForClipboard {
kind: "ask-for-clipboard";
}
interface MessageInCopiedText {
kind: "copied-text";
text: string;
}
type MessageIn = MessageInCopiedText | MessageInAskForClipboard;
interface MessageOutAskForClipboardResponse {
kind: "ask-for-clipboard-response";
text: string;
}
type MessageOut = MessageOutAskForClipboardResponse;
type Props = {
browserSessionId?: string;
interactive?: boolean;
@@ -102,6 +125,20 @@ function BrowserStream({
if (!browserSession || browserSession.completed_at) {
setHasBrowserSession(false);
return false;
// NOTE(jdo:streaming-local-dev): remove above and use this instead
// if (browserSession && browserSession.completed_at) {
// console.warn(
// "Completed at:",
// browserSession.completed_at,
// "continuing anyway!",
// );
// setHasBrowserSession(true);
// return true;
// } else {
// setHasBrowserSession(false);
// return false;
// }
}
setHasBrowserSession(true);
@@ -117,16 +154,16 @@ function BrowserStream({
const [hasBrowserSession, setHasBrowserSession] = useState(true); // be optimistic
const [userIsControlling, setUserIsControlling] = useState(false);
const [commandSocket, setCommandSocket] = useState<WebSocket | null>(null);
const [messageSocket, setMessageSocket] = useState<WebSocket | null>(null);
const [vncDisconnectedTrigger, setVncDisconnectedTrigger] = useState(0);
const prevVncConnectedRef = useRef<boolean>(false);
const [isVncConnected, setIsVncConnected] = useState<boolean>(false);
const [isCanvasReady, setIsCanvasReady] = useState<boolean>(false);
const [isReady, setIsReady] = useState(false);
const [commandDisconnectedTrigger, setCommandDisconnectedTrigger] =
const [messagesDisconnectedTrigger, setMessagesDisconnectedTrigger] =
useState(0);
const prevCommandConnectedRef = useRef<boolean>(false);
const [isCommandConnected, setIsCommandConnected] = useState<boolean>(false);
const prevMessageConnectedRef = useRef<boolean>(false);
const [isMessageConnected, setIsMessageConnected] = useState<boolean>(false);
const [canvasContainer, setCanvasContainer] = useState<HTMLDivElement | null>(
null,
);
@@ -168,14 +205,14 @@ function BrowserStream({
prevVncConnectedRef.current = isVncConnected;
}, [isVncConnected, onClose]);
// effect for command disconnects only
// effect for message disconnects only
useEffect(() => {
if (prevCommandConnectedRef.current && !isCommandConnected) {
setCommandDisconnectedTrigger((x) => x + 1);
if (prevMessageConnectedRef.current && !isMessageConnected) {
setMessagesDisconnectedTrigger((x) => x + 1);
onClose?.();
}
prevCommandConnectedRef.current = isCommandConnected;
}, [isCommandConnected, onClose]);
prevMessageConnectedRef.current = isMessageConnected;
}, [isMessageConnected, onClose]);
// vnc socket
useEffect(
@@ -289,7 +326,6 @@ function BrowserStream({
],
);
// command socket
useEffect(() => {
if (!showStream || !canvasContainer || !runId) {
return;
@@ -300,34 +336,52 @@ function BrowserStream({
const connect = async () => {
const wsParams = await getWebSocketParams();
const commandUrl =
const messageUrl =
entity === "browserSession"
? `${newWssBaseUrl}/stream/commands/browser_session/${runId}?${wsParams}`
? `${newWssBaseUrl}/stream/messages/browser_session/${runId}?${wsParams}`
: entity === "task"
? `${wssBaseUrl}/stream/commands/task/${runId}?${wsParams}`
? `${wssBaseUrl}/stream/messages/task/${runId}?${wsParams}`
: entity === "workflow"
? `${wssBaseUrl}/stream/commands/workflow_run/${runId}?${wsParams}`
? `${wssBaseUrl}/stream/messages/workflow_run/${runId}?${wsParams}`
: null;
if (!commandUrl) {
throw new Error("No command url");
if (!messageUrl) {
throw new Error("No message url");
}
if (!hasBrowserSession) {
setIsCommandConnected(false);
setIsMessageConnected(false);
return;
}
ws = new WebSocket(commandUrl);
ws = new WebSocket(messageUrl);
ws.onopen = () => {
setIsCommandConnected(true);
setCommandSocket(ws);
setIsMessageConnected(true);
setMessageSocket(ws);
};
ws.onmessage = (event) => {
const data = event.data;
try {
const message = JSON.parse(data);
handleMessage(message, ws);
// handle incoming messages if needed
} catch (e) {
console.error(
"Error parsing message from message channel:",
e,
event,
);
}
};
ws.onclose = () => {
setIsCommandConnected(false);
setCommandSocket(null);
setIsMessageConnected(false);
setMessageSocket(null);
};
};
@@ -340,31 +394,33 @@ function BrowserStream({
// pass
}
};
// NOTE: adding getWebSocketParams causes constant reconnects of message channel when,
// for instance, take-control or cede-control is clicked
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [
browserSessionId,
canvasContainer,
commandDisconnectedTrigger,
messagesDisconnectedTrigger,
entity,
getWebSocketParams,
hasBrowserSession,
runId,
showStream,
]);
// effect to send a command when the user is controlling, vs not controlling
// effect to send a message when the user is controlling, vs not controlling
useEffect(() => {
if (!isCommandConnected) {
if (!isMessageConnected) {
return;
}
const sendCommand = (command: Command) => {
if (!commandSocket) {
if (!messageSocket) {
console.warn("Cannot send command, as command socket is closed.");
console.warn(command);
return;
}
commandSocket.send(JSON.stringify(command));
messageSocket.send(JSON.stringify(command));
};
if (interactive || userIsControlling) {
@@ -373,7 +429,7 @@ function BrowserStream({
sendCommand({ kind: "cede-control" });
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [interactive, isCommandConnected, userIsControlling]);
}, [interactive, isMessageConnected, userIsControlling]);
// Effect to handle window resize trigger for NoVNC canvas
useEffect(() => {
@@ -420,6 +476,134 @@ function BrowserStream({
}
}, [task, workflow]);
/**
* TODO(jdo): could use zod or smth similar
*/
const getMessage = (data: unknown): MessageIn | undefined => {
if (!data) {
return;
}
if (typeof data !== "object") {
return;
}
if (!("kind" in data)) {
return;
}
const k = data.kind;
if (typeof k !== "string") {
return;
}
if (!messageInKinds.includes(k as MessageInKind)) {
return;
}
const kind = k as MessageInKind;
switch (kind) {
case "ask-for-clipboard": {
return data as MessageInAskForClipboard;
}
case "copied-text": {
if ("text" in data && typeof data.text === "string") {
return {
kind: "copied-text",
text: data.text,
};
}
break;
}
default: {
const _exhaustive: never = kind;
return _exhaustive;
}
}
};
const handleMessage = (data: unknown, ws: WebSocket | null) => {
const message = getMessage(data);
if (!message) {
console.warn("Unknown message received on message channel:", data);
return;
}
const kind = message.kind;
const respond = (message: MessageOut) => {
if (!ws) {
console.warn("Cannot send message, as message socket is null.");
console.warn(message);
return;
}
ws.send(JSON.stringify(message));
};
switch (kind) {
case "ask-for-clipboard": {
if (!navigator.clipboard) {
console.warn("Clipboard API not available.");
return;
}
navigator.clipboard
.readText()
.then((text) => {
toast({
title: "Pasting Into Browser",
description:
"Pasting your current clipboard text into the web page. NOTE: copy-paste only works in the web page - not in the browser (like the address bar).",
});
const response: MessageOutAskForClipboardResponse = {
kind: "ask-for-clipboard-response",
text,
};
respond(response);
})
.catch((err) => {
console.error("Failed to read clipboard contents: ", err);
});
break;
}
case "copied-text": {
const text = message.text;
navigator.clipboard
.writeText(text)
.then(() => {
toast({
title: "Copied to Clipboard",
description:
"The text has been copied to your clipboard. NOTE: copy-paste only works in the web page - not in the browser (like the address bar).",
});
})
.catch((err) => {
console.error("Failed to write to clipboard:", err);
toast({
variant: "destructive",
title: "Failed to write to Clipboard",
description: "The text could not be copied to your clipboard.",
});
});
break;
}
default: {
const _exhaustive: never = kind;
return _exhaustive;
}
}
};
const theUserIsControlling =
userIsControlling || (interactive && !showControlButtons);