FE portion of seamless clipboard transfer in browser stream (#3799)

skyvern-frontend/src/components/BrowserStream.tsx

@@ -39,8 +39,31 @@ interface CommandCedeControl {
   kind: "cede-control";
 }
 
+// a "Command" is an fire-n-forget out-message - it does not require a response
 type Command = CommandTakeControl | CommandCedeControl;
 
+const messageInKinds = ["ask-for-clipboard", "copied-text"] as const;
+
+type MessageInKind = (typeof messageInKinds)[number];
+
+interface MessageInAskForClipboard {
+  kind: "ask-for-clipboard";
+}
+
+interface MessageInCopiedText {
+  kind: "copied-text";
+  text: string;
+}
+
+type MessageIn = MessageInCopiedText | MessageInAskForClipboard;
+
+interface MessageOutAskForClipboardResponse {
+  kind: "ask-for-clipboard-response";
+  text: string;
+}
+
+type MessageOut = MessageOutAskForClipboardResponse;
+
 type Props = {
   browserSessionId?: string;
   interactive?: boolean;
@@ -102,6 +125,20 @@ function BrowserStream({
         if (!browserSession || browserSession.completed_at) {
           setHasBrowserSession(false);
           return false;
+
+          // NOTE(jdo:streaming-local-dev): remove above and use this instead
+          // if (browserSession && browserSession.completed_at) {
+          //   console.warn(
+          //     "Completed at:",
+          //     browserSession.completed_at,
+          //     "continuing anyway!",
+          //   );
+          //   setHasBrowserSession(true);
+          //   return true;
+          // } else {
+          //   setHasBrowserSession(false);
+          //   return false;
+          // }
         }
 
         setHasBrowserSession(true);
@@ -117,16 +154,16 @@ function BrowserStream({
 
   const [hasBrowserSession, setHasBrowserSession] = useState(true); // be optimistic
   const [userIsControlling, setUserIsControlling] = useState(false);
-  const [commandSocket, setCommandSocket] = useState<WebSocket | null>(null);
+  const [messageSocket, setMessageSocket] = useState<WebSocket | null>(null);
   const [vncDisconnectedTrigger, setVncDisconnectedTrigger] = useState(0);
   const prevVncConnectedRef = useRef<boolean>(false);
   const [isVncConnected, setIsVncConnected] = useState<boolean>(false);
   const [isCanvasReady, setIsCanvasReady] = useState<boolean>(false);
   const [isReady, setIsReady] = useState(false);
-  const [commandDisconnectedTrigger, setCommandDisconnectedTrigger] =
+  const [messagesDisconnectedTrigger, setMessagesDisconnectedTrigger] =
     useState(0);
-  const prevCommandConnectedRef = useRef<boolean>(false);
+  const prevMessageConnectedRef = useRef<boolean>(false);
-  const [isCommandConnected, setIsCommandConnected] = useState<boolean>(false);
+  const [isMessageConnected, setIsMessageConnected] = useState<boolean>(false);
   const [canvasContainer, setCanvasContainer] = useState<HTMLDivElement | null>(
     null,
   );
@@ -168,14 +205,14 @@ function BrowserStream({
     prevVncConnectedRef.current = isVncConnected;
   }, [isVncConnected, onClose]);
 
-  // effect for command disconnects only
+  // effect for message disconnects only
   useEffect(() => {
-    if (prevCommandConnectedRef.current && !isCommandConnected) {
+    if (prevMessageConnectedRef.current && !isMessageConnected) {
-      setCommandDisconnectedTrigger((x) => x + 1);
+      setMessagesDisconnectedTrigger((x) => x + 1);
       onClose?.();
     }
-    prevCommandConnectedRef.current = isCommandConnected;
+    prevMessageConnectedRef.current = isMessageConnected;
-  }, [isCommandConnected, onClose]);
+  }, [isMessageConnected, onClose]);
 
   // vnc socket
   useEffect(
@@ -289,7 +326,6 @@ function BrowserStream({
     ],
   );
 
-  // command socket
   useEffect(() => {
     if (!showStream || !canvasContainer || !runId) {
       return;
@@ -300,34 +336,52 @@ function BrowserStream({
     const connect = async () => {
       const wsParams = await getWebSocketParams();
 
-      const commandUrl =
+      const messageUrl =
         entity === "browserSession"
-          ? `${newWssBaseUrl}/stream/commands/browser_session/${runId}?${wsParams}`
+          ? `${newWssBaseUrl}/stream/messages/browser_session/${runId}?${wsParams}`
           : entity === "task"
-            ? `${wssBaseUrl}/stream/commands/task/${runId}?${wsParams}`
+            ? `${wssBaseUrl}/stream/messages/task/${runId}?${wsParams}`
             : entity === "workflow"
-              ? `${wssBaseUrl}/stream/commands/workflow_run/${runId}?${wsParams}`
+              ? `${wssBaseUrl}/stream/messages/workflow_run/${runId}?${wsParams}`
               : null;
 
-      if (!commandUrl) {
+      if (!messageUrl) {
-        throw new Error("No command url");
+        throw new Error("No message url");
       }
 
       if (!hasBrowserSession) {
-        setIsCommandConnected(false);
+        setIsMessageConnected(false);
         return;
       }
 
-      ws = new WebSocket(commandUrl);
+      ws = new WebSocket(messageUrl);
 
       ws.onopen = () => {
-        setIsCommandConnected(true);
+        setIsMessageConnected(true);
-        setCommandSocket(ws);
+        setMessageSocket(ws);
+      };
+
+      ws.onmessage = (event) => {
+        const data = event.data;
+
+        try {
+          const message = JSON.parse(data);
+
+          handleMessage(message, ws);
+
+          // handle incoming messages if needed
+        } catch (e) {
+          console.error(
+            "Error parsing message from message channel:",
+            e,
+            event,
+          );
+        }
       };
 
       ws.onclose = () => {
-        setIsCommandConnected(false);
+        setIsMessageConnected(false);
-        setCommandSocket(null);
+        setMessageSocket(null);
       };
     };
 
@@ -340,31 +394,33 @@ function BrowserStream({
         // pass
       }
     };
+    // NOTE: adding getWebSocketParams causes constant reconnects of message channel when,
+    // for instance, take-control or cede-control is clicked
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [
     browserSessionId,
     canvasContainer,
-    commandDisconnectedTrigger,
+    messagesDisconnectedTrigger,
     entity,
-    getWebSocketParams,
     hasBrowserSession,
     runId,
     showStream,
   ]);
 
-  // effect to send a command when the user is controlling, vs not controlling
+  // effect to send a message when the user is controlling, vs not controlling
   useEffect(() => {
-    if (!isCommandConnected) {
+    if (!isMessageConnected) {
       return;
     }
 
     const sendCommand = (command: Command) => {
-      if (!commandSocket) {
+      if (!messageSocket) {
         console.warn("Cannot send command, as command socket is closed.");
         console.warn(command);
         return;
       }
 
-      commandSocket.send(JSON.stringify(command));
+      messageSocket.send(JSON.stringify(command));
     };
 
     if (interactive || userIsControlling) {
@@ -373,7 +429,7 @@ function BrowserStream({
       sendCommand({ kind: "cede-control" });
     }
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [interactive, isCommandConnected, userIsControlling]);
+  }, [interactive, isMessageConnected, userIsControlling]);
 
   // Effect to handle window resize trigger for NoVNC canvas
   useEffect(() => {
@@ -420,6 +476,134 @@ function BrowserStream({
     }
   }, [task, workflow]);
 
+  /**
+   * TODO(jdo): could use zod or smth similar
+   */
+  const getMessage = (data: unknown): MessageIn | undefined => {
+    if (!data) {
+      return;
+    }
+
+    if (typeof data !== "object") {
+      return;
+    }
+
+    if (!("kind" in data)) {
+      return;
+    }
+
+    const k = data.kind;
+
+    if (typeof k !== "string") {
+      return;
+    }
+
+    if (!messageInKinds.includes(k as MessageInKind)) {
+      return;
+    }
+
+    const kind = k as MessageInKind;
+
+    switch (kind) {
+      case "ask-for-clipboard": {
+        return data as MessageInAskForClipboard;
+      }
+      case "copied-text": {
+        if ("text" in data && typeof data.text === "string") {
+          return {
+            kind: "copied-text",
+            text: data.text,
+          };
+        }
+        break;
+      }
+      default: {
+        const _exhaustive: never = kind;
+        return _exhaustive;
+      }
+    }
+  };
+
+  const handleMessage = (data: unknown, ws: WebSocket | null) => {
+    const message = getMessage(data);
+
+    if (!message) {
+      console.warn("Unknown message received on message channel:", data);
+      return;
+    }
+
+    const kind = message.kind;
+
+    const respond = (message: MessageOut) => {
+      if (!ws) {
+        console.warn("Cannot send message, as message socket is null.");
+        console.warn(message);
+        return;
+      }
+
+      ws.send(JSON.stringify(message));
+    };
+
+    switch (kind) {
+      case "ask-for-clipboard": {
+        if (!navigator.clipboard) {
+          console.warn("Clipboard API not available.");
+          return;
+        }
+
+        navigator.clipboard
+          .readText()
+          .then((text) => {
+            toast({
+              title: "Pasting Into Browser",
+              description:
+                "Pasting your current clipboard text into the web page. NOTE: copy-paste only works in the web page - not in the browser (like the address bar).",
+            });
+
+            const response: MessageOutAskForClipboardResponse = {
+              kind: "ask-for-clipboard-response",
+              text,
+            };
+
+            respond(response);
+          })
+          .catch((err) => {
+            console.error("Failed to read clipboard contents: ", err);
+          });
+
+        break;
+      }
+      case "copied-text": {
+        const text = message.text;
+
+        navigator.clipboard
+          .writeText(text)
+          .then(() => {
+            toast({
+              title: "Copied to Clipboard",
+              description:
+                "The text has been copied to your clipboard. NOTE: copy-paste only works in the web page - not in the browser (like the address bar).",
+            });
+          })
+          .catch((err) => {
+            console.error("Failed to write to clipboard:", err);
+
+            toast({
+              variant: "destructive",
+              title: "Failed to write to Clipboard",
+              description: "The text could not be copied to your clipboard.",
+            });
+          });
+
+        break;
+      }
+      default: {
+        const _exhaustive: never = kind;
+        return _exhaustive;
+      }
+    }
+  };
+
   const theUserIsControlling =
     userIsControlling || (interactive && !showControlButtons);