multiple UI fixes/updates (#3422)

skyvern-frontend/src/components/BrowserStream.tsx

@@ -343,9 +343,12 @@ function BrowserStream({
 
   return (
     <div
-      className={cn("browser-stream flex items-center justify-center", {
+      className={cn(
-        "user-is-controlling": theUserIsControlling,
+        "browser-stream relative flex items-center justify-center",
-      })}
+        {
+          "user-is-controlling": theUserIsControlling,
+        },
+      )}
       ref={setCanvasContainerRef}
     >
       {isVncConnected && (
@@ -384,7 +387,7 @@ function BrowserStream({
         </div>
       )}
       {!isVncConnected && (
-        <div className="flex aspect-video w-full flex-col items-center justify-center gap-2 rounded-md border border-slate-800 text-sm text-slate-400">
+        <div className="absolute left-0 top-1/2 flex aspect-video w-full -translate-y-1/2 flex-col items-center justify-center gap-2 rounded-md border border-slate-800 text-sm text-slate-400">
           <RotateThrough interval={7 * 1000}>
             <span>Hm, working on the connection...</span>
             <span>Hang tight, we're almost there...</span>

skyvern-frontend/src/components/Splitter.tsx

@@ -1,7 +1,7 @@
 import { useRef, useState, RefObject } from "react";
-import { useMountEffect } from "@/hooks/useMountEffect";
 import { cn } from "@/util/utils";
 import { useOnChange } from "@/hooks/useOnChange";
+import { useMountEffect } from "@/hooks/useMountEffect";
 
 function Handle({
   direction,
@@ -341,21 +341,30 @@ function Splitter({
   const [isDragging, setIsDragging] = useState(false);
 
   useMountEffect(() => {
-    if (containerRef.current) {
+    // small delay here, to allow for arbitrary layout thrashing to settle;
-      const newPosition = normalizeUnitsToPercent(
+    // otherwise we have to rely on an observer for the container size, and
-        containerRef,
+    // resetting whenever the container resizes it likely incorrect behaviour
-        direction,
+    setTimeout(() => {
-        firstSizingTarget,
+      if (containerRef.current) {
-        firstSizing,
+        const newPosition = normalizeUnitsToPercent(
-        storageKey,
+          containerRef,
-      );
+          direction,
+          firstSizingTarget,
+          firstSizing,
+          storageKey,
+        );
 
-      setSplitPosition(newPosition);
+        setSplitPosition(newPosition);
 
-      if (storageKey) {
+        if (storageKey) {
-        setStoredSizing(firstSizingTarget, storageKey, newPosition.toString());
+          setStoredSizing(
+            firstSizingTarget,
+            storageKey,
+            newPosition.toString(),
+          );
+        }
       }
-    }
+    }, 100);
   });
 
   useOnChange(isDragging, (newValue, oldValue) => {

skyvern-frontend/src/router.tsx

@@ -13,6 +13,7 @@ import { TaskDetails } from "./routes/tasks/detail/TaskDetails";
 import { TaskParameters } from "./routes/tasks/detail/TaskParameters";
 import { TaskRecording } from "./routes/tasks/detail/TaskRecording";
 import { TasksPage } from "./routes/tasks/list/TasksPage";
+import { Debugger } from "@/routes/workflows/debugger/Debugger";
 import { WorkflowPage } from "./routes/workflows/WorkflowPage";
 import { WorkflowRun } from "./routes/workflows/WorkflowRun";
 import { WorkflowRunParameters } from "./routes/workflows/WorkflowRunParameters";
@@ -110,11 +111,11 @@ const router = createBrowserRouter([
               },
               {
                 path: "debug",
-                element: <WorkflowEditor />,
+                element: <Debugger />,
               },
               {
                 path: ":workflowRunId/:blockLabel/debug",
-                element: <WorkflowEditor />,
+                element: <Debugger />,
               },
               {
                 path: "edit",

skyvern-frontend/src/routes/workflows/editor/Workspace.tsx

@@ -1,5 +1,5 @@
 import { AxiosError } from "axios";
-import { useEffect, useRef, useState, useMemo } from "react";
+import { useEffect, useRef, useState } from "react";
 import { nanoid } from "nanoid";
 import {
   ChevronRightIcon,
@@ -17,6 +17,7 @@ import { useCredentialGetter } from "@/hooks/useCredentialGetter";
 import { useMountEffect } from "@/hooks/useMountEffect";
 import { useDebugSessionQuery } from "../hooks/useDebugSessionQuery";
 import { useBlockScriptsQuery } from "@/routes/workflows/hooks/useBlockScriptsQuery";
+import { WorkflowRunStream } from "@/routes/workflows/workflowRun/WorkflowRunStream";
 import { useCacheKeyValuesQuery } from "../hooks/useCacheKeyValuesQuery";
 import { useBlockScriptStore } from "@/store/BlockScriptStore";
 import { useSidebarStore } from "@/store/SidebarStore";
@@ -137,12 +138,6 @@ function Workspace({
   const blockScriptStore = useBlockScriptStore();
   const cacheKey = workflow?.cache_key ?? "";
 
-  const enableDebugBrowser = useMemo(() => {
-    return (
-      showBrowser && (activeDebugSession?.vnc_streaming_supported ?? false)
-    );
-  }, [showBrowser, activeDebugSession?.vnc_streaming_supported]);
-
   const [cacheKeyValue, setCacheKeyValue] = useState(
     cacheKey === ""
       ? ""
@@ -215,10 +210,10 @@ function Workspace({
   const hasLoopBlock = nodes.some((node) => node.type === "loop");
   const hasHttpBlock = nodes.some((node) => node.type === "http_request");
   const workflowWidth = hasHttpBlock
-    ? "35.1rem"
+    ? "39rem"
     : hasLoopBlock
-      ? "31.25rem"
+      ? "34.25rem"
-      : "30rem";
+      : "34rem";
 
   /**
    * Open a new tab (not window) with the browser session URL.
@@ -705,7 +700,7 @@ function Workspace({
       </div>
 
       {/* infinite canvas and sub panels when not in debug mode */}
-      {!enableDebugBrowser && (
+      {!showBrowser && (
         <div className="relative flex h-full w-full overflow-hidden overflow-x-hidden">
           {/* infinite canvas */}
           <FlowRenderer
@@ -722,7 +717,7 @@ function Workspace({
           {/* sub panels */}
           {workflowPanelState.active && (
             <div
-              className="pointer-events-none absolute right-6 top-[5rem] z-30"
+              className="absolute right-6 top-[8.5rem] z-30"
               style={{
                 height:
                   workflowPanelState.content === "nodeLibrary"
@@ -750,12 +745,12 @@ function Workspace({
                 />
               )}
               {workflowPanelState.content === "parameters" && (
-                <div className="pointer-events-auto relative right-0 top-[3.5rem] z-30">
+                <div className="z-30">
                   <WorkflowParametersPanel />
                 </div>
               )}
               {workflowPanelState.content === "nodeLibrary" && (
-                <div className="pointer-events-auto relative right-0 top-[3.5rem] z-30 h-full w-[25rem]">
+                <div className="z-30 h-full w-[25rem]">
                   <WorkflowNodeLibraryPanel
                     onNodeClick={(props) => {
                       addNode(props);
@@ -768,8 +763,51 @@ function Workspace({
         </div>
       )}
 
-      {/* infinite canvas, sub panels, browser, and timeline when in debug mode */}
+      {/* sub panels when in debug mode */}
-      {enableDebugBrowser && (
+      {showBrowser && workflowPanelState.active && (
+        <div
+          className="absolute right-6 top-[8.5rem] z-30"
+          style={{
+            height:
+              workflowPanelState.content === "nodeLibrary"
+                ? "calc(100vh - 14rem)"
+                : "unset",
+          }}
+        >
+          {workflowPanelState.content === "cacheKeyValues" && (
+            <WorkflowCacheKeyValuesPanel
+              cacheKeyValues={cacheKeyValues}
+              pending={cacheKeyValuesLoading}
+              scriptKey={workflow.cache_key ?? "default"}
+              onDelete={(cacheKeyValue) => {
+                setToDeleteCacheKeyValue(cacheKeyValue);
+                setOpenConfirmCacheKeyValueDeleteDialogue(true);
+              }}
+              onPaginate={(page) => {
+                setPage(page);
+              }}
+              onSelect={(cacheKeyValue) => {
+                setCacheKeyValue(cacheKeyValue);
+                setCacheKeyValueFilter("");
+                closeWorkflowPanel();
+              }}
+            />
+          )}
+          {workflowPanelState.content === "parameters" && (
+            <WorkflowParametersPanel />
+          )}
+          {workflowPanelState.content === "nodeLibrary" && (
+            <WorkflowNodeLibraryPanel
+              onNodeClick={(props) => {
+                addNode(props);
+              }}
+            />
+          )}
+        </div>
+      )}
+
+      {/* infinite canvas, browser, and timeline when in debug mode */}
+      {showBrowser && (
         <div className="relative flex h-full w-full overflow-hidden overflow-x-hidden">
           <Splitter
             className="splittah"
@@ -795,96 +833,65 @@ function Workspace({
               />
             </div>
 
-            {/* browser & timeline & sub-panels in debug mode */}
+            {/* browser & timeline */}
             <div className="skyvern-split-right relative flex h-full items-end justify-center bg-[#020617] p-4 pl-6">
-              {/* sub panels */}
-              {workflowPanelState.active && (
-                <div
-                  className={cn("absolute right-6 top-[8.5rem] z-30", {
-                    "left-6": workflowPanelState.content === "nodeLibrary",
-                  })}
-                  style={{
-                    height:
-                      workflowPanelState.content === "nodeLibrary"
-                        ? "calc(100vh - 14rem)"
-                        : "unset",
-                  }}
-                >
-                  {workflowPanelState.content === "cacheKeyValues" && (
-                    <WorkflowCacheKeyValuesPanel
-                      cacheKeyValues={cacheKeyValues}
-                      pending={cacheKeyValuesLoading}
-                      scriptKey={workflow.cache_key ?? "default"}
-                      onDelete={(cacheKeyValue) => {
-                        setToDeleteCacheKeyValue(cacheKeyValue);
-                        setOpenConfirmCacheKeyValueDeleteDialogue(true);
-                      }}
-                      onPaginate={(page) => {
-                        setPage(page);
-                      }}
-                      onSelect={(cacheKeyValue) => {
-                        setCacheKeyValue(cacheKeyValue);
-                        setCacheKeyValueFilter("");
-                        closeWorkflowPanel();
-                      }}
-                    />
-                  )}
-                  {workflowPanelState.content === "parameters" && (
-                    <WorkflowParametersPanel />
-                  )}
-                  {workflowPanelState.content === "nodeLibrary" && (
-                    <WorkflowNodeLibraryPanel
-                      onNodeClick={(props) => {
-                        addNode(props);
-                      }}
-                    />
-                  )}
-                </div>
-              )}
-
-              {/* browser & timeline */}
               <div className="flex h-[calc(100%_-_8rem)] w-full gap-6">
-                {/* browser */}
+                {/* VNC browser */}
-                <div className="flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">
+                {!activeDebugSession ||
-                  <div key={reloadKey} className="w-full flex-1">
+                  (activeDebugSession.vnc_streaming_supported && (
-                    {activeDebugSession &&
+                    <div className="skyvern-vnc-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">
-                    activeDebugSession.browser_session_id &&
+                      <div key={reloadKey} className="w-full flex-1">
-                    !cycleBrowser.isPending ? (
+                        {activeDebugSession &&
-                      <BrowserStream
+                        activeDebugSession.browser_session_id &&
-                        interactive={interactor === "human"}
+                        !cycleBrowser.isPending ? (
-                        browserSessionId={activeDebugSession.browser_session_id}
+                          <BrowserStream
-                        showControlButtons={interactor === "human"}
+                            interactive={interactor === "human"}
-                        resizeTrigger={windowResizeTrigger}
+                            browserSessionId={
-                      />
+                              activeDebugSession.browser_session_id
-                    ) : (
+                            }
-                      <div className="flex aspect-video w-full flex-col items-center justify-center gap-2 rounded-md border border-slate-800 pb-2 pt-4 text-sm text-slate-400">
+                            showControlButtons={interactor === "human"}
-                        Connecting to your browser...
+                            resizeTrigger={windowResizeTrigger}
-                        <AnimatedWave text=".‧₊˚ ⋅ ✨★ ‧₊˚ ⋅" />
+                          />
+                        ) : (
+                          <div className="flex aspect-video w-full flex-col items-center justify-center gap-2 rounded-md border border-slate-800 pb-2 pt-4 text-sm text-slate-400">
+                            Connecting to your browser...
+                            <AnimatedWave text=".‧₊˚ ⋅ ✨★ ‧₊˚ ⋅" />
+                          </div>
+                        )}
                       </div>
-                    )}
+                      <footer className="flex h-[2rem] w-full items-center justify-start gap-4">
-                  </div>
+                        <div className="flex items-center gap-2">
-                  <footer className="flex h-[2rem] w-full items-center justify-start gap-4">
+                          <GlobeIcon /> Live Browser
-                    <div className="flex items-center gap-2">
+                        </div>
-                      <GlobeIcon /> Live Browser
+                        {showBreakoutButton && (
+                          <BreakoutButton onClick={() => breakout()} />
+                        )}
+                        <div
+                          className={cn("ml-auto flex items-center gap-2", {
+                            "mr-16": !blockLabel,
+                          })}
+                        >
+                          {showPowerButton && (
+                            <PowerButton onClick={() => cycle()} />
+                          )}
+                          <ReloadButton
+                            isReloading={isReloading}
+                            onClick={() => reload()}
+                          />
+                        </div>
+                      </footer>
                     </div>
-                    {showBreakoutButton && (
+                  ))}
-                      <BreakoutButton onClick={() => breakout()} />
+
-                    )}
+                {/* Screenshot browser} */}
-                    <div
+                {activeDebugSession &&
-                      className={cn("ml-auto flex items-center gap-2", {
+                  !activeDebugSession.vnc_streaming_supported && (
-                        "mr-16": !blockLabel,
+                    <div className="skyvern-screenshot-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">
-                      })}
+                      <div className="aspect-video w-full">
-                    >
+                        <WorkflowRunStream alwaysShowStream={true} />
-                      {showPowerButton && (
+                      </div>
-                        <PowerButton onClick={() => cycle()} />
-                      )}
-                      <ReloadButton
-                        isReloading={isReloading}
-                        onClick={() => reload()}
-                      />
                     </div>
-                  </footer>
+                  )}
-                </div>
 
                 {/* timeline */}
                 <div

skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsx

@@ -15,14 +15,20 @@ type StreamMessage = {
   screenshot?: string;
 };
 
+interface Props {
+  alwaysShowStream?: boolean;
+}
+
 let socket: WebSocket | null = null;
 
 const wssBaseUrl = import.meta.env.VITE_WSS_BASE_URL;
 
-function WorkflowRunStream() {
+function WorkflowRunStream(props?: Props) {
+  const alwaysShowStream = props?.alwaysShowStream ?? false;
   const { data: workflowRun } = useWorkflowRunQuery();
   const [streamImgSrc, setStreamImgSrc] = useState<string>("");
-  const showStream = workflowRun && statusIsNotFinalized(workflowRun);
+  const showStream =
+    alwaysShowStream || (workflowRun && statusIsNotFinalized(workflowRun));
   const credentialGetter = useCredentialGetter();
   const { workflowRunId, workflowPermanentId } = useParams();
   const queryClient = useQueryClient();
@@ -149,6 +155,26 @@ function WorkflowRunStream() {
       </div>
     );
   }
+
+  if (alwaysShowStream) {
+    if (streamImgSrc?.length > 0) {
+      return (
+        <div className="h-full w-full">
+          <ZoomableImage
+            src={`data:image/png;base64,${streamImgSrc}`}
+            className="rounded-md"
+          />
+        </div>
+      );
+    }
+
+    return (
+      <div className="flex h-full w-full items-center justify-center">
+        Waiting for stream...
+      </div>
+    );
+  }
+
   return null;
 }