multiple UI fixes/updates (#3422)

2025-09-12 16:13:05 -04:00
parent 012aec0cd5
commit d82eba77b6
5 changed files with 168 additions and 122 deletions
--- a/skyvern-frontend/src/components/BrowserStream.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.tsx
@@ -343,9 +343,12 @@ function BrowserStream({

  return (
    <div
-      className={cn("browser-stream flex items-center justify-center", {
-        "user-is-controlling": theUserIsControlling,
-      })}
+      className={cn(
+        "browser-stream relative flex items-center justify-center",
+        {
+          "user-is-controlling": theUserIsControlling,
+        },
+      )}
      ref={setCanvasContainerRef}
    >
      {isVncConnected && (
@@ -384,7 +387,7 @@ function BrowserStream({
        </div>
      )}
      {!isVncConnected && (
-        <div className="flex aspect-video w-full flex-col items-center justify-center gap-2 rounded-md border border-slate-800 text-sm text-slate-400">
+        <div className="absolute left-0 top-1/2 flex aspect-video w-full -translate-y-1/2 flex-col items-center justify-center gap-2 rounded-md border border-slate-800 text-sm text-slate-400">
          <RotateThrough interval={7 * 1000}>
            <span>Hm, working on the connection...</span>
            <span>Hang tight, we're almost there...</span>
--- a/skyvern-frontend/src/components/Splitter.tsx
+++ b/skyvern-frontend/src/components/Splitter.tsx
@@ -1,7 +1,7 @@
 import { useRef, useState, RefObject } from "react";
-import { useMountEffect } from "@/hooks/useMountEffect";
 import { cn } from "@/util/utils";
 import { useOnChange } from "@/hooks/useOnChange";
+import { useMountEffect } from "@/hooks/useMountEffect";

 function Handle({
  direction,
@@ -341,21 +341,30 @@ function Splitter({
  const [isDragging, setIsDragging] = useState(false);

  useMountEffect(() => {
-    if (containerRef.current) {
-      const newPosition = normalizeUnitsToPercent(
-        containerRef,
-        direction,
-        firstSizingTarget,
-        firstSizing,
-        storageKey,
-      );
+    // small delay here, to allow for arbitrary layout thrashing to settle;
+    // otherwise we have to rely on an observer for the container size, and
+    // resetting whenever the container resizes it likely incorrect behaviour
+    setTimeout(() => {
+      if (containerRef.current) {
+        const newPosition = normalizeUnitsToPercent(
+          containerRef,
+          direction,
+          firstSizingTarget,
+          firstSizing,
+          storageKey,
+        );

-      setSplitPosition(newPosition);
+        setSplitPosition(newPosition);

-      if (storageKey) {
-        setStoredSizing(firstSizingTarget, storageKey, newPosition.toString());
+        if (storageKey) {
+          setStoredSizing(
+            firstSizingTarget,
+            storageKey,
+            newPosition.toString(),
+          );
+        }
      }
-    }
+    }, 100);
  });

  useOnChange(isDragging, (newValue, oldValue) => {
--- a/skyvern-frontend/src/router.tsx
+++ b/skyvern-frontend/src/router.tsx
@@ -13,6 +13,7 @@ import { TaskDetails } from "./routes/tasks/detail/TaskDetails";
 import { TaskParameters } from "./routes/tasks/detail/TaskParameters";
 import { TaskRecording } from "./routes/tasks/detail/TaskRecording";
 import { TasksPage } from "./routes/tasks/list/TasksPage";
+import { Debugger } from "@/routes/workflows/debugger/Debugger";
 import { WorkflowPage } from "./routes/workflows/WorkflowPage";
 import { WorkflowRun } from "./routes/workflows/WorkflowRun";
 import { WorkflowRunParameters } from "./routes/workflows/WorkflowRunParameters";
@@ -110,11 +111,11 @@ const router = createBrowserRouter([
              },
              {
                path: "debug",
-                element: <WorkflowEditor />,
+                element: <Debugger />,
              },
              {
                path: ":workflowRunId/:blockLabel/debug",
-                element: <WorkflowEditor />,
+                element: <Debugger />,
              },
              {
                path: "edit",
--- a/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/Workspace.tsx
@@ -1,5 +1,5 @@
 import { AxiosError } from "axios";
-import { useEffect, useRef, useState, useMemo } from "react";
+import { useEffect, useRef, useState } from "react";
 import { nanoid } from "nanoid";
 import {
  ChevronRightIcon,
@@ -17,6 +17,7 @@ import { useCredentialGetter } from "@/hooks/useCredentialGetter";
 import { useMountEffect } from "@/hooks/useMountEffect";
 import { useDebugSessionQuery } from "../hooks/useDebugSessionQuery";
 import { useBlockScriptsQuery } from "@/routes/workflows/hooks/useBlockScriptsQuery";
+import { WorkflowRunStream } from "@/routes/workflows/workflowRun/WorkflowRunStream";
 import { useCacheKeyValuesQuery } from "../hooks/useCacheKeyValuesQuery";
 import { useBlockScriptStore } from "@/store/BlockScriptStore";
 import { useSidebarStore } from "@/store/SidebarStore";
@@ -137,12 +138,6 @@ function Workspace({
  const blockScriptStore = useBlockScriptStore();
  const cacheKey = workflow?.cache_key ?? "";

-  const enableDebugBrowser = useMemo(() => {
-    return (
-      showBrowser && (activeDebugSession?.vnc_streaming_supported ?? false)
-    );
-  }, [showBrowser, activeDebugSession?.vnc_streaming_supported]);
-
  const [cacheKeyValue, setCacheKeyValue] = useState(
    cacheKey === ""
      ? ""
@@ -215,10 +210,10 @@ function Workspace({
  const hasLoopBlock = nodes.some((node) => node.type === "loop");
  const hasHttpBlock = nodes.some((node) => node.type === "http_request");
  const workflowWidth = hasHttpBlock
-    ? "35.1rem"
+    ? "39rem"
    : hasLoopBlock
-      ? "31.25rem"
-      : "30rem";
+      ? "34.25rem"
+      : "34rem";

  /**
   * Open a new tab (not window) with the browser session URL.
@@ -705,7 +700,7 @@ function Workspace({
      </div>

      {/* infinite canvas and sub panels when not in debug mode */}
-      {!enableDebugBrowser && (
+      {!showBrowser && (
        <div className="relative flex h-full w-full overflow-hidden overflow-x-hidden">
          {/* infinite canvas */}
          <FlowRenderer
@@ -722,7 +717,7 @@ function Workspace({
          {/* sub panels */}
          {workflowPanelState.active && (
            <div
-              className="pointer-events-none absolute right-6 top-[5rem] z-30"
+              className="absolute right-6 top-[8.5rem] z-30"
              style={{
                height:
                  workflowPanelState.content === "nodeLibrary"
@@ -750,12 +745,12 @@ function Workspace({
                />
              )}
              {workflowPanelState.content === "parameters" && (
-                <div className="pointer-events-auto relative right-0 top-[3.5rem] z-30">
+                <div className="z-30">
                  <WorkflowParametersPanel />
                </div>
              )}
              {workflowPanelState.content === "nodeLibrary" && (
-                <div className="pointer-events-auto relative right-0 top-[3.5rem] z-30 h-full w-[25rem]">
+                <div className="z-30 h-full w-[25rem]">
                  <WorkflowNodeLibraryPanel
                    onNodeClick={(props) => {
                      addNode(props);
@@ -768,8 +763,51 @@ function Workspace({
        </div>
      )}

-      {/* infinite canvas, sub panels, browser, and timeline when in debug mode */}
-      {enableDebugBrowser && (
+      {/* sub panels when in debug mode */}
+      {showBrowser && workflowPanelState.active && (
+        <div
+          className="absolute right-6 top-[8.5rem] z-30"
+          style={{
+            height:
+              workflowPanelState.content === "nodeLibrary"
+                ? "calc(100vh - 14rem)"
+                : "unset",
+          }}
+        >
+          {workflowPanelState.content === "cacheKeyValues" && (
+            <WorkflowCacheKeyValuesPanel
+              cacheKeyValues={cacheKeyValues}
+              pending={cacheKeyValuesLoading}
+              scriptKey={workflow.cache_key ?? "default"}
+              onDelete={(cacheKeyValue) => {
+                setToDeleteCacheKeyValue(cacheKeyValue);
+                setOpenConfirmCacheKeyValueDeleteDialogue(true);
+              }}
+              onPaginate={(page) => {
+                setPage(page);
+              }}
+              onSelect={(cacheKeyValue) => {
+                setCacheKeyValue(cacheKeyValue);
+                setCacheKeyValueFilter("");
+                closeWorkflowPanel();
+              }}
+            />
+          )}
+          {workflowPanelState.content === "parameters" && (
+            <WorkflowParametersPanel />
+          )}
+          {workflowPanelState.content === "nodeLibrary" && (
+            <WorkflowNodeLibraryPanel
+              onNodeClick={(props) => {
+                addNode(props);
+              }}
+            />
+          )}
+        </div>
+      )}
+
+      {/* infinite canvas, browser, and timeline when in debug mode */}
+      {showBrowser && (
        <div className="relative flex h-full w-full overflow-hidden overflow-x-hidden">
          <Splitter
            className="splittah"
@@ -795,96 +833,65 @@ function Workspace({
              />
            </div>

-            {/* browser & timeline & sub-panels in debug mode */}
+            {/* browser & timeline */}
            <div className="skyvern-split-right relative flex h-full items-end justify-center bg-[#020617] p-4 pl-6">
-              {/* sub panels */}
-              {workflowPanelState.active && (
-                <div
-                  className={cn("absolute right-6 top-[8.5rem] z-30", {
-                    "left-6": workflowPanelState.content === "nodeLibrary",
-                  })}
-                  style={{
-                    height:
-                      workflowPanelState.content === "nodeLibrary"
-                        ? "calc(100vh - 14rem)"
-                        : "unset",
-                  }}
-                >
-                  {workflowPanelState.content === "cacheKeyValues" && (
-                    <WorkflowCacheKeyValuesPanel
-                      cacheKeyValues={cacheKeyValues}
-                      pending={cacheKeyValuesLoading}
-                      scriptKey={workflow.cache_key ?? "default"}
-                      onDelete={(cacheKeyValue) => {
-                        setToDeleteCacheKeyValue(cacheKeyValue);
-                        setOpenConfirmCacheKeyValueDeleteDialogue(true);
-                      }}
-                      onPaginate={(page) => {
-                        setPage(page);
-                      }}
-                      onSelect={(cacheKeyValue) => {
-                        setCacheKeyValue(cacheKeyValue);
-                        setCacheKeyValueFilter("");
-                        closeWorkflowPanel();
-                      }}
-                    />
-                  )}
-                  {workflowPanelState.content === "parameters" && (
-                    <WorkflowParametersPanel />
-                  )}
-                  {workflowPanelState.content === "nodeLibrary" && (
-                    <WorkflowNodeLibraryPanel
-                      onNodeClick={(props) => {
-                        addNode(props);
-                      }}
-                    />
-                  )}
-                </div>
-              )}
-
-              {/* browser & timeline */}
              <div className="flex h-[calc(100%_-_8rem)] w-full gap-6">
-                {/* browser */}
-                <div className="flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">
-                  <div key={reloadKey} className="w-full flex-1">
-                    {activeDebugSession &&
-                    activeDebugSession.browser_session_id &&
-                    !cycleBrowser.isPending ? (
-                      <BrowserStream
-                        interactive={interactor === "human"}
-                        browserSessionId={activeDebugSession.browser_session_id}
-                        showControlButtons={interactor === "human"}
-                        resizeTrigger={windowResizeTrigger}
-                      />
-                    ) : (
-                      <div className="flex aspect-video w-full flex-col items-center justify-center gap-2 rounded-md border border-slate-800 pb-2 pt-4 text-sm text-slate-400">
-                        Connecting to your browser...
-                        <AnimatedWave text=".‧₊˚ ⋅ ✨★ ‧₊˚ ⋅" />
+                {/* VNC browser */}
+                {!activeDebugSession ||
+                  (activeDebugSession.vnc_streaming_supported && (
+                    <div className="skyvern-vnc-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">
+                      <div key={reloadKey} className="w-full flex-1">
+                        {activeDebugSession &&
+                        activeDebugSession.browser_session_id &&
+                        !cycleBrowser.isPending ? (
+                          <BrowserStream
+                            interactive={interactor === "human"}
+                            browserSessionId={
+                              activeDebugSession.browser_session_id
+                            }
+                            showControlButtons={interactor === "human"}
+                            resizeTrigger={windowResizeTrigger}
+                          />
+                        ) : (
+                          <div className="flex aspect-video w-full flex-col items-center justify-center gap-2 rounded-md border border-slate-800 pb-2 pt-4 text-sm text-slate-400">
+                            Connecting to your browser...
+                            <AnimatedWave text=".‧₊˚ ⋅ ✨★ ‧₊˚ ⋅" />
+                          </div>
+                        )}
                      </div>
-                    )}
-                  </div>
-                  <footer className="flex h-[2rem] w-full items-center justify-start gap-4">
-                    <div className="flex items-center gap-2">
-                      <GlobeIcon /> Live Browser
+                      <footer className="flex h-[2rem] w-full items-center justify-start gap-4">
+                        <div className="flex items-center gap-2">
+                          <GlobeIcon /> Live Browser
+                        </div>
+                        {showBreakoutButton && (
+                          <BreakoutButton onClick={() => breakout()} />
+                        )}
+                        <div
+                          className={cn("ml-auto flex items-center gap-2", {
+                            "mr-16": !blockLabel,
+                          })}
+                        >
+                          {showPowerButton && (
+                            <PowerButton onClick={() => cycle()} />
+                          )}
+                          <ReloadButton
+                            isReloading={isReloading}
+                            onClick={() => reload()}
+                          />
+                        </div>
+                      </footer>
                    </div>
-                    {showBreakoutButton && (
-                      <BreakoutButton onClick={() => breakout()} />
-                    )}
-                    <div
-                      className={cn("ml-auto flex items-center gap-2", {
-                        "mr-16": !blockLabel,
-                      })}
-                    >
-                      {showPowerButton && (
-                        <PowerButton onClick={() => cycle()} />
-                      )}
-                      <ReloadButton
-                        isReloading={isReloading}
-                        onClick={() => reload()}
-                      />
+                  ))}
+
+                {/* Screenshot browser} */}
+                {activeDebugSession &&
+                  !activeDebugSession.vnc_streaming_supported && (
+                    <div className="skyvern-screenshot-browser flex h-full w-[calc(100%_-_6rem)] flex-1 flex-col items-center justify-center">
+                      <div className="aspect-video w-full">
+                        <WorkflowRunStream alwaysShowStream={true} />
+                      </div>
                    </div>
-                  </footer>
-                </div>
+                  )}

                {/* timeline */}
                <div
--- a/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsx
+++ b/skyvern-frontend/src/routes/workflows/workflowRun/WorkflowRunStream.tsx
@@ -15,14 +15,20 @@ type StreamMessage = {
  screenshot?: string;
 };

+interface Props {
+  alwaysShowStream?: boolean;
+}
+
 let socket: WebSocket | null = null;

 const wssBaseUrl = import.meta.env.VITE_WSS_BASE_URL;

-function WorkflowRunStream() {
+function WorkflowRunStream(props?: Props) {
+  const alwaysShowStream = props?.alwaysShowStream ?? false;
  const { data: workflowRun } = useWorkflowRunQuery();
  const [streamImgSrc, setStreamImgSrc] = useState<string>("");
-  const showStream = workflowRun && statusIsNotFinalized(workflowRun);
+  const showStream =
+    alwaysShowStream || (workflowRun && statusIsNotFinalized(workflowRun));
  const credentialGetter = useCredentialGetter();
  const { workflowRunId, workflowPermanentId } = useParams();
  const queryClient = useQueryClient();
@@ -149,6 +155,26 @@ function WorkflowRunStream() {
      </div>
    );
  }
+
+  if (alwaysShowStream) {
+    if (streamImgSrc?.length > 0) {
+      return (
+        <div className="h-full w-full">
+          <ZoomableImage
+            src={`data:image/png;base64,${streamImgSrc}`}
+            className="rounded-md"
+          />
+        </div>
+      );
+    }
+
+    return (
+      <div className="flex h-full w-full items-center justify-center">
+        Waiting for stream...
+      </div>
+    );
+  }
+
  return null;
 }