Workflow Run Timeline UI (#1433)

This commit is contained in:
Shuchang Zheng
2024-12-23 23:44:47 -08:00
committed by GitHub
parent 682bc717f9
commit 517de67811
42 changed files with 1517 additions and 301 deletions

View File

@@ -0,0 +1,63 @@
import { ActionsApiResponse, ActionTypes, Status } from "@/api/types";
import { Separator } from "@/components/ui/separator";
import { ActionTypePill } from "@/routes/tasks/detail/ActionTypePill";
import { cn } from "@/util/utils";
import { CheckCircledIcon, CrossCircledIcon } from "@radix-ui/react-icons";
type Props = {
action: ActionsApiResponse;
index: number;
active: boolean;
onClick: () => void;
};
function ActionCard({ action, onClick, active, index }: Props) {
const success = action.status === Status.Completed;
return (
<div
className={cn(
"flex cursor-pointer rounded-lg border-2 bg-slate-elevation3 hover:border-slate-50",
{
"border-l-destructive": !success,
"border-l-success": success,
"border-slate-50": active,
},
)}
onClick={onClick}
>
<div className="flex-1 space-y-2 p-4 pl-5">
<div className="flex justify-between">
<div className="flex items-center gap-2">
<span>#{index}</span>
</div>
<div className="flex items-center gap-2">
<ActionTypePill actionType={action.action_type} />
{success ? (
<div className="flex gap-1 rounded-sm bg-slate-elevation5 px-2 py-1">
<CheckCircledIcon className="h-4 w-4 text-success" />
<span className="text-xs">Success</span>
</div>
) : (
<div className="flex gap-1 rounded-sm bg-slate-elevation5 px-2 py-1">
<CrossCircledIcon className="h-4 w-4 text-destructive" />
<span className="text-xs">Fail</span>
</div>
)}
</div>
</div>
<div className="text-xs text-slate-400">{action.reasoning}</div>
{action.action_type === ActionTypes.InputText && (
<>
<Separator />
<div className="text-xs text-slate-400">
Input: {action.response}
</div>
</>
)}
</div>
</div>
);
}
export { ActionCard };

View File

@@ -0,0 +1,41 @@
import { CubeIcon } from "@radix-ui/react-icons";
import { WorkflowRunBlock } from "../types/workflowRunTypes";
import { cn } from "@/util/utils";
import { WorkflowBlockIcon } from "../editor/nodes/WorkflowBlockIcon";
import { workflowBlockTitle } from "../editor/nodes/types";
type Props = {
active: boolean;
block: WorkflowRunBlock;
onClick: () => void;
};
function BlockCard({ block, onClick, active }: Props) {
return (
<div
className={cn(
"cursor-pointer space-y-3 rounded-md border bg-slate-elevation3 p-4 hover:border-slate-50",
{
"border-slate-50": active,
},
)}
onClick={onClick}
>
<div className="flex justify-between">
<div className="flex gap-3">
<WorkflowBlockIcon
workflowBlockType={block.block_type}
className="size-6"
/>
<span>{workflowBlockTitle[block.block_type]}</span>
</div>
<div className="flex items-center gap-1 rounded bg-slate-elevation5 px-2 py-1">
<CubeIcon className="size-4" />
<span className="text-xs">Block</span>
</div>
</div>
</div>
);
}
export { BlockCard };

View File

@@ -0,0 +1,76 @@
import { getClient } from "@/api/AxiosClient";
import { ArtifactApiResponse, ArtifactType, Status } from "@/api/types";
import { ZoomableImage } from "@/components/ZoomableImage";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { useQuery } from "@tanstack/react-query";
import { ReloadIcon } from "@radix-ui/react-icons";
import { statusIsNotFinalized } from "@/routes/tasks/types";
import { getImageURL } from "@/routes/tasks/detail/artifactUtils";
type Props = {
observerThoughtId: string;
taskStatus?: Status; // to give a hint that screenshot may not be available if task is not finalized
};
function ObserverThoughtScreenshot({ observerThoughtId, taskStatus }: Props) {
const credentialGetter = useCredentialGetter();
const { data: artifacts, isLoading } = useQuery<Array<ArtifactApiResponse>>({
queryKey: ["observerThought", observerThoughtId, "artifacts"],
queryFn: async () => {
const client = await getClient(credentialGetter);
return client
.get(`/observer_thought/${observerThoughtId}/artifacts`)
.then((response) => response.data);
},
refetchInterval: (query) => {
const data = query.state.data;
const screenshot = data?.filter(
(artifact) => artifact.artifact_type === ArtifactType.LLMScreenshot,
)?.[0];
if (!screenshot) {
return 5000;
}
return false;
},
});
const llmScreenshots = artifacts?.filter(
(artifact) => artifact.artifact_type === ArtifactType.LLMScreenshot,
);
const screenshot = llmScreenshots?.[0];
if (isLoading) {
return (
<div className="flex h-full items-center justify-center gap-2 bg-slate-elevation1">
<ReloadIcon className="h-6 w-6 animate-spin" />
<div>Loading screenshot...</div>
</div>
);
}
if (
!screenshot &&
taskStatus &&
statusIsNotFinalized({ status: taskStatus })
) {
return <div>The screenshot for this action is not available yet.</div>;
}
if (!screenshot) {
return (
<div className="flex h-full items-center justify-center bg-slate-elevation1">
No screenshot found for this action.
</div>
);
}
return (
<figure className="mx-auto flex max-w-full flex-col items-center gap-2 overflow-hidden rounded">
<ZoomableImage src={getImageURL(screenshot)} alt="llm-screenshot" />
</figure>
);
}
export { ObserverThoughtScreenshot };

View File

@@ -0,0 +1,36 @@
import { PersonIcon } from "@radix-ui/react-icons";
import { ObserverThought } from "../types/workflowRunTypes";
import { cn } from "@/util/utils";
type Props = {
active: boolean;
thought: ObserverThought;
onClick: (thought: ObserverThought) => void;
};
function ThoughtCard({ thought, onClick, active }: Props) {
return (
<div
className={cn(
"space-y-3 rounded-md border bg-slate-elevation3 p-4 hover:border-slate-50",
{
"border-slate-50": active,
},
)}
onClick={() => {
onClick(thought);
}}
>
<div className="flex justify-between">
<span>Thought</span>
<div className="flex items-center gap-1 bg-slate-elevation5">
<PersonIcon className="size-4" />
<span className="text-xs">Decision</span>
</div>
</div>
<div className="text-xs text-slate-400">{thought.answer}</div>
</div>
);
}
export { ThoughtCard };

View File

@@ -0,0 +1,171 @@
import { AspectRatio } from "@/components/ui/aspect-ratio";
import { useWorkflowRunQuery } from "../hooks/useWorkflowRunQuery";
import { WorkflowRunOverviewSkeleton } from "./WorkflowRunOverviewSkeleton";
import { useState } from "react";
import { statusIsNotFinalized } from "@/routes/tasks/types";
import { useWorkflowRunTimelineQuery } from "../hooks/useWorkflowRunTimelineQuery";
import { WorkflowRunStream } from "./WorkflowRunStream";
import { ActionScreenshot } from "@/routes/tasks/detail/ActionScreenshot";
import { WorkflowRunTimelineBlockItem } from "./WorkflowRunTimelineBlockItem";
import { ThoughtCard } from "./ThoughtCard";
import {
isAction,
isBlockItem,
isObserverThought,
isThoughtItem,
isWorkflowRunBlock,
ObserverThought,
WorkflowRunBlock,
} from "../types/workflowRunTypes";
import { ActionsApiResponse } from "@/api/types";
import { cn } from "@/util/utils";
import { DotFilledIcon } from "@radix-ui/react-icons";
import { WorkflowRunTimelineItemInfoSection } from "./WorkflowRunTimelineItemInfoSection";
import { ObserverThoughtScreenshot } from "./ObserverThoughtScreenshot";
import { ScrollArea, ScrollAreaViewport } from "@/components/ui/scroll-area";
export type WorkflowRunOverviewActiveElement =
| ActionsApiResponse
| ObserverThought
| WorkflowRunBlock
| "stream"
| null;
function WorkflowRunOverview() {
const [active, setActive] = useState<WorkflowRunOverviewActiveElement>(null);
const { data: workflowRun, isLoading: workflowRunIsLoading } =
useWorkflowRunQuery();
const { data: workflowRunTimeline, isLoading: workflowRunTimelineIsLoading } =
useWorkflowRunTimelineQuery();
if (workflowRunIsLoading || workflowRunTimelineIsLoading) {
return <WorkflowRunOverviewSkeleton />;
}
if (!workflowRun) {
return null;
}
if (typeof workflowRunTimeline === "undefined") {
return null;
}
const workflowRunIsNotFinalized = statusIsNotFinalized(workflowRun);
const timeline = workflowRunTimeline.slice().reverse();
function getActiveSelection(): WorkflowRunOverviewActiveElement {
if (active === null) {
if (workflowRunIsNotFinalized) {
return "stream";
}
if (timeline!.length > 0) {
const timelineItem = timeline![0];
if (isBlockItem(timelineItem)) {
if (
timelineItem.block.actions &&
timelineItem.block.actions.length > 0
) {
return timelineItem.block
.actions[0] as WorkflowRunOverviewActiveElement;
}
return timelineItem.block;
}
if (isThoughtItem(timelineItem)) {
return timelineItem.thought;
}
}
}
return active;
}
const selection = getActiveSelection();
return (
<div className="flex h-[42rem] gap-6">
<div className="w-2/3 space-y-4">
<AspectRatio ratio={16 / 9} className="overflow-y-hidden">
{selection === "stream" && <WorkflowRunStream />}
{selection !== "stream" && isAction(selection) && (
<ActionScreenshot
index={selection.action_order ?? 0}
stepId={selection.step_id ?? ""}
/>
)}
{isWorkflowRunBlock(selection) && (
<div className="flex h-full w-full items-center justify-center bg-slate-elevation1">
No screenshot found for this block
</div>
)}
{isObserverThought(selection) && (
<ObserverThoughtScreenshot
observerThoughtId={selection.observer_thought_id}
/>
)}
</AspectRatio>
<WorkflowRunTimelineItemInfoSection item={selection} />
</div>
<div className="w-1/3 min-w-0 rounded bg-slate-elevation1 p-4">
<ScrollArea>
<ScrollAreaViewport className="max-h-[42rem]">
<div className="space-y-4">
<div className="gap-2"></div>
{workflowRunIsNotFinalized && (
<div
key="stream"
className={cn(
"flex cursor-pointer rounded-lg border-2 bg-slate-elevation3 p-4 hover:border-slate-50",
{
"border-slate-50": selection === "stream",
},
)}
onClick={() => setActive("stream")}
>
<div className="flex items-center gap-2">
<DotFilledIcon className="h-6 w-6 text-destructive" />
Live
</div>
</div>
)}
{timeline.length === 0 && <div>Workflow timeline is empty</div>}
{timeline?.map((timelineItem) => {
if (isBlockItem(timelineItem)) {
return (
<WorkflowRunTimelineBlockItem
key={timelineItem.block.workflow_run_block_id}
subBlocks={timelineItem.children
.filter((item) => item.type === "block")
.map((item) => item.block)}
activeItem={selection}
block={timelineItem.block}
onActionClick={setActive}
onBlockItemClick={setActive}
/>
);
}
if (isThoughtItem(timelineItem)) {
return (
<ThoughtCard
key={timelineItem.thought.observer_thought_id}
active={
isObserverThought(selection) &&
selection.observer_thought_id ===
timelineItem.thought.observer_thought_id
}
onClick={setActive}
thought={timelineItem.thought}
/>
);
}
})}
</div>
</ScrollAreaViewport>
</ScrollArea>
</div>
</div>
);
}
export { WorkflowRunOverview };

View File

@@ -0,0 +1,22 @@
import { AspectRatio } from "@/components/ui/aspect-ratio";
import { Skeleton } from "@/components/ui/skeleton";
function WorkflowRunOverviewSkeleton() {
return (
<div className="flex h-[42rem] gap-6">
<div className="w-2/3 space-y-4">
<AspectRatio ratio={16 / 9}>
<Skeleton className="h-full w-full" />
</AspectRatio>
<div className="h-[10rem]">
<Skeleton className="h-full w-full" />
</div>
</div>
<div className="w-1/3">
<Skeleton className="h-full w-full" />
</div>
</div>
);
}
export { WorkflowRunOverviewSkeleton };

View File

@@ -0,0 +1,152 @@
import { Status } from "@/api/types";
import { useWorkflowRunQuery } from "../hooks/useWorkflowRunQuery";
import { ZoomableImage } from "@/components/ZoomableImage";
import { useEffect, useState } from "react";
import { statusIsNotFinalized } from "@/routes/tasks/types";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
import { useParams } from "react-router-dom";
import { envCredential } from "@/util/env";
import { toast } from "@/components/ui/use-toast";
import { useQueryClient } from "@tanstack/react-query";
type StreamMessage = {
task_id: string;
status: string;
screenshot?: string;
};
let socket: WebSocket | null = null;
const wssBaseUrl = import.meta.env.VITE_WSS_BASE_URL;
function WorkflowRunStream() {
const { data: workflowRun } = useWorkflowRunQuery();
const [streamImgSrc, setStreamImgSrc] = useState<string>("");
const showStream = workflowRun && statusIsNotFinalized(workflowRun);
const credentialGetter = useCredentialGetter();
const { workflowRunId, workflowPermanentId } = useParams();
const queryClient = useQueryClient();
useEffect(() => {
if (!showStream) {
return;
}
async function run() {
// Create WebSocket connection.
let credential = null;
if (credentialGetter) {
const token = await credentialGetter();
credential = `?token=Bearer ${token}`;
} else {
credential = `?apikey=${envCredential}`;
}
if (socket) {
socket.close();
}
socket = new WebSocket(
`${wssBaseUrl}/stream/workflow_runs/${workflowRunId}${credential}`,
);
// Listen for messages
socket.addEventListener("message", (event) => {
try {
const message: StreamMessage = JSON.parse(event.data);
if (message.screenshot) {
setStreamImgSrc(message.screenshot);
}
if (
message.status === "completed" ||
message.status === "failed" ||
message.status === "terminated"
) {
socket?.close();
queryClient.invalidateQueries({
queryKey: ["workflowRuns"],
});
queryClient.invalidateQueries({
queryKey: ["workflowRun", workflowPermanentId, workflowRunId],
});
queryClient.invalidateQueries({
queryKey: ["workflowTasks", workflowRunId],
});
if (
message.status === "failed" ||
message.status === "terminated"
) {
toast({
title: "Run Failed",
description: "The workflow run has failed.",
variant: "destructive",
});
} else if (message.status === "completed") {
toast({
title: "Run Completed",
description: "The workflow run has been completed.",
variant: "success",
});
}
}
} catch (e) {
console.error("Failed to parse message", e);
}
});
socket.addEventListener("close", () => {
socket = null;
});
}
run();
return () => {
if (socket) {
socket.close();
socket = null;
}
};
}, [
credentialGetter,
workflowRunId,
showStream,
queryClient,
workflowPermanentId,
]);
if (workflowRun?.status === Status.Created) {
return (
<div className="flex h-full w-full flex-col items-center justify-center gap-8 rounded-md bg-slate-900 py-8 text-lg">
<span>Workflow has been created.</span>
<span>Stream will start when the workflow is running.</span>
</div>
);
}
if (workflowRun?.status === Status.Queued) {
return (
<div className="flex h-full w-full flex-col items-center justify-center gap-8 rounded-md bg-slate-900 py-8 text-lg">
<span>Your workflow run is queued.</span>
<span>Stream will start when the workflow is running.</span>
</div>
);
}
if (workflowRun?.status === Status.Running && streamImgSrc.length === 0) {
return (
<div className="flex h-full w-full items-center justify-center rounded-md bg-slate-900 py-8 text-lg">
Starting the stream...
</div>
);
}
if (workflowRun?.status === Status.Running && streamImgSrc.length > 0) {
return (
<div className="h-full w-full">
<ZoomableImage
src={`data:image/png;base64,${streamImgSrc}`}
className="rounded-md"
/>
</div>
);
}
return null;
}
export { WorkflowRunStream };

View File

@@ -0,0 +1,70 @@
import { ActionsApiResponse } from "@/api/types";
import {
isAction,
isWorkflowRunBlock,
WorkflowRunBlock,
} from "../types/workflowRunTypes";
import { ActionCard } from "./ActionCard";
import { BlockCard } from "./BlockCard";
import { WorkflowRunOverviewActiveElement } from "./WorkflowRunOverview";
type Props = {
activeItem: WorkflowRunOverviewActiveElement;
block: WorkflowRunBlock;
subBlocks: Array<WorkflowRunBlock>;
onBlockItemClick: (block: WorkflowRunBlock) => void;
onActionClick: (action: ActionsApiResponse) => void;
};
function WorkflowRunTimelineBlockItem({
activeItem,
block,
subBlocks,
onBlockItemClick,
onActionClick,
}: Props) {
const actions = block.actions ? [...block.actions].reverse() : [];
return (
<div className="space-y-4 rounded border border-slate-600 p-4">
{actions.map((action, index) => {
return (
<ActionCard
key={action.action_id}
action={action}
active={
isAction(activeItem) && activeItem.action_id === action.action_id
}
index={actions.length - index}
onClick={() => {
onActionClick(action);
}}
/>
);
})}
{subBlocks.map((block) => {
return (
<WorkflowRunTimelineBlockItem
block={block}
activeItem={activeItem}
onActionClick={onActionClick}
onBlockItemClick={onBlockItemClick}
subBlocks={[]}
/>
);
})}
<BlockCard
active={
isWorkflowRunBlock(activeItem) &&
activeItem.workflow_run_block_id === block.workflow_run_block_id
}
block={block}
onClick={() => {
onBlockItemClick(block);
}}
/>
</div>
);
}
export { WorkflowRunTimelineBlockItem };

View File

@@ -0,0 +1,211 @@
import { ActionsApiResponse, Status } from "@/api/types";
import {
hasExtractedInformation,
isAction,
isObserverThought,
isWorkflowRunBlock,
ObserverThought,
WorkflowRunBlock,
} from "../types/workflowRunTypes";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { CodeEditor } from "../components/CodeEditor";
import { AutoResizingTextarea } from "@/components/AutoResizingTextarea/AutoResizingTextarea";
import { WorkflowBlockTypes } from "../types/workflowTypes";
import { statusIsAFailureType } from "@/routes/tasks/types";
import { SendEmailBlockInfo } from "./blockInfo/SendEmailBlockInfo";
type Props = {
item:
| ActionsApiResponse
| ObserverThought
| WorkflowRunBlock
| "stream"
| null;
};
function WorkflowRunTimelineItemInfoSection({ item }: Props) {
if (!item) {
return null;
}
if (item === "stream") {
return null;
}
if (isAction(item)) {
return null;
}
if (isObserverThought(item)) {
return (
<div className="rounded bg-slate-elevation1 p-4">
<Tabs key="thought" defaultValue="observation">
<TabsList>
<TabsTrigger value="observation">Observation</TabsTrigger>
<TabsTrigger value="thought">Thought</TabsTrigger>
<TabsTrigger value="answer">Answer</TabsTrigger>
</TabsList>
<TabsContent value="observation">
<AutoResizingTextarea value={item.observation ?? ""} readOnly />
</TabsContent>
<TabsContent value="thought">
<AutoResizingTextarea value={item.thought ?? ""} readOnly />
</TabsContent>
<TabsContent value="answer">
<AutoResizingTextarea value={item.answer ?? ""} readOnly />
</TabsContent>
</Tabs>
</div>
);
}
if (isWorkflowRunBlock(item)) {
if (
item.block_type === WorkflowBlockTypes.Task ||
item.block_type === WorkflowBlockTypes.Navigation ||
item.block_type === WorkflowBlockTypes.Action ||
item.block_type === WorkflowBlockTypes.Extraction ||
item.block_type === WorkflowBlockTypes.Validation ||
item.block_type === WorkflowBlockTypes.Login ||
item.block_type === WorkflowBlockTypes.FileDownload
) {
return (
<div className="rounded bg-slate-elevation1 p-4">
<Tabs key={item.block_type} defaultValue="navigation_goal">
<TabsList>
<TabsTrigger value="navigation_goal">Navigation Goal</TabsTrigger>
{item.status === Status.Completed && (
<TabsTrigger value="extracted_information">
Extracted Information
</TabsTrigger>
)}
{item.status && statusIsAFailureType({ status: item.status }) && (
<TabsTrigger value="failure_reason">Failure Reason</TabsTrigger>
)}
<TabsTrigger value="parameters">Parameters</TabsTrigger>
</TabsList>
<TabsContent value="navigation_goal">
<AutoResizingTextarea
value={item.navigation_goal ?? ""}
readOnly
/>
</TabsContent>
{item.status === Status.Completed && (
<TabsContent value="extracted_information">
<CodeEditor
language="json"
value={JSON.stringify(
(hasExtractedInformation(item.output) &&
item.output.extracted_information) ??
null,
null,
2,
)}
minHeight="96px"
maxHeight="500px"
readOnly
/>
</TabsContent>
)}
{item.status && statusIsAFailureType({ status: item.status }) && (
<TabsContent value="failure_reason">
<AutoResizingTextarea
value={
item.status === "canceled"
? "This block was cancelled"
: item.failure_reason ?? ""
}
readOnly
/>
</TabsContent>
)}
<TabsContent value="parameters">
<CodeEditor
value={JSON.stringify(item.navigation_payload, null, 2)}
minHeight="96px"
maxHeight="500px"
language="json"
readOnly
/>
</TabsContent>
</Tabs>
</div>
);
}
if (item.block_type === WorkflowBlockTypes.SendEmail) {
if (
item.body !== null &&
typeof item.body !== "undefined" &&
item.recipients !== null &&
typeof item.recipients !== "undefined"
) {
return (
<SendEmailBlockInfo body={item.body} recipients={item.recipients} />
);
}
return null;
}
if (item.block_type === WorkflowBlockTypes.TextPrompt) {
if (item.prompt !== null) {
return (
<div className="rounded bg-slate-elevation1 p-4">
<Tabs key={item.block_type} defaultValue="prompt">
<TabsList>
<TabsTrigger value="prompt">Prompt</TabsTrigger>
<TabsTrigger value="output">Output</TabsTrigger>
</TabsList>
<TabsContent value="prompt">
<CodeEditor
value={item.prompt ?? ""}
minHeight="96px"
maxHeight="500px"
readOnly
/>
</TabsContent>
<TabsContent value="output">
<CodeEditor
value={JSON.stringify(item.output, null, 2)}
minHeight="96px"
maxHeight="500px"
language="json"
readOnly
/>
</TabsContent>
</Tabs>
</div>
);
}
return null;
}
if (item.block_type === WorkflowBlockTypes.Wait) {
if (item.wait_sec !== null && typeof item.wait_sec !== "undefined") {
return (
<div className="flex w-1/2 justify-between rounded bg-slate-elevation1 p-4">
<span className="text-sm text-slate-400">Wait Time</span>
<span className="text-sm">{item.wait_sec} Seconds</span>
</div>
);
}
return null;
}
return (
<div className="rounded bg-slate-elevation1 p-4">
<Tabs key={item.block_type} defaultValue="output">
<TabsList>
<TabsTrigger value="output">Output</TabsTrigger>
</TabsList>
<TabsContent value="output">
<CodeEditor
value={JSON.stringify(item.output, null, 2)}
minHeight="96px"
maxHeight="500px"
language="json"
readOnly
/>
</TabsContent>
</Tabs>
</div>
);
}
}
export { WorkflowRunTimelineItemInfoSection };

View File

@@ -0,0 +1,29 @@
type Props = {
recipients: Array<string>;
body: string;
};
function SendEmailBlockInfo({ recipients, body }: Props) {
return (
<div className="flex gap-2">
<div className="w-1/2 space-y-4 p-4">
<div className="flex justify-between">
<span className="text-sm text-slate-400">From</span>
<span className="text-sm">hello@skyvern.com</span>
</div>
<div className="flex justify-between">
<span className="text-sm text-slate-400">To</span>
{recipients.map((recipient) => {
return <span className="text-sm">{recipient}</span>;
})}
</div>
</div>
<div className="w-1/2 space-y-4 p-4">
<span className="text-sm text-slate-400">Body</span>
<p className="text-sm">{body}</p>
</div>
</div>
);
}
export { SendEmailBlockInfo };