Enable browser recording (#4182)

skyvern-frontend/src/store/RecordedBlocksStore.ts

@@ -1,5 +1,6 @@
 import { create } from "zustand";
 import type { WorkflowBlock } from "@/routes/workflows/types/workflowTypes";
+import type { WorkflowParameter } from "@/routes/workflows/types/workflowTypes";
 
 type InsertionPoint = {
   previous: string | null;
@@ -10,12 +11,16 @@ type InsertionPoint = {
 
 type RecordedBlocksState = {
   blocks: Array<WorkflowBlock> | null;
+  parameters: Array<WorkflowParameter> | null;
   insertionPoint: InsertionPoint | null;
 };
 
 type RecordedBlocksStore = RecordedBlocksState & {
   setRecordedBlocks: (
-    blocks: Array<WorkflowBlock>,
+    data: {
+      blocks: Array<WorkflowBlock>;
+      parameters: Array<WorkflowParameter>;
+    },
     insertionPoint: InsertionPoint,
   ) => void;
   clearRecordedBlocks: () => void;
@@ -23,9 +28,10 @@ type RecordedBlocksStore = RecordedBlocksState & {
 
 const useRecordedBlocksStore = create<RecordedBlocksStore>((set) => ({
   blocks: null,
+  parameters: null,
   insertionPoint: null,
-  setRecordedBlocks: (blocks, insertionPoint) => {
-    set({ blocks, insertionPoint });
+  setRecordedBlocks: ({ blocks, parameters }, insertionPoint) => {
+    set({ blocks, parameters, insertionPoint });
   },
   clearRecordedBlocks: () => {
     set({ blocks: null, insertionPoint: null });

skyvern-frontend/src/store/useRecordingStore.ts

@@ -81,6 +81,7 @@ export interface MessageInExfiltratedCdpEvent {
   event_name: string;
   params: ExfiltratedEventCdpParams;
   source: "cdp";
+  timestamp: number;
 }
 
 export interface MessageInExfiltratedConsoleEvent {
@@ -88,6 +89,7 @@ export interface MessageInExfiltratedConsoleEvent {
   event_name: string;
   params: ExfiltratedEventConsoleParams;
   source: "console";
+  timestamp: number;
 }
 
 export type MessageInExfiltratedEvent =
@@ -105,6 +107,11 @@ interface RecordingStore {
    * Each chunk contains up to CHUNK_SIZE events.
    */
   compressedChunks: string[];
+  /**
+   * The number of events to show the user. This elides noisy events, like
+   * `mousemove`.
+   */
+  exposedEventCount: number;
   /**
    * Buffer of events not yet compressed into a chunk.
    */
@@ -194,8 +201,25 @@ async function compressEventsToB64(jsonString: string): Promise<string> {
   return btoa(binary);
 }
 
+const isExposedEvent = (event: MessageInExfiltratedEvent): boolean => {
+  const exposedConsoleEventTypes = new Set(["focus", "click", "keypress"]);
+
+  if (event.source === "console") {
+    if (exposedConsoleEventTypes.has(event.params.type)) {
+      return true;
+    }
+  }
+
+  if (event.source === "cdp") {
+    return true;
+  }
+
+  return false;
+};
+
 export const useRecordingStore = create<RecordingStore>((set, get) => ({
   compressedChunks: [],
+  exposedEventCount: 0,
   pendingEvents: [],
   isCompressing: false,
   isRecording: false,
@@ -204,6 +228,10 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({
     const state = get();
     const newPendingEvents = [...state.pendingEvents, event];
 
+    if (isExposedEvent(event)) {
+      set({ exposedEventCount: state.exposedEventCount + 1 });
+    }
+
     if (newPendingEvents.length >= CHUNK_SIZE && !state.isCompressing) {
       const eventsToCompress = newPendingEvents.slice(0, CHUNK_SIZE);
       const remainingEvents = newPendingEvents.slice(CHUNK_SIZE);
@@ -241,6 +269,7 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({
   reset: () =>
     set({
       compressedChunks: [],
+      exposedEventCount: 0,
       pendingEvents: [],
       isCompressing: false,
       isRecording: false,