diff --git a/package.json b/package.json index 0f1e977d..666f9e00 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "moment-timezone": "^0.5.45", "node-cron": "^3.0.3", "pg": "^8.13.0", + "pg-boss": "^10.1.6", "pkce-challenge": "^4.1.0", "playwright": "^1.48.2", "playwright-extra": "^4.3.6", diff --git a/perf/performance.ts b/perf/performance.ts index c50ef850..c499ecd2 100644 --- a/perf/performance.ts +++ b/perf/performance.ts @@ -18,7 +18,7 @@ export class FrontendPerformanceMonitor { }; this.lastFrameTime = performance.now(); this.frameCount = 0; - + // Start monitoring this.startMonitoring(); } @@ -93,6 +93,262 @@ export class FrontendPerformanceMonitor { } } +// Enhanced Performance Monitor with Memory Management +export class EnhancedPerformanceMonitor extends FrontendPerformanceMonitor { + private memoryWarningThreshold: number = 100000000; // 100MB + private memoryAlertCallback?: (usage: MemoryInfo) => void; + private frameTimeHistory: number[] = []; + private maxMetricsHistory: number = 100; // Keep only the last 100 readings + private isThrottled: boolean = false; + private rafHandle: number | null = null; + private memoryCheckInterval: NodeJS.Timeout | null = null; + + constructor(options?: { + memoryWarningThreshold?: number, + maxMetricsHistory?: number, + memoryAlertCallback?: (usage: MemoryInfo) => void + }) { + super(); + + if (options) { + if (options.memoryWarningThreshold) { + this.memoryWarningThreshold = options.memoryWarningThreshold; + } + + if (options.maxMetricsHistory) { + this.maxMetricsHistory = options.maxMetricsHistory; + } + + if (options.memoryAlertCallback) { + this.memoryAlertCallback = options.memoryAlertCallback; + } + } + + // Override the parent's monitoring with our enhanced version + this.startEnhancedMonitoring(); + } + + private startEnhancedMonitoring(): void { + // Stop any existing monitoring from parent class + if (this.rafHandle) { + cancelAnimationFrame(this.rafHandle); + } + + if (this.memoryCheckInterval) { + clearInterval(this.memoryCheckInterval); + } + + // Enhanced FPS monitoring with frame time tracking + let lastFrameTime = performance.now(); + let frameCount = 0; + let frameTimes: number[] = []; + + const measureFPS = () => { + const now = performance.now(); + const frameTime = now - lastFrameTime; + lastFrameTime = now; + + // Track individual frame times for jank detection + frameTimes.push(frameTime); + if (frameTimes.length > 60) { // Keep only last 60 frame times + frameTimes.shift(); + } + + frameCount++; + this.frameTimeHistory.push(frameTime); + + // Calculate FPS every second + if (this.frameTimeHistory.length >= 60) { + const totalTime = this.frameTimeHistory.reduce((sum, time) => sum + time, 0); + const fps = Math.round((this.frameTimeHistory.length * 1000) / totalTime); + + // Get metrics from parent class + const metrics = this.getMetrics(); + metrics.fps.push(fps); + + // Limit metrics history + if (metrics.fps.length > this.maxMetricsHistory) { + metrics.fps.shift(); + } + + // Detect jank (long frames) + const jankThreshold = 16.7 * 2; // 2x normal frame time at 60fps + const jankFrames = frameTimes.filter(time => time > jankThreshold); + + if (jankFrames.length > 10) { // If more than 10 out of 60 frames are janky + this.detectPerformanceIssue('jank', { + jankFrames: jankFrames.length, + averageJankTime: jankFrames.reduce((sum, time) => sum + time, 0) / jankFrames.length + }); + } + + // Reset for next measurement + this.frameTimeHistory = []; + frameTimes = []; + } + + this.rafHandle = requestAnimationFrame(measureFPS); + }; + + this.rafHandle = requestAnimationFrame(measureFPS); + + // Enhanced memory monitoring + if (window.performance && (performance as any).memory) { + this.memoryCheckInterval = setInterval(() => { + const memory = (performance as any).memory; + const memoryInfo = { + usedJSHeapSize: memory.usedJSHeapSize, + totalJSHeapSize: memory.totalJSHeapSize, + timestamp: Date.now() + }; + + // Get metrics from parent class + const metrics = this.getMetrics(); + metrics.memoryUsage.push(memoryInfo); + + // Limit metrics history + if (metrics.memoryUsage.length > this.maxMetricsHistory) { + metrics.memoryUsage.shift(); + } + + // Check for memory warnings + if (memoryInfo.usedJSHeapSize > this.memoryWarningThreshold) { + this.detectPerformanceIssue('memory', memoryInfo); + + if (this.memoryAlertCallback) { + this.memoryAlertCallback(memoryInfo); + } + } + + // Check for memory leaks (steady increase) + if (metrics.memoryUsage.length >= 10) { + const recentMemory = metrics.memoryUsage.slice(-10); + let increasingCount = 0; + + for (let i = 1; i < recentMemory.length; i++) { + if (recentMemory[i].usedJSHeapSize > recentMemory[i - 1].usedJSHeapSize) { + increasingCount++; + } + } + + // If memory increased in 8 out of 9 consecutive readings + if (increasingCount >= 8) { + this.detectPerformanceIssue('memoryLeak', { + startMemory: recentMemory[0].usedJSHeapSize, + currentMemory: recentMemory[recentMemory.length - 1].usedJSHeapSize, + increaseRate: (recentMemory[recentMemory.length - 1].usedJSHeapSize - recentMemory[0].usedJSHeapSize) / + (recentMemory[recentMemory.length - 1].timestamp - recentMemory[0].timestamp) * 1000 // bytes per second + }); + } + } + }, 1000); + } + } + + // Method to detect various performance issues + private detectPerformanceIssue(type: 'jank' | 'memory' | 'memoryLeak', data: any): void { + console.warn(`Performance issue detected: ${type}`, data); + + if (type === 'memory' || type === 'memoryLeak') { + // Auto-throttle rendering if memory issues detected + if (!this.isThrottled) { + this.throttleRendering(); + } + + // Suggest garbage collection + this.suggestGarbageCollection(); + } + } + + // Get access to the metrics from parent class + private getMetrics(): any { + return (this as any).metrics; + } + + // Throttle rendering to reduce memory pressure + private throttleRendering(): void { + this.isThrottled = true; + console.info('Throttling rendering due to memory pressure'); + // Application code would implement throttling behavior + } + + // Un-throttle rendering when memory pressure is reduced + public unthrottleRendering(): void { + if (this.isThrottled) { + this.isThrottled = false; + console.info('Resuming normal rendering'); + } + } + + // Suggest garbage collection to the browser + private suggestGarbageCollection(): void { + if (window.gc) { + try { + window.gc(); + } catch (e) { + // gc() might not be available without special flags + } + } + + // Alternative approach to encourage garbage collection + const largeArray = new Array(1000000).fill(0); + largeArray.length = 0; + } + + // Enhanced performance report with more detailed metrics + public getEnhancedPerformanceReport(): EnhancedPerformanceReport { + const baseReport = super.getPerformanceReport(); + const metrics = this.getMetrics(); + + // Calculate 95th percentile render time + const sortedRenderTimes = [...metrics.renderTime].sort((a, b) => a - b); + const idx95 = Math.floor(sortedRenderTimes.length * 0.95); + const renderTime95Percentile = sortedRenderTimes[idx95] || 0; + + // Calculate memory growth rate + let memoryGrowthRate = 0; + if (metrics.memoryUsage.length >= 2) { + const first = metrics.memoryUsage[0]; + const last = metrics.memoryUsage[metrics.memoryUsage.length - 1]; + const timeDiffInSeconds = (last.timestamp - first.timestamp) / 1000; + memoryGrowthRate = timeDiffInSeconds > 0 + ? (last.usedJSHeapSize - first.usedJSHeapSize) / timeDiffInSeconds + : 0; + } + + return { + ...baseReport, + renderTime95Percentile, + memoryGrowthRate, + isThrottled: this.isThrottled, + heapUsagePercentage: baseReport.lastMemoryUsage + ? (baseReport.lastMemoryUsage.usedJSHeapSize / baseReport.lastMemoryUsage.totalJSHeapSize) * 100 + : 0 + }; + } + + // Clean up resources when no longer needed + public dispose(): void { + if (this.rafHandle) { + cancelAnimationFrame(this.rafHandle); + this.rafHandle = null; + } + + if (this.memoryCheckInterval) { + clearInterval(this.memoryCheckInterval); + this.memoryCheckInterval = null; + } + } +} + +// Extended types +interface EnhancedPerformanceReport extends PerformanceReport { + renderTime95Percentile: number; + memoryGrowthRate: number; // bytes per second + isThrottled: boolean; + heapUsagePercentage: number; +} + // Backend Performance Monitoring export class BackendPerformanceMonitor { private metrics: { diff --git a/public/locales/de.json b/public/locales/de.json index 3db0c64a..363aebbf 100644 --- a/public/locales/de.json +++ b/public/locales/de.json @@ -512,6 +512,7 @@ "success": "Erfolg", "running": "Läuft", "scheduled": "Geplant", + "queued": "In Warteschlange", "failed": "Fehlgeschlagen" }, "run_settings_modal": { diff --git a/public/locales/en.json b/public/locales/en.json index 4f7ebdc4..53ac4184 100644 --- a/public/locales/en.json +++ b/public/locales/en.json @@ -54,6 +54,12 @@ "label":"URL", "button":"Start Recording" }, + "warning_modal":{ + "title":"Active Browser Detected", + "message": "There is already a browser recording session running. Would you like to discard it and create a new recording?", + "discard_and_create":"Discard & Create New", + "cancel":"Cancel" + }, "edit":"Edit", "delete":"Delete", "duplicate":"Duplicate", @@ -514,6 +520,7 @@ "success": "Success", "running": "Running", "scheduled": "Scheduled", + "queued": "Queued", "failed": "Failed" }, "run_settings_modal": { diff --git a/public/locales/es.json b/public/locales/es.json index a13c69b6..14db7b5d 100644 --- a/public/locales/es.json +++ b/public/locales/es.json @@ -513,6 +513,7 @@ "success": "Éxito", "running": "Ejecutando", "scheduled": "Programado", + "queued": "En cola", "failed": "Fallido" }, "run_settings_modal": { diff --git a/public/locales/ja.json b/public/locales/ja.json index 1b38b933..54f7e788 100644 --- a/public/locales/ja.json +++ b/public/locales/ja.json @@ -513,6 +513,7 @@ "success": "成功", "running": "実行中", "scheduled": "スケジュール済み", + "queued": "キューに入れました", "failed": "失敗" }, "run_settings_modal": { diff --git a/public/locales/zh.json b/public/locales/zh.json index 5db9eed5..cc23556e 100644 --- a/public/locales/zh.json +++ b/public/locales/zh.json @@ -513,6 +513,7 @@ "success": "成功", "running": "运行中", "scheduled": "已计划", + "queued": "排队", "failed": "失败" }, "run_settings_modal": { diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 4bbc97af..c1f0f557 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -4,6 +4,12 @@ import logger from "../../logger"; /** * @category Types */ +/** + * Represents the possible states of a remote browser. + * @category Types + */ +type BrowserState = "recording" | "run"; + interface BrowserPoolInfo { /** * The instance of remote browser. @@ -19,6 +25,12 @@ interface BrowserPoolInfo { * The user ID that owns this browser instance. */ userId: string, + /** + * The current state of the browser. + * Can be "recording" or "run". + * @default "recording" + */ + state: BrowserState, } /** @@ -33,7 +45,7 @@ interface PoolDictionary { /** * A browser pool is a collection of remote browsers that are initialized and ready to be used. - * Enforces a "1 User - 1 Browser" policy, while allowing multiple users to have their own browser instances. + * Enforces a "1 User - 2 Browser" policy, while allowing multiple users to have their own browser instances. * Adds the possibility to add, remove and retrieve remote browsers from the pool. * @category BrowserManagement */ @@ -45,12 +57,13 @@ export class BrowserPool { /** * Maps user IDs to their browser IDs. + * A user can have up to 2 browsers. */ - private userToBrowserMap: Map = new Map(); + private userToBrowserMap: Map = new Map(); /** * Adds a remote browser instance to the pool for a specific user. - * If the user already has a browser, the existing browser will be closed and replaced. + * If the user already has two browsers, the oldest browser will be closed and replaced. * * @param id remote browser instance's id * @param browser remote browser instance @@ -62,38 +75,58 @@ export class BrowserPool { id: string, browser: RemoteBrowser, userId: string, - active: boolean = false + active: boolean = false, + state: BrowserState = "recording" ): boolean => { - // Check if user already has a browser - const existingBrowserId = this.userToBrowserMap.get(userId); + // Check if browser with this ID already exists and belongs to this user + if (this.pool[id] && this.pool[id].userId === userId) { + // Just update the existing browser + this.pool[id] = { + browser, + active, + userId, + state: this.pool[id].state || state, + }; + logger.log('debug', `Updated existing browser with id: ${id} for user: ${userId}`); + return false; + } + + // Get existing browsers for this user + let userBrowserIds = this.userToBrowserMap.get(userId) || []; let replaced = false; - if (existingBrowserId) { - // Close and remove the existing browser - if (existingBrowserId !== id) { - this.closeAndDeleteBrowser(existingBrowserId); - replaced = true; - } else { - // If it's the same browser ID, just update the info - this.pool[id] = { - browser, - active, - userId, - }; - logger.log('debug', `Updated existing browser with id: ${id} for user: ${userId}`); + // If trying to add a "recording" browser, check if one already exists + if (state === "recording") { + // Check if user already has a recording browser + const hasRecordingBrowser = userBrowserIds.some(browserId => + this.pool[browserId] && this.pool[browserId].state === "recording" + ); + + if (hasRecordingBrowser) { + logger.log('debug', `User ${userId} already has a browser in "recording" state`); return false; } } + + // For "run" state, check if the user already has the maximum number of browsers (2) + if (userBrowserIds.length >= 2 && !userBrowserIds.includes(id)) { + logger.log('debug', "User already has the maximum number of browsers (2)"); + return false; + } // Add the new browser to the pool this.pool[id] = { browser, active, userId, + state, }; // Update the user-to-browser mapping - this.userToBrowserMap.set(userId, id); + if (!userBrowserIds.includes(id)) { + userBrowserIds.push(id); + } + this.userToBrowserMap.set(userId, userBrowserIds); logger.log('debug', `Remote browser with id: ${id} added to the pool for user: ${userId}`); return !replaced; @@ -115,8 +148,16 @@ export class BrowserPool { // Remove the user-to-browser mapping const userId = this.pool[id].userId; - if (this.userToBrowserMap.get(userId) === id) { - this.userToBrowserMap.delete(userId); + const userBrowserIds = this.userToBrowserMap.get(userId) || []; + + if (userBrowserIds.includes(id)) { + const updatedBrowserIds = userBrowserIds.filter(bid => bid !== id); + + if (updatedBrowserIds.length === 0) { + this.userToBrowserMap.delete(userId); + } else { + this.userToBrowserMap.set(userId, updatedBrowserIds); + } } // Remove from pool @@ -139,8 +180,16 @@ export class BrowserPool { // Remove the user-to-browser mapping const userId = this.pool[id].userId; - if (this.userToBrowserMap.get(userId) === id) { - this.userToBrowserMap.delete(userId); + const userBrowserIds = this.userToBrowserMap.get(userId) || []; + + if (userBrowserIds.includes(id)) { + const updatedBrowserIds = userBrowserIds.filter(bid => bid !== id); + + if (updatedBrowserIds.length === 0) { + this.userToBrowserMap.delete(userId); + } else { + this.userToBrowserMap.set(userId, updatedBrowserIds); + } } // Remove from pool @@ -162,25 +211,73 @@ export class BrowserPool { /** * Returns the active browser's instance id for a specific user. + * If state is specified, only returns a browser with that exact state. * * @param userId the user ID to find the browser for - * @returns the browser ID for the user, or null if no browser exists + * @param state optional browser state filter ("recording" or "run") + * @returns the browser ID for the user, or null if no browser exists with the required state */ - public getActiveBrowserId = (userId: string): string | null => { - const browserId = this.userToBrowserMap.get(userId); - if (!browserId) { + public getActiveBrowserId = (userId: string, state?: BrowserState): string | null => { + const browserIds = this.userToBrowserMap.get(userId); + if (!browserIds || browserIds.length === 0) { logger.log('debug', `No browser found for user: ${userId}`); return null; } - // Verify the browser still exists in the pool - if (!this.pool[browserId]) { - this.userToBrowserMap.delete(userId); - logger.log('warn', `Browser mapping found for user: ${userId}, but browser doesn't exist in pool`); + // If state is specified, only return browsers with that exact state + if (state) { + // Check browsers in reverse order (newest first) to find one with the specified state + for (let i = browserIds.length - 1; i >= 0; i--) { + const browserId = browserIds[i]; + + // Verify the browser still exists in the pool + if (!this.pool[browserId]) { + browserIds.splice(i, 1); + continue; + } + + // Check if browser matches state filter + if (this.pool[browserId].state === state) { + return browserId; + } + } + + // If no browser with matching state, return null + logger.log('debug', `No browser with state ${state} found for user: ${userId}`); return null; } - console.log(`Browser Id ${browserId} found for user: ${userId}`); - return browserId; + + // If no state specified, return any browser + for (let i = browserIds.length - 1; i >= 0; i--) { + const browserId = browserIds[i]; + + // Verify the browser still exists in the pool + if (!this.pool[browserId]) { + browserIds.splice(i, 1); + continue; + } + + // Return the first browser found + if (this.pool[browserId]) { + console.log(`Active browser Id ${browserId} found for user: ${userId}`); + return browserId; + } + } + + // If no active browser, return the most recent one + if (browserIds.length > 0) { + const mostRecentId = browserIds[browserIds.length - 1]; + console.log(`No active browser found, returning most recent browser Id ${mostRecentId} for user: ${userId}`); + return mostRecentId; + } + + // Clean up the mapping if all browsers were invalid + if (browserIds.length === 0) { + this.userToBrowserMap.delete(userId); + } + + logger.log('warn', `Browser mapping found for user: ${userId}, but no valid browsers exist in pool`); + return null; }; /** @@ -213,10 +310,62 @@ export class BrowserPool { logger.log('debug', `Remote browser with id: ${id} set to ${active ? 'active' : 'inactive'}`); return true; }; + + /** + * Sets the state of a browser. + * Only allows one browser in "recording" state per user. + * + * @param id the browser ID + * @param state the new state ("recording" or "run") + * @returns true if successful, false if the browser wasn't found or state change not allowed + */ + public setBrowserState = (id: string, state: BrowserState): boolean => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return false; + } + + // If trying to set to "recording" state, check if another browser is already recording + if (state === "recording") { + const userId = this.pool[id].userId; + const userBrowserIds = this.userToBrowserMap.get(userId) || []; + + // Check if any other browser for this user is already in recording state + const hasAnotherRecordingBrowser = userBrowserIds.some(browserId => + browserId !== id && + this.pool[browserId] && + this.pool[browserId].state === "recording" + ); + + if (hasAnotherRecordingBrowser) { + logger.log('warn', `Cannot set browser ${id} to "recording" state: User ${userId} already has a browser in recording state`); + return false; + } + } + + this.pool[id].state = state; + logger.log('debug', `Remote browser with id: ${id} state set to ${state}`); + return true; + }; + + /** + * Gets the current state of a browser. + * + * @param id the browser ID + * @returns the current state or null if the browser wasn't found + */ + public getBrowserState = (id: string): BrowserState | null => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return null; + } + + return this.pool[id].state; + }; /** * Returns all browser instances for a specific user. - * Should only be one per the "1 User - 1 Browser" policy, but included for flexibility. + * With the "1 User - 2 Browser" policy, this can return up to 2 browsers. * * @param userId the user ID to find browsers for * @returns an array of browser IDs belonging to the user @@ -224,24 +373,31 @@ export class BrowserPool { public getAllBrowserIdsForUser = (userId: string): string[] => { const browserIds: string[] = []; - // Normally this would just return the one browser from the map - const mappedBrowserId = this.userToBrowserMap.get(userId); - if (mappedBrowserId && this.pool[mappedBrowserId]) { - browserIds.push(mappedBrowserId); + // Get browser IDs from the map + const mappedBrowserIds = this.userToBrowserMap.get(userId) || []; + + // Filter to only include IDs that exist in the pool + for (const id of mappedBrowserIds) { + if (this.pool[id]) { + browserIds.push(id); + } } - // But as a safeguard, also check the entire pool for any browsers assigned to this user + // As a safeguard, also check the entire pool for any browsers assigned to this user // This helps detect and fix any inconsistencies in the maps for (const [id, info] of Object.entries(this.pool)) { if (info.userId === userId && !browserIds.includes(id)) { browserIds.push(id); - // Fix the map if it's inconsistent - if (!mappedBrowserId) { - this.userToBrowserMap.set(userId, id); - } } } + // Update the map if inconsistencies were found + if (browserIds.length > 0 && JSON.stringify(browserIds) !== JSON.stringify(mappedBrowserIds)) { + // Limit to 2 browsers if more were found + const limitedBrowserIds = browserIds.slice(-2); + this.userToBrowserMap.set(userId, limitedBrowserIds); + } + return browserIds; }; @@ -264,24 +420,81 @@ export class BrowserPool { * This is a migration helper to support code that hasn't been updated to the user-browser model yet. * * @param currentUserId The ID of the current user, which will be prioritized if multiple browsers exist + * @param state Optional state filter to find browsers in a specific state * @returns A browser ID if one can be determined, or null */ - public getActiveBrowserForMigration = (currentUserId?: string): string | null => { + public getActiveBrowserForMigration = (currentUserId?: string, state?: BrowserState): string | null => { // If a current user ID is provided and they have a browser, return that if (currentUserId) { - const browserForUser = this.getActiveBrowserId(currentUserId); + const browserForUser = this.getActiveBrowserId(currentUserId, state); if (browserForUser) { return browserForUser; } + + // If state is specified and no matching browser was found, return null + if (state) { + return null; + } } - // If only one user has a browser, return that + // If only one user has a browser, try to find a matching browser if (this.userToBrowserMap.size === 1) { const userId = Array.from(this.userToBrowserMap.keys())[0]; - return this.userToBrowserMap.get(userId) || null; + const browserIds = this.userToBrowserMap.get(userId) || []; + + // If state is specified, only look for that state + if (state) { + // Return the active browser that matches the state + for (let i = browserIds.length - 1; i >= 0; i--) { + const bid = browserIds[i]; + if (this.pool[bid]?.active && this.pool[bid].state === state) { + return bid; + } + } + + // If no active browser with matching state, try to find any browser with matching state + for (let i = browserIds.length - 1; i >= 0; i--) { + const bid = browserIds[i]; + if (this.pool[bid] && this.pool[bid].state === state) { + return bid; + } + } + + // If still no matching browser, return null + return null; + } + + // If no state filter, find any active browser + for (let i = browserIds.length - 1; i >= 0; i--) { + if (this.pool[browserIds[i]]?.active) { + return browserIds[i]; + } + } + + return browserIds.length > 0 ? browserIds[browserIds.length - 1] : null; } - // Fall back to the first active browser if any + // Fall back to checking all browsers if no user was specified + if (state) { + // Look for active browsers with the specific state + for (const id of Object.keys(this.pool)) { + if (this.pool[id].active && this.pool[id].state === state) { + return id; + } + } + + // Then look for any browser with the specific state + for (const id of Object.keys(this.pool)) { + if (this.pool[id].state === state) { + return id; + } + } + + // If no browser with the requested state is found, return null + return null; + } + + // If no state filter, find any active browser for (const id of Object.keys(this.pool)) { if (this.pool[id].active) { return id; @@ -299,7 +512,7 @@ export class BrowserPool { * If there are multiple active browsers, it returns the first one. * * @returns the first remote active browser instance's id from the pool - * @deprecated Use getBrowserIdForUser instead to enforce the 1 User - 1 Browser policy + * @deprecated Use getBrowserIdForUser instead to enforce the 1 User - 2 Browser policy */ public getActiveBrowserIdLegacy = (): string | null => { for (const id of Object.keys(this.pool)) { diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index e6619d77..2cd9498d 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -21,12 +21,12 @@ import logger from "../logger"; * @category BrowserManagement-Controller */ export const initializeRemoteBrowserForRecording = (userId: string): string => { - const id = getActiveBrowserId(userId) || uuid(); + const id = getActiveBrowserIdByState(userId, "recording") || uuid(); createSocketConnection( io.of(id), async (socket: Socket) => { // browser is already active - const activeId = getActiveBrowserId(userId); + const activeId = getActiveBrowserIdByState(userId, "recording"); if (activeId) { const remoteBrowser = browserPool.getRemoteBrowser(activeId); remoteBrowser?.updateSocket(socket); @@ -37,7 +37,7 @@ export const initializeRemoteBrowserForRecording = (userId: string): string => { await browserSession.initialize(userId); await browserSession.registerEditorEvents(); await browserSession.subscribeToScreencast(); - browserPool.addRemoteBrowser(id, browserSession, userId); + browserPool.addRemoteBrowser(id, browserSession, userId, false, "recording"); } socket.emit('loaded'); }); @@ -59,7 +59,7 @@ export const createRemoteBrowserForRun = (userId: string): string => { async (socket: Socket) => { const browserSession = new RemoteBrowser(socket, userId); await browserSession.initialize(userId); - browserPool.addRemoteBrowser(id, browserSession, userId); + browserPool.addRemoteBrowser(id, browserSession, userId, false, "run"); socket.emit('ready-for-run'); }); return id; @@ -92,6 +92,17 @@ export const getActiveBrowserId = (userId: string): string | null => { return browserPool.getActiveBrowserId(userId); }; +/** + * Returns the id of an active browser with the specified state or null. + * @param userId the user ID to find the browser for + * @param state the browser state to filter by ("recording" or "run") + * @returns {string | null} + * @category BrowserManagement-Controller + */ +export const getActiveBrowserIdByState = (userId: string, state: "recording" | "run"): string | null => { + return browserPool.getActiveBrowserId(userId, state); +}; + /** * Returns the url string from a remote browser if exists in the browser pool. * @param id instance id of the remote browser @@ -127,7 +138,7 @@ export const getRemoteBrowserCurrentTabs = (id: string, userId: string): string[ * @category BrowserManagement-Controller */ export const interpretWholeWorkflow = async (userId: string) => { - const id = getActiveBrowserId(userId); + const id = getActiveBrowserIdByState(userId, "recording"); if (id) { const browser = browserPool.getRemoteBrowser(id); if (browser) { @@ -147,7 +158,7 @@ export const interpretWholeWorkflow = async (userId: string) => { * @category BrowserManagement-Controller */ export const stopRunningInterpretation = async (userId: string) => { - const id = getActiveBrowserId(userId); + const id = getActiveBrowserIdByState(userId, "recording"); if (id) { const browser = browserPool.getRemoteBrowser(id); await browser?.stopCurrentInterpretation(); diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 92c75e1d..8f4161b5 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -56,7 +56,7 @@ const handleWrapper = async ( return; } - const id = browserPool.getActiveBrowserId(userId); + const id = browserPool.getActiveBrowserId(userId, "recording"); if (id) { const activeBrowser = browserPool.getRemoteBrowser(id); if (activeBrowser?.interpreter.interpretationInProgress() && !activeBrowser.interpreter.interpretationIsPaused) { diff --git a/server/src/pgboss-worker.ts b/server/src/pgboss-worker.ts new file mode 100644 index 00000000..3566b4f8 --- /dev/null +++ b/server/src/pgboss-worker.ts @@ -0,0 +1,488 @@ +/** + * Recording worker using PgBoss for asynchronous browser recording operations + */ +import PgBoss, { Job } from 'pg-boss'; +import logger from './logger'; +import { + initializeRemoteBrowserForRecording, + destroyRemoteBrowser, + interpretWholeWorkflow, + stopRunningInterpretation, + createRemoteBrowserForRun +} from './browser-management/controller'; +import { WorkflowFile } from 'maxun-core'; +import Run from './models/Run'; +import Robot from './models/Robot'; +import { browserPool } from './server'; +import { Page } from 'playwright'; +import { BinaryOutputService } from './storage/mino'; +import { capture } from './utils/analytics'; +import { googleSheetUpdateTasks, processGoogleSheetUpdates } from './workflow-management/integrations/gsheet'; +import { airtableUpdateTasks, processAirtableUpdates } from './workflow-management/integrations/airtable'; +import { RemoteBrowser } from './browser-management/classes/RemoteBrowser'; +import { io as serverIo } from "./server"; + +const pgBossConnectionString = `postgres://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`; + +interface InitializeBrowserData { + userId: string; +} + +interface InterpretWorkflow { + userId: string; +} + +interface StopInterpretWorkflow { + userId: string; +} + +interface DestroyBrowserData { + browserId: string; + userId: string; +} + +interface ExecuteRunData { + userId: string; + runId: string; + browserId: string; +} + +const pgBoss = new PgBoss({connectionString: pgBossConnectionString, schema: 'public'}); + +/** + * Extract data safely from a job (single job or job array) + */ +function extractJobData(job: Job | Job[]): T { + if (Array.isArray(job)) { + if (job.length === 0) { + throw new Error('Empty job array received'); + } + return job[0].data; + } + return job.data; +} + +function AddGeneratedFlags(workflow: WorkflowFile) { + const copy = JSON.parse(JSON.stringify(workflow)); + for (let i = 0; i < workflow.workflow.length; i++) { + copy.workflow[i].what.unshift({ + action: 'flag', + args: ['generated'], + }); + } + return copy; +}; + +/** + * Function to reset browser state without creating a new browser + */ +async function resetBrowserState(browser: RemoteBrowser): Promise { + try { + const currentPage = browser.getCurrentPage(); + if (!currentPage) { + logger.log('error', 'No current page available to reset browser state'); + return false; + } + + // Navigate to blank page to reset state + await currentPage.goto('about:blank'); + + // Clear browser storage + await currentPage.evaluate(() => { + try { + localStorage.clear(); + sessionStorage.clear(); + } catch (e) { + // Ignore errors in cleanup + } + }); + + // Clear cookies + const context = currentPage.context(); + await context.clearCookies(); + + return true; + } catch (error) { + logger.log('error', `Failed to reset browser state`); + return false; + } +} + +/** + * Modified checkAndProcessQueuedRun function - only changes browser reset logic + */ +async function checkAndProcessQueuedRun(userId: string, browserId: string): Promise { + try { + // Find the oldest queued run for this specific browser + const queuedRun = await Run.findOne({ + where: { + browserId: browserId, + status: 'queued' + }, + order: [['startedAt', 'ASC']] + }); + + if (!queuedRun) { + logger.log('info', `No queued runs found for browser ${browserId}`); + return false; + } + + // Reset the browser state before next run + const browser = browserPool.getRemoteBrowser(browserId); + if (browser) { + logger.log('info', `Resetting browser state for browser ${browserId} before next run`); + await resetBrowserState(browser); + } + + // Update the queued run to running status + await queuedRun.update({ + status: 'running', + log: 'Run started - using browser from previous run' + }); + + // Schedule the run execution + await pgBoss.createQueue('execute-run'); + const executeJobId = await pgBoss.send('execute-run', { + userId: userId, + runId: queuedRun.runId, + browserId: browserId + }); + + logger.log('info', `Scheduled queued run ${queuedRun.runId} to use browser ${browserId}, job ID: ${executeJobId}`); + return true; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Error checking for queued runs: ${errorMessage}`); + return false; + } +} + +/** + * Modified processRunExecution function - only add browser reset + */ +async function processRunExecution(job: Job) { + try { + const data = job.data; + logger.log('info', `Processing run execution job for runId: ${data.runId}, browserId: ${data.browserId}`); + + // Find the run + const run = await Run.findOne({ where: { runId: data.runId } }); + if (!run) { + logger.log('error', `Run ${data.runId} not found in database`); + return { success: false }; + } + + const plainRun = run.toJSON(); + + // Find the recording + const recording = await Robot.findOne({ where: { 'recording_meta.id': plainRun.robotMetaId }, raw: true }); + if (!recording) { + logger.log('error', `Recording for run ${data.runId} not found`); + + // Update run status to failed + await run.update({ + status: 'failed', + finishedAt: new Date().toLocaleString(), + log: 'Failed: Recording not found', + }); + + // Check for queued runs even if this one failed + await checkAndProcessQueuedRun(data.userId, data.browserId); + + return { success: false }; + } + + // Get the browser and execute the run + const browser = browserPool.getRemoteBrowser(plainRun.browserId); + let currentPage = browser?.getCurrentPage(); + + if (!browser || !currentPage) { + logger.log('error', `Browser or page not available for run ${data.runId}`); + + await pgBoss.fail(job.id, "Failed to get browser or page for run"); + + // Even if this run failed, check for queued runs + await checkAndProcessQueuedRun(data.userId, data.browserId); + + return { success: false }; + } + + try { + // Reset the browser state before executing this run + await resetBrowserState(browser); + + // Execute the workflow + const workflow = AddGeneratedFlags(recording.recording); + const interpretationInfo = await browser.interpreter.InterpretRecording( + workflow, + currentPage, + (newPage: Page) => currentPage = newPage, + plainRun.interpreterSettings + ); + + // Process the results + const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); + const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); + + // Update the run record with results + await run.update({ + ...run, + status: 'success', + finishedAt: new Date().toLocaleString(), + browserId: plainRun.browserId, + log: interpretationInfo.log.join('\n'), + serializableOutput: interpretationInfo.serializableOutput, + binaryOutput: uploadedBinaryOutput, + }); + + // Track extraction metrics + let totalRowsExtracted = 0; + let extractedScreenshotsCount = 0; + let extractedItemsCount = 0; + + if (run.dataValues.binaryOutput && run.dataValues.binaryOutput["item-0"]) { + extractedScreenshotsCount = 1; + } + + if (run.dataValues.serializableOutput && run.dataValues.serializableOutput["item-0"]) { + const itemsArray = run.dataValues.serializableOutput["item-0"]; + extractedItemsCount = itemsArray.length; + + totalRowsExtracted = itemsArray.reduce((total, item) => { + return total + Object.keys(item).length; + }, 0); + } + + console.log(`Extracted Items Count: ${extractedItemsCount}`); + console.log(`Extracted Screenshots Count: ${extractedScreenshotsCount}`); + console.log(`Total Rows Extracted: ${totalRowsExtracted}`); + + // Capture metrics + capture( + 'maxun-oss-run-created-manual', + { + runId: data.runId, + user_id: data.userId, + created_at: new Date().toISOString(), + status: 'success', + totalRowsExtracted, + extractedItemsCount, + extractedScreenshotsCount, + } + ); + + // Schedule updates for Google Sheets and Airtable + try { + googleSheetUpdateTasks[plainRun.runId] = { + robotId: plainRun.robotMetaId, + runId: plainRun.runId, + status: 'pending', + retries: 5, + }; + + airtableUpdateTasks[plainRun.runId] = { + robotId: plainRun.robotMetaId, + runId: plainRun.runId, + status: 'pending', + retries: 5, + }; + + processAirtableUpdates(); + processGoogleSheetUpdates(); + } catch (err: any) { + logger.log('error', `Failed to update Google Sheet for run: ${plainRun.runId}: ${err.message}`); + } + + serverIo.of(plainRun.browserId).emit('run-completed', { + runId: data.runId, + robotMetaId: plainRun.robotMetaId, + robotName: recording.recording_meta.name, + status: 'success', + finishedAt: new Date().toLocaleString() + });; + + // Check for and process queued runs before destroying the browser + const queuedRunProcessed = await checkAndProcessQueuedRun(data.userId, plainRun.browserId); + + // Only destroy the browser if no queued run was found + if (!queuedRunProcessed) { + await destroyRemoteBrowser(plainRun.browserId, data.userId); + logger.log('info', `No queued runs found for browser ${plainRun.browserId}, browser destroyed`); + } + + return { success: true }; + } catch (executionError: any) { + logger.log('error', `Run execution failed for run ${data.runId}: ${executionError.message}`); + + await run.update({ + status: 'failed', + finishedAt: new Date().toLocaleString(), + log: `Failed: ${executionError.message}`, + }); + + // Check for queued runs before destroying the browser + const queuedRunProcessed = await checkAndProcessQueuedRun(data.userId, plainRun.browserId); + + // Only destroy the browser if no queued run was found + if (!queuedRunProcessed) { + try { + await destroyRemoteBrowser(plainRun.browserId, data.userId); + logger.log('info', `No queued runs found for browser ${plainRun.browserId}, browser destroyed`); + } catch (cleanupError: any) { + logger.log('warn', `Failed to clean up browser for failed run ${data.runId}: ${cleanupError.message}`); + } + } + + // Capture failure metrics + capture( + 'maxun-oss-run-created-manual', + { + runId: data.runId, + user_id: data.userId, + created_at: new Date().toISOString(), + status: 'failed', + error_message: executionError.message, + } + ); + + return { success: false }; + } + + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Failed to process run execution job: ${errorMessage}`); + return { success: false }; + } +} + + +async function registerRunExecutionWorker() { + try { + // Worker for executing runs + await pgBoss.work('execute-run', async (job: Job | Job[]) => { + try { + const singleJob = Array.isArray(job) ? job[0] : job; + return await processRunExecution(singleJob); + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Run execution job failed: ${errorMessage}`); + throw error; + } + }); + + // setInterval(checkForStuckQueuedRuns, 30000); + + logger.log('info', 'Run execution worker registered successfully'); + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Failed to register run execution worker: ${errorMessage}`); + } +} + + +/** + * Initialize PgBoss and register all workers + */ +async function startWorkers() { + try { + logger.log('info', 'Starting PgBoss worker...'); + await pgBoss.start(); + logger.log('info', 'PgBoss worker started successfully'); + + // Worker for initializing browser recording + await pgBoss.work('initialize-browser-recording', async (job: Job | Job[]) => { + try { + const data = extractJobData(job); + const userId = data.userId; + + logger.log('info', `Starting browser initialization job for user: ${userId}`); + const browserId = initializeRemoteBrowserForRecording(userId); + logger.log('info', `Browser recording job completed with browserId: ${browserId}`); + return { browserId }; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Browser recording job failed: ${errorMessage}`); + throw error; + } + }); + + // Worker for stopping a browser + await pgBoss.work('destroy-browser', async (job: Job | Job[]) => { + try { + const data = extractJobData(job); + const { browserId, userId } = data; + + logger.log('info', `Starting browser destruction job for browser: ${browserId}`); + const success = await destroyRemoteBrowser(browserId, userId); + logger.log('info', `Browser destruction job completed with result: ${success}`); + return { success }; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Destroy browser job failed: ${errorMessage}`); + throw error; + } + }); + + // Worker for interpreting workflow + await pgBoss.work('interpret-workflow', async (job: Job | Job[]) => { + try { + const data = extractJobData(job); + const userId = data.userId; + + logger.log('info', 'Starting workflow interpretation job'); + await interpretWholeWorkflow(userId); + logger.log('info', 'Workflow interpretation job completed'); + return { success: true }; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Interpret workflow job failed: ${errorMessage}`); + throw error; + } + }); + + // Worker for stopping workflow interpretation + await pgBoss.work('stop-interpretation', async (job: Job | Job[]) => { + try { + const data = extractJobData(job); + const userId = data.userId; + + logger.log('info', 'Starting stop interpretation job'); + await stopRunningInterpretation(userId); + logger.log('info', 'Stop interpretation job completed'); + return { success: true }; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Stop interpretation job failed: ${errorMessage}`); + throw error; + } + }); + + // Register the run execution worker + await registerRunExecutionWorker(); + + logger.log('info', 'All recording workers registered successfully'); + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Failed to start PgBoss workers: ${errorMessage}`); + process.exit(1); + } +} + +// Start all workers +startWorkers(); + +// Handle graceful shutdown +process.on('SIGTERM', async () => { + logger.log('info', 'SIGTERM received, shutting down PgBoss...'); + await pgBoss.stop(); + process.exit(0); +}); + +process.on('SIGINT', async () => { + logger.log('info', 'SIGINT received, shutting down PgBoss...'); + await pgBoss.stop(); + process.exit(0); +}); + +// For use in other files +export { pgBoss }; \ No newline at end of file diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index 495dda5d..fee5039a 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -9,22 +9,58 @@ import { getActiveBrowserId, interpretWholeWorkflow, stopRunningInterpretation, - getRemoteBrowserCurrentUrl, getRemoteBrowserCurrentTabs, -} from '../browser-management/controller' + getRemoteBrowserCurrentUrl, + getRemoteBrowserCurrentTabs, + getActiveBrowserIdByState, +} from '../browser-management/controller'; import { chromium } from 'playwright-extra'; import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import logger from "../logger"; import { getDecryptedProxyConfig } from './proxy'; import { requireSignIn } from '../middlewares/auth'; +import { pgBoss } from '../pgboss-worker'; export const router = Router(); chromium.use(stealthPlugin()); - export interface AuthenticatedRequest extends Request { user?: any; } +async function waitForJobCompletion(jobId: string, queueName: string, timeout = 15000): Promise { + return new Promise((resolve, reject) => { + const startTime = Date.now(); + + const checkJobStatus = async () => { + if (Date.now() - startTime > timeout) { + return reject(new Error(`Timeout waiting for job ${jobId} to complete`)); + } + + try { + const job = await pgBoss.getJobById(queueName, jobId); + + if (!job) { + return reject(new Error(`Job ${jobId} not found`)); + } + + if (job.state === 'completed') { + return resolve(job.output); + } + + if (job.state === 'failed') { + return reject(new Error(`Job ${jobId} failed.`)); + } + + setTimeout(checkJobStatus, 200); + } catch (error) { + reject(error); + } + }; + + checkJobStatus(); + }); +} + /** * Logs information about remote browser recording session. */ @@ -33,31 +69,53 @@ router.all('/', requireSignIn, (req, res, next) => { next() // pass control to the next handler }) + /** - * GET endpoint for starting the remote browser recording session. - * returns session's id + * GET endpoint for starting the remote browser recording session + * Waits for job completion */ router.get('/start', requireSignIn, async (req: AuthenticatedRequest, res: Response) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const proxyConfig = await getDecryptedProxyConfig(req.user.id); - // Prepare the proxy options dynamically based on the user's proxy configuration - let proxyOptions: any = {}; // Default to no proxy - - if (proxyConfig.proxy_url) { - // Set the server, and if username & password exist, set those as well - proxyOptions = { - server: proxyConfig.proxy_url, - ...(proxyConfig.proxy_username && proxyConfig.proxy_password && { - username: proxyConfig.proxy_username, - password: proxyConfig.proxy_password, - }), - }; + + try { + await pgBoss.createQueue('initialize-browser-recording'); + + const jobId = await pgBoss.send('initialize-browser-recording', { + userId: req.user.id, + timestamp: new Date().toISOString() + }); + + if (!jobId) { + const browserId = initializeRemoteBrowserForRecording(req.user.id); + return res.send(browserId); + } + + logger.log('info', `Queued browser initialization job: ${jobId}, waiting for completion...`); + + try { + const result = await waitForJobCompletion(jobId, 'initialize-browser-recording', 15000); + + if (result && result.browserId) { + return res.send(result.browserId); + } else { + return res.send(jobId); + } + } catch (waitError: any) { + return res.send(jobId); + } + } catch (error: any) { + logger.log('error', `Failed to queue browser initialization job: ${error.message}`); + + try { + const browserId = initializeRemoteBrowserForRecording(req.user.id); + return res.send( browserId ); + } catch (directError: any) { + logger.log('error', `Direct initialization also failed: ${directError.message}`); + return res.status(500).send('Failed to start recording'); + } } - - const id = initializeRemoteBrowserForRecording(req.user.id); - return res.send(id); }); /** @@ -80,8 +138,38 @@ router.get('/stop/:browserId', requireSignIn, async (req: AuthenticatedRequest, if (!req.user) { return res.status(401).send('User not authenticated'); } - const success = await destroyRemoteBrowser(req.params.browserId, req.user?.id); - return res.send(success); + + try { + await pgBoss.createQueue('destroy-browser'); + + const jobId = await pgBoss.send('destroy-browser', { + browserId: req.params.browserId, + userId: req.user.id, + timestamp: new Date().toISOString() + }); + + if (!jobId) { + const browserId = initializeRemoteBrowserForRecording(req.user.id); + return res.send( browserId ); + } + + logger.log('info', `Queued browser destruction job: ${jobId}, waiting for completion...`); + + try { + const result = await waitForJobCompletion(jobId, 'destroy-browser', 15000); + + if (result) { + return res.send(result.success); + } else { + return res.send(false); + } + } catch (waitError: any) { + return res.send(false); + } + } catch (error: any) { + logger.log('error', `Failed to stop browser: ${error.message}`); + return res.status(500).send(false); + } }); /** @@ -91,7 +179,7 @@ router.get('/active', requireSignIn, (req: AuthenticatedRequest, res) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = getActiveBrowserId(req.user?.id); + const id = getActiveBrowserIdByState(req.user?.id, "recording"); return res.send(id); }); @@ -102,7 +190,7 @@ router.get('/active/url', requireSignIn, (req: AuthenticatedRequest, res) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = getActiveBrowserId(req.user?.id); + const id = getActiveBrowserIdByState(req.user?.id, "recording"); if (id) { const url = getRemoteBrowserCurrentUrl(id, req.user?.id); return res.send(url); @@ -117,7 +205,7 @@ router.get('/active/tabs', requireSignIn, (req: AuthenticatedRequest, res) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = getActiveBrowserId(req.user?.id); + const id = getActiveBrowserIdByState(req.user?.id, "recording"); if (id) { const hosts = getRemoteBrowserCurrentTabs(id, req.user?.id); return res.send(hosts); @@ -129,24 +217,77 @@ router.get('/active/tabs', requireSignIn, (req: AuthenticatedRequest, res) => { * GET endpoint for starting an interpretation of the currently generated workflow. */ router.get('/interpret', requireSignIn, async (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + try { - if (!req.user) { - return res.status(401).send('User not authenticated'); + await pgBoss.createQueue('interpret-workflow'); + + const jobId = await pgBoss.send('interpret-workflow', { + userId: req.user.id, + timestamp: new Date().toISOString() + }); + + if (!jobId) { + await interpretWholeWorkflow(req.user?.id); + return res.send('interpretation done'); } - await interpretWholeWorkflow(req.user?.id); - return res.send('interpretation done'); - } catch (e) { - return res.send('interpretation failed'); + + logger.log('info', `Queued interpret workflow job: ${jobId}, waiting for completion...`); + + try { + const result = await waitForJobCompletion(jobId, 'interpret-workflow', 15000); + + if (result) { + return res.send('interpretation done'); + } else { + return res.send('interpretation failed'); + } + } catch (waitError: any) { + return res.send('interpretation failed'); + } + } catch (error: any) { + logger.log('error', `Failed to stop interpret workflow: ${error.message}`); + return res.status(500).send('interpretation failed'); } }); -/** - * GET endpoint for stopping an ongoing interpretation of the currently generated workflow. - */ router.get('/interpret/stop', requireSignIn, async (req: AuthenticatedRequest, res) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - await stopRunningInterpretation(req.user?.id); - return res.send('interpretation stopped'); + + try { + await pgBoss.createQueue('stop-interpretation'); + + const jobId = await pgBoss.send('stop-interpretation', { + userId: req.user.id, + timestamp: new Date().toISOString() + }); + + if (!jobId) { + await stopRunningInterpretation(req.user?.id); + return res.send('interpretation stopped'); + } + + logger.log('info', `Queued stop interpret workflow job: ${jobId}, waiting for completion...`); + + try { + const result = await waitForJobCompletion(jobId, 'stop-interpretation', 15000); + + if (result) { + return res.send('interpretation stopped'); + } else { + return res.send('interpretation failed to stop'); + } + } catch (waitError: any) { + return res.send('interpretation failed to stop'); + } + } catch (error: any) { + logger.log('error', `Failed to stop interpretation: ${error.message}`); + return res.status(500).send('interpretation failed to stop'); + } }); + +export default router; \ No newline at end of file diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index aa3f4e12..ffdf0149 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -1,6 +1,6 @@ import { Router } from 'express'; import logger from "../logger"; -import { createRemoteBrowserForRun, destroyRemoteBrowser } from "../browser-management/controller"; +import { createRemoteBrowserForRun, destroyRemoteBrowser, getActiveBrowserIdByState } from "../browser-management/controller"; import { chromium } from 'playwright-extra'; import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import { browserPool } from "../server"; @@ -22,6 +22,7 @@ import { encrypt, decrypt } from '../utils/auth'; import { WorkflowFile } from 'maxun-core'; import { Page } from 'playwright'; import { airtableUpdateTasks, processAirtableUpdates } from '../workflow-management/integrations/airtable'; +import { pgBoss } from '../pgboss-worker'; chromium.use(stealthPlugin()); export const router = Router(); @@ -494,6 +495,8 @@ router.delete('/runs/:id', requireSignIn, async (req: AuthenticatedRequest, res) /** * PUT endpoint for starting a remote browser instance and saving run metadata to the storage. * Making it ready for interpretation and returning a runId. + * + * If the user has reached their browser limit, the run will be queued using PgBoss. */ router.put('/runs/:id', requireSignIn, async (req: AuthenticatedRequest, res) => { try { @@ -525,35 +528,81 @@ router.put('/runs/:id', requireSignIn, async (req: AuthenticatedRequest, res) => }; } - console.log(`Proxy config for run: ${JSON.stringify(proxyOptions)}`) - - const id = createRemoteBrowserForRun(req.user.id); + console.log(`Proxy config for run: ${JSON.stringify(proxyOptions)}`); + // Generate runId first const runId = uuid(); + + // Check if user has reached browser limit + const userBrowserIds = browserPool.getAllBrowserIdsForUser(req.user.id); + const canCreateBrowser = userBrowserIds.length < 2; + + if (canCreateBrowser) { + // User has available browser slots, create it directly + const id = createRemoteBrowserForRun(req.user.id); - const run = await Run.create({ - status: 'running', - name: recording.recording_meta.name, - robotId: recording.id, - robotMetaId: recording.recording_meta.id, - startedAt: new Date().toLocaleString(), - finishedAt: '', - browserId: id, - interpreterSettings: req.body, - log: '', - runId, - runByUserId: req.user.id, - serializableOutput: {}, - binaryOutput: {}, - }); + const run = await Run.create({ + status: 'running', + name: recording.recording_meta.name, + robotId: recording.id, + robotMetaId: recording.recording_meta.id, + startedAt: new Date().toLocaleString(), + finishedAt: '', + browserId: id, + interpreterSettings: req.body, + log: '', + runId, + runByUserId: req.user.id, + serializableOutput: {}, + binaryOutput: {}, + }); - const plainRun = run.toJSON(); + const plainRun = run.toJSON(); - return res.send({ - browserId: id, - runId: plainRun.runId, - robotMetaId: recording.recording_meta.id, - }); + return res.send({ + browserId: id, + runId: plainRun.runId, + robotMetaId: recording.recording_meta.id, + queued: false + }); + } else { + const browserId = getActiveBrowserIdByState(req.user.id, "run") + + if (browserId) { + // User has reached the browser limit, queue the run + try { + // Create the run record with 'queued' status + await Run.create({ + status: 'queued', + name: recording.recording_meta.name, + robotId: recording.id, + robotMetaId: recording.recording_meta.id, + startedAt: new Date().toLocaleString(), + finishedAt: '', + browserId: browserId, // Random will be updated later + interpreterSettings: req.body, + log: 'Run queued - waiting for available browser slot', + runId, + runByUserId: req.user.id, + serializableOutput: {}, + binaryOutput: {}, + }); + + return res.send({ + browserId: browserId, + runId: runId, + robotMetaId: recording.recording_meta.id, + queued: true, + }); + } catch (queueError: any) { + logger.log('error', `Failed to queue run job: ${queueError.message}`); + return res.status(503).send({ error: 'Unable to queue run, please try again later' }); + } + } else { + logger.log('info', "Browser id does not exist"); + return res.send(''); + } + } } catch (e) { const { message } = e as Error; logger.log('info', `Error while creating a run with robot id: ${req.params.id} - ${message}`); @@ -608,82 +657,20 @@ router.post('/runs/run/:id', requireSignIn, async (req: AuthenticatedRequest, re return res.status(404).send(false); } - // interpret the run in active browser - const browser = browserPool.getRemoteBrowser(plainRun.browserId); - let currentPage = browser?.getCurrentPage(); - if (browser && currentPage) { - const workflow = AddGeneratedFlags(recording.recording); - const interpretationInfo = await browser.interpreter.InterpretRecording( - workflow, currentPage, (newPage: Page) => currentPage = newPage, plainRun.interpreterSettings); - const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); - const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(plainRun.browserId, req.user?.id); - await run.update({ - ...run, - status: 'success', - finishedAt: new Date().toLocaleString(), - browserId: plainRun.browserId, - log: interpretationInfo.log.join('\n'), - serializableOutput: interpretationInfo.serializableOutput, - binaryOutput: uploadedBinaryOutput, + try { + // Queue the execution job + await pgBoss.createQueue('execute-run'); + + const jobId = await pgBoss.send('execute-run', { + userId: req.user.id, + runId: req.params.id, + browserId: plainRun.browserId }); - - let totalRowsExtracted = 0; - let extractedScreenshotsCount = 0; - let extractedItemsCount = 0; - - if (run.dataValues.binaryOutput && run.dataValues.binaryOutput["item-0"]) { - extractedScreenshotsCount = 1; - } - - if (run.dataValues.serializableOutput && run.dataValues.serializableOutput["item-0"]) { - const itemsArray = run.dataValues.serializableOutput["item-0"]; - extractedItemsCount = itemsArray.length; - - totalRowsExtracted = itemsArray.reduce((total, item) => { - return total + Object.keys(item).length; - }, 0); - } - - console.log(`Extracted Items Count: ${extractedItemsCount}`); - console.log(`Extracted Screenshots Count: ${extractedScreenshotsCount}`); - console.log(`Total Rows Extracted: ${totalRowsExtracted}`); - - capture( - 'maxun-oss-run-created-manual', - { - runId: req.params.id, - user_id: req.user?.id, - created_at: new Date().toISOString(), - status: 'success', - totalRowsExtracted, - extractedItemsCount, - extractedScreenshotsCount, - } - ) - try { - googleSheetUpdateTasks[plainRun.runId] = { - robotId: plainRun.robotMetaId, - runId: plainRun.runId, - status: 'pending', - retries: 5, - }; - - airtableUpdateTasks[plainRun.runId] = { - robotId: plainRun.robotMetaId, - runId: plainRun.runId, - status: 'pending', - retries: 5, - }; - - processAirtableUpdates(); - processGoogleSheetUpdates(); - } catch (err: any) { - logger.log('error', `Failed to update Google Sheet for run: ${plainRun.runId}: ${err.message}`); - } - return res.send(true); - } else { - throw new Error('Could not destroy browser'); + + logger.log('info', `Queued run execution job with ID: ${jobId} for run: ${req.params.id}`); + } catch (queueError: any) { + logger.log('error', `Failed to queue run execution`); + } } catch (e) { const { message } = e as Error; diff --git a/server/src/routes/workflow.ts b/server/src/routes/workflow.ts index 3a117ce1..c8ad92d5 100644 --- a/server/src/routes/workflow.ts +++ b/server/src/routes/workflow.ts @@ -49,7 +49,7 @@ router.get('/params/:browserId', requireSignIn, (req, res) => { */ router.delete('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = browserPool.getActiveBrowserId(req.user?.id); + const id = browserPool.getActiveBrowserId(req.user?.id, "recording"); if (id) { const browser = browserPool.getRemoteBrowser(id); if (browser) { @@ -66,7 +66,7 @@ router.delete('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => */ router.post('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = browserPool.getActiveBrowserId(req.user?.id); + const id = browserPool.getActiveBrowserId(req.user?.id, "recording"); if (id) { const browser = browserPool.getRemoteBrowser(id); logger.log('debug', `Adding pair to workflow`); @@ -87,7 +87,7 @@ router.post('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { */ router.put('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = browserPool.getActiveBrowserId(req.user?.id); + const id = browserPool.getActiveBrowserId(req.user?.id, "recording"); if (id) { const browser = browserPool.getRemoteBrowser(id); logger.log('debug', `Updating pair in workflow`); diff --git a/server/src/server.ts b/server/src/server.ts index 9c0eff23..0bd46b38 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -23,7 +23,6 @@ import session from 'express-session'; import Run from './models/Run'; - const app = express(); app.use(cors({ origin: process.env.PUBLIC_URL ? process.env.PUBLIC_URL : 'http://localhost:5173', @@ -79,8 +78,11 @@ readdirSync(path.join(__dirname, 'api')).forEach((r) => { const isProduction = process.env.NODE_ENV === 'production'; const workerPath = path.resolve(__dirname, isProduction ? './worker.js' : './worker.ts'); +const recordingWorkerPath = path.resolve(__dirname, isProduction ? './pgboss-worker.js' : './pgboss-worker.ts'); let workerProcess: any; +let recordingWorkerProcess: any; + if (!isProduction) { workerProcess = fork(workerPath, [], { execArgv: ['--inspect=5859'], @@ -94,6 +96,19 @@ if (!isProduction) { workerProcess.on('exit', (code: any) => { console.log(`Worker exited with code: ${code}`); }); + + recordingWorkerProcess = fork(recordingWorkerPath, [], { + execArgv: ['--inspect=5860'], + }); + recordingWorkerProcess.on('message', (message: any) => { + console.log(`Message from recording worker: ${message}`); + }); + recordingWorkerProcess.on('error', (error: any) => { + console.error(`Error in recording worker: ${error}`); + }); + recordingWorkerProcess.on('exit', (code: any) => { + console.log(`Recording worker exited with code: ${code}`); + }); } app.get('/', function (req, res) { @@ -146,7 +161,8 @@ process.on('SIGINT', async () => { } if (!isProduction) { - workerProcess.kill(); + if (workerProcess) workerProcess.kill(); + if (recordingWorkerProcess) recordingWorkerProcess.kill(); } process.exit(); }); diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 8cdc233c..aea37126 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -152,7 +152,7 @@ export class WorkflowGenerator { }); socket.on('activeIndex', (data) => this.generatedData.lastIndex = parseInt(data)); socket.on('decision', async ({ pair, actionType, decision, userId }) => { - const id = browserPool.getActiveBrowserId(userId); + const id = browserPool.getActiveBrowserId(userId, "recording"); if (id) { // const activeBrowser = browserPool.getRemoteBrowser(id); // const currentPage = activeBrowser?.getCurrentPage(); @@ -708,6 +708,7 @@ export class WorkflowGenerator { public updateSocket = (socket: Socket): void => { this.socket = socket; this.registerEventHandlers(socket); + this.initializeSocketListeners(); }; /** diff --git a/src/components/browser/BrowserNavBar.tsx b/src/components/browser/BrowserNavBar.tsx index c3db51c1..72456931 100644 --- a/src/components/browser/BrowserNavBar.tsx +++ b/src/components/browser/BrowserNavBar.tsx @@ -54,6 +54,7 @@ const BrowserNavBar: FC = ({ const handleCurrentUrlChange = useCallback((url: string) => { handleUrlChanged(url); setRecordingUrl(url); + window.sessionStorage.setItem('recordingUrl', url); }, [handleUrlChanged, recordingUrl]); useEffect(() => { diff --git a/src/components/browser/BrowserRecordingSave.tsx b/src/components/browser/BrowserRecordingSave.tsx index 81719251..d4fd54fb 100644 --- a/src/components/browser/BrowserRecordingSave.tsx +++ b/src/components/browser/BrowserRecordingSave.tsx @@ -14,9 +14,9 @@ import { MoreHoriz } from '@mui/icons-material'; const BrowserRecordingSave = () => { const { t } = useTranslation(); - const [openDiscardModal, setOpenDiscardModal] = useState(false); - const [openResetModal, setOpenResetModal] = useState(false); - const [anchorEl, setAnchorEl] = React.useState(null); + const [openDiscardModal, setOpenDiscardModal] = useState(false); + const [openResetModal, setOpenResetModal] = useState(false); + const [anchorEl, setAnchorEl] = React.useState(null); const { recordingName, browserId, initialUrl, setRecordingUrl, setBrowserId, notify, setCurrentWorkflowActionsState, resetInterpretationLog } = useGlobalInfoStore(); const navigate = useNavigate(); @@ -42,10 +42,25 @@ const BrowserRecordingSave = () => { const goToMainMenu = async () => { if (browserId) { await stopRecording(browserId); - notify('warning', t('browser_recording.notifications.terminated')); + + const notificationData = { + type: 'warning', + message: t('browser_recording.notifications.terminated'), + timestamp: Date.now() + }; + window.sessionStorage.setItem('pendingNotification', JSON.stringify(notificationData)); + + if (window.opener) { + window.opener.postMessage({ + type: 'recording-notification', + notification: notificationData + }, '*'); + } + setBrowserId(null); + + window.close(); } - navigate('/'); }; const performReset = () => { @@ -82,7 +97,13 @@ const BrowserRecordingSave = () => { socket?.emit('new-recording'); socket.emit('input:url', initialUrl); // Update the URL in the navbar to match - setRecordingUrl(initialUrl); + let sessionInitialUrl = window.sessionStorage.getItem('initialUrl'); + if (sessionInitialUrl) { + setRecordingUrl(sessionInitialUrl); + window.sessionStorage.setItem('recordingUrl', sessionInitialUrl); + } else { + setRecordingUrl(initialUrl); + } } // Close the reset confirmation modal @@ -92,7 +113,7 @@ const BrowserRecordingSave = () => { notify('info', t('browser_recording.notifications.environment_reset')); }; - const handleClick = (event: React.MouseEvent) => { + const handleClick = (event: any) => { setAnchorEl(event.currentTarget); }; diff --git a/src/components/browser/BrowserWindow.tsx b/src/components/browser/BrowserWindow.tsx index 05c93c1e..8edae179 100644 --- a/src/components/browser/BrowserWindow.tsx +++ b/src/components/browser/BrowserWindow.tsx @@ -78,9 +78,24 @@ export const BrowserWindow = () => { const { notify } = useGlobalInfoStore(); const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext(); const { addTextStep, addListStep } = useBrowserSteps(); - + const { state } = useContext(AuthContext); - const { user } = state; + const { user } = state; + + useEffect(() => { + if (listSelector) { + window.sessionStorage.setItem('recordingListSelector', listSelector); + } + }, [listSelector]); + + useEffect(() => { + const storedListSelector = window.sessionStorage.getItem('recordingListSelector'); + + // Only restore state if it exists in sessionStorage + if (storedListSelector && !listSelector) { + setListSelector(storedListSelector); + } + }, []); const onMouseMove = (e: MouseEvent) => { if (canvasRef && canvasRef.current && highlighterData) { @@ -211,13 +226,25 @@ export const BrowserWindow = () => { useEffect(() => { document.addEventListener('mousemove', onMouseMove, false); if (socket) { - socket.on("highlighter", highlighterHandler); + socket.off("highlighter", highlighterHandler); + + socket.on("highlighter", highlighterHandler); } return () => { - document.removeEventListener('mousemove', onMouseMove); - socket?.off("highlighter", highlighterHandler); + document.removeEventListener('mousemove', onMouseMove); + if (socket) { + socket.off("highlighter", highlighterHandler); + } }; - }, [socket, onMouseMove]); + }, [socket, highlighterHandler, onMouseMove, getList, listSelector]); + + useEffect(() => { + if (socket && listSelector) { + console.log('Syncing list selector with server:', listSelector); + socket.emit('setGetList', { getList: true }); + socket.emit('listSelector', { selector: listSelector }); + } + }, [socket, listSelector]); useEffect(() => { if (captureStage === 'initial' && listSelector) { diff --git a/src/components/recorder/Renderer.tsx b/src/components/recorder/Renderer.tsx new file mode 100644 index 00000000..bff63b08 --- /dev/null +++ b/src/components/recorder/Renderer.tsx @@ -0,0 +1,241 @@ +export class CanvasRenderer { + private canvas: HTMLCanvasElement; + private ctx: CanvasRenderingContext2D; + private offscreenCanvas: OffscreenCanvas | null = null; + private offscreenCtx: CanvasRenderingContext2D | null = null; + private lastFrameRequest: number | null = null; + private imageCache: Map = new Map(); + private consecutiveFrameCount: number = 0; + private lastDrawTime: number = 0; + private memoryCheckCounter: number = 0; + private lastMemoryCheck: number = 0; + private memoryThreshold: number = 100000000; // 100MB + + constructor(canvas: HTMLCanvasElement) { + this.canvas = canvas; + + // Get 2D context with optimized settings + const ctx = canvas.getContext('2d', { + alpha: false, // Disable alpha for better performance + desynchronized: true, // Reduce latency when possible + }); + + if (!ctx) { + throw new Error('Could not get 2D context from canvas'); + } + + this.ctx = ctx; + + // Apply performance optimizations + this.ctx.imageSmoothingEnabled = false; + + // Set up offscreen canvas if supported + if (typeof OffscreenCanvas !== 'undefined') { + this.offscreenCanvas = new OffscreenCanvas(canvas.width, canvas.height); + const offCtx = this.offscreenCanvas.getContext('2d', { + alpha: false + }); + + if (offCtx) { + this.offscreenCtx = offCtx as unknown as CanvasRenderingContext2D; + this.offscreenCtx.imageSmoothingEnabled = false; + } + } + + // Initial timestamp + this.lastDrawTime = performance.now(); + this.lastMemoryCheck = performance.now(); + } + + /** + * Renders a screenshot to the canvas, optimized for performance + */ + public drawScreenshot( + screenshot: string | ImageBitmap | HTMLImageElement, + x: number = 0, + y: number = 0, + width?: number, + height?: number + ): void { + // Cancel any pending frame request + if (this.lastFrameRequest !== null) { + cancelAnimationFrame(this.lastFrameRequest); + } + + // Check memory usage periodically + this.memoryCheckCounter++; + const now = performance.now(); + + if (this.memoryCheckCounter >= 30 || now - this.lastMemoryCheck > 5000) { + this.checkMemoryUsage(); + this.memoryCheckCounter = 0; + this.lastMemoryCheck = now; + } + + // Request a new frame + this.lastFrameRequest = requestAnimationFrame(() => { + this.renderFrame(screenshot, x, y, width, height); + }); + } + + private renderFrame( + screenshot: string | ImageBitmap | HTMLImageElement, + x: number, + y: number, + width?: number, + height?: number + ): void { + // Target context (offscreen if available, otherwise main) + const targetCtx = this.offscreenCtx || this.ctx; + + // Start timing the render + const startTime = performance.now(); + const timeSinceLastDraw = startTime - this.lastDrawTime; + + // Adaptive frame skipping for high-frequency updates + // If we're getting updates faster than 60fps and this isn't the first frame + if (timeSinceLastDraw < 16 && this.consecutiveFrameCount > 5) { + this.consecutiveFrameCount++; + + // Skip some frames when we're getting excessive updates + if (this.consecutiveFrameCount % 2 !== 0) { + return; + } + } else { + this.consecutiveFrameCount = 0; + } + + try { + if (typeof screenshot === 'string') { + // Check if we have this image in cache + let img = this.imageCache.get(screenshot); + + if (!img) { + img = new Image(); + img.src = screenshot; + this.imageCache.set(screenshot, img); + + // If image isn't loaded yet, draw when it loads + if (!img.complete) { + img.onload = () => { + if (img) { + this.drawScreenshot(img, x, y, width, height); + } + }; + return; + } + } + + targetCtx.drawImage( + img, + x, y, + width || img.width, + height || img.height + ); + } else { + // Draw ImageBitmap or HTMLImageElement directly + targetCtx.drawImage( + screenshot, + x, y, + width || screenshot.width, + height || screenshot.height + ); + } + + // If using offscreen canvas, copy to main canvas + if (this.offscreenCanvas && this.offscreenCtx) { + if ('transferToImageBitmap' in this.offscreenCanvas) { + // Use more efficient transfer when available + const bitmap = this.offscreenCanvas.transferToImageBitmap(); + this.ctx.drawImage(bitmap, 0, 0); + } else { + // Fallback to drawImage + this.ctx.drawImage(this.offscreenCanvas, 0, 0); + } + } + + // Update timestamp + this.lastDrawTime = performance.now(); + } catch (error) { + console.error('Error rendering frame:', error); + } + } + + /** + * Checks current memory usage and cleans up if necessary + */ + private checkMemoryUsage(): void { + if (window.performance && (performance as any).memory) { + const memory = (performance as any).memory; + + if (memory.usedJSHeapSize > this.memoryThreshold) { + this.cleanupMemory(); + } + } + } + + /** + * Cleans up resources to reduce memory usage + */ + private cleanupMemory(): void { + // Limit image cache size + if (this.imageCache.size > 20) { + // Keep only the most recent 10 images + const keysToDelete = Array.from(this.imageCache.keys()).slice(0, this.imageCache.size - 10); + keysToDelete.forEach(key => { + this.imageCache.delete(key); + }); + } + + // Suggest garbage collection + if (window.gc) { + try { + window.gc(); + } catch (e) { + // GC not available, ignore + } + } + } + + /** + * Update canvas dimensions + */ + public updateCanvasSize(width: number, height: number): void { + this.canvas.width = width; + this.canvas.height = height; + + // Re-apply context settings + this.ctx.imageSmoothingEnabled = false; + + // Update offscreen canvas if available + if (this.offscreenCanvas) { + this.offscreenCanvas.width = width; + this.offscreenCanvas.height = height; + + if (this.offscreenCtx) { + this.offscreenCtx.imageSmoothingEnabled = false; + } + } + } + + /** + * Clean up resources + */ + public dispose(): void { + // Cancel any pending frame requests + if (this.lastFrameRequest !== null) { + cancelAnimationFrame(this.lastFrameRequest); + this.lastFrameRequest = null; + } + + // Clear the image cache + this.imageCache.clear(); + + // Clear canvases + this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); + + if (this.offscreenCtx && this.offscreenCanvas) { + this.offscreenCtx.clearRect(0, 0, this.offscreenCanvas.width, this.offscreenCanvas.height); + } + } +} \ No newline at end of file diff --git a/src/components/recorder/SaveRecording.tsx b/src/components/recorder/SaveRecording.tsx index c6b5687b..f7020b44 100644 --- a/src/components/recorder/SaveRecording.tsx +++ b/src/components/recorder/SaveRecording.tsx @@ -47,13 +47,27 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => { }; const exitRecording = useCallback(async () => { - notify('success', t('save_recording.notifications.save_success')); + const notificationData = { + type: 'success', + message: t('save_recording.notifications.save_success'), + timestamp: Date.now() + }; + window.sessionStorage.setItem('pendingNotification', JSON.stringify(notificationData)); + + if (window.opener) { + window.opener.postMessage({ + type: 'recording-notification', + notification: notificationData + }, '*'); + } + if (browserId) { await stopRecording(browserId); } setBrowserId(null); - navigate('/'); - }, [setBrowserId, browserId, notify]); + + window.close(); + }, [setBrowserId, browserId]); // notifies backed to save the recording in progress, // releases resources and changes the view for main page by clearing the global browserId @@ -142,4 +156,4 @@ const modalStyle = { height: 'fit-content', display: 'block', padding: '20px', -}; +}; \ No newline at end of file diff --git a/src/components/recorder/canvas.tsx b/src/components/recorder/canvas.tsx index cc75a6ba..4a015c87 100644 --- a/src/components/recorder/canvas.tsx +++ b/src/components/recorder/canvas.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useEffect, useRef } from 'react'; +import React, { memo, useCallback, useEffect, useRef } from 'react'; import { useSocketStore } from '../../context/socket'; import { useGlobalInfoStore } from "../../context/globalInfo"; import { useActionContext } from '../../context/browserActions'; @@ -6,6 +6,7 @@ import DatePicker from '../pickers/DatePicker'; import Dropdown from '../pickers/Dropdown'; import TimePicker from '../pickers/TimePicker'; import DateTimeLocalPicker from '../pickers/DateTimeLocalPicker'; +import { EnhancedPerformanceMonitor } from '../../../perf/performance'; interface CreateRefCallback { (ref: React.RefObject): void; @@ -27,6 +28,9 @@ export interface Coordinates { const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { + const performanceMonitor = useRef(new EnhancedPerformanceMonitor()); + console.log('Frontend Performance Report:', performanceMonitor.current.getPerformanceReport()); + const canvasRef = useRef(null); const { socket } = useSocketStore(); const { setLastAction, lastAction } = useGlobalInfoStore(); @@ -34,6 +38,9 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { const getTextRef = useRef(getText); const getListRef = useRef(getList); + const MOUSE_MOVE_THROTTLE = 16; // ~60fps + const lastMouseMoveTime = useRef(0); + const [datePickerInfo, setDatePickerInfo] = React.useState<{ coordinates: Coordinates; selector: string; @@ -110,6 +117,7 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { }, [socket]); const onMouseEvent = useCallback((event: MouseEvent) => { + performanceMonitor.current.measureEventLatency(event); if (socket && canvasRef.current) { // Get the canvas bounding rectangle const rect = canvasRef.current.getBoundingClientRect(); @@ -129,29 +137,38 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { } notifyLastAction('click'); break; - case 'mousemove': - if (lastMousePosition.current.x !== clickCoordinates.x || - lastMousePosition.current.y !== clickCoordinates.y) { + case 'mousemove': { + const now = performance.now(); + if (now - lastMouseMoveTime.current < MOUSE_MOVE_THROTTLE) { + return; + } + lastMouseMoveTime.current = now; + + const dx = Math.abs(lastMousePosition.current.x - clickCoordinates.x); + const dy = Math.abs(lastMousePosition.current.y - clickCoordinates.y); + if (dx > 1 || dy > 1) { lastMousePosition.current = { x: clickCoordinates.x, y: clickCoordinates.y, }; - socket.emit('input:mousemove', { - x: clickCoordinates.x, - y: clickCoordinates.y, - }); + socket.emit('input:mousemove', clickCoordinates); notifyLastAction('move'); } break; - case 'wheel': + } + + // Optimize wheel events + case 'wheel': { const wheelEvent = event as WheelEvent; - const deltas = { - deltaX: Math.round(wheelEvent.deltaX), - deltaY: Math.round(wheelEvent.deltaY), - }; - socket.emit('input:wheel', deltas); - notifyLastAction('scroll'); + const deltaX = Math.round(wheelEvent.deltaX / 10) * 10; + const deltaY = Math.round(wheelEvent.deltaY / 10) * 10; + + if (Math.abs(deltaX) > 5 || Math.abs(deltaY) > 5) { + socket.emit('input:wheel', { deltaX, deltaY }); + notifyLastAction('scroll'); + } break; + } default: console.log('Default mouseEvent registered'); return; @@ -159,6 +176,16 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { } }, [socket]); + // performance logging + useEffect(() => { + const intervalId = setInterval(() => { + const report = performanceMonitor.current.getPerformanceReport(); + console.log('Frontend Performance Report:', report); + }, 5000); + + return () => clearInterval(intervalId); + }, []); + const onKeyboardEvent = useCallback((event: KeyboardEvent) => { if (socket) { switch (event.type) { @@ -207,9 +234,14 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { {datePickerInfo && ( { }; -export default Canvas; \ No newline at end of file +export default memo(Canvas); \ No newline at end of file diff --git a/src/components/robot/RecordingsTable.tsx b/src/components/robot/RecordingsTable.tsx index 429c53c5..2fc4f26e 100644 --- a/src/components/robot/RecordingsTable.tsx +++ b/src/components/robot/RecordingsTable.tsx @@ -41,9 +41,14 @@ import { useGlobalInfoStore } from "../../context/globalInfo"; import { checkRunsForRecording, deleteRecordingFromStorage, getStoredRecordings } from "../../api/storage"; import { Add } from "@mui/icons-material"; import { useNavigate } from 'react-router-dom'; -import { stopRecording } from "../../api/recording"; +import { getActiveBrowserId, stopRecording } from "../../api/recording"; import { GenericModal } from '../ui/GenericModal'; +declare global { + interface Window { + openedRecordingWindow?: Window | null; + } +} /** TODO: * 1. allow editing existing robot after persisting browser steps @@ -148,6 +153,8 @@ export const RecordingsTable = ({ const [rows, setRows] = React.useState([]); const [isModalOpen, setModalOpen] = React.useState(false); const [searchTerm, setSearchTerm] = React.useState(''); + const [isWarningModalOpen, setWarningModalOpen] = React.useState(false); + const [activeBrowserId, setActiveBrowserId] = React.useState(''); const columns = useMemo(() => [ { id: 'interpret', label: t('recordingtable.run'), minWidth: 80 }, @@ -176,6 +183,30 @@ export const RecordingsTable = ({ setRecordingId } = useGlobalInfoStore(); const navigate = useNavigate(); + useEffect(() => { + const handleMessage = (event: any) => { + if (event.data && event.data.type === 'recording-notification') { + const notificationData = event.data.notification; + if (notificationData) { + notify(notificationData.type, notificationData.message); + + if ((notificationData.type === 'success' && + notificationData.message.includes('saved')) || + (notificationData.type === 'warning' && + notificationData.message.includes('terminated'))) { + setRerenderRobots(true); + } + } + } + }; + + window.addEventListener('message', handleMessage); + + return () => { + window.removeEventListener('message', handleMessage); + }; + }, [notify, setRerenderRobots]); + const handleChangePage = useCallback((event: unknown, newPage: number) => { setPage(newPage); }, []); @@ -233,28 +264,69 @@ export const RecordingsTable = ({ }, [setRecordings, notify, t]); const handleNewRecording = useCallback(async () => { - if (browserId) { - setBrowserId(null); - await stopRecording(browserId); + const activeBrowserId = await getActiveBrowserId(); + + if (activeBrowserId) { + setActiveBrowserId(activeBrowserId); + setWarningModalOpen(true); + } else { + setModalOpen(true); } - setModalOpen(true); - }, [browserId]); + }, []); - const handleStartRecording = useCallback(() => { - setBrowserId('new-recording'); - setRecordingName(''); - setRecordingId(''); - navigate('/recording'); - }, [navigate]); + const notifyRecordingTabsToClose = (browserId: string) => { + const closeMessage = { + action: 'close-recording-tab', + browserId: browserId, + timestamp: Date.now() + }; + window.sessionStorage.setItem('recordingTabCloseMessage', JSON.stringify(closeMessage)); + + if (window.openedRecordingWindow && !window.openedRecordingWindow.closed) { + try { + window.openedRecordingWindow.close(); + } catch (e) { + console.log('Could not directly close recording window:', e); + } + } + }; + + const handleDiscardAndCreate = async () => { + if (activeBrowserId) { + await stopRecording(activeBrowserId); + notify('warning', t('browser_recording.notifications.terminated')); + + notifyRecordingTabsToClose(activeBrowserId); + } + + setWarningModalOpen(false); + setModalOpen(true); + }; const startRecording = () => { setModalOpen(false); - handleStartRecording(); + + // Set local state + setBrowserId('new-recording'); + setRecordingName(''); + setRecordingId(''); + + window.sessionStorage.setItem('browserId', 'new-recording'); + + const sessionId = Date.now().toString(); + window.sessionStorage.setItem('recordingSessionId', sessionId); + window.sessionStorage.setItem('recordingUrl', recordingUrl); + + window.openedRecordingWindow = window.open(`/recording-setup?session=${sessionId}`, '_blank'); + + window.sessionStorage.setItem('nextTabIsRecording', 'true'); }; const setBrowserRecordingUrl = (event: React.ChangeEvent) => { setInitialUrl(event.target.value); setRecordingUrl(event.target.value); + + window.sessionStorage.setItem('initialUrl', event.target.value); } useEffect(() => { @@ -401,6 +473,30 @@ export const RecordingsTable = ({ onPageChange={handleChangePage} onRowsPerPageChange={handleChangeRowsPerPage} /> + setWarningModalOpen(false)} modalStyle={modalStyle}> +
+ {t('recordingtable.warning_modal.title')} + + {t('recordingtable.warning_modal.message')} + + + + + + +
+ setModalOpen(false)} modalStyle={modalStyle}>
{t('recordingtable.modal.title')} diff --git a/src/components/run/ColapsibleRow.tsx b/src/components/run/ColapsibleRow.tsx index 87e38684..4d5fe822 100644 --- a/src/components/run/ColapsibleRow.tsx +++ b/src/components/run/ColapsibleRow.tsx @@ -123,6 +123,7 @@ export const CollapsibleRow = ({ row, handleDelete, isOpen, currentLog, abortRun {row.status === 'success' && } {row.status === 'running' && } {row.status === 'scheduled' && } + {row.status === 'queued' && } {row.status === 'failed' && } ) diff --git a/src/components/run/RunContent.tsx b/src/components/run/RunContent.tsx index cb3b63a6..77bce544 100644 --- a/src/components/run/RunContent.tsx +++ b/src/components/run/RunContent.tsx @@ -134,13 +134,13 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe }}>
- {interpretationInProgress ? currentLog : row.log} + {row.status === 'running' ? currentLog : row.log}
- {interpretationInProgress ? : null} - {interpretationInProgress ? ( + {row.status === 'running' || row.status === 'queued' ? ( {t('run_content.loading')} @@ -248,4 +248,4 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe ); -}; +}; \ No newline at end of file diff --git a/src/pages/MainPage.tsx b/src/pages/MainPage.tsx index 0801a933..60990291 100644 --- a/src/pages/MainPage.tsx +++ b/src/pages/MainPage.tsx @@ -103,6 +103,17 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps) setSockets(sockets => [...sockets, socket]); socket.on('ready-for-run', () => readyForRunHandler(browserId, runId)); socket.on('debugMessage', debugMessageHandler); + socket.on('run-completed', (data) => { + setRerenderRuns(true); + + const robotName = data.robotName; + + if (data.status === 'success') { + notify('success', t('main_page.notifications.interpretation_success', { name: robotName })); + } else { + notify('error', t('main_page.notifications.interpretation_failed', { name: robotName })); + } + }); setContent('runs'); if (browserId) { notify('info', t('main_page.notifications.run_started', { name: runningRecordingName })); diff --git a/src/pages/PageWrapper.tsx b/src/pages/PageWrapper.tsx index 1e73f26a..16f2e50d 100644 --- a/src/pages/PageWrapper.tsx +++ b/src/pages/PageWrapper.tsx @@ -16,10 +16,11 @@ import { NotFoundPage } from '../components/dashboard/NotFound'; export const PageWrapper = () => { const [open, setOpen] = useState(false); + const [isRecordingMode, setIsRecordingMode] = useState(false); const navigate = useNavigate(); - const { browserId, setBrowserId, notification, recordingName, setRecordingName, recordingId, setRecordingId } = useGlobalInfoStore(); + const { browserId, setBrowserId, notification, recordingName, setRecordingName, recordingId, setRecordingId, setRecordingUrl } = useGlobalInfoStore(); const handleEditRecording = (recordingId: string, fileName: string) => { setRecordingName(fileName); @@ -35,23 +36,62 @@ export const PageWrapper = () => { return notification.isOpen; } + /** + * Get the current tab's state from session storage + */ + const getTabState = (key: string): string | null => { + try { + const value = window.sessionStorage.getItem(key); + return value; + } catch (error) { + return null; + } + }; + useEffect(() => { - const isRecordingInProgress = async () => { - const id = await getActiveBrowserId(); - if (id) { - setBrowserId(id); + const tabMode = getTabState('tabMode'); + const urlParams = new URLSearchParams(window.location.search); + const sessionParam = urlParams.get('session'); + const storedSessionId = getTabState('recordingSessionId'); + const storedRecordingUrl = getTabState('recordingUrl'); + + if (location.pathname === '/recording-setup' && sessionParam && sessionParam === storedSessionId) { + setBrowserId('new-recording'); + setRecordingName(''); + setRecordingId(''); + + if (storedRecordingUrl) { + setRecordingUrl(storedRecordingUrl); + } + + navigate('/recording'); + } + else if (location.pathname === '/recording' || + (getTabState('nextTabIsRecording') === 'true' && sessionParam === storedSessionId)) { + setIsRecordingMode(true); + + if (location.pathname !== '/recording') { navigate('/recording'); } + + window.sessionStorage.removeItem('nextTabIsRecording'); + } else if (tabMode === 'main') { + console.log('Tab is in main application mode'); + } else { + const id = getTabState('browserId'); + if (id === 'new-recording' || location.pathname === '/recording') { + setIsRecordingMode(true); + } } - isRecordingInProgress(); - }, []); - + }, [location.pathname, navigate, setBrowserId, setRecordingId, setRecordingName, setRecordingUrl]); + return (
- {!browserId && } + {/* {!browserId && location.pathname !== '/recording' && } */} + {location.pathname !== '/recording' && } }> } /> diff --git a/src/pages/RecordingPage.tsx b/src/pages/RecordingPage.tsx index 259e6a0d..034b2ea8 100644 --- a/src/pages/RecordingPage.tsx +++ b/src/pages/RecordingPage.tsx @@ -44,7 +44,7 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => { const { setId, socket } = useSocketStore(); const { setWidth } = useBrowserDimensionsStore(); - const { browserId, setBrowserId, recordingId, recordingUrl } = useGlobalInfoStore(); + const { browserId, setBrowserId, recordingId, recordingUrl, setRecordingUrl } = useGlobalInfoStore(); const handleShowOutputData = useCallback(() => { setShowOutputData(true); @@ -78,6 +78,11 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => { useEffect(() => { let isCancelled = false; const handleRecording = async () => { + const storedUrl = window.sessionStorage.getItem('recordingUrl'); + if (storedUrl && !recordingUrl) { + setRecordingUrl(storedUrl); + } + const id = await getActiveBrowserId(); if (!isCancelled) { if (id) { @@ -91,13 +96,13 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => { } } }; - + handleRecording(); - + return () => { isCancelled = true; } - }, [setId]); + }, [setId, recordingUrl, setRecordingUrl]); const changeBrowserDimensions = useCallback(() => { if (browserContentRef.current) { diff --git a/src/routes/userRoute.tsx b/src/routes/userRoute.tsx index 4e9bb7c2..fe0a31ff 100644 --- a/src/routes/userRoute.tsx +++ b/src/routes/userRoute.tsx @@ -1,12 +1,56 @@ -import React from 'react'; -import { Navigate, Outlet } from 'react-router-dom'; +import React, { useEffect, useState } from 'react'; +import { Navigate, Outlet, useLocation } from 'react-router-dom'; import { useContext } from 'react'; import { AuthContext } from '../context/auth'; +import { useGlobalInfoStore } from '../context/globalInfo'; const UserRoute = () => { const { state } = useContext(AuthContext); + const location = useLocation(); + const [isCheckingAuth, setIsCheckingAuth] = useState(true); + const { setRecordingUrl } = useGlobalInfoStore(); + + useEffect(() => { + if (location.pathname === '/recording') { + const hasRecordingSession = + window.sessionStorage.getItem('browserId') || + window.sessionStorage.getItem('recordingSessionId'); + + const recordingUrl = window.sessionStorage.getItem('recordingUrl'); + if (recordingUrl) { + setRecordingUrl(recordingUrl); + } + + if (hasRecordingSession) { + console.log('UserRoute: Valid recording session detected, bypassing auth check'); + setIsCheckingAuth(false); + return; + } + } + + const timer = setTimeout(() => { + setIsCheckingAuth(false); + }, 100); + + return () => clearTimeout(timer); + }, [location.pathname]); + + if (isCheckingAuth) { + return null; + } + + if (location.pathname === '/recording') { + const hasRecordingSession = + window.sessionStorage.getItem('browserId') || + window.sessionStorage.getItem('recordingSessionId'); + + if (hasRecordingSession) { + return ; + } + } + return state.user ? : ; }; -export default UserRoute; +export default UserRoute; \ No newline at end of file