From b17397d0c84686b7ce7605511221ac683b411aaa Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 1 Mar 2025 20:24:57 +0530 Subject: [PATCH 01/51] wip: performance --- src/components/recorder/canvas.tsx | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/components/recorder/canvas.tsx b/src/components/recorder/canvas.tsx index cc75a6ba..9ec41f91 100644 --- a/src/components/recorder/canvas.tsx +++ b/src/components/recorder/canvas.tsx @@ -6,6 +6,7 @@ import DatePicker from '../pickers/DatePicker'; import Dropdown from '../pickers/Dropdown'; import TimePicker from '../pickers/TimePicker'; import DateTimeLocalPicker from '../pickers/DateTimeLocalPicker'; +import { FrontendPerformanceMonitor } from '../../../perf/performance'; interface CreateRefCallback { (ref: React.RefObject): void; @@ -27,6 +28,9 @@ export interface Coordinates { const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { + const performanceMonitor = useRef(new FrontendPerformanceMonitor()); + console.log('Frontend Performance Report:', performanceMonitor.current.getPerformanceReport()); + const canvasRef = useRef(null); const { socket } = useSocketStore(); const { setLastAction, lastAction } = useGlobalInfoStore(); @@ -110,6 +114,7 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { }, [socket]); const onMouseEvent = useCallback((event: MouseEvent) => { + performanceMonitor.current.measureEventLatency(event); if (socket && canvasRef.current) { // Get the canvas bounding rectangle const rect = canvasRef.current.getBoundingClientRect(); @@ -159,6 +164,16 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { } }, [socket]); + // performance logging + useEffect(() => { + const intervalId = setInterval(() => { + const report = performanceMonitor.current.getPerformanceReport(); + console.log('Frontend Performance Report:', report); + }, 5000); + + return () => clearInterval(intervalId); + }, []); + const onKeyboardEvent = useCallback((event: KeyboardEvent) => { if (socket) { switch (event.type) { From 26688b4b4592b1a8072d695266e35067fba17274 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 4 Mar 2025 22:36:42 +0530 Subject: [PATCH 02/51] feat: enhanced fe performance monitor --- perf/performance.ts | 256 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 256 insertions(+) diff --git a/perf/performance.ts b/perf/performance.ts index c50ef850..c8dce097 100644 --- a/perf/performance.ts +++ b/perf/performance.ts @@ -93,6 +93,262 @@ export class FrontendPerformanceMonitor { } } +// Enhanced Performance Monitor with Memory Management +export class EnhancedPerformanceMonitor extends FrontendPerformanceMonitor { + private memoryWarningThreshold: number = 100000000; // 100MB + private memoryAlertCallback?: (usage: MemoryInfo) => void; + private frameTimeHistory: number[] = []; + private maxMetricsHistory: number = 100; // Keep only the last 100 readings + private isThrottled: boolean = false; + private rafHandle: number | null = null; + private memoryCheckInterval: NodeJS.Timeout | null = null; + + constructor(options?: { + memoryWarningThreshold?: number, + maxMetricsHistory?: number, + memoryAlertCallback?: (usage: MemoryInfo) => void + }) { + super(); + + if (options) { + if (options.memoryWarningThreshold) { + this.memoryWarningThreshold = options.memoryWarningThreshold; + } + + if (options.maxMetricsHistory) { + this.maxMetricsHistory = options.maxMetricsHistory; + } + + if (options.memoryAlertCallback) { + this.memoryAlertCallback = options.memoryAlertCallback; + } + } + + // Override the parent's monitoring with our enhanced version + this.startEnhancedMonitoring(); + } + + private startEnhancedMonitoring(): void { + // Stop any existing monitoring from parent class + if (this.rafHandle) { + cancelAnimationFrame(this.rafHandle); + } + + if (this.memoryCheckInterval) { + clearInterval(this.memoryCheckInterval); + } + + // Enhanced FPS monitoring with frame time tracking + let lastFrameTime = performance.now(); + let frameCount = 0; + let frameTimes: number[] = []; + + const measureFPS = () => { + const now = performance.now(); + const frameTime = now - lastFrameTime; + lastFrameTime = now; + + // Track individual frame times for jank detection + frameTimes.push(frameTime); + if (frameTimes.length > 60) { // Keep only last 60 frame times + frameTimes.shift(); + } + + frameCount++; + this.frameTimeHistory.push(frameTime); + + // Calculate FPS every second + if (this.frameTimeHistory.length >= 60) { + const totalTime = this.frameTimeHistory.reduce((sum, time) => sum + time, 0); + const fps = Math.round((this.frameTimeHistory.length * 1000) / totalTime); + + // Get metrics from parent class + const metrics = this.getMetrics(); + metrics.fps.push(fps); + + // Limit metrics history + if (metrics.fps.length > this.maxMetricsHistory) { + metrics.fps.shift(); + } + + // Detect jank (long frames) + const jankThreshold = 16.7 * 2; // 2x normal frame time at 60fps + const jankFrames = frameTimes.filter(time => time > jankThreshold); + + if (jankFrames.length > 10) { // If more than 10 out of 60 frames are janky + this.detectPerformanceIssue('jank', { + jankFrames: jankFrames.length, + averageJankTime: jankFrames.reduce((sum, time) => sum + time, 0) / jankFrames.length + }); + } + + // Reset for next measurement + this.frameTimeHistory = []; + frameTimes = []; + } + + this.rafHandle = requestAnimationFrame(measureFPS); + }; + + this.rafHandle = requestAnimationFrame(measureFPS); + + // Enhanced memory monitoring + if (window.performance && (performance as any).memory) { + this.memoryCheckInterval = setInterval(() => { + const memory = (performance as any).memory; + const memoryInfo = { + usedJSHeapSize: memory.usedJSHeapSize, + totalJSHeapSize: memory.totalJSHeapSize, + timestamp: Date.now() + }; + + // Get metrics from parent class + const metrics = this.getMetrics(); + metrics.memoryUsage.push(memoryInfo); + + // Limit metrics history + if (metrics.memoryUsage.length > this.maxMetricsHistory) { + metrics.memoryUsage.shift(); + } + + // Check for memory warnings + if (memoryInfo.usedJSHeapSize > this.memoryWarningThreshold) { + this.detectPerformanceIssue('memory', memoryInfo); + + if (this.memoryAlertCallback) { + this.memoryAlertCallback(memoryInfo); + } + } + + // Check for memory leaks (steady increase) + if (metrics.memoryUsage.length >= 10) { + const recentMemory = metrics.memoryUsage.slice(-10); + let increasingCount = 0; + + for (let i = 1; i < recentMemory.length; i++) { + if (recentMemory[i].usedJSHeapSize > recentMemory[i-1].usedJSHeapSize) { + increasingCount++; + } + } + + // If memory increased in 8 out of 9 consecutive readings + if (increasingCount >= 8) { + this.detectPerformanceIssue('memoryLeak', { + startMemory: recentMemory[0].usedJSHeapSize, + currentMemory: recentMemory[recentMemory.length - 1].usedJSHeapSize, + increaseRate: (recentMemory[recentMemory.length - 1].usedJSHeapSize - recentMemory[0].usedJSHeapSize) / + (recentMemory[recentMemory.length - 1].timestamp - recentMemory[0].timestamp) * 1000 // bytes per second + }); + } + } + }, 1000); + } + } + + // Method to detect various performance issues + private detectPerformanceIssue(type: 'jank' | 'memory' | 'memoryLeak', data: any): void { + console.warn(`Performance issue detected: ${type}`, data); + + if (type === 'memory' || type === 'memoryLeak') { + // Auto-throttle rendering if memory issues detected + if (!this.isThrottled) { + this.throttleRendering(); + } + + // Suggest garbage collection + this.suggestGarbageCollection(); + } + } + + // Get access to the metrics from parent class + private getMetrics(): any { + return (this as any).metrics; + } + + // Throttle rendering to reduce memory pressure + private throttleRendering(): void { + this.isThrottled = true; + console.info('Throttling rendering due to memory pressure'); + // Application code would implement throttling behavior + } + + // Un-throttle rendering when memory pressure is reduced + public unthrottleRendering(): void { + if (this.isThrottled) { + this.isThrottled = false; + console.info('Resuming normal rendering'); + } + } + + // Suggest garbage collection to the browser + private suggestGarbageCollection(): void { + if (window.gc) { + try { + window.gc(); + } catch (e) { + // gc() might not be available without special flags + } + } + + // Alternative approach to encourage garbage collection + const largeArray = new Array(1000000).fill(0); + largeArray.length = 0; + } + + // Enhanced performance report with more detailed metrics + public getEnhancedPerformanceReport(): EnhancedPerformanceReport { + const baseReport = super.getPerformanceReport(); + const metrics = this.getMetrics(); + + // Calculate 95th percentile render time + const sortedRenderTimes = [...metrics.renderTime].sort((a, b) => a - b); + const idx95 = Math.floor(sortedRenderTimes.length * 0.95); + const renderTime95Percentile = sortedRenderTimes[idx95] || 0; + + // Calculate memory growth rate + let memoryGrowthRate = 0; + if (metrics.memoryUsage.length >= 2) { + const first = metrics.memoryUsage[0]; + const last = metrics.memoryUsage[metrics.memoryUsage.length - 1]; + const timeDiffInSeconds = (last.timestamp - first.timestamp) / 1000; + memoryGrowthRate = timeDiffInSeconds > 0 + ? (last.usedJSHeapSize - first.usedJSHeapSize) / timeDiffInSeconds + : 0; + } + + return { + ...baseReport, + renderTime95Percentile, + memoryGrowthRate, + isThrottled: this.isThrottled, + heapUsagePercentage: baseReport.lastMemoryUsage + ? (baseReport.lastMemoryUsage.usedJSHeapSize / baseReport.lastMemoryUsage.totalJSHeapSize) * 100 + : 0 + }; + } + + // Clean up resources when no longer needed + public dispose(): void { + if (this.rafHandle) { + cancelAnimationFrame(this.rafHandle); + this.rafHandle = null; + } + + if (this.memoryCheckInterval) { + clearInterval(this.memoryCheckInterval); + this.memoryCheckInterval = null; + } + } + } + + // Extended types + interface EnhancedPerformanceReport extends PerformanceReport { + renderTime95Percentile: number; + memoryGrowthRate: number; // bytes per second + isThrottled: boolean; + heapUsagePercentage: number; + } + // Backend Performance Monitoring export class BackendPerformanceMonitor { private metrics: { From 941ae93648c9721c522c7cc26c9e8239ce18d706 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 4 Mar 2025 22:37:03 +0530 Subject: [PATCH 03/51] chore: lint --- perf/performance.ts | 422 ++++++++++++++++++++++---------------------- 1 file changed, 211 insertions(+), 211 deletions(-) diff --git a/perf/performance.ts b/perf/performance.ts index c8dce097..c499ecd2 100644 --- a/perf/performance.ts +++ b/perf/performance.ts @@ -18,7 +18,7 @@ export class FrontendPerformanceMonitor { }; this.lastFrameTime = performance.now(); this.frameCount = 0; - + // Start monitoring this.startMonitoring(); } @@ -102,252 +102,252 @@ export class EnhancedPerformanceMonitor extends FrontendPerformanceMonitor { private isThrottled: boolean = false; private rafHandle: number | null = null; private memoryCheckInterval: NodeJS.Timeout | null = null; - + constructor(options?: { - memoryWarningThreshold?: number, - maxMetricsHistory?: number, - memoryAlertCallback?: (usage: MemoryInfo) => void + memoryWarningThreshold?: number, + maxMetricsHistory?: number, + memoryAlertCallback?: (usage: MemoryInfo) => void }) { - super(); - - if (options) { - if (options.memoryWarningThreshold) { - this.memoryWarningThreshold = options.memoryWarningThreshold; + super(); + + if (options) { + if (options.memoryWarningThreshold) { + this.memoryWarningThreshold = options.memoryWarningThreshold; + } + + if (options.maxMetricsHistory) { + this.maxMetricsHistory = options.maxMetricsHistory; + } + + if (options.memoryAlertCallback) { + this.memoryAlertCallback = options.memoryAlertCallback; + } } - - if (options.maxMetricsHistory) { - this.maxMetricsHistory = options.maxMetricsHistory; - } - - if (options.memoryAlertCallback) { - this.memoryAlertCallback = options.memoryAlertCallback; - } - } - - // Override the parent's monitoring with our enhanced version - this.startEnhancedMonitoring(); + + // Override the parent's monitoring with our enhanced version + this.startEnhancedMonitoring(); } - + private startEnhancedMonitoring(): void { - // Stop any existing monitoring from parent class - if (this.rafHandle) { - cancelAnimationFrame(this.rafHandle); - } - - if (this.memoryCheckInterval) { - clearInterval(this.memoryCheckInterval); - } - - // Enhanced FPS monitoring with frame time tracking - let lastFrameTime = performance.now(); - let frameCount = 0; - let frameTimes: number[] = []; - - const measureFPS = () => { - const now = performance.now(); - const frameTime = now - lastFrameTime; - lastFrameTime = now; - - // Track individual frame times for jank detection - frameTimes.push(frameTime); - if (frameTimes.length > 60) { // Keep only last 60 frame times - frameTimes.shift(); + // Stop any existing monitoring from parent class + if (this.rafHandle) { + cancelAnimationFrame(this.rafHandle); } - - frameCount++; - this.frameTimeHistory.push(frameTime); - - // Calculate FPS every second - if (this.frameTimeHistory.length >= 60) { - const totalTime = this.frameTimeHistory.reduce((sum, time) => sum + time, 0); - const fps = Math.round((this.frameTimeHistory.length * 1000) / totalTime); - - // Get metrics from parent class - const metrics = this.getMetrics(); - metrics.fps.push(fps); - - // Limit metrics history - if (metrics.fps.length > this.maxMetricsHistory) { - metrics.fps.shift(); - } - - // Detect jank (long frames) - const jankThreshold = 16.7 * 2; // 2x normal frame time at 60fps - const jankFrames = frameTimes.filter(time => time > jankThreshold); - - if (jankFrames.length > 10) { // If more than 10 out of 60 frames are janky - this.detectPerformanceIssue('jank', { - jankFrames: jankFrames.length, - averageJankTime: jankFrames.reduce((sum, time) => sum + time, 0) / jankFrames.length - }); - } - - // Reset for next measurement - this.frameTimeHistory = []; - frameTimes = []; + + if (this.memoryCheckInterval) { + clearInterval(this.memoryCheckInterval); } - + + // Enhanced FPS monitoring with frame time tracking + let lastFrameTime = performance.now(); + let frameCount = 0; + let frameTimes: number[] = []; + + const measureFPS = () => { + const now = performance.now(); + const frameTime = now - lastFrameTime; + lastFrameTime = now; + + // Track individual frame times for jank detection + frameTimes.push(frameTime); + if (frameTimes.length > 60) { // Keep only last 60 frame times + frameTimes.shift(); + } + + frameCount++; + this.frameTimeHistory.push(frameTime); + + // Calculate FPS every second + if (this.frameTimeHistory.length >= 60) { + const totalTime = this.frameTimeHistory.reduce((sum, time) => sum + time, 0); + const fps = Math.round((this.frameTimeHistory.length * 1000) / totalTime); + + // Get metrics from parent class + const metrics = this.getMetrics(); + metrics.fps.push(fps); + + // Limit metrics history + if (metrics.fps.length > this.maxMetricsHistory) { + metrics.fps.shift(); + } + + // Detect jank (long frames) + const jankThreshold = 16.7 * 2; // 2x normal frame time at 60fps + const jankFrames = frameTimes.filter(time => time > jankThreshold); + + if (jankFrames.length > 10) { // If more than 10 out of 60 frames are janky + this.detectPerformanceIssue('jank', { + jankFrames: jankFrames.length, + averageJankTime: jankFrames.reduce((sum, time) => sum + time, 0) / jankFrames.length + }); + } + + // Reset for next measurement + this.frameTimeHistory = []; + frameTimes = []; + } + + this.rafHandle = requestAnimationFrame(measureFPS); + }; + this.rafHandle = requestAnimationFrame(measureFPS); - }; - - this.rafHandle = requestAnimationFrame(measureFPS); - - // Enhanced memory monitoring - if (window.performance && (performance as any).memory) { - this.memoryCheckInterval = setInterval(() => { - const memory = (performance as any).memory; - const memoryInfo = { - usedJSHeapSize: memory.usedJSHeapSize, - totalJSHeapSize: memory.totalJSHeapSize, - timestamp: Date.now() - }; - - // Get metrics from parent class - const metrics = this.getMetrics(); - metrics.memoryUsage.push(memoryInfo); - - // Limit metrics history - if (metrics.memoryUsage.length > this.maxMetricsHistory) { - metrics.memoryUsage.shift(); - } - - // Check for memory warnings - if (memoryInfo.usedJSHeapSize > this.memoryWarningThreshold) { - this.detectPerformanceIssue('memory', memoryInfo); - - if (this.memoryAlertCallback) { - this.memoryAlertCallback(memoryInfo); - } - } - - // Check for memory leaks (steady increase) - if (metrics.memoryUsage.length >= 10) { - const recentMemory = metrics.memoryUsage.slice(-10); - let increasingCount = 0; - - for (let i = 1; i < recentMemory.length; i++) { - if (recentMemory[i].usedJSHeapSize > recentMemory[i-1].usedJSHeapSize) { - increasingCount++; - } - } - - // If memory increased in 8 out of 9 consecutive readings - if (increasingCount >= 8) { - this.detectPerformanceIssue('memoryLeak', { - startMemory: recentMemory[0].usedJSHeapSize, - currentMemory: recentMemory[recentMemory.length - 1].usedJSHeapSize, - increaseRate: (recentMemory[recentMemory.length - 1].usedJSHeapSize - recentMemory[0].usedJSHeapSize) / - (recentMemory[recentMemory.length - 1].timestamp - recentMemory[0].timestamp) * 1000 // bytes per second - }); - } - } - }, 1000); - } + + // Enhanced memory monitoring + if (window.performance && (performance as any).memory) { + this.memoryCheckInterval = setInterval(() => { + const memory = (performance as any).memory; + const memoryInfo = { + usedJSHeapSize: memory.usedJSHeapSize, + totalJSHeapSize: memory.totalJSHeapSize, + timestamp: Date.now() + }; + + // Get metrics from parent class + const metrics = this.getMetrics(); + metrics.memoryUsage.push(memoryInfo); + + // Limit metrics history + if (metrics.memoryUsage.length > this.maxMetricsHistory) { + metrics.memoryUsage.shift(); + } + + // Check for memory warnings + if (memoryInfo.usedJSHeapSize > this.memoryWarningThreshold) { + this.detectPerformanceIssue('memory', memoryInfo); + + if (this.memoryAlertCallback) { + this.memoryAlertCallback(memoryInfo); + } + } + + // Check for memory leaks (steady increase) + if (metrics.memoryUsage.length >= 10) { + const recentMemory = metrics.memoryUsage.slice(-10); + let increasingCount = 0; + + for (let i = 1; i < recentMemory.length; i++) { + if (recentMemory[i].usedJSHeapSize > recentMemory[i - 1].usedJSHeapSize) { + increasingCount++; + } + } + + // If memory increased in 8 out of 9 consecutive readings + if (increasingCount >= 8) { + this.detectPerformanceIssue('memoryLeak', { + startMemory: recentMemory[0].usedJSHeapSize, + currentMemory: recentMemory[recentMemory.length - 1].usedJSHeapSize, + increaseRate: (recentMemory[recentMemory.length - 1].usedJSHeapSize - recentMemory[0].usedJSHeapSize) / + (recentMemory[recentMemory.length - 1].timestamp - recentMemory[0].timestamp) * 1000 // bytes per second + }); + } + } + }, 1000); + } } - + // Method to detect various performance issues private detectPerformanceIssue(type: 'jank' | 'memory' | 'memoryLeak', data: any): void { - console.warn(`Performance issue detected: ${type}`, data); - - if (type === 'memory' || type === 'memoryLeak') { - // Auto-throttle rendering if memory issues detected - if (!this.isThrottled) { - this.throttleRendering(); + console.warn(`Performance issue detected: ${type}`, data); + + if (type === 'memory' || type === 'memoryLeak') { + // Auto-throttle rendering if memory issues detected + if (!this.isThrottled) { + this.throttleRendering(); + } + + // Suggest garbage collection + this.suggestGarbageCollection(); } - - // Suggest garbage collection - this.suggestGarbageCollection(); - } } - + // Get access to the metrics from parent class private getMetrics(): any { - return (this as any).metrics; + return (this as any).metrics; } - + // Throttle rendering to reduce memory pressure private throttleRendering(): void { - this.isThrottled = true; - console.info('Throttling rendering due to memory pressure'); - // Application code would implement throttling behavior + this.isThrottled = true; + console.info('Throttling rendering due to memory pressure'); + // Application code would implement throttling behavior } - + // Un-throttle rendering when memory pressure is reduced public unthrottleRendering(): void { - if (this.isThrottled) { - this.isThrottled = false; - console.info('Resuming normal rendering'); - } + if (this.isThrottled) { + this.isThrottled = false; + console.info('Resuming normal rendering'); + } } - + // Suggest garbage collection to the browser private suggestGarbageCollection(): void { - if (window.gc) { - try { - window.gc(); - } catch (e) { - // gc() might not be available without special flags + if (window.gc) { + try { + window.gc(); + } catch (e) { + // gc() might not be available without special flags + } } - } - - // Alternative approach to encourage garbage collection - const largeArray = new Array(1000000).fill(0); - largeArray.length = 0; + + // Alternative approach to encourage garbage collection + const largeArray = new Array(1000000).fill(0); + largeArray.length = 0; } - + // Enhanced performance report with more detailed metrics public getEnhancedPerformanceReport(): EnhancedPerformanceReport { - const baseReport = super.getPerformanceReport(); - const metrics = this.getMetrics(); - - // Calculate 95th percentile render time - const sortedRenderTimes = [...metrics.renderTime].sort((a, b) => a - b); - const idx95 = Math.floor(sortedRenderTimes.length * 0.95); - const renderTime95Percentile = sortedRenderTimes[idx95] || 0; - - // Calculate memory growth rate - let memoryGrowthRate = 0; - if (metrics.memoryUsage.length >= 2) { - const first = metrics.memoryUsage[0]; - const last = metrics.memoryUsage[metrics.memoryUsage.length - 1]; - const timeDiffInSeconds = (last.timestamp - first.timestamp) / 1000; - memoryGrowthRate = timeDiffInSeconds > 0 - ? (last.usedJSHeapSize - first.usedJSHeapSize) / timeDiffInSeconds - : 0; - } - - return { - ...baseReport, - renderTime95Percentile, - memoryGrowthRate, - isThrottled: this.isThrottled, - heapUsagePercentage: baseReport.lastMemoryUsage - ? (baseReport.lastMemoryUsage.usedJSHeapSize / baseReport.lastMemoryUsage.totalJSHeapSize) * 100 - : 0 - }; + const baseReport = super.getPerformanceReport(); + const metrics = this.getMetrics(); + + // Calculate 95th percentile render time + const sortedRenderTimes = [...metrics.renderTime].sort((a, b) => a - b); + const idx95 = Math.floor(sortedRenderTimes.length * 0.95); + const renderTime95Percentile = sortedRenderTimes[idx95] || 0; + + // Calculate memory growth rate + let memoryGrowthRate = 0; + if (metrics.memoryUsage.length >= 2) { + const first = metrics.memoryUsage[0]; + const last = metrics.memoryUsage[metrics.memoryUsage.length - 1]; + const timeDiffInSeconds = (last.timestamp - first.timestamp) / 1000; + memoryGrowthRate = timeDiffInSeconds > 0 + ? (last.usedJSHeapSize - first.usedJSHeapSize) / timeDiffInSeconds + : 0; + } + + return { + ...baseReport, + renderTime95Percentile, + memoryGrowthRate, + isThrottled: this.isThrottled, + heapUsagePercentage: baseReport.lastMemoryUsage + ? (baseReport.lastMemoryUsage.usedJSHeapSize / baseReport.lastMemoryUsage.totalJSHeapSize) * 100 + : 0 + }; } - + // Clean up resources when no longer needed public dispose(): void { - if (this.rafHandle) { - cancelAnimationFrame(this.rafHandle); - this.rafHandle = null; - } - - if (this.memoryCheckInterval) { - clearInterval(this.memoryCheckInterval); - this.memoryCheckInterval = null; - } + if (this.rafHandle) { + cancelAnimationFrame(this.rafHandle); + this.rafHandle = null; + } + + if (this.memoryCheckInterval) { + clearInterval(this.memoryCheckInterval); + this.memoryCheckInterval = null; + } } - } - - // Extended types - interface EnhancedPerformanceReport extends PerformanceReport { +} + +// Extended types +interface EnhancedPerformanceReport extends PerformanceReport { renderTime95Percentile: number; memoryGrowthRate: number; // bytes per second isThrottled: boolean; heapUsagePercentage: number; - } +} // Backend Performance Monitoring export class BackendPerformanceMonitor { From 9456688074446f394b4061864d7fda19c86fb81b Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 4 Mar 2025 22:38:04 +0530 Subject: [PATCH 04/51] feat --- src/components/recorder/Renderer.tsx | 51 ++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 src/components/recorder/Renderer.tsx diff --git a/src/components/recorder/Renderer.tsx b/src/components/recorder/Renderer.tsx new file mode 100644 index 00000000..00859858 --- /dev/null +++ b/src/components/recorder/Renderer.tsx @@ -0,0 +1,51 @@ +export class CanvasRenderer { + private canvas: HTMLCanvasElement; + private ctx: CanvasRenderingContext2D; + private offscreenCanvas: OffscreenCanvas | null = null; + private offscreenCtx: CanvasRenderingContext2D | null = null; + private lastFrameRequest: number | null = null; + private imageCache: Map = new Map(); + private consecutiveFrameCount: number = 0; + private lastDrawTime: number = 0; + private memoryCheckCounter: number = 0; + private lastMemoryCheck: number = 0; + private memoryThreshold: number = 100000000; // 100MB + + constructor(canvas: HTMLCanvasElement) { + this.canvas = canvas; + + // Get 2D context with optimized settings + const ctx = canvas.getContext('2d', { + alpha: false, // Disable alpha for better performance + desynchronized: true, // Reduce latency when possible + }); + + if (!ctx) { + throw new Error('Could not get 2D context from canvas'); + } + + this.ctx = ctx; + + // Apply performance optimizations + this.ctx.imageSmoothingEnabled = false; + + // Set up offscreen canvas if supported + if (typeof OffscreenCanvas !== 'undefined') { + this.offscreenCanvas = new OffscreenCanvas(canvas.width, canvas.height); + const offCtx = this.offscreenCanvas.getContext('2d', { + alpha: false + }); + + if (offCtx) { + this.offscreenCtx = offCtx as unknown as CanvasRenderingContext2D; + this.offscreenCtx.imageSmoothingEnabled = false; + } + } + + // Initial timestamp + this.lastDrawTime = performance.now(); + this.lastMemoryCheck = performance.now(); + } + + + } \ No newline at end of file From bcee0e9174c09ae5340d4dd7c7aacbda8336d72b Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 4 Mar 2025 22:38:45 +0530 Subject: [PATCH 05/51] feat: draw screenshot --- src/components/recorder/Renderer.tsx | 31 +++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/components/recorder/Renderer.tsx b/src/components/recorder/Renderer.tsx index 00859858..2b88ae06 100644 --- a/src/components/recorder/Renderer.tsx +++ b/src/components/recorder/Renderer.tsx @@ -47,5 +47,34 @@ export class CanvasRenderer { this.lastMemoryCheck = performance.now(); } - + /** + * Renders a screenshot to the canvas, optimized for performance + */ + public drawScreenshot( + screenshot: string | ImageBitmap | HTMLImageElement, + x: number = 0, + y: number = 0, + width?: number, + height?: number + ): void { + // Cancel any pending frame request + if (this.lastFrameRequest !== null) { + cancelAnimationFrame(this.lastFrameRequest); + } + + // Check memory usage periodically + this.memoryCheckCounter++; + const now = performance.now(); + + if (this.memoryCheckCounter >= 30 || now - this.lastMemoryCheck > 5000) { + this.checkMemoryUsage(); + this.memoryCheckCounter = 0; + this.lastMemoryCheck = now; + } + + // Request a new frame + this.lastFrameRequest = requestAnimationFrame(() => { + this.renderFrame(screenshot, x, y, width, height); + }); + } } \ No newline at end of file From 0c94132c7438988c15dcb544dafae1ace90220f7 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 4 Mar 2025 22:39:17 +0530 Subject: [PATCH 06/51] feat: render frame --- src/components/recorder/Renderer.tsx | 85 ++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/src/components/recorder/Renderer.tsx b/src/components/recorder/Renderer.tsx index 2b88ae06..2d968498 100644 --- a/src/components/recorder/Renderer.tsx +++ b/src/components/recorder/Renderer.tsx @@ -77,4 +77,89 @@ export class CanvasRenderer { this.renderFrame(screenshot, x, y, width, height); }); } + + private renderFrame( + screenshot: string | ImageBitmap | HTMLImageElement, + x: number, + y: number, + width?: number, + height?: number + ): void { + // Target context (offscreen if available, otherwise main) + const targetCtx = this.offscreenCtx || this.ctx; + + // Start timing the render + const startTime = performance.now(); + const timeSinceLastDraw = startTime - this.lastDrawTime; + + // Adaptive frame skipping for high-frequency updates + // If we're getting updates faster than 60fps and this isn't the first frame + if (timeSinceLastDraw < 16 && this.consecutiveFrameCount > 5) { + this.consecutiveFrameCount++; + + // Skip some frames when we're getting excessive updates + if (this.consecutiveFrameCount % 2 !== 0) { + return; + } + } else { + this.consecutiveFrameCount = 0; + } + + try { + if (typeof screenshot === 'string') { + // Check if we have this image in cache + let img = this.imageCache.get(screenshot); + + if (!img) { + img = new Image(); + img.src = screenshot; + this.imageCache.set(screenshot, img); + + // If image isn't loaded yet, draw when it loads + if (!img.complete) { + img.onload = () => { + if (img) { + this.drawScreenshot(img, x, y, width, height); + } + }; + return; + } + } + + targetCtx.drawImage( + img, + x, y, + width || img.width, + height || img.height + ); + } else { + // Draw ImageBitmap or HTMLImageElement directly + targetCtx.drawImage( + screenshot, + x, y, + width || screenshot.width, + height || screenshot.height + ); + } + + // If using offscreen canvas, copy to main canvas + if (this.offscreenCanvas && this.offscreenCtx) { + if ('transferToImageBitmap' in this.offscreenCanvas) { + // Use more efficient transfer when available + const bitmap = this.offscreenCanvas.transferToImageBitmap(); + this.ctx.drawImage(bitmap, 0, 0); + } else { + // Fallback to drawImage + this.ctx.drawImage(this.offscreenCanvas, 0, 0); + } + } + + // Update timestamp + this.lastDrawTime = performance.now(); + } catch (error) { + console.error('Error rendering frame:', error); + } + } + + } \ No newline at end of file From 34057f9b13d026de016e2eedc0a28d3aa7f40b19 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 4 Mar 2025 22:39:36 +0530 Subject: [PATCH 07/51] feat: check memory usage --- src/components/recorder/Renderer.tsx | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/components/recorder/Renderer.tsx b/src/components/recorder/Renderer.tsx index 2d968498..212a60dc 100644 --- a/src/components/recorder/Renderer.tsx +++ b/src/components/recorder/Renderer.tsx @@ -161,5 +161,18 @@ export class CanvasRenderer { } } - + /** + * Checks current memory usage and cleans up if necessary + */ + private checkMemoryUsage(): void { + if (window.performance && (performance as any).memory) { + const memory = (performance as any).memory; + + if (memory.usedJSHeapSize > this.memoryThreshold) { + this.cleanupMemory(); + } + } + } + + } \ No newline at end of file From 2d638a623931e6dc69ade2dbff94c227f8c8e575 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 4 Mar 2025 22:40:03 +0530 Subject: [PATCH 08/51] feat: cleanup memory --- src/components/recorder/Renderer.tsx | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/components/recorder/Renderer.tsx b/src/components/recorder/Renderer.tsx index 212a60dc..1d778d5b 100644 --- a/src/components/recorder/Renderer.tsx +++ b/src/components/recorder/Renderer.tsx @@ -174,5 +174,28 @@ export class CanvasRenderer { } } + /** + * Cleans up resources to reduce memory usage + */ + private cleanupMemory(): void { + // Limit image cache size + if (this.imageCache.size > 20) { + // Keep only the most recent 10 images + const keysToDelete = Array.from(this.imageCache.keys()).slice(0, this.imageCache.size - 10); + keysToDelete.forEach(key => { + this.imageCache.delete(key); + }); + } + + // Suggest garbage collection + if (window.gc) { + try { + window.gc(); + } catch (e) { + // GC not available, ignore + } + } + } + } \ No newline at end of file From bad60077d44e468e8453bd0e2b484f87bca9efae Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 4 Mar 2025 22:40:26 +0530 Subject: [PATCH 09/51] feat: update canvas size --- src/components/recorder/Renderer.tsx | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/components/recorder/Renderer.tsx b/src/components/recorder/Renderer.tsx index 1d778d5b..e042595b 100644 --- a/src/components/recorder/Renderer.tsx +++ b/src/components/recorder/Renderer.tsx @@ -197,5 +197,25 @@ export class CanvasRenderer { } } - + /** + * Update canvas dimensions + */ + public updateCanvasSize(width: number, height: number): void { + this.canvas.width = width; + this.canvas.height = height; + + // Re-apply context settings + this.ctx.imageSmoothingEnabled = false; + + // Update offscreen canvas if available + if (this.offscreenCanvas) { + this.offscreenCanvas.width = width; + this.offscreenCanvas.height = height; + + if (this.offscreenCtx) { + this.offscreenCtx.imageSmoothingEnabled = false; + } + } + } + } \ No newline at end of file From bcff0e5d83cb78dc06ebb3700b7de61010fa0ac6 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 4 Mar 2025 22:40:39 +0530 Subject: [PATCH 10/51] feat: cleanup resources --- src/components/recorder/Renderer.tsx | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/components/recorder/Renderer.tsx b/src/components/recorder/Renderer.tsx index e042595b..6e69c5a1 100644 --- a/src/components/recorder/Renderer.tsx +++ b/src/components/recorder/Renderer.tsx @@ -218,4 +218,24 @@ export class CanvasRenderer { } } + /** + * Clean up resources + */ + public dispose(): void { + // Cancel any pending frame requests + if (this.lastFrameRequest !== null) { + cancelAnimationFrame(this.lastFrameRequest); + this.lastFrameRequest = null; + } + + // Clear the image cache + this.imageCache.clear(); + + // Clear canvases + this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); + + if (this.offscreenCtx && this.offscreenCanvas) { + this.offscreenCtx.clearRect(0, 0, this.offscreenCanvas.width, this.offscreenCanvas.height); + } + } } \ No newline at end of file From 2f1aca94d9c89e31d93ae4d3a8bb0d4346ac33b8 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 4 Mar 2025 22:40:51 +0530 Subject: [PATCH 11/51] chore: lint --- src/components/recorder/Renderer.tsx | 448 +++++++++++++-------------- 1 file changed, 224 insertions(+), 224 deletions(-) diff --git a/src/components/recorder/Renderer.tsx b/src/components/recorder/Renderer.tsx index 6e69c5a1..bff63b08 100644 --- a/src/components/recorder/Renderer.tsx +++ b/src/components/recorder/Renderer.tsx @@ -1,241 +1,241 @@ export class CanvasRenderer { - private canvas: HTMLCanvasElement; - private ctx: CanvasRenderingContext2D; - private offscreenCanvas: OffscreenCanvas | null = null; - private offscreenCtx: CanvasRenderingContext2D | null = null; - private lastFrameRequest: number | null = null; - private imageCache: Map = new Map(); - private consecutiveFrameCount: number = 0; - private lastDrawTime: number = 0; - private memoryCheckCounter: number = 0; - private lastMemoryCheck: number = 0; - private memoryThreshold: number = 100000000; // 100MB - - constructor(canvas: HTMLCanvasElement) { - this.canvas = canvas; - - // Get 2D context with optimized settings - const ctx = canvas.getContext('2d', { - alpha: false, // Disable alpha for better performance - desynchronized: true, // Reduce latency when possible - }); - - if (!ctx) { - throw new Error('Could not get 2D context from canvas'); - } - - this.ctx = ctx; - - // Apply performance optimizations - this.ctx.imageSmoothingEnabled = false; - - // Set up offscreen canvas if supported - if (typeof OffscreenCanvas !== 'undefined') { - this.offscreenCanvas = new OffscreenCanvas(canvas.width, canvas.height); - const offCtx = this.offscreenCanvas.getContext('2d', { - alpha: false - }); - - if (offCtx) { - this.offscreenCtx = offCtx as unknown as CanvasRenderingContext2D; - this.offscreenCtx.imageSmoothingEnabled = false; - } - } - - // Initial timestamp - this.lastDrawTime = performance.now(); - this.lastMemoryCheck = performance.now(); + private canvas: HTMLCanvasElement; + private ctx: CanvasRenderingContext2D; + private offscreenCanvas: OffscreenCanvas | null = null; + private offscreenCtx: CanvasRenderingContext2D | null = null; + private lastFrameRequest: number | null = null; + private imageCache: Map = new Map(); + private consecutiveFrameCount: number = 0; + private lastDrawTime: number = 0; + private memoryCheckCounter: number = 0; + private lastMemoryCheck: number = 0; + private memoryThreshold: number = 100000000; // 100MB + + constructor(canvas: HTMLCanvasElement) { + this.canvas = canvas; + + // Get 2D context with optimized settings + const ctx = canvas.getContext('2d', { + alpha: false, // Disable alpha for better performance + desynchronized: true, // Reduce latency when possible + }); + + if (!ctx) { + throw new Error('Could not get 2D context from canvas'); } - - /** - * Renders a screenshot to the canvas, optimized for performance - */ - public drawScreenshot( - screenshot: string | ImageBitmap | HTMLImageElement, - x: number = 0, - y: number = 0, - width?: number, - height?: number - ): void { - // Cancel any pending frame request - if (this.lastFrameRequest !== null) { - cancelAnimationFrame(this.lastFrameRequest); - } - - // Check memory usage periodically - this.memoryCheckCounter++; - const now = performance.now(); - - if (this.memoryCheckCounter >= 30 || now - this.lastMemoryCheck > 5000) { - this.checkMemoryUsage(); - this.memoryCheckCounter = 0; - this.lastMemoryCheck = now; - } - - // Request a new frame - this.lastFrameRequest = requestAnimationFrame(() => { - this.renderFrame(screenshot, x, y, width, height); + + this.ctx = ctx; + + // Apply performance optimizations + this.ctx.imageSmoothingEnabled = false; + + // Set up offscreen canvas if supported + if (typeof OffscreenCanvas !== 'undefined') { + this.offscreenCanvas = new OffscreenCanvas(canvas.width, canvas.height); + const offCtx = this.offscreenCanvas.getContext('2d', { + alpha: false }); + + if (offCtx) { + this.offscreenCtx = offCtx as unknown as CanvasRenderingContext2D; + this.offscreenCtx.imageSmoothingEnabled = false; + } } - - private renderFrame( - screenshot: string | ImageBitmap | HTMLImageElement, - x: number, - y: number, - width?: number, - height?: number - ): void { - // Target context (offscreen if available, otherwise main) - const targetCtx = this.offscreenCtx || this.ctx; - - // Start timing the render - const startTime = performance.now(); - const timeSinceLastDraw = startTime - this.lastDrawTime; - - // Adaptive frame skipping for high-frequency updates - // If we're getting updates faster than 60fps and this isn't the first frame - if (timeSinceLastDraw < 16 && this.consecutiveFrameCount > 5) { - this.consecutiveFrameCount++; - - // Skip some frames when we're getting excessive updates - if (this.consecutiveFrameCount % 2 !== 0) { - return; + + // Initial timestamp + this.lastDrawTime = performance.now(); + this.lastMemoryCheck = performance.now(); + } + + /** + * Renders a screenshot to the canvas, optimized for performance + */ + public drawScreenshot( + screenshot: string | ImageBitmap | HTMLImageElement, + x: number = 0, + y: number = 0, + width?: number, + height?: number + ): void { + // Cancel any pending frame request + if (this.lastFrameRequest !== null) { + cancelAnimationFrame(this.lastFrameRequest); + } + + // Check memory usage periodically + this.memoryCheckCounter++; + const now = performance.now(); + + if (this.memoryCheckCounter >= 30 || now - this.lastMemoryCheck > 5000) { + this.checkMemoryUsage(); + this.memoryCheckCounter = 0; + this.lastMemoryCheck = now; + } + + // Request a new frame + this.lastFrameRequest = requestAnimationFrame(() => { + this.renderFrame(screenshot, x, y, width, height); + }); + } + + private renderFrame( + screenshot: string | ImageBitmap | HTMLImageElement, + x: number, + y: number, + width?: number, + height?: number + ): void { + // Target context (offscreen if available, otherwise main) + const targetCtx = this.offscreenCtx || this.ctx; + + // Start timing the render + const startTime = performance.now(); + const timeSinceLastDraw = startTime - this.lastDrawTime; + + // Adaptive frame skipping for high-frequency updates + // If we're getting updates faster than 60fps and this isn't the first frame + if (timeSinceLastDraw < 16 && this.consecutiveFrameCount > 5) { + this.consecutiveFrameCount++; + + // Skip some frames when we're getting excessive updates + if (this.consecutiveFrameCount % 2 !== 0) { + return; + } + } else { + this.consecutiveFrameCount = 0; + } + + try { + if (typeof screenshot === 'string') { + // Check if we have this image in cache + let img = this.imageCache.get(screenshot); + + if (!img) { + img = new Image(); + img.src = screenshot; + this.imageCache.set(screenshot, img); + + // If image isn't loaded yet, draw when it loads + if (!img.complete) { + img.onload = () => { + if (img) { + this.drawScreenshot(img, x, y, width, height); + } + }; + return; + } } + + targetCtx.drawImage( + img, + x, y, + width || img.width, + height || img.height + ); } else { - this.consecutiveFrameCount = 0; + // Draw ImageBitmap or HTMLImageElement directly + targetCtx.drawImage( + screenshot, + x, y, + width || screenshot.width, + height || screenshot.height + ); } - - try { - if (typeof screenshot === 'string') { - // Check if we have this image in cache - let img = this.imageCache.get(screenshot); - - if (!img) { - img = new Image(); - img.src = screenshot; - this.imageCache.set(screenshot, img); - - // If image isn't loaded yet, draw when it loads - if (!img.complete) { - img.onload = () => { - if (img) { - this.drawScreenshot(img, x, y, width, height); - } - }; - return; - } - } - - targetCtx.drawImage( - img, - x, y, - width || img.width, - height || img.height - ); + + // If using offscreen canvas, copy to main canvas + if (this.offscreenCanvas && this.offscreenCtx) { + if ('transferToImageBitmap' in this.offscreenCanvas) { + // Use more efficient transfer when available + const bitmap = this.offscreenCanvas.transferToImageBitmap(); + this.ctx.drawImage(bitmap, 0, 0); } else { - // Draw ImageBitmap or HTMLImageElement directly - targetCtx.drawImage( - screenshot, - x, y, - width || screenshot.width, - height || screenshot.height - ); - } - - // If using offscreen canvas, copy to main canvas - if (this.offscreenCanvas && this.offscreenCtx) { - if ('transferToImageBitmap' in this.offscreenCanvas) { - // Use more efficient transfer when available - const bitmap = this.offscreenCanvas.transferToImageBitmap(); - this.ctx.drawImage(bitmap, 0, 0); - } else { - // Fallback to drawImage - this.ctx.drawImage(this.offscreenCanvas, 0, 0); - } - } - - // Update timestamp - this.lastDrawTime = performance.now(); - } catch (error) { - console.error('Error rendering frame:', error); - } - } - - /** - * Checks current memory usage and cleans up if necessary - */ - private checkMemoryUsage(): void { - if (window.performance && (performance as any).memory) { - const memory = (performance as any).memory; - - if (memory.usedJSHeapSize > this.memoryThreshold) { - this.cleanupMemory(); + // Fallback to drawImage + this.ctx.drawImage(this.offscreenCanvas, 0, 0); } } + + // Update timestamp + this.lastDrawTime = performance.now(); + } catch (error) { + console.error('Error rendering frame:', error); } - - /** - * Cleans up resources to reduce memory usage - */ - private cleanupMemory(): void { - // Limit image cache size - if (this.imageCache.size > 20) { - // Keep only the most recent 10 images - const keysToDelete = Array.from(this.imageCache.keys()).slice(0, this.imageCache.size - 10); - keysToDelete.forEach(key => { - this.imageCache.delete(key); - }); - } - - // Suggest garbage collection - if (window.gc) { - try { - window.gc(); - } catch (e) { - // GC not available, ignore - } + } + + /** + * Checks current memory usage and cleans up if necessary + */ + private checkMemoryUsage(): void { + if (window.performance && (performance as any).memory) { + const memory = (performance as any).memory; + + if (memory.usedJSHeapSize > this.memoryThreshold) { + this.cleanupMemory(); } } - - /** - * Update canvas dimensions - */ - public updateCanvasSize(width: number, height: number): void { - this.canvas.width = width; - this.canvas.height = height; - - // Re-apply context settings - this.ctx.imageSmoothingEnabled = false; - - // Update offscreen canvas if available - if (this.offscreenCanvas) { - this.offscreenCanvas.width = width; - this.offscreenCanvas.height = height; - - if (this.offscreenCtx) { - this.offscreenCtx.imageSmoothingEnabled = false; - } + } + + /** + * Cleans up resources to reduce memory usage + */ + private cleanupMemory(): void { + // Limit image cache size + if (this.imageCache.size > 20) { + // Keep only the most recent 10 images + const keysToDelete = Array.from(this.imageCache.keys()).slice(0, this.imageCache.size - 10); + keysToDelete.forEach(key => { + this.imageCache.delete(key); + }); + } + + // Suggest garbage collection + if (window.gc) { + try { + window.gc(); + } catch (e) { + // GC not available, ignore } } - - /** - * Clean up resources - */ - public dispose(): void { - // Cancel any pending frame requests - if (this.lastFrameRequest !== null) { - cancelAnimationFrame(this.lastFrameRequest); - this.lastFrameRequest = null; - } - - // Clear the image cache - this.imageCache.clear(); - - // Clear canvases - this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); - - if (this.offscreenCtx && this.offscreenCanvas) { - this.offscreenCtx.clearRect(0, 0, this.offscreenCanvas.width, this.offscreenCanvas.height); + } + + /** + * Update canvas dimensions + */ + public updateCanvasSize(width: number, height: number): void { + this.canvas.width = width; + this.canvas.height = height; + + // Re-apply context settings + this.ctx.imageSmoothingEnabled = false; + + // Update offscreen canvas if available + if (this.offscreenCanvas) { + this.offscreenCanvas.width = width; + this.offscreenCanvas.height = height; + + if (this.offscreenCtx) { + this.offscreenCtx.imageSmoothingEnabled = false; } } - } \ No newline at end of file + } + + /** + * Clean up resources + */ + public dispose(): void { + // Cancel any pending frame requests + if (this.lastFrameRequest !== null) { + cancelAnimationFrame(this.lastFrameRequest); + this.lastFrameRequest = null; + } + + // Clear the image cache + this.imageCache.clear(); + + // Clear canvases + this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); + + if (this.offscreenCtx && this.offscreenCanvas) { + this.offscreenCtx.clearRect(0, 0, this.offscreenCanvas.width, this.offscreenCanvas.height); + } + } +} \ No newline at end of file From 6903ef2c32c0ae2a44f0474888d0a0c2b896acd0 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 4 Mar 2025 22:47:35 +0530 Subject: [PATCH 12/51] feat: use enhanced performance monitor --- src/components/recorder/canvas.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/recorder/canvas.tsx b/src/components/recorder/canvas.tsx index 9ec41f91..29054773 100644 --- a/src/components/recorder/canvas.tsx +++ b/src/components/recorder/canvas.tsx @@ -6,7 +6,7 @@ import DatePicker from '../pickers/DatePicker'; import Dropdown from '../pickers/Dropdown'; import TimePicker from '../pickers/TimePicker'; import DateTimeLocalPicker from '../pickers/DateTimeLocalPicker'; -import { FrontendPerformanceMonitor } from '../../../perf/performance'; +import { EnhancedPerformanceMonitor } from '../../../perf/performance'; interface CreateRefCallback { (ref: React.RefObject): void; @@ -28,7 +28,7 @@ export interface Coordinates { const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { - const performanceMonitor = useRef(new FrontendPerformanceMonitor()); + const performanceMonitor = useRef(new EnhancedPerformanceMonitor()); console.log('Frontend Performance Report:', performanceMonitor.current.getPerformanceReport()); const canvasRef = useRef(null); From 0d9092367cfd20950c411c753491128cc83c730d Mon Sep 17 00:00:00 2001 From: Rohit Date: Thu, 6 Mar 2025 16:37:22 +0530 Subject: [PATCH 13/51] feat: add jobs for recording --- server/src/routes/record.ts | 206 ++++++++++++++++++++++++++++++------ 1 file changed, 174 insertions(+), 32 deletions(-) diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index 51d3ff92..055099ab 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -9,18 +9,19 @@ import { getActiveBrowserId, interpretWholeWorkflow, stopRunningInterpretation, - getRemoteBrowserCurrentUrl, getRemoteBrowserCurrentTabs, -} from '../browser-management/controller' + getRemoteBrowserCurrentUrl, + getRemoteBrowserCurrentTabs, +} from '../browser-management/controller'; import { chromium } from 'playwright-extra'; import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import logger from "../logger"; import { getDecryptedProxyConfig } from './proxy'; import { requireSignIn } from '../middlewares/auth'; +import { pgBoss } from '../server'; // Import pgBoss reference export const router = Router(); chromium.use(stealthPlugin()); - export interface AuthenticatedRequest extends Request { user?: any; } @@ -35,41 +36,106 @@ router.all('/', requireSignIn, (req, res, next) => { /** * GET endpoint for starting the remote browser recording session. - * returns session's id + * returns session's id or job id */ router.get('/start', requireSignIn, async (req: AuthenticatedRequest, res: Response) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const proxyConfig = await getDecryptedProxyConfig(req.user.id); - // Prepare the proxy options dynamically based on the user's proxy configuration - let proxyOptions: any = {}; // Default to no proxy - - if (proxyConfig.proxy_url) { - // Set the server, and if username & password exist, set those as well - proxyOptions = { - server: proxyConfig.proxy_url, - ...(proxyConfig.proxy_username && proxyConfig.proxy_password && { - username: proxyConfig.proxy_username, - password: proxyConfig.proxy_password, - }), - }; + try { + const job = await pgBoss.send('initialize-browser-recording', { + userId: req.user.id + }); + + if (!job) { + logger.log('warn', 'pgBoss.send returned null, falling back to direct initialization'); + const browserId = initializeRemoteBrowserForRecording(req.user.id); + return res.send(browserId); + } + + logger.log('info', `Queued browser initialization job: ${job}`); + return res.send(job); + } catch (error: any) { + logger.log('error', `Failed to queue browser initialization job: ${error.message}`); + + try { + const browserId = initializeRemoteBrowserForRecording(req.user.id); + return res.send(browserId); + } catch (directError: any) { + logger.log('error', `Direct initialization also failed: ${directError.message}`); + return res.status(500).send('Failed to start recording'); + } } - - const id = initializeRemoteBrowserForRecording(req.user.id); - return res.send(id); }); /** * POST endpoint for starting the remote browser recording session accepting browser launch options. - * returns session's id + * returns session's id or job id */ -router.post('/start', requireSignIn, (req: AuthenticatedRequest, res:Response) => { +router.post('/start', requireSignIn, async (req: AuthenticatedRequest, res: Response) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = initializeRemoteBrowserForRecording(req.user.id); - return res.send(id); + try { + const job = await pgBoss.send('initialize-browser-recording', { + userId: req.user.id + }); + + if (!job) { + logger.log('warn', 'pgBoss.send returned null, falling back to direct initialization'); + const browserId = initializeRemoteBrowserForRecording(req.user.id); + return res.send(browserId); + } + + logger.log('info', `Queued browser initialization job: ${job}`); + return res.send(job); + } catch (error: any) { + logger.log('error', `Failed to queue browser initialization job: ${error.message}`); + + try { + const browserId = initializeRemoteBrowserForRecording(req.user.id); + return res.send(browserId); + } catch (directError: any) { + logger.log('error', `Direct initialization also failed: ${directError.message}`); + return res.status(500).send('Failed to start recording'); + } + } +}); + +/** + * GET endpoint for getting the job status of a browser initialization job + */ +router.get('/job-status/:jobId', requireSignIn, async (req, res) => { + try { + logger.log('debug', `Checking status for job ${req.params.jobId}`); + const job = await pgBoss.getJobById("job-status", req.params.jobId); + + if (!job) { + logger.log('warn', `Job ${req.params.jobId} not found`); + return res.status(404).send('Job not found'); + } + + logger.log('debug', `Job state: ${job.state}, hasOutput: ${!!job.output}`); + + if (job.state === 'completed' && job.output) { + const output = job.output as { browserId?: string }; + if (output.browserId) { + logger.log('info', `Job completed with browserId: ${output.browserId}`); + } else { + logger.log('warn', `Job completed but missing browserId in output`); + } + return res.send(output); // Return the browser ID from the completed job + } + + return res.send({ + state: job.state, + createdAt: job.createdOn, + startedAt: job.startedOn || null + }); + } catch (error: any) { + logger.log('error', `Failed to get job status: ${error.message}`); + return res.status(500).send('Failed to get job status'); + } }); /** @@ -77,8 +143,48 @@ router.post('/start', requireSignIn, (req: AuthenticatedRequest, res:Response) = * returns whether the termination was successful */ router.get('/stop/:browserId', requireSignIn, async (req, res) => { - const success = await destroyRemoteBrowser(req.params.browserId); - return res.send(success); + try { + if (req.params.browserId.startsWith('job_')) { + logger.log('debug', `Stopping job ${req.params.browserId}`); + + try { + const job = await pgBoss.getJobById("stop", req.params.browserId); + if (job && job.state === 'completed' && job.output) { + const output = job.output as { browserId?: string }; + if (output.browserId) { + await pgBoss.send('destroy-browser', { + browserId: output.browserId + }); + logger.log('info', `Queued destroy job for browser ${output.browserId}`); + return res.send(true); + } + } else if (job && (job.state === 'created' || job.state === 'active')) { + await pgBoss.cancel("cancel", req.params.browserId); + logger.log('info', `Cancelled job ${req.params.browserId}`); + return res.send(true); + } + } catch (jobError: any) { + logger.log('error', `Error handling job termination: ${jobError.message}`); + } + } + + try { + await pgBoss.send('destroy-browser', { + browserId: req.params.browserId + }); + logger.log('info', `Queued destroy job for browser ${req.params.browserId}`); + return res.send(true); + } catch (queueError: any) { + logger.log('error', `Failed to queue destroy job: ${queueError.message}`); + + const success = await destroyRemoteBrowser(req.params.browserId); + logger.log('info', `Direct browser destruction result: ${success}`); + return res.send(success); + } + } catch (error: any) { + logger.log('error', `Failed to stop browser: ${error.message}`); + return res.status(500).send(false); + } }); /** @@ -118,10 +224,25 @@ router.get('/active/tabs', requireSignIn, (req, res) => { */ router.get('/interpret', requireSignIn, async (req, res) => { try { - await interpretWholeWorkflow(); - return res.send('interpretation done'); - } catch (e) { - return res.send('interpretation failed'); + const job = await pgBoss.send('interpret-workflow', {}); + + if (!job) { + logger.log('warn', 'pgBoss.send returned null for interpret, falling back to direct interpretation'); + await interpretWholeWorkflow(); + return res.send('interpretation complete (direct)'); + } + + logger.log('info', `Queued interpretation job: ${job}`); + return res.send('interpretation queued'); + } catch (error: any) { + logger.log('error', `Failed to queue interpretation job: ${error.message}`); + + try { + await interpretWholeWorkflow(); + return res.send('interpretation complete (fallback)'); + } catch (directError: any) { + return res.status(500).send('interpretation failed'); + } } }); @@ -129,6 +250,27 @@ router.get('/interpret', requireSignIn, async (req, res) => { * GET endpoint for stopping an ongoing interpretation of the currently generated workflow. */ router.get('/interpret/stop', requireSignIn, async (req, res) => { - await stopRunningInterpretation(); - return res.send('interpretation stopped'); + try { + const job = await pgBoss.send('stop-interpretation', {}); + + if (!job) { + logger.log('warn', 'pgBoss.send returned null for stop-interpretation, falling back to direct stop'); + await stopRunningInterpretation(); + return res.send('interpretation stopped (direct)'); + } + + logger.log('info', `Queued stop interpretation job: ${job}`); + return res.send('interpretation stop queued'); + } catch (error: any) { + logger.log('error', `Failed to queue stop interpretation job: ${error.message}`); + + try { + await stopRunningInterpretation(); + return res.send('interpretation stopped (fallback)'); + } catch (directError: any) { + return res.status(500).send('interpretation stop failed'); + } + } }); + +export default router; \ No newline at end of file From 4e5c3b1bb1e6a7eed6c949becd54ee890b95e52c Mon Sep 17 00:00:00 2001 From: Rohit Date: Thu, 6 Mar 2025 16:38:16 +0530 Subject: [PATCH 14/51] feat: register recording worker path --- server/src/server.ts | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/server/src/server.ts b/server/src/server.ts index 9c0eff23..85d05240 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -22,7 +22,7 @@ import swaggerSpec from './swagger/config'; import session from 'express-session'; import Run from './models/Run'; - +import PgBoss from 'pg-boss'; const app = express(); app.use(cors({ @@ -54,6 +54,10 @@ export const io = new Server(server); */ export const browserPool = new BrowserPool(); +const pgBossConnectionString = 'postgres://postgres:admin1234@localhost:5432/maxun'; + +export const pgBoss = new PgBoss({connectionString: pgBossConnectionString, schema: 'public'}); + // app.use(bodyParser.json({ limit: '10mb' })) // app.use(bodyParser.urlencoded({ extended: true, limit: '10mb', parameterLimit: 9000 })); // parse cookies - "cookie" is true in csrfProtection @@ -79,8 +83,11 @@ readdirSync(path.join(__dirname, 'api')).forEach((r) => { const isProduction = process.env.NODE_ENV === 'production'; const workerPath = path.resolve(__dirname, isProduction ? './worker.js' : './worker.ts'); +const recordingWorkerPath = path.resolve(__dirname, isProduction ? './pgboss-worker.js' : './pgboss-worker.ts'); let workerProcess: any; +let recordingWorkerProcess: any; + if (!isProduction) { workerProcess = fork(workerPath, [], { execArgv: ['--inspect=5859'], @@ -94,6 +101,19 @@ if (!isProduction) { workerProcess.on('exit', (code: any) => { console.log(`Worker exited with code: ${code}`); }); + + recordingWorkerProcess = fork(recordingWorkerPath, [], { + execArgv: ['--inspect=5860'], + }); + recordingWorkerProcess.on('message', (message: any) => { + console.log(`Message from recording worker: ${message}`); + }); + recordingWorkerProcess.on('error', (error: any) => { + console.error(`Error in recording worker: ${error}`); + }); + recordingWorkerProcess.on('exit', (code: any) => { + console.log(`Recording worker exited with code: ${code}`); + }); } app.get('/', function (req, res) { @@ -146,7 +166,8 @@ process.on('SIGINT', async () => { } if (!isProduction) { - workerProcess.kill(); + if (workerProcess) workerProcess.kill(); + if (recordingWorkerProcess) recordingWorkerProcess.kill(); } process.exit(); }); From 360f863aab227e6aab5e0bfacc0272810b3b3766 Mon Sep 17 00:00:00 2001 From: Rohit Date: Thu, 6 Mar 2025 16:38:42 +0530 Subject: [PATCH 15/51] feat: add pgboss recording worker --- server/src/pgboss-worker.ts | 142 ++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 server/src/pgboss-worker.ts diff --git a/server/src/pgboss-worker.ts b/server/src/pgboss-worker.ts new file mode 100644 index 00000000..51db3bcf --- /dev/null +++ b/server/src/pgboss-worker.ts @@ -0,0 +1,142 @@ +/** + * Recording worker using PgBoss for asynchronous browser recording operations + */ +import PgBoss, { Job } from 'pg-boss'; +import logger from './logger'; +import { + initializeRemoteBrowserForRecording, + destroyRemoteBrowser, + interpretWholeWorkflow, + stopRunningInterpretation +} from './browser-management/controller'; +import dotenv from 'dotenv'; + +// Load environment variables +dotenv.config(); + +// Define connection string +const pgBossConnectionString = 'postgres://postgres:admin1234@localhost:5432/maxun'; + +// Define interfaces for job data structures +interface InitializeBrowserData { + userId: string; +} + +interface DestroyBrowserData { + browserId: string; +} + +// Initialize pg-boss instance +const pgBoss = new PgBoss({connectionString: pgBossConnectionString, schema: 'public'}); + +// Start pg-boss +pgBoss.start() + .then(() => { + logger.log('info', 'Recording worker started successfully'); + + // Register all workers + registerWorkers(); + }) + .catch((error: Error) => { + logger.log('error', `Failed to start recording worker: ${error.message}`); + process.exit(1); + }); + +/** + * Extract data safely from a job (single job or job array) + */ +function extractJobData(job: Job | Job[]): T { + if (Array.isArray(job)) { + if (job.length === 0) { + throw new Error('Empty job array received'); + } + return job[0].data; + } + return job.data; +} + +/** + * Register all browser operation workers + */ +function registerWorkers(): void { + // Worker for initializing browser recording + pgBoss.work('initialize-browser-recording', async (job: Job | Job[]) => { + try { + const data = extractJobData(job); + const userId = data.userId; + + logger.log('info', `Starting browser initialization job for user: ${userId}`); + const browserId = initializeRemoteBrowserForRecording(userId); + logger.log('info', `Browser recording job completed with browserId: ${browserId}`); + return { browserId }; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Browser recording job failed: ${errorMessage}`); + throw error; + } + }); + + // Worker for stopping a browser + pgBoss.work('destroy-browser', async (job: Job | Job[]) => { + try { + const data = extractJobData(job); + const browserId = data.browserId; + + logger.log('info', `Starting browser destruction job for browser: ${browserId}`); + const success = await destroyRemoteBrowser(browserId); + logger.log('info', `Browser destruction job completed with result: ${success}`); + return { success }; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Destroy browser job failed: ${errorMessage}`); + throw error; + } + }); + + // Worker for interpreting workflow + pgBoss.work('interpret-workflow', async () => { + try { + logger.log('info', 'Starting workflow interpretation job'); + await interpretWholeWorkflow(); + logger.log('info', 'Workflow interpretation job completed'); + return { success: true }; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Interpret workflow job failed: ${errorMessage}`); + throw error; + } + }); + + // Worker for stopping workflow interpretation + pgBoss.work('stop-interpretation', async () => { + try { + logger.log('info', 'Starting stop interpretation job'); + await stopRunningInterpretation(); + logger.log('info', 'Stop interpretation job completed'); + return { success: true }; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Stop interpretation job failed: ${errorMessage}`); + throw error; + } + }); + + logger.log('info', 'All recording workers registered successfully'); +} + +// Handle shutdown +process.on('SIGINT', async () => { + logger.log('info', 'Recording worker shutting down...'); + + try { + await pgBoss.stop(); + logger.log('info', 'PgBoss stopped gracefully'); + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Error stopping PgBoss: ${errorMessage}`); + } + + process.exit(0); +}); + +export { pgBoss }; \ No newline at end of file From cc12f19aa03ac22abefa9ffd6225927b5aad1a5e Mon Sep 17 00:00:00 2001 From: Rohit Date: Mon, 10 Mar 2025 16:01:38 +0530 Subject: [PATCH 16/51] feat: add recording worker process --- server/src/server.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/server/src/server.ts b/server/src/server.ts index 85d05240..0bd46b38 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -22,7 +22,6 @@ import swaggerSpec from './swagger/config'; import session from 'express-session'; import Run from './models/Run'; -import PgBoss from 'pg-boss'; const app = express(); app.use(cors({ @@ -54,10 +53,6 @@ export const io = new Server(server); */ export const browserPool = new BrowserPool(); -const pgBossConnectionString = 'postgres://postgres:admin1234@localhost:5432/maxun'; - -export const pgBoss = new PgBoss({connectionString: pgBossConnectionString, schema: 'public'}); - // app.use(bodyParser.json({ limit: '10mb' })) // app.use(bodyParser.urlencoded({ extended: true, limit: '10mb', parameterLimit: 9000 })); // parse cookies - "cookie" is true in csrfProtection From 65cb201474f16529dca2dce82fc9754ce2960023 Mon Sep 17 00:00:00 2001 From: Rohit Date: Mon, 10 Mar 2025 16:02:21 +0530 Subject: [PATCH 17/51] feat: start pgboss workers --- server/src/pgboss-worker.ts | 184 +++++++++++++++++------------------- 1 file changed, 88 insertions(+), 96 deletions(-) diff --git a/server/src/pgboss-worker.ts b/server/src/pgboss-worker.ts index 51db3bcf..5d2ea86d 100644 --- a/server/src/pgboss-worker.ts +++ b/server/src/pgboss-worker.ts @@ -9,15 +9,9 @@ import { interpretWholeWorkflow, stopRunningInterpretation } from './browser-management/controller'; -import dotenv from 'dotenv'; -// Load environment variables -dotenv.config(); +const pgBossConnectionString = `postgres://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`; -// Define connection string -const pgBossConnectionString = 'postgres://postgres:admin1234@localhost:5432/maxun'; - -// Define interfaces for job data structures interface InitializeBrowserData { userId: string; } @@ -26,22 +20,8 @@ interface DestroyBrowserData { browserId: string; } -// Initialize pg-boss instance const pgBoss = new PgBoss({connectionString: pgBossConnectionString, schema: 'public'}); -// Start pg-boss -pgBoss.start() - .then(() => { - logger.log('info', 'Recording worker started successfully'); - - // Register all workers - registerWorkers(); - }) - .catch((error: Error) => { - logger.log('error', `Failed to start recording worker: ${error.message}`); - process.exit(1); - }); - /** * Extract data safely from a job (single job or job array) */ @@ -56,87 +36,99 @@ function extractJobData(job: Job | Job[]): T { } /** - * Register all browser operation workers + * Initialize PgBoss and register all workers */ -function registerWorkers(): void { - // Worker for initializing browser recording - pgBoss.work('initialize-browser-recording', async (job: Job | Job[]) => { - try { - const data = extractJobData(job); - const userId = data.userId; - - logger.log('info', `Starting browser initialization job for user: ${userId}`); - const browserId = initializeRemoteBrowserForRecording(userId); - logger.log('info', `Browser recording job completed with browserId: ${browserId}`); - return { browserId }; - } catch (error: unknown) { - const errorMessage = error instanceof Error ? error.message : String(error); - logger.log('error', `Browser recording job failed: ${errorMessage}`); - throw error; - } - }); - - // Worker for stopping a browser - pgBoss.work('destroy-browser', async (job: Job | Job[]) => { - try { - const data = extractJobData(job); - const browserId = data.browserId; - - logger.log('info', `Starting browser destruction job for browser: ${browserId}`); - const success = await destroyRemoteBrowser(browserId); - logger.log('info', `Browser destruction job completed with result: ${success}`); - return { success }; - } catch (error: unknown) { - const errorMessage = error instanceof Error ? error.message : String(error); - logger.log('error', `Destroy browser job failed: ${errorMessage}`); - throw error; - } - }); - - // Worker for interpreting workflow - pgBoss.work('interpret-workflow', async () => { - try { - logger.log('info', 'Starting workflow interpretation job'); - await interpretWholeWorkflow(); - logger.log('info', 'Workflow interpretation job completed'); - return { success: true }; - } catch (error: unknown) { - const errorMessage = error instanceof Error ? error.message : String(error); - logger.log('error', `Interpret workflow job failed: ${errorMessage}`); - throw error; - } - }); - - // Worker for stopping workflow interpretation - pgBoss.work('stop-interpretation', async () => { - try { - logger.log('info', 'Starting stop interpretation job'); - await stopRunningInterpretation(); - logger.log('info', 'Stop interpretation job completed'); - return { success: true }; - } catch (error: unknown) { - const errorMessage = error instanceof Error ? error.message : String(error); - logger.log('error', `Stop interpretation job failed: ${errorMessage}`); - throw error; - } - }); - - logger.log('info', 'All recording workers registered successfully'); -} - -// Handle shutdown -process.on('SIGINT', async () => { - logger.log('info', 'Recording worker shutting down...'); - +async function startWorkers() { try { - await pgBoss.stop(); - logger.log('info', 'PgBoss stopped gracefully'); + logger.log('info', 'Starting PgBoss worker...'); + await pgBoss.start(); + logger.log('info', 'PgBoss worker started successfully'); + + // Worker for initializing browser recording + await pgBoss.work('initialize-browser-recording', async (job: Job | Job[]) => { + try { + const data = extractJobData(job); + const userId = data.userId; + + logger.log('info', `Starting browser initialization job for user: ${userId}`); + const browserId = initializeRemoteBrowserForRecording(userId); + logger.log('info', `Browser recording job completed with browserId: ${browserId}`); + return { browserId }; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Browser recording job failed: ${errorMessage}`); + throw error; + } + }); + + // Worker for stopping a browser + await pgBoss.work('destroy-browser', async (job: Job | Job[]) => { + try { + const data = extractJobData(job); + const browserId = data.browserId; + + logger.log('info', `Starting browser destruction job for browser: ${browserId}`); + const success = await destroyRemoteBrowser(browserId); + logger.log('info', `Browser destruction job completed with result: ${success}`); + return { success }; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Destroy browser job failed: ${errorMessage}`); + throw error; + } + }); + + // Worker for interpreting workflow + await pgBoss.work('interpret-workflow', async () => { + try { + logger.log('info', 'Starting workflow interpretation job'); + await interpretWholeWorkflow(); + logger.log('info', 'Workflow interpretation job completed'); + return { success: true }; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Interpret workflow job failed: ${errorMessage}`); + throw error; + } + }); + + // Worker for stopping workflow interpretation + await pgBoss.work('stop-interpretation', async () => { + try { + logger.log('info', 'Starting stop interpretation job'); + await stopRunningInterpretation(); + logger.log('info', 'Stop interpretation job completed'); + return { success: true }; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Stop interpretation job failed: ${errorMessage}`); + throw error; + } + }); + + logger.log('info', 'All recording workers registered successfully'); } catch (error: unknown) { const errorMessage = error instanceof Error ? error.message : String(error); - logger.log('error', `Error stopping PgBoss: ${errorMessage}`); + logger.log('error', `Failed to start PgBoss workers: ${errorMessage}`); + process.exit(1); } - +} + +// Start all workers +startWorkers(); + +// Handle graceful shutdown +process.on('SIGTERM', async () => { + logger.log('info', 'SIGTERM received, shutting down PgBoss...'); + await pgBoss.stop(); process.exit(0); }); +process.on('SIGINT', async () => { + logger.log('info', 'SIGINT received, shutting down PgBoss...'); + await pgBoss.stop(); + process.exit(0); +}); + +// For use in other files export { pgBoss }; \ No newline at end of file From be1b90feefe58173d000e1f776edcd18464dccde Mon Sep 17 00:00:00 2001 From: Rohit Date: Mon, 10 Mar 2025 16:04:12 +0530 Subject: [PATCH 18/51] feat: send jobs and wait for completion --- server/src/routes/record.ts | 256 ++++++++++++++++-------------------- 1 file changed, 110 insertions(+), 146 deletions(-) diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index 055099ab..676254be 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -17,7 +17,7 @@ import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import logger from "../logger"; import { getDecryptedProxyConfig } from './proxy'; import { requireSignIn } from '../middlewares/auth'; -import { pgBoss } from '../server'; // Import pgBoss reference +import { pgBoss } from '../pgboss-worker'; export const router = Router(); chromium.use(stealthPlugin()); @@ -26,6 +26,41 @@ export interface AuthenticatedRequest extends Request { user?: any; } +async function waitForJobCompletion(jobId: string, queueName: string, timeout = 15000): Promise { + return new Promise((resolve, reject) => { + const startTime = Date.now(); + + const checkJobStatus = async () => { + if (Date.now() - startTime > timeout) { + return reject(new Error(`Timeout waiting for job ${jobId} to complete`)); + } + + try { + const job = await pgBoss.getJobById(queueName, jobId); + + if (!job) { + return reject(new Error(`Job ${jobId} not found`)); + } + + if (job.state === 'completed') { + return resolve(job.output); + } + + if (job.state === 'failed') { + return reject(new Error(`Job ${jobId} failed.`)); + } + + setTimeout(checkJobStatus, 200); + } catch (error) { + reject(error); + } + }; + + // Start checking + checkJobStatus(); + }); +} + /** * Logs information about remote browser recording session. */ @@ -34,33 +69,52 @@ router.all('/', requireSignIn, (req, res, next) => { next() // pass control to the next handler }) + /** - * GET endpoint for starting the remote browser recording session. - * returns session's id or job id + * GET endpoint for starting the remote browser recording session + * Waits for job completion */ router.get('/start', requireSignIn, async (req: AuthenticatedRequest, res: Response) => { if (!req.user) { return res.status(401).send('User not authenticated'); } + try { - const job = await pgBoss.send('initialize-browser-recording', { - userId: req.user.id + await pgBoss.createQueue('initialize-browser-recording'); + + const jobId = await pgBoss.send('initialize-browser-recording', { + userId: req.user.id, + timestamp: new Date().toISOString() }); - if (!job) { + if (!jobId) { logger.log('warn', 'pgBoss.send returned null, falling back to direct initialization'); const browserId = initializeRemoteBrowserForRecording(req.user.id); - return res.send(browserId); + return res.send( browserId ); } - logger.log('info', `Queued browser initialization job: ${job}`); - return res.send(job); + logger.log('info', `Queued browser initialization job: ${jobId}, waiting for completion...`); + + try { + const result = await waitForJobCompletion(jobId, 'initialize-browser-recording', 15000); + + if (result && result.browserId) { + logger.log('info', `Job completed with browserId: ${result.browserId}`); + return res.send(result.browserId); + } else { + logger.log('warn', 'Job completed but returned unexpected result'); + return res.send(jobId); + } + } catch (waitError: any) { + logger.log('warn', `Error waiting for job completion: ${waitError.message}`); + return res.send(jobId); + } } catch (error: any) { logger.log('error', `Failed to queue browser initialization job: ${error.message}`); try { const browserId = initializeRemoteBrowserForRecording(req.user.id); - return res.send(browserId); + return res.send( browserId ); } catch (directError: any) { logger.log('error', `Direct initialization also failed: ${directError.message}`); return res.status(500).send('Failed to start recording'); @@ -70,116 +124,54 @@ router.get('/start', requireSignIn, async (req: AuthenticatedRequest, res: Respo /** * POST endpoint for starting the remote browser recording session accepting browser launch options. - * returns session's id or job id + * returns session's id */ -router.post('/start', requireSignIn, async (req: AuthenticatedRequest, res: Response) => { +router.post('/start', requireSignIn, (req: AuthenticatedRequest, res:Response) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - try { - const job = await pgBoss.send('initialize-browser-recording', { - userId: req.user.id - }); - - if (!job) { - logger.log('warn', 'pgBoss.send returned null, falling back to direct initialization'); - const browserId = initializeRemoteBrowserForRecording(req.user.id); - return res.send(browserId); - } - - logger.log('info', `Queued browser initialization job: ${job}`); - return res.send(job); - } catch (error: any) { - logger.log('error', `Failed to queue browser initialization job: ${error.message}`); - - try { - const browserId = initializeRemoteBrowserForRecording(req.user.id); - return res.send(browserId); - } catch (directError: any) { - logger.log('error', `Direct initialization also failed: ${directError.message}`); - return res.status(500).send('Failed to start recording'); - } - } -}); - -/** - * GET endpoint for getting the job status of a browser initialization job - */ -router.get('/job-status/:jobId', requireSignIn, async (req, res) => { - try { - logger.log('debug', `Checking status for job ${req.params.jobId}`); - const job = await pgBoss.getJobById("job-status", req.params.jobId); - - if (!job) { - logger.log('warn', `Job ${req.params.jobId} not found`); - return res.status(404).send('Job not found'); - } - - logger.log('debug', `Job state: ${job.state}, hasOutput: ${!!job.output}`); - - if (job.state === 'completed' && job.output) { - const output = job.output as { browserId?: string }; - if (output.browserId) { - logger.log('info', `Job completed with browserId: ${output.browserId}`); - } else { - logger.log('warn', `Job completed but missing browserId in output`); - } - return res.send(output); // Return the browser ID from the completed job - } - - return res.send({ - state: job.state, - createdAt: job.createdOn, - startedAt: job.startedOn || null - }); - } catch (error: any) { - logger.log('error', `Failed to get job status: ${error.message}`); - return res.status(500).send('Failed to get job status'); - } + const id = initializeRemoteBrowserForRecording(req.user.id); + return res.send(id); }); /** * GET endpoint for terminating the remote browser recording session. * returns whether the termination was successful */ -router.get('/stop/:browserId', requireSignIn, async (req, res) => { +router.get('/stop/:browserId', requireSignIn, async (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + try { - if (req.params.browserId.startsWith('job_')) { - logger.log('debug', `Stopping job ${req.params.browserId}`); - - try { - const job = await pgBoss.getJobById("stop", req.params.browserId); - if (job && job.state === 'completed' && job.output) { - const output = job.output as { browserId?: string }; - if (output.browserId) { - await pgBoss.send('destroy-browser', { - browserId: output.browserId - }); - logger.log('info', `Queued destroy job for browser ${output.browserId}`); - return res.send(true); - } - } else if (job && (job.state === 'created' || job.state === 'active')) { - await pgBoss.cancel("cancel", req.params.browserId); - logger.log('info', `Cancelled job ${req.params.browserId}`); - return res.send(true); - } - } catch (jobError: any) { - logger.log('error', `Error handling job termination: ${jobError.message}`); - } - } + await pgBoss.createQueue('destroy-browser'); + const jobId = await pgBoss.send('destroy-browser', { + browserId: req.params.browserId, + timestamp: new Date().toISOString() + }); + + if (!jobId) { + logger.log('warn', 'pgBoss.send returned null, falling back to direct destruction'); + const browserId = initializeRemoteBrowserForRecording(req.user.id); + return res.send( browserId ); + } + + logger.log('info', `Queued browser destruction job: ${jobId}, waiting for completion...`); + try { - await pgBoss.send('destroy-browser', { - browserId: req.params.browserId - }); - logger.log('info', `Queued destroy job for browser ${req.params.browserId}`); - return res.send(true); - } catch (queueError: any) { - logger.log('error', `Failed to queue destroy job: ${queueError.message}`); + const result = await waitForJobCompletion(jobId, 'destroy-browser', 15000); - const success = await destroyRemoteBrowser(req.params.browserId); - logger.log('info', `Direct browser destruction result: ${success}`); - return res.send(success); + if (result) { + logger.log('info', `Browser destruction job completed with result: ${result.success}`); + return res.send(result.success); + } else { + logger.log('warn', 'Job completed but returned unexpected result'); + return res.send(false); + } + } catch (waitError: any) { + logger.log('warn', `Error waiting for job completion: ${waitError.message}`); + return res.send(false); } } catch (error: any) { logger.log('error', `Failed to stop browser: ${error.message}`); @@ -222,55 +214,27 @@ router.get('/active/tabs', requireSignIn, (req, res) => { /** * GET endpoint for starting an interpretation of the currently generated workflow. */ -router.get('/interpret', requireSignIn, async (req, res) => { +router.get('/interpret', requireSignIn, async (req: AuthenticatedRequest, res) => { try { - const job = await pgBoss.send('interpret-workflow', {}); - - if (!job) { - logger.log('warn', 'pgBoss.send returned null for interpret, falling back to direct interpretation'); - await interpretWholeWorkflow(); - return res.send('interpretation complete (direct)'); - } - - logger.log('info', `Queued interpretation job: ${job}`); - return res.send('interpretation queued'); - } catch (error: any) { - logger.log('error', `Failed to queue interpretation job: ${error.message}`); - - try { - await interpretWholeWorkflow(); - return res.send('interpretation complete (fallback)'); - } catch (directError: any) { - return res.status(500).send('interpretation failed'); + if (!req.user) { + return res.status(401).send('User not authenticated'); } + await interpretWholeWorkflow(); + return res.send('interpretation done'); + } catch (e) { + return res.send('interpretation failed'); } }); /** * GET endpoint for stopping an ongoing interpretation of the currently generated workflow. */ -router.get('/interpret/stop', requireSignIn, async (req, res) => { - try { - const job = await pgBoss.send('stop-interpretation', {}); - - if (!job) { - logger.log('warn', 'pgBoss.send returned null for stop-interpretation, falling back to direct stop'); - await stopRunningInterpretation(); - return res.send('interpretation stopped (direct)'); - } - - logger.log('info', `Queued stop interpretation job: ${job}`); - return res.send('interpretation stop queued'); - } catch (error: any) { - logger.log('error', `Failed to queue stop interpretation job: ${error.message}`); - - try { - await stopRunningInterpretation(); - return res.send('interpretation stopped (fallback)'); - } catch (directError: any) { - return res.status(500).send('interpretation stop failed'); - } +router.get('/interpret/stop', requireSignIn, async (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); } + await stopRunningInterpretation(); + return res.send('interpretation stopped'); }); export default router; \ No newline at end of file From 40bc2929cc1a8b721370eaa1e9e12fec7b3c2141 Mon Sep 17 00:00:00 2001 From: Rohit Date: Mon, 10 Mar 2025 22:53:18 +0530 Subject: [PATCH 19/51] feat: optimize mouse events and memoize canvas --- src/components/recorder/canvas.tsx | 55 +++++++++++++++++++----------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/src/components/recorder/canvas.tsx b/src/components/recorder/canvas.tsx index 29054773..4a015c87 100644 --- a/src/components/recorder/canvas.tsx +++ b/src/components/recorder/canvas.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useEffect, useRef } from 'react'; +import React, { memo, useCallback, useEffect, useRef } from 'react'; import { useSocketStore } from '../../context/socket'; import { useGlobalInfoStore } from "../../context/globalInfo"; import { useActionContext } from '../../context/browserActions'; @@ -38,6 +38,9 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { const getTextRef = useRef(getText); const getListRef = useRef(getList); + const MOUSE_MOVE_THROTTLE = 16; // ~60fps + const lastMouseMoveTime = useRef(0); + const [datePickerInfo, setDatePickerInfo] = React.useState<{ coordinates: Coordinates; selector: string; @@ -134,29 +137,38 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { } notifyLastAction('click'); break; - case 'mousemove': - if (lastMousePosition.current.x !== clickCoordinates.x || - lastMousePosition.current.y !== clickCoordinates.y) { + case 'mousemove': { + const now = performance.now(); + if (now - lastMouseMoveTime.current < MOUSE_MOVE_THROTTLE) { + return; + } + lastMouseMoveTime.current = now; + + const dx = Math.abs(lastMousePosition.current.x - clickCoordinates.x); + const dy = Math.abs(lastMousePosition.current.y - clickCoordinates.y); + if (dx > 1 || dy > 1) { lastMousePosition.current = { x: clickCoordinates.x, y: clickCoordinates.y, }; - socket.emit('input:mousemove', { - x: clickCoordinates.x, - y: clickCoordinates.y, - }); + socket.emit('input:mousemove', clickCoordinates); notifyLastAction('move'); } break; - case 'wheel': + } + + // Optimize wheel events + case 'wheel': { const wheelEvent = event as WheelEvent; - const deltas = { - deltaX: Math.round(wheelEvent.deltaX), - deltaY: Math.round(wheelEvent.deltaY), - }; - socket.emit('input:wheel', deltas); - notifyLastAction('scroll'); + const deltaX = Math.round(wheelEvent.deltaX / 10) * 10; + const deltaY = Math.round(wheelEvent.deltaY / 10) * 10; + + if (Math.abs(deltaX) > 5 || Math.abs(deltaY) > 5) { + socket.emit('input:wheel', { deltaX, deltaY }); + notifyLastAction('scroll'); + } break; + } default: console.log('Default mouseEvent registered'); return; @@ -222,9 +234,14 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { {datePickerInfo && ( { }; -export default Canvas; \ No newline at end of file +export default memo(Canvas); \ No newline at end of file From a47042f8fe8240805127a855789f2fa2b534a912 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 10 Mar 2025 23:26:17 +0530 Subject: [PATCH 20/51] chore(deps): install pg-boss --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 0f1e977d..666f9e00 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "moment-timezone": "^0.5.45", "node-cron": "^3.0.3", "pg": "^8.13.0", + "pg-boss": "^10.1.6", "pkce-challenge": "^4.1.0", "playwright": "^1.48.2", "playwright-extra": "^4.3.6", From 1789c789668037c41e9b7c1dd929fc34f0b24947 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 11 Mar 2025 12:52:40 +0530 Subject: [PATCH 21/51] feat: check state and navigate to recording --- src/pages/PageWrapper.tsx | 58 +++++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/src/pages/PageWrapper.tsx b/src/pages/PageWrapper.tsx index 1e73f26a..16f2e50d 100644 --- a/src/pages/PageWrapper.tsx +++ b/src/pages/PageWrapper.tsx @@ -16,10 +16,11 @@ import { NotFoundPage } from '../components/dashboard/NotFound'; export const PageWrapper = () => { const [open, setOpen] = useState(false); + const [isRecordingMode, setIsRecordingMode] = useState(false); const navigate = useNavigate(); - const { browserId, setBrowserId, notification, recordingName, setRecordingName, recordingId, setRecordingId } = useGlobalInfoStore(); + const { browserId, setBrowserId, notification, recordingName, setRecordingName, recordingId, setRecordingId, setRecordingUrl } = useGlobalInfoStore(); const handleEditRecording = (recordingId: string, fileName: string) => { setRecordingName(fileName); @@ -35,23 +36,62 @@ export const PageWrapper = () => { return notification.isOpen; } + /** + * Get the current tab's state from session storage + */ + const getTabState = (key: string): string | null => { + try { + const value = window.sessionStorage.getItem(key); + return value; + } catch (error) { + return null; + } + }; + useEffect(() => { - const isRecordingInProgress = async () => { - const id = await getActiveBrowserId(); - if (id) { - setBrowserId(id); + const tabMode = getTabState('tabMode'); + const urlParams = new URLSearchParams(window.location.search); + const sessionParam = urlParams.get('session'); + const storedSessionId = getTabState('recordingSessionId'); + const storedRecordingUrl = getTabState('recordingUrl'); + + if (location.pathname === '/recording-setup' && sessionParam && sessionParam === storedSessionId) { + setBrowserId('new-recording'); + setRecordingName(''); + setRecordingId(''); + + if (storedRecordingUrl) { + setRecordingUrl(storedRecordingUrl); + } + + navigate('/recording'); + } + else if (location.pathname === '/recording' || + (getTabState('nextTabIsRecording') === 'true' && sessionParam === storedSessionId)) { + setIsRecordingMode(true); + + if (location.pathname !== '/recording') { navigate('/recording'); } + + window.sessionStorage.removeItem('nextTabIsRecording'); + } else if (tabMode === 'main') { + console.log('Tab is in main application mode'); + } else { + const id = getTabState('browserId'); + if (id === 'new-recording' || location.pathname === '/recording') { + setIsRecordingMode(true); + } } - isRecordingInProgress(); - }, []); - + }, [location.pathname, navigate, setBrowserId, setRecordingId, setRecordingName, setRecordingUrl]); + return (
- {!browserId && } + {/* {!browserId && location.pathname !== '/recording' && } */} + {location.pathname !== '/recording' && } }> } /> From 1ff226f15854c58b8b4a0335b083f4c22bfc0247 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 11 Mar 2025 12:54:06 +0530 Subject: [PATCH 22/51] feat: prevent refresh on reload --- src/routes/userRoute.tsx | 50 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/src/routes/userRoute.tsx b/src/routes/userRoute.tsx index 4e9bb7c2..fe0a31ff 100644 --- a/src/routes/userRoute.tsx +++ b/src/routes/userRoute.tsx @@ -1,12 +1,56 @@ -import React from 'react'; -import { Navigate, Outlet } from 'react-router-dom'; +import React, { useEffect, useState } from 'react'; +import { Navigate, Outlet, useLocation } from 'react-router-dom'; import { useContext } from 'react'; import { AuthContext } from '../context/auth'; +import { useGlobalInfoStore } from '../context/globalInfo'; const UserRoute = () => { const { state } = useContext(AuthContext); + const location = useLocation(); + const [isCheckingAuth, setIsCheckingAuth] = useState(true); + const { setRecordingUrl } = useGlobalInfoStore(); + + useEffect(() => { + if (location.pathname === '/recording') { + const hasRecordingSession = + window.sessionStorage.getItem('browserId') || + window.sessionStorage.getItem('recordingSessionId'); + + const recordingUrl = window.sessionStorage.getItem('recordingUrl'); + if (recordingUrl) { + setRecordingUrl(recordingUrl); + } + + if (hasRecordingSession) { + console.log('UserRoute: Valid recording session detected, bypassing auth check'); + setIsCheckingAuth(false); + return; + } + } + + const timer = setTimeout(() => { + setIsCheckingAuth(false); + }, 100); + + return () => clearTimeout(timer); + }, [location.pathname]); + + if (isCheckingAuth) { + return null; + } + + if (location.pathname === '/recording') { + const hasRecordingSession = + window.sessionStorage.getItem('browserId') || + window.sessionStorage.getItem('recordingSessionId'); + + if (hasRecordingSession) { + return ; + } + } + return state.user ? : ; }; -export default UserRoute; +export default UserRoute; \ No newline at end of file From 9b126a673bdc15f3c0042a07728691ab44dc5d6f Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 11 Mar 2025 12:56:30 +0530 Subject: [PATCH 23/51] feat: set recording url on reload --- src/pages/RecordingPage.tsx | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/pages/RecordingPage.tsx b/src/pages/RecordingPage.tsx index 259e6a0d..034b2ea8 100644 --- a/src/pages/RecordingPage.tsx +++ b/src/pages/RecordingPage.tsx @@ -44,7 +44,7 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => { const { setId, socket } = useSocketStore(); const { setWidth } = useBrowserDimensionsStore(); - const { browserId, setBrowserId, recordingId, recordingUrl } = useGlobalInfoStore(); + const { browserId, setBrowserId, recordingId, recordingUrl, setRecordingUrl } = useGlobalInfoStore(); const handleShowOutputData = useCallback(() => { setShowOutputData(true); @@ -78,6 +78,11 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => { useEffect(() => { let isCancelled = false; const handleRecording = async () => { + const storedUrl = window.sessionStorage.getItem('recordingUrl'); + if (storedUrl && !recordingUrl) { + setRecordingUrl(storedUrl); + } + const id = await getActiveBrowserId(); if (!isCancelled) { if (id) { @@ -91,13 +96,13 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => { } } }; - + handleRecording(); - + return () => { isCancelled = true; } - }, [setId]); + }, [setId, recordingUrl, setRecordingUrl]); const changeBrowserDimensions = useCallback(() => { if (browserContentRef.current) { From 08da78c4177190487f2eaf0f4e5872b1d67d0542 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 11 Mar 2025 12:58:36 +0530 Subject: [PATCH 24/51] feat: modify highlighter events --- src/components/browser/BrowserWindow.tsx | 35 +++++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/src/components/browser/BrowserWindow.tsx b/src/components/browser/BrowserWindow.tsx index ffeb0df5..cb7e44d2 100644 --- a/src/components/browser/BrowserWindow.tsx +++ b/src/components/browser/BrowserWindow.tsx @@ -72,6 +72,21 @@ export const BrowserWindow = () => { const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext(); const { addTextStep, addListStep } = useBrowserSteps(); + useEffect(() => { + if (listSelector) { + window.sessionStorage.setItem('recordingListSelector', listSelector); + } + }, [listSelector]); + + useEffect(() => { + const storedListSelector = window.sessionStorage.getItem('recordingListSelector'); + + // Only restore state if it exists in sessionStorage + if (storedListSelector && !listSelector) { + setListSelector(storedListSelector); + } + }, []); + const onMouseMove = (e: MouseEvent) => { if (canvasRef && canvasRef.current && highlighterData) { const canvasRect = canvasRef.current.getBoundingClientRect(); @@ -195,13 +210,25 @@ export const BrowserWindow = () => { useEffect(() => { document.addEventListener('mousemove', onMouseMove, false); if (socket) { - socket.on("highlighter", highlighterHandler); + socket.off("highlighter", highlighterHandler); + + socket.on("highlighter", highlighterHandler); } return () => { - document.removeEventListener('mousemove', onMouseMove); - socket?.off("highlighter", highlighterHandler); + document.removeEventListener('mousemove', onMouseMove); + if (socket) { + socket.off("highlighter", highlighterHandler); + } }; - }, [socket, onMouseMove]); + }, [socket, highlighterHandler, onMouseMove, getList, listSelector]); + + useEffect(() => { + if (socket && listSelector) { + console.log('Syncing list selector with server:', listSelector); + socket.emit('setGetList', { getList: true }); + socket.emit('listSelector', { selector: listSelector }); + } + }, [socket, listSelector]); useEffect(() => { if (captureStage === 'initial' && listSelector) { From b7d06cd8ccbb209d499232a71c20fcd2be283b11 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 11 Mar 2025 12:59:30 +0530 Subject: [PATCH 25/51] feat: set recording url if session exists --- src/components/browser/BrowserRecordingSave.tsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/components/browser/BrowserRecordingSave.tsx b/src/components/browser/BrowserRecordingSave.tsx index 81719251..dc63eedf 100644 --- a/src/components/browser/BrowserRecordingSave.tsx +++ b/src/components/browser/BrowserRecordingSave.tsx @@ -82,7 +82,13 @@ const BrowserRecordingSave = () => { socket?.emit('new-recording'); socket.emit('input:url', initialUrl); // Update the URL in the navbar to match - setRecordingUrl(initialUrl); + let sessionInitialUrl = window.sessionStorage.getItem('initialUrl'); + if (sessionInitialUrl) { + setRecordingUrl(sessionInitialUrl); + window.sessionStorage.setItem('recordingUrl', sessionInitialUrl); + } else { + setRecordingUrl(initialUrl); + } } // Close the reset confirmation modal From 2d9d9c95382f5532d1af6ad637cb07e91c44b217 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 11 Mar 2025 13:00:13 +0530 Subject: [PATCH 26/51] feat: set session recording url on url change --- src/components/browser/BrowserNavBar.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/components/browser/BrowserNavBar.tsx b/src/components/browser/BrowserNavBar.tsx index c3db51c1..72456931 100644 --- a/src/components/browser/BrowserNavBar.tsx +++ b/src/components/browser/BrowserNavBar.tsx @@ -54,6 +54,7 @@ const BrowserNavBar: FC = ({ const handleCurrentUrlChange = useCallback((url: string) => { handleUrlChanged(url); setRecordingUrl(url); + window.sessionStorage.setItem('recordingUrl', url); }, [handleUrlChanged, recordingUrl]); useEffect(() => { From 3cd42f7a1c92a914a72d6d7743041a000d8db69d Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 11 Mar 2025 13:01:16 +0530 Subject: [PATCH 27/51] feat: set initial url as session storage --- src/components/robot/RecordingsTable.tsx | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/components/robot/RecordingsTable.tsx b/src/components/robot/RecordingsTable.tsx index 429c53c5..75b68717 100644 --- a/src/components/robot/RecordingsTable.tsx +++ b/src/components/robot/RecordingsTable.tsx @@ -240,21 +240,30 @@ export const RecordingsTable = ({ setModalOpen(true); }, [browserId]); - const handleStartRecording = useCallback(() => { + const startRecording = () => { + setModalOpen(false); + + // Set local state setBrowserId('new-recording'); setRecordingName(''); setRecordingId(''); - navigate('/recording'); - }, [navigate]); - - const startRecording = () => { - setModalOpen(false); - handleStartRecording(); + + window.sessionStorage.setItem('browserId', 'new-recording'); + + const sessionId = Date.now().toString(); + window.sessionStorage.setItem('recordingSessionId', sessionId); + window.sessionStorage.setItem('recordingUrl', recordingUrl); + + window.open(`/recording-setup?session=${sessionId}`, '_blank'); + + window.sessionStorage.setItem('nextTabIsRecording', 'true'); }; const setBrowserRecordingUrl = (event: React.ChangeEvent) => { setInitialUrl(event.target.value); setRecordingUrl(event.target.value); + + window.sessionStorage.setItem('initialUrl', event.target.value); } useEffect(() => { From 3315c53d0036c0aea7a7ce222260f779fa6be5bb Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 11 Mar 2025 16:37:34 +0530 Subject: [PATCH 28/51] feat: init socket listeners on update socket --- server/src/workflow-management/classes/Generator.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 2eb5f130..107dc500 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -708,6 +708,7 @@ export class WorkflowGenerator { public updateSocket = (socket: Socket): void => { this.socket = socket; this.registerEventHandlers(socket); + this.initializeSocketListeners(); }; /** From efd5d023a4c29bc57f85a244ddf4cee87f769749 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 11 Mar 2025 20:27:27 +0530 Subject: [PATCH 29/51] feat: discard notif and close tab --- .../browser/BrowserRecordingSave.tsx | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/components/browser/BrowserRecordingSave.tsx b/src/components/browser/BrowserRecordingSave.tsx index dc63eedf..d4fd54fb 100644 --- a/src/components/browser/BrowserRecordingSave.tsx +++ b/src/components/browser/BrowserRecordingSave.tsx @@ -14,9 +14,9 @@ import { MoreHoriz } from '@mui/icons-material'; const BrowserRecordingSave = () => { const { t } = useTranslation(); - const [openDiscardModal, setOpenDiscardModal] = useState(false); - const [openResetModal, setOpenResetModal] = useState(false); - const [anchorEl, setAnchorEl] = React.useState(null); + const [openDiscardModal, setOpenDiscardModal] = useState(false); + const [openResetModal, setOpenResetModal] = useState(false); + const [anchorEl, setAnchorEl] = React.useState(null); const { recordingName, browserId, initialUrl, setRecordingUrl, setBrowserId, notify, setCurrentWorkflowActionsState, resetInterpretationLog } = useGlobalInfoStore(); const navigate = useNavigate(); @@ -42,10 +42,25 @@ const BrowserRecordingSave = () => { const goToMainMenu = async () => { if (browserId) { await stopRecording(browserId); - notify('warning', t('browser_recording.notifications.terminated')); + + const notificationData = { + type: 'warning', + message: t('browser_recording.notifications.terminated'), + timestamp: Date.now() + }; + window.sessionStorage.setItem('pendingNotification', JSON.stringify(notificationData)); + + if (window.opener) { + window.opener.postMessage({ + type: 'recording-notification', + notification: notificationData + }, '*'); + } + setBrowserId(null); + + window.close(); } - navigate('/'); }; const performReset = () => { @@ -98,7 +113,7 @@ const BrowserRecordingSave = () => { notify('info', t('browser_recording.notifications.environment_reset')); }; - const handleClick = (event: React.MouseEvent) => { + const handleClick = (event: any) => { setAnchorEl(event.currentTarget); }; From 3289a7203d9e57ed440aace68897bac188bdb706 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 11 Mar 2025 20:28:01 +0530 Subject: [PATCH 30/51] feat: save notif and close tab --- src/components/recorder/SaveRecording.tsx | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/components/recorder/SaveRecording.tsx b/src/components/recorder/SaveRecording.tsx index c6b5687b..f7020b44 100644 --- a/src/components/recorder/SaveRecording.tsx +++ b/src/components/recorder/SaveRecording.tsx @@ -47,13 +47,27 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => { }; const exitRecording = useCallback(async () => { - notify('success', t('save_recording.notifications.save_success')); + const notificationData = { + type: 'success', + message: t('save_recording.notifications.save_success'), + timestamp: Date.now() + }; + window.sessionStorage.setItem('pendingNotification', JSON.stringify(notificationData)); + + if (window.opener) { + window.opener.postMessage({ + type: 'recording-notification', + notification: notificationData + }, '*'); + } + if (browserId) { await stopRecording(browserId); } setBrowserId(null); - navigate('/'); - }, [setBrowserId, browserId, notify]); + + window.close(); + }, [setBrowserId, browserId]); // notifies backed to save the recording in progress, // releases resources and changes the view for main page by clearing the global browserId @@ -142,4 +156,4 @@ const modalStyle = { height: 'fit-content', display: 'block', padding: '20px', -}; +}; \ No newline at end of file From e3a8441d1ef3c50375516dd02e74a46299b6f26a Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 11 Mar 2025 20:28:33 +0530 Subject: [PATCH 31/51] feat: read notif and rerender robots --- src/components/robot/RecordingsTable.tsx | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/components/robot/RecordingsTable.tsx b/src/components/robot/RecordingsTable.tsx index 75b68717..4fd1d59a 100644 --- a/src/components/robot/RecordingsTable.tsx +++ b/src/components/robot/RecordingsTable.tsx @@ -176,6 +176,32 @@ export const RecordingsTable = ({ setRecordingId } = useGlobalInfoStore(); const navigate = useNavigate(); + useEffect(() => { + const handleMessage = (event: any) => { + if (event.data && event.data.type === 'recording-notification') { + console.log('Received message from recording tab:', event.data); + + const notificationData = event.data.notification; + if (notificationData) { + notify(notificationData.type, notificationData.message); + + if ((notificationData.type === 'success' && + notificationData.message.includes('saved')) || + (notificationData.type === 'warning' && + notificationData.message.includes('terminated'))) { + setRerenderRobots(true); + } + } + } + }; + + window.addEventListener('message', handleMessage); + + return () => { + window.removeEventListener('message', handleMessage); + }; + }, [notify, setRerenderRobots]); + const handleChangePage = useCallback((event: unknown, newPage: number) => { setPage(newPage); }, []); From 085e2e0ef6231d35302cc806362f02e2a67ada19 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 11 Mar 2025 20:29:19 +0530 Subject: [PATCH 32/51] feat: clear log --- src/components/robot/RecordingsTable.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/components/robot/RecordingsTable.tsx b/src/components/robot/RecordingsTable.tsx index 4fd1d59a..e0f180d4 100644 --- a/src/components/robot/RecordingsTable.tsx +++ b/src/components/robot/RecordingsTable.tsx @@ -179,8 +179,6 @@ export const RecordingsTable = ({ useEffect(() => { const handleMessage = (event: any) => { if (event.data && event.data.type === 'recording-notification') { - console.log('Received message from recording tab:', event.data); - const notificationData = event.data.notification; if (notificationData) { notify(notificationData.type, notificationData.message); From 8a451889a926a80fcd43e0ca348b22d286980bd0 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 11 Mar 2025 21:23:06 +0530 Subject: [PATCH 33/51] feat: discard and create new recording --- src/components/robot/RecordingsTable.tsx | 75 ++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 6 deletions(-) diff --git a/src/components/robot/RecordingsTable.tsx b/src/components/robot/RecordingsTable.tsx index e0f180d4..2fc4f26e 100644 --- a/src/components/robot/RecordingsTable.tsx +++ b/src/components/robot/RecordingsTable.tsx @@ -41,9 +41,14 @@ import { useGlobalInfoStore } from "../../context/globalInfo"; import { checkRunsForRecording, deleteRecordingFromStorage, getStoredRecordings } from "../../api/storage"; import { Add } from "@mui/icons-material"; import { useNavigate } from 'react-router-dom'; -import { stopRecording } from "../../api/recording"; +import { getActiveBrowserId, stopRecording } from "../../api/recording"; import { GenericModal } from '../ui/GenericModal'; +declare global { + interface Window { + openedRecordingWindow?: Window | null; + } +} /** TODO: * 1. allow editing existing robot after persisting browser steps @@ -148,6 +153,8 @@ export const RecordingsTable = ({ const [rows, setRows] = React.useState([]); const [isModalOpen, setModalOpen] = React.useState(false); const [searchTerm, setSearchTerm] = React.useState(''); + const [isWarningModalOpen, setWarningModalOpen] = React.useState(false); + const [activeBrowserId, setActiveBrowserId] = React.useState(''); const columns = useMemo(() => [ { id: 'interpret', label: t('recordingtable.run'), minWidth: 80 }, @@ -257,12 +264,44 @@ export const RecordingsTable = ({ }, [setRecordings, notify, t]); const handleNewRecording = useCallback(async () => { - if (browserId) { - setBrowserId(null); - await stopRecording(browserId); + const activeBrowserId = await getActiveBrowserId(); + + if (activeBrowserId) { + setActiveBrowserId(activeBrowserId); + setWarningModalOpen(true); + } else { + setModalOpen(true); } + }, []); + + const notifyRecordingTabsToClose = (browserId: string) => { + const closeMessage = { + action: 'close-recording-tab', + browserId: browserId, + timestamp: Date.now() + }; + window.sessionStorage.setItem('recordingTabCloseMessage', JSON.stringify(closeMessage)); + + if (window.openedRecordingWindow && !window.openedRecordingWindow.closed) { + try { + window.openedRecordingWindow.close(); + } catch (e) { + console.log('Could not directly close recording window:', e); + } + } + }; + + const handleDiscardAndCreate = async () => { + if (activeBrowserId) { + await stopRecording(activeBrowserId); + notify('warning', t('browser_recording.notifications.terminated')); + + notifyRecordingTabsToClose(activeBrowserId); + } + + setWarningModalOpen(false); setModalOpen(true); - }, [browserId]); + }; const startRecording = () => { setModalOpen(false); @@ -278,7 +317,7 @@ export const RecordingsTable = ({ window.sessionStorage.setItem('recordingSessionId', sessionId); window.sessionStorage.setItem('recordingUrl', recordingUrl); - window.open(`/recording-setup?session=${sessionId}`, '_blank'); + window.openedRecordingWindow = window.open(`/recording-setup?session=${sessionId}`, '_blank'); window.sessionStorage.setItem('nextTabIsRecording', 'true'); }; @@ -434,6 +473,30 @@ export const RecordingsTable = ({ onPageChange={handleChangePage} onRowsPerPageChange={handleChangeRowsPerPage} /> + setWarningModalOpen(false)} modalStyle={modalStyle}> +
+ {t('recordingtable.warning_modal.title')} + + {t('recordingtable.warning_modal.message')} + + + + + + +
+ setModalOpen(false)} modalStyle={modalStyle}>
{t('recordingtable.modal.title')} From e187520efd5bea6508339bc78e8a1f050a8104c7 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 11 Mar 2025 21:26:29 +0530 Subject: [PATCH 34/51] feat: add en translation for modal --- public/locales/en.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/public/locales/en.json b/public/locales/en.json index 4f7ebdc4..41932f1f 100644 --- a/public/locales/en.json +++ b/public/locales/en.json @@ -54,6 +54,12 @@ "label":"URL", "button":"Start Recording" }, + "warning_modal":{ + "title":"Active Browser Detected", + "message": "There is already a browser recording session running. Would you like to discard it and create a new recording?", + "discard_and_create":"Discard & Create New", + "cancel":"Cancel" + }, "edit":"Edit", "delete":"Delete", "duplicate":"Duplicate", From 7b940ab965c3d1ce8f529684f44e7531a4f5b5a3 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 11 Mar 2025 21:55:27 +0530 Subject: [PATCH 35/51] feat: add support for stop and interpret workflow --- server/src/pgboss-worker.ts | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/server/src/pgboss-worker.ts b/server/src/pgboss-worker.ts index 5d2ea86d..5fd87518 100644 --- a/server/src/pgboss-worker.ts +++ b/server/src/pgboss-worker.ts @@ -16,8 +16,17 @@ interface InitializeBrowserData { userId: string; } +interface InterpretWorkflow { + userId: string; +} + +interface StopInterpretWorkflow { + userId: string; +} + interface DestroyBrowserData { browserId: string; + userId: string; } const pgBoss = new PgBoss({connectionString: pgBossConnectionString, schema: 'public'}); @@ -65,10 +74,10 @@ async function startWorkers() { await pgBoss.work('destroy-browser', async (job: Job | Job[]) => { try { const data = extractJobData(job); - const browserId = data.browserId; + const { browserId, userId } = data; logger.log('info', `Starting browser destruction job for browser: ${browserId}`); - const success = await destroyRemoteBrowser(browserId); + const success = await destroyRemoteBrowser(browserId, userId); logger.log('info', `Browser destruction job completed with result: ${success}`); return { success }; } catch (error: unknown) { @@ -79,10 +88,13 @@ async function startWorkers() { }); // Worker for interpreting workflow - await pgBoss.work('interpret-workflow', async () => { + await pgBoss.work('interpret-workflow', async (job: Job | Job[]) => { try { + const data = extractJobData(job); + const userId = data.userId; + logger.log('info', 'Starting workflow interpretation job'); - await interpretWholeWorkflow(); + await interpretWholeWorkflow(userId); logger.log('info', 'Workflow interpretation job completed'); return { success: true }; } catch (error: unknown) { @@ -93,10 +105,13 @@ async function startWorkers() { }); // Worker for stopping workflow interpretation - await pgBoss.work('stop-interpretation', async () => { + await pgBoss.work('stop-interpretation', async (job: Job | Job[]) => { try { + const data = extractJobData(job); + const userId = data.userId; + logger.log('info', 'Starting stop interpretation job'); - await stopRunningInterpretation(); + await stopRunningInterpretation(userId); logger.log('info', 'Stop interpretation job completed'); return { success: true }; } catch (error: unknown) { From 10e1e8a87c096afe3caf4b57962646b99760bb22 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 11 Mar 2025 21:56:12 +0530 Subject: [PATCH 36/51] feat: queue stop and interpret workflow jobs --- server/src/routes/record.ts | 83 ++++++++++++++++++++++++++++++++++--- 1 file changed, 77 insertions(+), 6 deletions(-) diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index de468ef7..17fd56d2 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -148,6 +148,7 @@ router.get('/stop/:browserId', requireSignIn, async (req: AuthenticatedRequest, const jobId = await pgBoss.send('destroy-browser', { browserId: req.params.browserId, + userId: req.user.id, timestamp: new Date().toISOString() }); @@ -224,14 +225,84 @@ router.get('/active/tabs', requireSignIn, (req: AuthenticatedRequest, res) => { * GET endpoint for starting an interpretation of the currently generated workflow. */ router.get('/interpret', requireSignIn, async (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + try { - if (!req.user) { - return res.status(401).send('User not authenticated'); + await pgBoss.createQueue('interpret-workflow'); + + const jobId = await pgBoss.send('interpret-workflow', { + userId: req.user.id, + timestamp: new Date().toISOString() + }); + + if (!jobId) { + logger.log('warn', 'pgBoss.send returned null, falling back to direct destruction'); + await interpretWholeWorkflow(req.user?.id); + return res.send('interpretation done'); } - await interpretWholeWorkflow(req.user?.id); - return res.send('interpretation done'); - } catch (e) { - return res.send('interpretation failed'); + + logger.log('info', `Queued interpret workflow job: ${jobId}, waiting for completion...`); + + try { + const result = await waitForJobCompletion(jobId, 'interpret-workflow', 15000); + + if (result) { + logger.log('info', `Browser destruction job completed with result: ${result.success}`); + return res.send('interpretation done'); + } else { + logger.log('warn', 'Job completed but returned unexpected result'); + return res.send('interpretation failed'); + } + } catch (waitError: any) { + logger.log('warn', `Error waiting for job completion: ${waitError.message}`); + return res.send('interpretation failed'); + } + } catch (error: any) { + logger.log('error', `Failed to stop browser: ${error.message}`); + return res.status(500).send('interpretation failed'); + } +}); + +router.get('/interpret/stop', requireSignIn, async (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + + try { + await pgBoss.createQueue('stop-interpretation'); + + const jobId = await pgBoss.send('stop-interpretation', { + userId: req.user.id, + timestamp: new Date().toISOString() + }); + + if (!jobId) { + logger.log('warn', 'pgBoss.send returned null, falling back to direct destruction'); + await interpretWholeWorkflow(req.user?.id); + return res.send('interpretation done'); + } + + logger.log('info', `Queued stop interpret workflow job: ${jobId}, waiting for completion...`); + + try { + const result = await waitForJobCompletion(jobId, 'stop-interpretation', 15000); + + if (result) { + logger.log('info', `Browser destruction job completed with result: ${result.success}`); + return res.send('interpretation stopped'); + } else { + logger.log('warn', 'Job completed but returned unexpected result'); + return res.send('interpretation failed to stop'); + } + } catch (waitError: any) { + logger.log('warn', `Error waiting for job completion: ${waitError.message}`); + return res.send('interpretation failed to stop'); + } + } catch (error: any) { + logger.log('error', `Failed to stop browser: ${error.message}`); + return res.status(500).send('interpretation failed to stop'); } }); From e09d3179e71b27dcd904bf407db287d99cdb4ec9 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 11 Mar 2025 22:03:44 +0530 Subject: [PATCH 37/51] feat: log cleanup, better error messages --- server/src/routes/record.ts | 38 +++++-------------------------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index 17fd56d2..1a00fdb7 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -56,7 +56,6 @@ async function waitForJobCompletion(jobId: string, queueName: string, timeout = } }; - // Start checking checkJobStatus(); }); } @@ -88,9 +87,8 @@ router.get('/start', requireSignIn, async (req: AuthenticatedRequest, res: Respo }); if (!jobId) { - logger.log('warn', 'pgBoss.send returned null, falling back to direct initialization'); const browserId = initializeRemoteBrowserForRecording(req.user.id); - return res.send( browserId ); + return res.send(browserId); } logger.log('info', `Queued browser initialization job: ${jobId}, waiting for completion...`); @@ -99,14 +97,11 @@ router.get('/start', requireSignIn, async (req: AuthenticatedRequest, res: Respo const result = await waitForJobCompletion(jobId, 'initialize-browser-recording', 15000); if (result && result.browserId) { - logger.log('info', `Job completed with browserId: ${result.browserId}`); return res.send(result.browserId); } else { - logger.log('warn', 'Job completed but returned unexpected result'); return res.send(jobId); } } catch (waitError: any) { - logger.log('warn', `Error waiting for job completion: ${waitError.message}`); return res.send(jobId); } } catch (error: any) { @@ -153,7 +148,6 @@ router.get('/stop/:browserId', requireSignIn, async (req: AuthenticatedRequest, }); if (!jobId) { - logger.log('warn', 'pgBoss.send returned null, falling back to direct destruction'); const browserId = initializeRemoteBrowserForRecording(req.user.id); return res.send( browserId ); } @@ -164,14 +158,11 @@ router.get('/stop/:browserId', requireSignIn, async (req: AuthenticatedRequest, const result = await waitForJobCompletion(jobId, 'destroy-browser', 15000); if (result) { - logger.log('info', `Browser destruction job completed with result: ${result.success}`); return res.send(result.success); } else { - logger.log('warn', 'Job completed but returned unexpected result'); return res.send(false); } } catch (waitError: any) { - logger.log('warn', `Error waiting for job completion: ${waitError.message}`); return res.send(false); } } catch (error: any) { @@ -238,7 +229,6 @@ router.get('/interpret', requireSignIn, async (req: AuthenticatedRequest, res) = }); if (!jobId) { - logger.log('warn', 'pgBoss.send returned null, falling back to direct destruction'); await interpretWholeWorkflow(req.user?.id); return res.send('interpretation done'); } @@ -249,18 +239,15 @@ router.get('/interpret', requireSignIn, async (req: AuthenticatedRequest, res) = const result = await waitForJobCompletion(jobId, 'interpret-workflow', 15000); if (result) { - logger.log('info', `Browser destruction job completed with result: ${result.success}`); return res.send('interpretation done'); } else { - logger.log('warn', 'Job completed but returned unexpected result'); return res.send('interpretation failed'); } } catch (waitError: any) { - logger.log('warn', `Error waiting for job completion: ${waitError.message}`); return res.send('interpretation failed'); } } catch (error: any) { - logger.log('error', `Failed to stop browser: ${error.message}`); + logger.log('error', `Failed to stop interpret workflow: ${error.message}`); return res.status(500).send('interpretation failed'); } }); @@ -279,9 +266,8 @@ router.get('/interpret/stop', requireSignIn, async (req: AuthenticatedRequest, r }); if (!jobId) { - logger.log('warn', 'pgBoss.send returned null, falling back to direct destruction'); - await interpretWholeWorkflow(req.user?.id); - return res.send('interpretation done'); + await stopRunningInterpretation(req.user?.id); + return res.send('interpretation stopped'); } logger.log('info', `Queued stop interpret workflow job: ${jobId}, waiting for completion...`); @@ -290,31 +276,17 @@ router.get('/interpret/stop', requireSignIn, async (req: AuthenticatedRequest, r const result = await waitForJobCompletion(jobId, 'stop-interpretation', 15000); if (result) { - logger.log('info', `Browser destruction job completed with result: ${result.success}`); return res.send('interpretation stopped'); } else { - logger.log('warn', 'Job completed but returned unexpected result'); return res.send('interpretation failed to stop'); } } catch (waitError: any) { - logger.log('warn', `Error waiting for job completion: ${waitError.message}`); return res.send('interpretation failed to stop'); } } catch (error: any) { - logger.log('error', `Failed to stop browser: ${error.message}`); + logger.log('error', `Failed to stop interpretation: ${error.message}`); return res.status(500).send('interpretation failed to stop'); } }); -/** - * GET endpoint for stopping an ongoing interpretation of the currently generated workflow. - */ -router.get('/interpret/stop', requireSignIn, async (req: AuthenticatedRequest, res) => { - if (!req.user) { - return res.status(401).send('User not authenticated'); - } - await stopRunningInterpretation(req.user?.id); - return res.send('interpretation stopped'); -}); - export default router; \ No newline at end of file From 9a2b14b9380ae7d967d22adf40c974678f70869a Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 12 Mar 2025 16:56:04 +0530 Subject: [PATCH 38/51] feat: 1 user 2 browser changes --- .../browser-management/classes/BrowserPool.ts | 313 +++++++++++++++--- 1 file changed, 263 insertions(+), 50 deletions(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 4bbc97af..c1f0f557 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -4,6 +4,12 @@ import logger from "../../logger"; /** * @category Types */ +/** + * Represents the possible states of a remote browser. + * @category Types + */ +type BrowserState = "recording" | "run"; + interface BrowserPoolInfo { /** * The instance of remote browser. @@ -19,6 +25,12 @@ interface BrowserPoolInfo { * The user ID that owns this browser instance. */ userId: string, + /** + * The current state of the browser. + * Can be "recording" or "run". + * @default "recording" + */ + state: BrowserState, } /** @@ -33,7 +45,7 @@ interface PoolDictionary { /** * A browser pool is a collection of remote browsers that are initialized and ready to be used. - * Enforces a "1 User - 1 Browser" policy, while allowing multiple users to have their own browser instances. + * Enforces a "1 User - 2 Browser" policy, while allowing multiple users to have their own browser instances. * Adds the possibility to add, remove and retrieve remote browsers from the pool. * @category BrowserManagement */ @@ -45,12 +57,13 @@ export class BrowserPool { /** * Maps user IDs to their browser IDs. + * A user can have up to 2 browsers. */ - private userToBrowserMap: Map = new Map(); + private userToBrowserMap: Map = new Map(); /** * Adds a remote browser instance to the pool for a specific user. - * If the user already has a browser, the existing browser will be closed and replaced. + * If the user already has two browsers, the oldest browser will be closed and replaced. * * @param id remote browser instance's id * @param browser remote browser instance @@ -62,38 +75,58 @@ export class BrowserPool { id: string, browser: RemoteBrowser, userId: string, - active: boolean = false + active: boolean = false, + state: BrowserState = "recording" ): boolean => { - // Check if user already has a browser - const existingBrowserId = this.userToBrowserMap.get(userId); + // Check if browser with this ID already exists and belongs to this user + if (this.pool[id] && this.pool[id].userId === userId) { + // Just update the existing browser + this.pool[id] = { + browser, + active, + userId, + state: this.pool[id].state || state, + }; + logger.log('debug', `Updated existing browser with id: ${id} for user: ${userId}`); + return false; + } + + // Get existing browsers for this user + let userBrowserIds = this.userToBrowserMap.get(userId) || []; let replaced = false; - if (existingBrowserId) { - // Close and remove the existing browser - if (existingBrowserId !== id) { - this.closeAndDeleteBrowser(existingBrowserId); - replaced = true; - } else { - // If it's the same browser ID, just update the info - this.pool[id] = { - browser, - active, - userId, - }; - logger.log('debug', `Updated existing browser with id: ${id} for user: ${userId}`); + // If trying to add a "recording" browser, check if one already exists + if (state === "recording") { + // Check if user already has a recording browser + const hasRecordingBrowser = userBrowserIds.some(browserId => + this.pool[browserId] && this.pool[browserId].state === "recording" + ); + + if (hasRecordingBrowser) { + logger.log('debug', `User ${userId} already has a browser in "recording" state`); return false; } } + + // For "run" state, check if the user already has the maximum number of browsers (2) + if (userBrowserIds.length >= 2 && !userBrowserIds.includes(id)) { + logger.log('debug', "User already has the maximum number of browsers (2)"); + return false; + } // Add the new browser to the pool this.pool[id] = { browser, active, userId, + state, }; // Update the user-to-browser mapping - this.userToBrowserMap.set(userId, id); + if (!userBrowserIds.includes(id)) { + userBrowserIds.push(id); + } + this.userToBrowserMap.set(userId, userBrowserIds); logger.log('debug', `Remote browser with id: ${id} added to the pool for user: ${userId}`); return !replaced; @@ -115,8 +148,16 @@ export class BrowserPool { // Remove the user-to-browser mapping const userId = this.pool[id].userId; - if (this.userToBrowserMap.get(userId) === id) { - this.userToBrowserMap.delete(userId); + const userBrowserIds = this.userToBrowserMap.get(userId) || []; + + if (userBrowserIds.includes(id)) { + const updatedBrowserIds = userBrowserIds.filter(bid => bid !== id); + + if (updatedBrowserIds.length === 0) { + this.userToBrowserMap.delete(userId); + } else { + this.userToBrowserMap.set(userId, updatedBrowserIds); + } } // Remove from pool @@ -139,8 +180,16 @@ export class BrowserPool { // Remove the user-to-browser mapping const userId = this.pool[id].userId; - if (this.userToBrowserMap.get(userId) === id) { - this.userToBrowserMap.delete(userId); + const userBrowserIds = this.userToBrowserMap.get(userId) || []; + + if (userBrowserIds.includes(id)) { + const updatedBrowserIds = userBrowserIds.filter(bid => bid !== id); + + if (updatedBrowserIds.length === 0) { + this.userToBrowserMap.delete(userId); + } else { + this.userToBrowserMap.set(userId, updatedBrowserIds); + } } // Remove from pool @@ -162,25 +211,73 @@ export class BrowserPool { /** * Returns the active browser's instance id for a specific user. + * If state is specified, only returns a browser with that exact state. * * @param userId the user ID to find the browser for - * @returns the browser ID for the user, or null if no browser exists + * @param state optional browser state filter ("recording" or "run") + * @returns the browser ID for the user, or null if no browser exists with the required state */ - public getActiveBrowserId = (userId: string): string | null => { - const browserId = this.userToBrowserMap.get(userId); - if (!browserId) { + public getActiveBrowserId = (userId: string, state?: BrowserState): string | null => { + const browserIds = this.userToBrowserMap.get(userId); + if (!browserIds || browserIds.length === 0) { logger.log('debug', `No browser found for user: ${userId}`); return null; } - // Verify the browser still exists in the pool - if (!this.pool[browserId]) { - this.userToBrowserMap.delete(userId); - logger.log('warn', `Browser mapping found for user: ${userId}, but browser doesn't exist in pool`); + // If state is specified, only return browsers with that exact state + if (state) { + // Check browsers in reverse order (newest first) to find one with the specified state + for (let i = browserIds.length - 1; i >= 0; i--) { + const browserId = browserIds[i]; + + // Verify the browser still exists in the pool + if (!this.pool[browserId]) { + browserIds.splice(i, 1); + continue; + } + + // Check if browser matches state filter + if (this.pool[browserId].state === state) { + return browserId; + } + } + + // If no browser with matching state, return null + logger.log('debug', `No browser with state ${state} found for user: ${userId}`); return null; } - console.log(`Browser Id ${browserId} found for user: ${userId}`); - return browserId; + + // If no state specified, return any browser + for (let i = browserIds.length - 1; i >= 0; i--) { + const browserId = browserIds[i]; + + // Verify the browser still exists in the pool + if (!this.pool[browserId]) { + browserIds.splice(i, 1); + continue; + } + + // Return the first browser found + if (this.pool[browserId]) { + console.log(`Active browser Id ${browserId} found for user: ${userId}`); + return browserId; + } + } + + // If no active browser, return the most recent one + if (browserIds.length > 0) { + const mostRecentId = browserIds[browserIds.length - 1]; + console.log(`No active browser found, returning most recent browser Id ${mostRecentId} for user: ${userId}`); + return mostRecentId; + } + + // Clean up the mapping if all browsers were invalid + if (browserIds.length === 0) { + this.userToBrowserMap.delete(userId); + } + + logger.log('warn', `Browser mapping found for user: ${userId}, but no valid browsers exist in pool`); + return null; }; /** @@ -213,10 +310,62 @@ export class BrowserPool { logger.log('debug', `Remote browser with id: ${id} set to ${active ? 'active' : 'inactive'}`); return true; }; + + /** + * Sets the state of a browser. + * Only allows one browser in "recording" state per user. + * + * @param id the browser ID + * @param state the new state ("recording" or "run") + * @returns true if successful, false if the browser wasn't found or state change not allowed + */ + public setBrowserState = (id: string, state: BrowserState): boolean => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return false; + } + + // If trying to set to "recording" state, check if another browser is already recording + if (state === "recording") { + const userId = this.pool[id].userId; + const userBrowserIds = this.userToBrowserMap.get(userId) || []; + + // Check if any other browser for this user is already in recording state + const hasAnotherRecordingBrowser = userBrowserIds.some(browserId => + browserId !== id && + this.pool[browserId] && + this.pool[browserId].state === "recording" + ); + + if (hasAnotherRecordingBrowser) { + logger.log('warn', `Cannot set browser ${id} to "recording" state: User ${userId} already has a browser in recording state`); + return false; + } + } + + this.pool[id].state = state; + logger.log('debug', `Remote browser with id: ${id} state set to ${state}`); + return true; + }; + + /** + * Gets the current state of a browser. + * + * @param id the browser ID + * @returns the current state or null if the browser wasn't found + */ + public getBrowserState = (id: string): BrowserState | null => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return null; + } + + return this.pool[id].state; + }; /** * Returns all browser instances for a specific user. - * Should only be one per the "1 User - 1 Browser" policy, but included for flexibility. + * With the "1 User - 2 Browser" policy, this can return up to 2 browsers. * * @param userId the user ID to find browsers for * @returns an array of browser IDs belonging to the user @@ -224,24 +373,31 @@ export class BrowserPool { public getAllBrowserIdsForUser = (userId: string): string[] => { const browserIds: string[] = []; - // Normally this would just return the one browser from the map - const mappedBrowserId = this.userToBrowserMap.get(userId); - if (mappedBrowserId && this.pool[mappedBrowserId]) { - browserIds.push(mappedBrowserId); + // Get browser IDs from the map + const mappedBrowserIds = this.userToBrowserMap.get(userId) || []; + + // Filter to only include IDs that exist in the pool + for (const id of mappedBrowserIds) { + if (this.pool[id]) { + browserIds.push(id); + } } - // But as a safeguard, also check the entire pool for any browsers assigned to this user + // As a safeguard, also check the entire pool for any browsers assigned to this user // This helps detect and fix any inconsistencies in the maps for (const [id, info] of Object.entries(this.pool)) { if (info.userId === userId && !browserIds.includes(id)) { browserIds.push(id); - // Fix the map if it's inconsistent - if (!mappedBrowserId) { - this.userToBrowserMap.set(userId, id); - } } } + // Update the map if inconsistencies were found + if (browserIds.length > 0 && JSON.stringify(browserIds) !== JSON.stringify(mappedBrowserIds)) { + // Limit to 2 browsers if more were found + const limitedBrowserIds = browserIds.slice(-2); + this.userToBrowserMap.set(userId, limitedBrowserIds); + } + return browserIds; }; @@ -264,24 +420,81 @@ export class BrowserPool { * This is a migration helper to support code that hasn't been updated to the user-browser model yet. * * @param currentUserId The ID of the current user, which will be prioritized if multiple browsers exist + * @param state Optional state filter to find browsers in a specific state * @returns A browser ID if one can be determined, or null */ - public getActiveBrowserForMigration = (currentUserId?: string): string | null => { + public getActiveBrowserForMigration = (currentUserId?: string, state?: BrowserState): string | null => { // If a current user ID is provided and they have a browser, return that if (currentUserId) { - const browserForUser = this.getActiveBrowserId(currentUserId); + const browserForUser = this.getActiveBrowserId(currentUserId, state); if (browserForUser) { return browserForUser; } + + // If state is specified and no matching browser was found, return null + if (state) { + return null; + } } - // If only one user has a browser, return that + // If only one user has a browser, try to find a matching browser if (this.userToBrowserMap.size === 1) { const userId = Array.from(this.userToBrowserMap.keys())[0]; - return this.userToBrowserMap.get(userId) || null; + const browserIds = this.userToBrowserMap.get(userId) || []; + + // If state is specified, only look for that state + if (state) { + // Return the active browser that matches the state + for (let i = browserIds.length - 1; i >= 0; i--) { + const bid = browserIds[i]; + if (this.pool[bid]?.active && this.pool[bid].state === state) { + return bid; + } + } + + // If no active browser with matching state, try to find any browser with matching state + for (let i = browserIds.length - 1; i >= 0; i--) { + const bid = browserIds[i]; + if (this.pool[bid] && this.pool[bid].state === state) { + return bid; + } + } + + // If still no matching browser, return null + return null; + } + + // If no state filter, find any active browser + for (let i = browserIds.length - 1; i >= 0; i--) { + if (this.pool[browserIds[i]]?.active) { + return browserIds[i]; + } + } + + return browserIds.length > 0 ? browserIds[browserIds.length - 1] : null; } - // Fall back to the first active browser if any + // Fall back to checking all browsers if no user was specified + if (state) { + // Look for active browsers with the specific state + for (const id of Object.keys(this.pool)) { + if (this.pool[id].active && this.pool[id].state === state) { + return id; + } + } + + // Then look for any browser with the specific state + for (const id of Object.keys(this.pool)) { + if (this.pool[id].state === state) { + return id; + } + } + + // If no browser with the requested state is found, return null + return null; + } + + // If no state filter, find any active browser for (const id of Object.keys(this.pool)) { if (this.pool[id].active) { return id; @@ -299,7 +512,7 @@ export class BrowserPool { * If there are multiple active browsers, it returns the first one. * * @returns the first remote active browser instance's id from the pool - * @deprecated Use getBrowserIdForUser instead to enforce the 1 User - 1 Browser policy + * @deprecated Use getBrowserIdForUser instead to enforce the 1 User - 2 Browser policy */ public getActiveBrowserIdLegacy = (): string | null => { for (const id of Object.keys(this.pool)) { From db4669ab8c2cdf00422808c4de819d0fc92d8049 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 12 Mar 2025 16:57:23 +0530 Subject: [PATCH 39/51] feat: add browser state --- server/src/browser-management/controller.ts | 23 +++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index e6619d77..2cd9498d 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -21,12 +21,12 @@ import logger from "../logger"; * @category BrowserManagement-Controller */ export const initializeRemoteBrowserForRecording = (userId: string): string => { - const id = getActiveBrowserId(userId) || uuid(); + const id = getActiveBrowserIdByState(userId, "recording") || uuid(); createSocketConnection( io.of(id), async (socket: Socket) => { // browser is already active - const activeId = getActiveBrowserId(userId); + const activeId = getActiveBrowserIdByState(userId, "recording"); if (activeId) { const remoteBrowser = browserPool.getRemoteBrowser(activeId); remoteBrowser?.updateSocket(socket); @@ -37,7 +37,7 @@ export const initializeRemoteBrowserForRecording = (userId: string): string => { await browserSession.initialize(userId); await browserSession.registerEditorEvents(); await browserSession.subscribeToScreencast(); - browserPool.addRemoteBrowser(id, browserSession, userId); + browserPool.addRemoteBrowser(id, browserSession, userId, false, "recording"); } socket.emit('loaded'); }); @@ -59,7 +59,7 @@ export const createRemoteBrowserForRun = (userId: string): string => { async (socket: Socket) => { const browserSession = new RemoteBrowser(socket, userId); await browserSession.initialize(userId); - browserPool.addRemoteBrowser(id, browserSession, userId); + browserPool.addRemoteBrowser(id, browserSession, userId, false, "run"); socket.emit('ready-for-run'); }); return id; @@ -92,6 +92,17 @@ export const getActiveBrowserId = (userId: string): string | null => { return browserPool.getActiveBrowserId(userId); }; +/** + * Returns the id of an active browser with the specified state or null. + * @param userId the user ID to find the browser for + * @param state the browser state to filter by ("recording" or "run") + * @returns {string | null} + * @category BrowserManagement-Controller + */ +export const getActiveBrowserIdByState = (userId: string, state: "recording" | "run"): string | null => { + return browserPool.getActiveBrowserId(userId, state); +}; + /** * Returns the url string from a remote browser if exists in the browser pool. * @param id instance id of the remote browser @@ -127,7 +138,7 @@ export const getRemoteBrowserCurrentTabs = (id: string, userId: string): string[ * @category BrowserManagement-Controller */ export const interpretWholeWorkflow = async (userId: string) => { - const id = getActiveBrowserId(userId); + const id = getActiveBrowserIdByState(userId, "recording"); if (id) { const browser = browserPool.getRemoteBrowser(id); if (browser) { @@ -147,7 +158,7 @@ export const interpretWholeWorkflow = async (userId: string) => { * @category BrowserManagement-Controller */ export const stopRunningInterpretation = async (userId: string) => { - const id = getActiveBrowserId(userId); + const id = getActiveBrowserIdByState(userId, "recording"); if (id) { const browser = browserPool.getRemoteBrowser(id); await browser?.stopCurrentInterpretation(); From 22b640b8d75a78aaee3626eb8b4ccf9412c4bd94 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 12 Mar 2025 16:58:32 +0530 Subject: [PATCH 40/51] feat: add browser state for routes --- server/src/routes/record.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index 1a00fdb7..fee5039a 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -11,6 +11,7 @@ import { stopRunningInterpretation, getRemoteBrowserCurrentUrl, getRemoteBrowserCurrentTabs, + getActiveBrowserIdByState, } from '../browser-management/controller'; import { chromium } from 'playwright-extra'; import stealthPlugin from 'puppeteer-extra-plugin-stealth'; @@ -178,7 +179,7 @@ router.get('/active', requireSignIn, (req: AuthenticatedRequest, res) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = getActiveBrowserId(req.user?.id); + const id = getActiveBrowserIdByState(req.user?.id, "recording"); return res.send(id); }); @@ -189,7 +190,7 @@ router.get('/active/url', requireSignIn, (req: AuthenticatedRequest, res) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = getActiveBrowserId(req.user?.id); + const id = getActiveBrowserIdByState(req.user?.id, "recording"); if (id) { const url = getRemoteBrowserCurrentUrl(id, req.user?.id); return res.send(url); @@ -204,7 +205,7 @@ router.get('/active/tabs', requireSignIn, (req: AuthenticatedRequest, res) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = getActiveBrowserId(req.user?.id); + const id = getActiveBrowserIdByState(req.user?.id, "recording"); if (id) { const hosts = getRemoteBrowserCurrentTabs(id, req.user?.id); return res.send(hosts); From 750cd11752ae904ab73adba9754bc6cc536a713f Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 12 Mar 2025 16:59:16 +0530 Subject: [PATCH 41/51] feat: add browser state for input handler --- server/src/browser-management/inputHandlers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 92c75e1d..8f4161b5 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -56,7 +56,7 @@ const handleWrapper = async ( return; } - const id = browserPool.getActiveBrowserId(userId); + const id = browserPool.getActiveBrowserId(userId, "recording"); if (id) { const activeBrowser = browserPool.getRemoteBrowser(id); if (activeBrowser?.interpreter.interpretationInProgress() && !activeBrowser.interpreter.interpretationIsPaused) { From 8dcbb84c01d2eebcc24ab5aa0d24f7eb77672f17 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 12 Mar 2025 16:59:49 +0530 Subject: [PATCH 42/51] feat: add browser state for workflow routes --- server/src/routes/workflow.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/routes/workflow.ts b/server/src/routes/workflow.ts index 3a117ce1..c8ad92d5 100644 --- a/server/src/routes/workflow.ts +++ b/server/src/routes/workflow.ts @@ -49,7 +49,7 @@ router.get('/params/:browserId', requireSignIn, (req, res) => { */ router.delete('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = browserPool.getActiveBrowserId(req.user?.id); + const id = browserPool.getActiveBrowserId(req.user?.id, "recording"); if (id) { const browser = browserPool.getRemoteBrowser(id); if (browser) { @@ -66,7 +66,7 @@ router.delete('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => */ router.post('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = browserPool.getActiveBrowserId(req.user?.id); + const id = browserPool.getActiveBrowserId(req.user?.id, "recording"); if (id) { const browser = browserPool.getRemoteBrowser(id); logger.log('debug', `Adding pair to workflow`); @@ -87,7 +87,7 @@ router.post('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { */ router.put('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = browserPool.getActiveBrowserId(req.user?.id); + const id = browserPool.getActiveBrowserId(req.user?.id, "recording"); if (id) { const browser = browserPool.getRemoteBrowser(id); logger.log('debug', `Updating pair in workflow`); From 8e80067b1dae9c22d22299b9e789cf4ebc613bec Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 12 Mar 2025 17:00:25 +0530 Subject: [PATCH 43/51] feat: add browser state for generator --- server/src/workflow-management/classes/Generator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 4899ac3b..aea37126 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -152,7 +152,7 @@ export class WorkflowGenerator { }); socket.on('activeIndex', (data) => this.generatedData.lastIndex = parseInt(data)); socket.on('decision', async ({ pair, actionType, decision, userId }) => { - const id = browserPool.getActiveBrowserId(userId); + const id = browserPool.getActiveBrowserId(userId, "recording"); if (id) { // const activeBrowser = browserPool.getRemoteBrowser(id); // const currentPage = activeBrowser?.getCurrentPage(); From ea6730208d76849ca6bd651908aa574aecde1478 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 12 Mar 2025 19:25:18 +0530 Subject: [PATCH 44/51] feat: queue robot run --- server/src/routes/storage.ts | 187 ++++++++++++++++------------------- 1 file changed, 87 insertions(+), 100 deletions(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index aa3f4e12..ffdf0149 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -1,6 +1,6 @@ import { Router } from 'express'; import logger from "../logger"; -import { createRemoteBrowserForRun, destroyRemoteBrowser } from "../browser-management/controller"; +import { createRemoteBrowserForRun, destroyRemoteBrowser, getActiveBrowserIdByState } from "../browser-management/controller"; import { chromium } from 'playwright-extra'; import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import { browserPool } from "../server"; @@ -22,6 +22,7 @@ import { encrypt, decrypt } from '../utils/auth'; import { WorkflowFile } from 'maxun-core'; import { Page } from 'playwright'; import { airtableUpdateTasks, processAirtableUpdates } from '../workflow-management/integrations/airtable'; +import { pgBoss } from '../pgboss-worker'; chromium.use(stealthPlugin()); export const router = Router(); @@ -494,6 +495,8 @@ router.delete('/runs/:id', requireSignIn, async (req: AuthenticatedRequest, res) /** * PUT endpoint for starting a remote browser instance and saving run metadata to the storage. * Making it ready for interpretation and returning a runId. + * + * If the user has reached their browser limit, the run will be queued using PgBoss. */ router.put('/runs/:id', requireSignIn, async (req: AuthenticatedRequest, res) => { try { @@ -525,35 +528,81 @@ router.put('/runs/:id', requireSignIn, async (req: AuthenticatedRequest, res) => }; } - console.log(`Proxy config for run: ${JSON.stringify(proxyOptions)}`) - - const id = createRemoteBrowserForRun(req.user.id); + console.log(`Proxy config for run: ${JSON.stringify(proxyOptions)}`); + // Generate runId first const runId = uuid(); + + // Check if user has reached browser limit + const userBrowserIds = browserPool.getAllBrowserIdsForUser(req.user.id); + const canCreateBrowser = userBrowserIds.length < 2; + + if (canCreateBrowser) { + // User has available browser slots, create it directly + const id = createRemoteBrowserForRun(req.user.id); - const run = await Run.create({ - status: 'running', - name: recording.recording_meta.name, - robotId: recording.id, - robotMetaId: recording.recording_meta.id, - startedAt: new Date().toLocaleString(), - finishedAt: '', - browserId: id, - interpreterSettings: req.body, - log: '', - runId, - runByUserId: req.user.id, - serializableOutput: {}, - binaryOutput: {}, - }); + const run = await Run.create({ + status: 'running', + name: recording.recording_meta.name, + robotId: recording.id, + robotMetaId: recording.recording_meta.id, + startedAt: new Date().toLocaleString(), + finishedAt: '', + browserId: id, + interpreterSettings: req.body, + log: '', + runId, + runByUserId: req.user.id, + serializableOutput: {}, + binaryOutput: {}, + }); - const plainRun = run.toJSON(); + const plainRun = run.toJSON(); - return res.send({ - browserId: id, - runId: plainRun.runId, - robotMetaId: recording.recording_meta.id, - }); + return res.send({ + browserId: id, + runId: plainRun.runId, + robotMetaId: recording.recording_meta.id, + queued: false + }); + } else { + const browserId = getActiveBrowserIdByState(req.user.id, "run") + + if (browserId) { + // User has reached the browser limit, queue the run + try { + // Create the run record with 'queued' status + await Run.create({ + status: 'queued', + name: recording.recording_meta.name, + robotId: recording.id, + robotMetaId: recording.recording_meta.id, + startedAt: new Date().toLocaleString(), + finishedAt: '', + browserId: browserId, // Random will be updated later + interpreterSettings: req.body, + log: 'Run queued - waiting for available browser slot', + runId, + runByUserId: req.user.id, + serializableOutput: {}, + binaryOutput: {}, + }); + + return res.send({ + browserId: browserId, + runId: runId, + robotMetaId: recording.recording_meta.id, + queued: true, + }); + } catch (queueError: any) { + logger.log('error', `Failed to queue run job: ${queueError.message}`); + return res.status(503).send({ error: 'Unable to queue run, please try again later' }); + } + } else { + logger.log('info', "Browser id does not exist"); + return res.send(''); + } + } } catch (e) { const { message } = e as Error; logger.log('info', `Error while creating a run with robot id: ${req.params.id} - ${message}`); @@ -608,82 +657,20 @@ router.post('/runs/run/:id', requireSignIn, async (req: AuthenticatedRequest, re return res.status(404).send(false); } - // interpret the run in active browser - const browser = browserPool.getRemoteBrowser(plainRun.browserId); - let currentPage = browser?.getCurrentPage(); - if (browser && currentPage) { - const workflow = AddGeneratedFlags(recording.recording); - const interpretationInfo = await browser.interpreter.InterpretRecording( - workflow, currentPage, (newPage: Page) => currentPage = newPage, plainRun.interpreterSettings); - const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); - const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(plainRun.browserId, req.user?.id); - await run.update({ - ...run, - status: 'success', - finishedAt: new Date().toLocaleString(), - browserId: plainRun.browserId, - log: interpretationInfo.log.join('\n'), - serializableOutput: interpretationInfo.serializableOutput, - binaryOutput: uploadedBinaryOutput, + try { + // Queue the execution job + await pgBoss.createQueue('execute-run'); + + const jobId = await pgBoss.send('execute-run', { + userId: req.user.id, + runId: req.params.id, + browserId: plainRun.browserId }); - - let totalRowsExtracted = 0; - let extractedScreenshotsCount = 0; - let extractedItemsCount = 0; - - if (run.dataValues.binaryOutput && run.dataValues.binaryOutput["item-0"]) { - extractedScreenshotsCount = 1; - } - - if (run.dataValues.serializableOutput && run.dataValues.serializableOutput["item-0"]) { - const itemsArray = run.dataValues.serializableOutput["item-0"]; - extractedItemsCount = itemsArray.length; - - totalRowsExtracted = itemsArray.reduce((total, item) => { - return total + Object.keys(item).length; - }, 0); - } - - console.log(`Extracted Items Count: ${extractedItemsCount}`); - console.log(`Extracted Screenshots Count: ${extractedScreenshotsCount}`); - console.log(`Total Rows Extracted: ${totalRowsExtracted}`); - - capture( - 'maxun-oss-run-created-manual', - { - runId: req.params.id, - user_id: req.user?.id, - created_at: new Date().toISOString(), - status: 'success', - totalRowsExtracted, - extractedItemsCount, - extractedScreenshotsCount, - } - ) - try { - googleSheetUpdateTasks[plainRun.runId] = { - robotId: plainRun.robotMetaId, - runId: plainRun.runId, - status: 'pending', - retries: 5, - }; - - airtableUpdateTasks[plainRun.runId] = { - robotId: plainRun.robotMetaId, - runId: plainRun.runId, - status: 'pending', - retries: 5, - }; - - processAirtableUpdates(); - processGoogleSheetUpdates(); - } catch (err: any) { - logger.log('error', `Failed to update Google Sheet for run: ${plainRun.runId}: ${err.message}`); - } - return res.send(true); - } else { - throw new Error('Could not destroy browser'); + + logger.log('info', `Queued run execution job with ID: ${jobId} for run: ${req.params.id}`); + } catch (queueError: any) { + logger.log('error', `Failed to queue run execution`); + } } catch (e) { const { message } = e as Error; From 7b4e689a7cc4ff079ae0f002f746262a5546724f Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 12 Mar 2025 19:26:08 +0530 Subject: [PATCH 45/51] feat: queue and execute robot run via worker --- server/src/pgboss-worker.ts | 339 +++++++++++++++++++++++++++++++++++- 1 file changed, 338 insertions(+), 1 deletion(-) diff --git a/server/src/pgboss-worker.ts b/server/src/pgboss-worker.ts index 5fd87518..a05b7959 100644 --- a/server/src/pgboss-worker.ts +++ b/server/src/pgboss-worker.ts @@ -7,8 +7,19 @@ import { initializeRemoteBrowserForRecording, destroyRemoteBrowser, interpretWholeWorkflow, - stopRunningInterpretation + stopRunningInterpretation, + createRemoteBrowserForRun } from './browser-management/controller'; +import { WorkflowFile } from 'maxun-core'; +import Run from './models/Run'; +import Robot from './models/Robot'; +import { browserPool } from './server'; +import { Page } from 'playwright'; +import { BinaryOutputService } from './storage/mino'; +import { capture } from './utils/analytics'; +import { googleSheetUpdateTasks, processGoogleSheetUpdates } from './workflow-management/integrations/gsheet'; +import { airtableUpdateTasks, processAirtableUpdates } from './workflow-management/integrations/airtable'; +import { RemoteBrowser } from './browser-management/classes/RemoteBrowser'; const pgBossConnectionString = `postgres://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`; @@ -29,6 +40,12 @@ interface DestroyBrowserData { userId: string; } +interface ExecuteRunData { + userId: string; + runId: string; + browserId: string; +} + const pgBoss = new PgBoss({connectionString: pgBossConnectionString, schema: 'public'}); /** @@ -44,6 +61,323 @@ function extractJobData(job: Job | Job[]): T { return job.data; } +function AddGeneratedFlags(workflow: WorkflowFile) { + const copy = JSON.parse(JSON.stringify(workflow)); + for (let i = 0; i < workflow.workflow.length; i++) { + copy.workflow[i].what.unshift({ + action: 'flag', + args: ['generated'], + }); + } + return copy; +}; + +/** + * Function to reset browser state without creating a new browser + */ +async function resetBrowserState(browser: RemoteBrowser): Promise { + try { + const currentPage = browser.getCurrentPage(); + if (!currentPage) { + logger.log('error', 'No current page available to reset browser state'); + return false; + } + + // Navigate to blank page to reset state + await currentPage.goto('about:blank'); + + // Clear browser storage + await currentPage.evaluate(() => { + try { + localStorage.clear(); + sessionStorage.clear(); + } catch (e) { + // Ignore errors in cleanup + } + }); + + // Clear cookies + const context = currentPage.context(); + await context.clearCookies(); + + return true; + } catch (error) { + logger.log('error', `Failed to reset browser state`); + return false; + } +} + +/** + * Modified checkAndProcessQueuedRun function - only changes browser reset logic + */ +async function checkAndProcessQueuedRun(userId: string, browserId: string): Promise { + try { + // Find the oldest queued run for this specific browser + const queuedRun = await Run.findOne({ + where: { + browserId: browserId, + status: 'queued' + }, + order: [['startedAt', 'ASC']] + }); + + if (!queuedRun) { + logger.log('info', `No queued runs found for browser ${browserId}`); + return false; + } + + // Reset the browser state before next run + const browser = browserPool.getRemoteBrowser(browserId); + if (browser) { + logger.log('info', `Resetting browser state for browser ${browserId} before next run`); + await resetBrowserState(browser); + } + + // Update the queued run to running status + await queuedRun.update({ + status: 'running', + log: 'Run started - using browser from previous run' + }); + + // Schedule the run execution + await pgBoss.createQueue('execute-run'); + const executeJobId = await pgBoss.send('execute-run', { + userId: userId, + runId: queuedRun.runId, + browserId: browserId + }); + + logger.log('info', `Scheduled queued run ${queuedRun.runId} to use browser ${browserId}, job ID: ${executeJobId}`); + return true; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Error checking for queued runs: ${errorMessage}`); + return false; + } +} + +/** + * Modified processRunExecution function - only add browser reset + */ +async function processRunExecution(job: Job) { + try { + const data = job.data; + logger.log('info', `Processing run execution job for runId: ${data.runId}, browserId: ${data.browserId}`); + + // Find the run + const run = await Run.findOne({ where: { runId: data.runId } }); + if (!run) { + logger.log('error', `Run ${data.runId} not found in database`); + return { success: false }; + } + + const plainRun = run.toJSON(); + + // Find the recording + const recording = await Robot.findOne({ where: { 'recording_meta.id': plainRun.robotMetaId }, raw: true }); + if (!recording) { + logger.log('error', `Recording for run ${data.runId} not found`); + + // Update run status to failed + await run.update({ + status: 'failed', + finishedAt: new Date().toLocaleString(), + log: 'Failed: Recording not found', + }); + + // Check for queued runs even if this one failed + await checkAndProcessQueuedRun(data.userId, data.browserId); + + return { success: false }; + } + + // Get the browser and execute the run + const browser = browserPool.getRemoteBrowser(plainRun.browserId); + let currentPage = browser?.getCurrentPage(); + + if (!browser || !currentPage) { + logger.log('error', `Browser or page not available for run ${data.runId}`); + + // Update run status to failed + await run.update({ + status: 'failed', + finishedAt: new Date().toLocaleString(), + log: 'Failed: Browser or page not available', + }); + + await pgBoss.fail(job.id, "Failed to get browser or page for run"); + + // Even if this run failed, check for queued runs + await checkAndProcessQueuedRun(data.userId, data.browserId); + + return { success: false }; + } + + try { + // Reset the browser state before executing this run + await resetBrowserState(browser); + + // Execute the workflow + const workflow = AddGeneratedFlags(recording.recording); + const interpretationInfo = await browser.interpreter.InterpretRecording( + workflow, + currentPage, + (newPage: Page) => currentPage = newPage, + plainRun.interpreterSettings + ); + + // Process the results + const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); + const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); + + // Update the run record with results + await run.update({ + ...run, + status: 'success', + finishedAt: new Date().toLocaleString(), + browserId: plainRun.browserId, + log: interpretationInfo.log.join('\n'), + serializableOutput: interpretationInfo.serializableOutput, + binaryOutput: uploadedBinaryOutput, + }); + + // Track extraction metrics + let totalRowsExtracted = 0; + let extractedScreenshotsCount = 0; + let extractedItemsCount = 0; + + if (run.dataValues.binaryOutput && run.dataValues.binaryOutput["item-0"]) { + extractedScreenshotsCount = 1; + } + + if (run.dataValues.serializableOutput && run.dataValues.serializableOutput["item-0"]) { + const itemsArray = run.dataValues.serializableOutput["item-0"]; + extractedItemsCount = itemsArray.length; + + totalRowsExtracted = itemsArray.reduce((total, item) => { + return total + Object.keys(item).length; + }, 0); + } + + console.log(`Extracted Items Count: ${extractedItemsCount}`); + console.log(`Extracted Screenshots Count: ${extractedScreenshotsCount}`); + console.log(`Total Rows Extracted: ${totalRowsExtracted}`); + + // Capture metrics + capture( + 'maxun-oss-run-created-manual', + { + runId: data.runId, + user_id: data.userId, + created_at: new Date().toISOString(), + status: 'success', + totalRowsExtracted, + extractedItemsCount, + extractedScreenshotsCount, + } + ); + + // Schedule updates for Google Sheets and Airtable + try { + googleSheetUpdateTasks[plainRun.runId] = { + robotId: plainRun.robotMetaId, + runId: plainRun.runId, + status: 'pending', + retries: 5, + }; + + airtableUpdateTasks[plainRun.runId] = { + robotId: plainRun.robotMetaId, + runId: plainRun.runId, + status: 'pending', + retries: 5, + }; + + processAirtableUpdates(); + processGoogleSheetUpdates(); + } catch (err: any) { + logger.log('error', `Failed to update Google Sheet for run: ${plainRun.runId}: ${err.message}`); + } + + // Check for and process queued runs before destroying the browser + const queuedRunProcessed = await checkAndProcessQueuedRun(data.userId, plainRun.browserId); + + // Only destroy the browser if no queued run was found + if (!queuedRunProcessed) { + await destroyRemoteBrowser(plainRun.browserId, data.userId); + logger.log('info', `No queued runs found for browser ${plainRun.browserId}, browser destroyed`); + } + + return { success: true }; + } catch (executionError: any) { + logger.log('error', `Run execution failed for run ${data.runId}: ${executionError.message}`); + + await run.update({ + status: 'failed', + finishedAt: new Date().toLocaleString(), + log: `Failed: ${executionError.message}`, + }); + + // Check for queued runs before destroying the browser + const queuedRunProcessed = await checkAndProcessQueuedRun(data.userId, plainRun.browserId); + + // Only destroy the browser if no queued run was found + if (!queuedRunProcessed) { + try { + await destroyRemoteBrowser(plainRun.browserId, data.userId); + logger.log('info', `No queued runs found for browser ${plainRun.browserId}, browser destroyed`); + } catch (cleanupError: any) { + logger.log('warn', `Failed to clean up browser for failed run ${data.runId}: ${cleanupError.message}`); + } + } + + // Capture failure metrics + capture( + 'maxun-oss-run-created-manual', + { + runId: data.runId, + user_id: data.userId, + created_at: new Date().toISOString(), + status: 'failed', + error_message: executionError.message, + } + ); + + return { success: false }; + } + + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Failed to process run execution job: ${errorMessage}`); + return { success: false }; + } +} + + +async function registerRunExecutionWorker() { + try { + // Worker for executing runs + await pgBoss.work('execute-run', async (job: Job | Job[]) => { + try { + const singleJob = Array.isArray(job) ? job[0] : job; + return await processRunExecution(singleJob); + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Run execution job failed: ${errorMessage}`); + throw error; + } + }); + + // setInterval(checkForStuckQueuedRuns, 30000); + + logger.log('info', 'Run execution worker registered successfully'); + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Failed to register run execution worker: ${errorMessage}`); + } +} + + /** * Initialize PgBoss and register all workers */ @@ -120,6 +454,9 @@ async function startWorkers() { throw error; } }); + + // Register the run execution worker + await registerRunExecutionWorker(); logger.log('info', 'All recording workers registered successfully'); } catch (error: unknown) { From 8163c46fb854a88d6b715558e13c5fcbfa0e3e57 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 12 Mar 2025 19:30:41 +0530 Subject: [PATCH 46/51] feat: add run status chip translation --- public/locales/de.json | 1 + public/locales/en.json | 1 + public/locales/es.json | 1 + public/locales/ja.json | 1 + public/locales/zh.json | 1 + 5 files changed, 5 insertions(+) diff --git a/public/locales/de.json b/public/locales/de.json index 3db0c64a..363aebbf 100644 --- a/public/locales/de.json +++ b/public/locales/de.json @@ -512,6 +512,7 @@ "success": "Erfolg", "running": "Läuft", "scheduled": "Geplant", + "queued": "In Warteschlange", "failed": "Fehlgeschlagen" }, "run_settings_modal": { diff --git a/public/locales/en.json b/public/locales/en.json index 41932f1f..53ac4184 100644 --- a/public/locales/en.json +++ b/public/locales/en.json @@ -520,6 +520,7 @@ "success": "Success", "running": "Running", "scheduled": "Scheduled", + "queued": "Queued", "failed": "Failed" }, "run_settings_modal": { diff --git a/public/locales/es.json b/public/locales/es.json index a13c69b6..14db7b5d 100644 --- a/public/locales/es.json +++ b/public/locales/es.json @@ -513,6 +513,7 @@ "success": "Éxito", "running": "Ejecutando", "scheduled": "Programado", + "queued": "En cola", "failed": "Fallido" }, "run_settings_modal": { diff --git a/public/locales/ja.json b/public/locales/ja.json index 1b38b933..54f7e788 100644 --- a/public/locales/ja.json +++ b/public/locales/ja.json @@ -513,6 +513,7 @@ "success": "成功", "running": "実行中", "scheduled": "スケジュール済み", + "queued": "キューに入れました", "failed": "失敗" }, "run_settings_modal": { diff --git a/public/locales/zh.json b/public/locales/zh.json index 5db9eed5..cc23556e 100644 --- a/public/locales/zh.json +++ b/public/locales/zh.json @@ -513,6 +513,7 @@ "success": "成功", "running": "运行中", "scheduled": "已计划", + "queued": "排队", "failed": "失败" }, "run_settings_modal": { From 767c8ac85c16883f973e68169fda1fccbe4205b8 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 12 Mar 2025 19:31:19 +0530 Subject: [PATCH 47/51] feat: add queued status chip --- src/components/run/ColapsibleRow.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/components/run/ColapsibleRow.tsx b/src/components/run/ColapsibleRow.tsx index 87e38684..4d5fe822 100644 --- a/src/components/run/ColapsibleRow.tsx +++ b/src/components/run/ColapsibleRow.tsx @@ -123,6 +123,7 @@ export const CollapsibleRow = ({ row, handleDelete, isOpen, currentLog, abortRun {row.status === 'success' && } {row.status === 'running' && } {row.status === 'scheduled' && } + {row.status === 'queued' && } {row.status === 'failed' && } ) From 0b758d772ca94a7f2b96244aa2ee5f02e08349b4 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 12 Mar 2025 20:03:10 +0530 Subject: [PATCH 48/51] feat: load based on run status queued and running --- src/components/run/RunContent.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/components/run/RunContent.tsx b/src/components/run/RunContent.tsx index cb3b63a6..77bce544 100644 --- a/src/components/run/RunContent.tsx +++ b/src/components/run/RunContent.tsx @@ -134,13 +134,13 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe }}>
- {interpretationInProgress ? currentLog : row.log} + {row.status === 'running' ? currentLog : row.log}
- {interpretationInProgress ? : null} - {interpretationInProgress ? ( + {row.status === 'running' || row.status === 'queued' ? ( {t('run_content.loading')} @@ -248,4 +248,4 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe ); -}; +}; \ No newline at end of file From a3f15b13e4ff4b3a1fe26a16fa6dfd3ff3e3dbc5 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 12 Mar 2025 20:47:39 +0530 Subject: [PATCH 49/51] feat: emit socket event for notif --- server/src/pgboss-worker.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/src/pgboss-worker.ts b/server/src/pgboss-worker.ts index a05b7959..633d5efa 100644 --- a/server/src/pgboss-worker.ts +++ b/server/src/pgboss-worker.ts @@ -20,6 +20,7 @@ import { capture } from './utils/analytics'; import { googleSheetUpdateTasks, processGoogleSheetUpdates } from './workflow-management/integrations/gsheet'; import { airtableUpdateTasks, processAirtableUpdates } from './workflow-management/integrations/airtable'; import { RemoteBrowser } from './browser-management/classes/RemoteBrowser'; +import { io as serverIo } from "./server"; const pgBossConnectionString = `postgres://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`; @@ -298,6 +299,14 @@ async function processRunExecution(job: Job) { } catch (err: any) { logger.log('error', `Failed to update Google Sheet for run: ${plainRun.runId}: ${err.message}`); } + + serverIo.of(plainRun.browserId).emit('run-completed', { + runId: data.runId, + robotMetaId: plainRun.robotMetaId, + robotName: recording.recording_meta.name, + status: 'success', + finishedAt: new Date().toLocaleString() + });; // Check for and process queued runs before destroying the browser const queuedRunProcessed = await checkAndProcessQueuedRun(data.userId, plainRun.browserId); From 351084f20e6e24e9a98a0d8761288072a23c58e3 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 12 Mar 2025 20:48:22 +0530 Subject: [PATCH 50/51] feat: check run complete and show notif --- src/pages/MainPage.tsx | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/pages/MainPage.tsx b/src/pages/MainPage.tsx index 0801a933..60990291 100644 --- a/src/pages/MainPage.tsx +++ b/src/pages/MainPage.tsx @@ -103,6 +103,17 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps) setSockets(sockets => [...sockets, socket]); socket.on('ready-for-run', () => readyForRunHandler(browserId, runId)); socket.on('debugMessage', debugMessageHandler); + socket.on('run-completed', (data) => { + setRerenderRuns(true); + + const robotName = data.robotName; + + if (data.status === 'success') { + notify('success', t('main_page.notifications.interpretation_success', { name: robotName })); + } else { + notify('error', t('main_page.notifications.interpretation_failed', { name: robotName })); + } + }); setContent('runs'); if (browserId) { notify('info', t('main_page.notifications.run_started', { name: runningRecordingName })); From f7938469598f250f5f87953eb1f13c0383891129 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 12 Mar 2025 21:09:49 +0530 Subject: [PATCH 51/51] feat: rm run update browser not exist --- server/src/pgboss-worker.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/server/src/pgboss-worker.ts b/server/src/pgboss-worker.ts index 633d5efa..3566b4f8 100644 --- a/server/src/pgboss-worker.ts +++ b/server/src/pgboss-worker.ts @@ -199,13 +199,6 @@ async function processRunExecution(job: Job) { if (!browser || !currentPage) { logger.log('error', `Browser or page not available for run ${data.runId}`); - // Update run status to failed - await run.update({ - status: 'failed', - finishedAt: new Date().toLocaleString(), - log: 'Failed: Browser or page not available', - }); - await pgBoss.fail(job.id, "Failed to get browser or page for run"); // Even if this run failed, check for queued runs