From edfcd8f869f194f7525744d050e30ba81a8bafef Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 2 Jan 2025 23:15:03 +0530 Subject: [PATCH 01/32] fix: format --- src/components/organisms/BrowserWindow.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/components/organisms/BrowserWindow.tsx b/src/components/organisms/BrowserWindow.tsx index b69a0921..421bb680 100644 --- a/src/components/organisms/BrowserWindow.tsx +++ b/src/components/organisms/BrowserWindow.tsx @@ -326,7 +326,6 @@ export const BrowserWindow = () => { } }, [paginationMode, resetPaginationSelector]); - return (
{ From 230f40bc9a8356215be7999ad47c59ba3a72b52c Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 4 Jan 2025 15:09:09 +0530 Subject: [PATCH 02/32] feat: monitor web recorder performance --- perf/performance.ts | 181 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 perf/performance.ts diff --git a/perf/performance.ts b/perf/performance.ts new file mode 100644 index 00000000..c50ef850 --- /dev/null +++ b/perf/performance.ts @@ -0,0 +1,181 @@ +// Frontend Performance Monitoring +export class FrontendPerformanceMonitor { + private metrics: { + fps: number[]; + memoryUsage: MemoryInfo[]; + renderTime: number[]; + eventLatency: number[]; + }; + private lastFrameTime: number; + private frameCount: number; + + constructor() { + this.metrics = { + fps: [], + memoryUsage: [], + renderTime: [], + eventLatency: [], + }; + this.lastFrameTime = performance.now(); + this.frameCount = 0; + + // Start monitoring + this.startMonitoring(); + } + + private startMonitoring(): void { + // Monitor FPS + const measureFPS = () => { + const currentTime = performance.now(); + const elapsed = currentTime - this.lastFrameTime; + this.frameCount++; + + if (elapsed >= 1000) { // Calculate FPS every second + const fps = Math.round((this.frameCount * 1000) / elapsed); + this.metrics.fps.push(fps); + this.frameCount = 0; + this.lastFrameTime = currentTime; + } + requestAnimationFrame(measureFPS); + }; + requestAnimationFrame(measureFPS); + + // Monitor Memory Usage + if (window.performance && (performance as any).memory) { + setInterval(() => { + const memory = (performance as any).memory; + this.metrics.memoryUsage.push({ + usedJSHeapSize: memory.usedJSHeapSize, + totalJSHeapSize: memory.totalJSHeapSize, + timestamp: Date.now() + }); + }, 1000); + } + } + + // Monitor Canvas Render Time + public measureRenderTime(renderFunction: () => void): void { + const startTime = performance.now(); + renderFunction(); + const endTime = performance.now(); + this.metrics.renderTime.push(endTime - startTime); + } + + // Monitor Event Latency + public measureEventLatency(event: MouseEvent | KeyboardEvent): void { + const latency = performance.now() - event.timeStamp; + this.metrics.eventLatency.push(latency); + } + + // Get Performance Report + public getPerformanceReport(): PerformanceReport { + return { + averageFPS: this.calculateAverage(this.metrics.fps), + averageRenderTime: this.calculateAverage(this.metrics.renderTime), + averageEventLatency: this.calculateAverage(this.metrics.eventLatency), + memoryTrend: this.getMemoryTrend(), + lastMemoryUsage: this.metrics.memoryUsage[this.metrics.memoryUsage.length - 1] + }; + } + + private calculateAverage(array: number[]): number { + return array.length ? array.reduce((a, b) => a + b) / array.length : 0; + } + + private getMemoryTrend(): MemoryTrend { + if (this.metrics.memoryUsage.length < 2) return 'stable'; + const latest = this.metrics.memoryUsage[this.metrics.memoryUsage.length - 1]; + const previous = this.metrics.memoryUsage[this.metrics.memoryUsage.length - 2]; + const change = latest.usedJSHeapSize - previous.usedJSHeapSize; + if (change > 1000000) return 'increasing'; // 1MB threshold + if (change < -1000000) return 'decreasing'; + return 'stable'; + } +} + +// Backend Performance Monitoring +export class BackendPerformanceMonitor { + private metrics: { + screenshotTimes: number[]; + emitTimes: number[]; + memoryUsage: NodeJS.MemoryUsage[]; + }; + + constructor() { + this.metrics = { + screenshotTimes: [], + emitTimes: [], + memoryUsage: [] + }; + this.startMonitoring(); + } + + private startMonitoring(): void { + // Monitor Memory Usage + setInterval(() => { + this.metrics.memoryUsage.push(process.memoryUsage()); + }, 1000); + } + + public async measureScreenshotPerformance( + makeScreenshot: () => Promise + ): Promise { + const startTime = process.hrtime(); + await makeScreenshot(); + const [seconds, nanoseconds] = process.hrtime(startTime); + this.metrics.screenshotTimes.push(seconds * 1000 + nanoseconds / 1000000); + } + + public measureEmitPerformance(emitFunction: () => void): void { + const startTime = process.hrtime(); + emitFunction(); + const [seconds, nanoseconds] = process.hrtime(startTime); + this.metrics.emitTimes.push(seconds * 1000 + nanoseconds / 1000000); + } + + public getPerformanceReport(): BackendPerformanceReport { + return { + averageScreenshotTime: this.calculateAverage(this.metrics.screenshotTimes), + averageEmitTime: this.calculateAverage(this.metrics.emitTimes), + currentMemoryUsage: this.metrics.memoryUsage[this.metrics.memoryUsage.length - 1], + memoryTrend: this.getMemoryTrend() + }; + } + + private calculateAverage(array: number[]): number { + return array.length ? array.reduce((a, b) => a + b) / array.length : 0; + } + + private getMemoryTrend(): MemoryTrend { + if (this.metrics.memoryUsage.length < 2) return 'stable'; + const latest = this.metrics.memoryUsage[this.metrics.memoryUsage.length - 1]; + const previous = this.metrics.memoryUsage[this.metrics.memoryUsage.length - 2]; + const change = latest.heapUsed - previous.heapUsed; + if (change > 1000000) return 'increasing'; + if (change < -1000000) return 'decreasing'; + return 'stable'; + } +} + +interface MemoryInfo { + usedJSHeapSize: number; + totalJSHeapSize: number; + timestamp: number; +} + +type MemoryTrend = 'increasing' | 'decreasing' | 'stable'; + +interface PerformanceReport { + averageFPS: number; + averageRenderTime: number; + averageEventLatency: number; + memoryTrend: MemoryTrend; + lastMemoryUsage: MemoryInfo; +} + +interface BackendPerformanceReport { + averageScreenshotTime: number; + averageEmitTime: number; + currentMemoryUsage: NodeJS.MemoryUsage; + memoryTrend: MemoryTrend; +} \ No newline at end of file From e20a4073de08931b947a8461db0c8c4eba681ad3 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 4 Jan 2025 15:57:18 +0530 Subject: [PATCH 03/32] feat: monitor web recorder frontend performance --- src/components/atoms/canvas.tsx | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/components/atoms/canvas.tsx b/src/components/atoms/canvas.tsx index e71a4d93..24e03bae 100644 --- a/src/components/atoms/canvas.tsx +++ b/src/components/atoms/canvas.tsx @@ -7,6 +7,7 @@ import DatePicker from './DatePicker'; import Dropdown from './Dropdown'; import TimePicker from './TimePicker'; import DateTimeLocalPicker from './DateTimeLocalPicker'; +import { FrontendPerformanceMonitor } from '../../../perf/performance'; interface CreateRefCallback { (ref: React.RefObject): void; @@ -28,6 +29,9 @@ export interface Coordinates { const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { + const performanceMonitor = useRef(new FrontendPerformanceMonitor()); + console.log('Frontend Performance Report:', performanceMonitor.current.getPerformanceReport()); + const canvasRef = useRef(null); const { socket } = useSocketStore(); const { setLastAction, lastAction } = useGlobalInfoStore(); @@ -111,6 +115,7 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { }, [socket]); const onMouseEvent = useCallback((event: MouseEvent) => { + performanceMonitor.current.measureEventLatency(event); if (socket && canvasRef.current) { // Get the canvas bounding rectangle const rect = canvasRef.current.getBoundingClientRect(); @@ -177,6 +182,15 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { } }, [socket]); + // performance logging + useEffect(() => { + const intervalId = setInterval(() => { + const report = performanceMonitor.current.getPerformanceReport(); + console.log('Frontend Performance Report:', report); + }, 5000); + + return () => clearInterval(intervalId); + }, []); useEffect(() => { if (canvasRef.current) { From 0da5cb0f67575226d8a2dbab8cfeb114e7778678 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 4 Jan 2025 15:57:41 +0530 Subject: [PATCH 04/32] feat: monitor web recorder backend performance --- .../classes/RemoteBrowser.ts | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 2c45d146..2b80b037 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -16,6 +16,7 @@ import { WorkflowGenerator } from "../../workflow-management/classes/Generator"; import { WorkflowInterpreter } from "../../workflow-management/classes/Interpreter"; import { getDecryptedProxyConfig } from '../../routes/proxy'; import { getInjectableScript } from 'idcac-playwright'; +import { BackendPerformanceMonitor } from '../../../../perf/performance' chromium.use(stealthPlugin()); @@ -78,6 +79,17 @@ export class RemoteBrowser { */ public interpreter: WorkflowInterpreter; + private performanceMonitor: BackendPerformanceMonitor; + + private startPerformanceReporting() { + setInterval(() => { + const report = this.performanceMonitor.getPerformanceReport(); + + console.log('Backend Performance Report:', report); + + }, 5000); + } + /** * Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and * assigns the socket instance everywhere. @@ -88,6 +100,8 @@ export class RemoteBrowser { this.socket = socket; this.interpreter = new WorkflowInterpreter(socket); this.generator = new WorkflowGenerator(socket); + this.performanceMonitor = new BackendPerformanceMonitor(); + this.startPerformanceReporting(); } /** @@ -519,8 +533,10 @@ export class RemoteBrowser { * @returns void */ private emitScreenshot = (payload: any): void => { - const dataWithMimeType = ('data:image/jpeg;base64,').concat(payload); - this.socket.emit('screencast', dataWithMimeType); - logger.log('debug', `Screenshot emitted`); + this.performanceMonitor.measureEmitPerformance(() => { + const dataWithMimeType = ('data:image/jpeg;base64,').concat(payload); + this.socket.emit('screencast', dataWithMimeType); + logger.log('debug', `Screenshot emitted`); + }); }; } From e74616a17734d8c9854f04e4e21b6a79f912fb6e Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sun, 5 Jan 2025 22:23:52 +0530 Subject: [PATCH 05/32] feat: reduce memory usage by canvas --- src/components/atoms/canvas.tsx | 434 ++++++++++++++++++-------------- 1 file changed, 251 insertions(+), 183 deletions(-) diff --git a/src/components/atoms/canvas.tsx b/src/components/atoms/canvas.tsx index 24e03bae..3cb8c090 100644 --- a/src/components/atoms/canvas.tsx +++ b/src/components/atoms/canvas.tsx @@ -1,6 +1,6 @@ -import React, { useCallback, useEffect, useRef } from 'react'; +// Canvas.tsx +import React, { useCallback, useEffect, useRef, useMemo } from 'react'; import { useSocketStore } from '../../context/socket'; -import { getMappedCoordinates } from "../../helpers/inputHelpers"; import { useGlobalInfoStore } from "../../context/globalInfo"; import { useActionContext } from '../../context/browserActions'; import DatePicker from './DatePicker'; @@ -9,6 +9,7 @@ import TimePicker from './TimePicker'; import DateTimeLocalPicker from './DateTimeLocalPicker'; import { FrontendPerformanceMonitor } from '../../../perf/performance'; +// Types interface CreateRefCallback { (ref: React.RefObject): void; } @@ -19,246 +20,313 @@ interface CanvasProps { onCreateRef: CreateRefCallback; } -/** - * Interface for mouse's x,y coordinates - */ export interface Coordinates { x: number; y: number; +} + +interface DropdownOption { + value: string; + text: string; + disabled: boolean; + selected: boolean; +} + +interface CanvasState { + datePickerInfo: { + coordinates: Coordinates; + selector: string; + } | null; + dropdownInfo: { + coordinates: Coordinates; + selector: string; + options: DropdownOption[]; + } | null; + timePickerInfo: { + coordinates: Coordinates; + selector: string; + } | null; + dateTimeLocalInfo: { + coordinates: Coordinates; + selector: string; + } | null; +} + +type CanvasAction = + | { type: 'SET_DATE_PICKER'; payload: CanvasState['datePickerInfo'] } + | { type: 'SET_DROPDOWN'; payload: CanvasState['dropdownInfo'] } + | { type: 'SET_TIME_PICKER'; payload: CanvasState['timePickerInfo'] } + | { type: 'SET_DATETIME_PICKER'; payload: CanvasState['dateTimeLocalInfo'] }; + +// Helper functions +const throttle = any>(func: T, limit: number): T => { + let inThrottle = false; + return ((...args: Parameters): ReturnType | void => { + if (!inThrottle) { + func.apply(null, args); + inThrottle = true; + setTimeout(() => inThrottle = false, limit); + } + }) as T; }; -const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { +const createOffscreenCanvas = (width: number, height: number) => { + if (typeof OffscreenCanvas !== 'undefined') { + return new OffscreenCanvas(width, height); + } + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + return canvas; +}; +// Reducer +const canvasReducer = (state: CanvasState, action: CanvasAction): CanvasState => { + switch (action.type) { + case 'SET_DATE_PICKER': + return { ...state, datePickerInfo: action.payload }; + case 'SET_DROPDOWN': + return { ...state, dropdownInfo: action.payload }; + case 'SET_TIME_PICKER': + return { ...state, timePickerInfo: action.payload }; + case 'SET_DATETIME_PICKER': + return { ...state, dateTimeLocalInfo: action.payload }; + default: + return state; + } +}; + +// Main Component +const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { + // Refs const performanceMonitor = useRef(new FrontendPerformanceMonitor()); - console.log('Frontend Performance Report:', performanceMonitor.current.getPerformanceReport()); - const canvasRef = useRef(null); + const lastMousePosition = useRef({ x: 0, y: 0 }); + const frameRequest = useRef(); + const renderingContext = useRef(null); + const offscreenCanvas = useRef( + createOffscreenCanvas(width || 900, height || 400) + ); + + // Hooks const { socket } = useSocketStore(); const { setLastAction, lastAction } = useGlobalInfoStore(); const { getText, getList } = useActionContext(); const getTextRef = useRef(getText); const getListRef = useRef(getList); - const [datePickerInfo, setDatePickerInfo] = React.useState<{ - coordinates: Coordinates; - selector: string; - } | null>(null); + // State + const [state, dispatch] = React.useReducer(canvasReducer, { + datePickerInfo: null, + dropdownInfo: null, + timePickerInfo: null, + dateTimeLocalInfo: null + }); - const [dropdownInfo, setDropdownInfo] = React.useState<{ - coordinates: Coordinates; - selector: string; - options: Array<{ - value: string; - text: string; - disabled: boolean; - selected: boolean; - }>; - } | null>(null); + // Memoized values + const canvasSize = useMemo(() => ({ + width: width || 900, + height: height || 400 + }), [width, height]); - const [timePickerInfo, setTimePickerInfo] = React.useState<{ - coordinates: Coordinates; - selector: string; - } | null>(null); - - const [dateTimeLocalInfo, setDateTimeLocalInfo] = React.useState<{ - coordinates: Coordinates; - selector: string; - } | null>(null); - - const notifyLastAction = (action: string) => { + const notifyLastAction = useCallback((action: string) => { if (lastAction !== action) { setLastAction(action); } - }; + }, [lastAction, setLastAction]); - const lastMousePosition = useRef({ x: 0, y: 0 }); + // Socket event handlers + const socketHandlers = useMemo(() => ({ + showDatePicker: (info: CanvasState['datePickerInfo']) => { + dispatch({ type: 'SET_DATE_PICKER', payload: info }); + }, + showDropdown: (info: CanvasState['dropdownInfo']) => { + dispatch({ type: 'SET_DROPDOWN', payload: info }); + }, + showTimePicker: (info: CanvasState['timePickerInfo']) => { + dispatch({ type: 'SET_TIME_PICKER', payload: info }); + }, + showDateTimePicker: (info: CanvasState['dateTimeLocalInfo']) => { + dispatch({ type: 'SET_DATETIME_PICKER', payload: info }); + } + }), []); + // Event handlers + const handleMouseMove = useCallback( + throttle((coordinates: Coordinates) => { + if (!socket) return; + + if ( + lastMousePosition.current.x !== coordinates.x || + lastMousePosition.current.y !== coordinates.y + ) { + lastMousePosition.current = coordinates; + socket.emit('input:mousemove', coordinates); + notifyLastAction('move'); + } + }, 16), + [socket, notifyLastAction] + ); + + const onMouseEvent = useCallback((event: MouseEvent) => { + performanceMonitor.current.measureEventLatency(event); + if (!socket || !canvasRef.current) return; + + const rect = canvasRef.current.getBoundingClientRect(); + const clickCoordinates = { + x: event.clientX - rect.left, + y: event.clientY - rect.top, + }; + + switch (event.type) { + case 'mousedown': + if (getTextRef.current) { + console.log('Capturing Text...'); + } else if (getListRef.current) { + console.log('Capturing List...'); + } else { + socket.emit('input:mousedown', clickCoordinates); + } + notifyLastAction('click'); + break; + + case 'mousemove': + handleMouseMove(clickCoordinates); + break; + + case 'wheel': + if (frameRequest.current) { + cancelAnimationFrame(frameRequest.current); + } + frameRequest.current = requestAnimationFrame(() => { + const wheelEvent = event as WheelEvent; + socket.emit('input:wheel', { + deltaX: Math.round(wheelEvent.deltaX), + deltaY: Math.round(wheelEvent.deltaY), + }); + notifyLastAction('scroll'); + }); + break; + } + }, [socket, handleMouseMove, notifyLastAction]); + + const onKeyboardEvent = useCallback((event: KeyboardEvent) => { + if (!socket) return; + + switch (event.type) { + case 'keydown': + socket.emit('input:keydown', { + key: event.key, + coordinates: lastMousePosition.current + }); + notifyLastAction(`${event.key} pressed`); + break; + + case 'keyup': + socket.emit('input:keyup', event.key); + break; + } + }, [socket, notifyLastAction]); + + // Effects useEffect(() => { getTextRef.current = getText; getListRef.current = getList; }, [getText, getList]); useEffect(() => { - if (socket) { - socket.on('showDatePicker', (info: {coordinates: Coordinates, selector: string}) => { - setDatePickerInfo(info); + if (!socket) return; + + Object.entries(socketHandlers).forEach(([event, handler]) => { + socket.on(event, handler); + }); + + return () => { + Object.keys(socketHandlers).forEach(event => { + socket.off(event); }); + }; + }, [socket, socketHandlers]); - socket.on('showDropdown', (info: { - coordinates: Coordinates, - selector: string, - options: Array<{ - value: string; - text: string; - disabled: boolean; - selected: boolean; - }>; - }) => { - setDropdownInfo(info); - }); - - socket.on('showTimePicker', (info: {coordinates: Coordinates, selector: string}) => { - setTimePickerInfo(info); - }); - - socket.on('showDateTimePicker', (info: {coordinates: Coordinates, selector: string}) => { - setDateTimeLocalInfo(info); - }); - - return () => { - socket.off('showDatePicker'); - socket.off('showDropdown'); - socket.off('showTimePicker'); - socket.off('showDateTimePicker'); - }; - } - }, [socket]); - - const onMouseEvent = useCallback((event: MouseEvent) => { - performanceMonitor.current.measureEventLatency(event); - if (socket && canvasRef.current) { - // Get the canvas bounding rectangle - const rect = canvasRef.current.getBoundingClientRect(); - const clickCoordinates = { - x: event.clientX - rect.left, // Use relative x coordinate - y: event.clientY - rect.top, // Use relative y coordinate - }; - - switch (event.type) { - case 'mousedown': - if (getTextRef.current === true) { - console.log('Capturing Text...'); - } else if (getListRef.current === true) { - console.log('Capturing List...'); - } else { - socket.emit('input:mousedown', clickCoordinates); - } - notifyLastAction('click'); - break; - case 'mousemove': - if (lastMousePosition.current.x !== clickCoordinates.x || - lastMousePosition.current.y !== clickCoordinates.y) { - lastMousePosition.current = { - x: clickCoordinates.x, - y: clickCoordinates.y, - }; - socket.emit('input:mousemove', { - x: clickCoordinates.x, - y: clickCoordinates.y, - }); - notifyLastAction('move'); - } - break; - case 'wheel': - const wheelEvent = event as WheelEvent; - const deltas = { - deltaX: Math.round(wheelEvent.deltaX), - deltaY: Math.round(wheelEvent.deltaY), - }; - socket.emit('input:wheel', deltas); - notifyLastAction('scroll'); - break; - default: - console.log('Default mouseEvent registered'); - return; - } - } - }, [socket]); - - const onKeyboardEvent = useCallback((event: KeyboardEvent) => { - if (socket) { - switch (event.type) { - case 'keydown': - socket.emit('input:keydown', { key: event.key, coordinates: lastMousePosition.current }); - notifyLastAction(`${event.key} pressed`); - break; - case 'keyup': - socket.emit('input:keyup', event.key); - break; - default: - console.log('Default keyEvent registered'); - return; - } - } - }, [socket]); - - // performance logging useEffect(() => { + const monitor = performanceMonitor.current; const intervalId = setInterval(() => { - const report = performanceMonitor.current.getPerformanceReport(); + const report = monitor.getPerformanceReport(); console.log('Frontend Performance Report:', report); - }, 5000); + }, 10000); - return () => clearInterval(intervalId); + return () => { + clearInterval(intervalId); + if (frameRequest.current) { + cancelAnimationFrame(frameRequest.current); + } + }; }, []); useEffect(() => { - if (canvasRef.current) { - onCreateRef(canvasRef); - canvasRef.current.addEventListener('mousedown', onMouseEvent); - canvasRef.current.addEventListener('mousemove', onMouseEvent); - canvasRef.current.addEventListener('wheel', onMouseEvent, { passive: true }); - canvasRef.current.addEventListener('keydown', onKeyboardEvent); - canvasRef.current.addEventListener('keyup', onKeyboardEvent); + if (!canvasRef.current) return; - return () => { - if (canvasRef.current) { - canvasRef.current.removeEventListener('mousedown', onMouseEvent); - canvasRef.current.removeEventListener('mousemove', onMouseEvent); - canvasRef.current.removeEventListener('wheel', onMouseEvent); - canvasRef.current.removeEventListener('keydown', onKeyboardEvent); - canvasRef.current.removeEventListener('keyup', onKeyboardEvent); - } + renderingContext.current = canvasRef.current.getContext('2d'); + onCreateRef(canvasRef); - }; - } else { - console.log('Canvas not initialized'); - } + const canvas = canvasRef.current; + canvas.addEventListener('mousedown', onMouseEvent); + canvas.addEventListener('mousemove', onMouseEvent); + canvas.addEventListener('wheel', onMouseEvent, { passive: true }); + canvas.addEventListener('keydown', onKeyboardEvent); + canvas.addEventListener('keyup', onKeyboardEvent); - }, [onMouseEvent]); + return () => { + canvas.removeEventListener('mousedown', onMouseEvent); + canvas.removeEventListener('mousemove', onMouseEvent); + canvas.removeEventListener('wheel', onMouseEvent); + canvas.removeEventListener('keydown', onKeyboardEvent); + canvas.removeEventListener('keyup', onKeyboardEvent); + }; + }, [onMouseEvent, onKeyboardEvent, onCreateRef]); return ( -
+
- {datePickerInfo && ( + {state.datePickerInfo && ( setDatePickerInfo(null)} + coordinates={state.datePickerInfo.coordinates} + selector={state.datePickerInfo.selector} + onClose={() => dispatch({ type: 'SET_DATE_PICKER', payload: null })} /> )} - {dropdownInfo && ( + {state.dropdownInfo && ( setDropdownInfo(null)} + coordinates={state.dropdownInfo.coordinates} + selector={state.dropdownInfo.selector} + options={state.dropdownInfo.options} + onClose={() => dispatch({ type: 'SET_DROPDOWN', payload: null })} /> )} - {timePickerInfo && ( + {state.timePickerInfo && ( setTimePickerInfo(null)} + coordinates={state.timePickerInfo.coordinates} + selector={state.timePickerInfo.selector} + onClose={() => dispatch({ type: 'SET_TIME_PICKER', payload: null })} /> )} - {dateTimeLocalInfo && ( + {state.dateTimeLocalInfo && ( setDateTimeLocalInfo(null)} + coordinates={state.dateTimeLocalInfo.coordinates} + selector={state.dateTimeLocalInfo.selector} + onClose={() => dispatch({ type: 'SET_DATETIME_PICKER', payload: null })} /> )}
); +}); -}; - +Canvas.displayName = 'Canvas'; export default Canvas; \ No newline at end of file From f26718f7e2b1ab56eb0c0ab949ea455f5334fb7a Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sun, 5 Jan 2025 23:31:11 +0530 Subject: [PATCH 06/32] feat: decreasing memory trend & latency --- src/components/atoms/canvas.tsx | 349 +++++++++++++++----------------- 1 file changed, 165 insertions(+), 184 deletions(-) diff --git a/src/components/atoms/canvas.tsx b/src/components/atoms/canvas.tsx index 3cb8c090..cd6536a4 100644 --- a/src/components/atoms/canvas.tsx +++ b/src/components/atoms/canvas.tsx @@ -1,5 +1,5 @@ -// Canvas.tsx import React, { useCallback, useEffect, useRef, useMemo } from 'react'; +import { unstable_batchedUpdates } from 'react-dom'; import { useSocketStore } from '../../context/socket'; import { useGlobalInfoStore } from "../../context/globalInfo"; import { useActionContext } from '../../context/browserActions'; @@ -9,7 +9,36 @@ import TimePicker from './TimePicker'; import DateTimeLocalPicker from './DateTimeLocalPicker'; import { FrontendPerformanceMonitor } from '../../../perf/performance'; -// Types +// Optimized throttle with RAF +const rafThrottle = any>(callback: T) => { + let requestId: number | null = null; + let lastArgs: Parameters; + + const later = () => { + requestId = null; + callback.apply(null, lastArgs); + }; + + return (...args: Parameters) => { + lastArgs = args; + if (requestId === null) { + requestId = requestAnimationFrame(later); + } + }; +}; + +// Cache DOM measurements +let measurementCache = new WeakMap(); +const getBoundingClientRectCached = (element: HTMLElement) => { + let rect = measurementCache.get(element); + if (!rect) { + rect = element.getBoundingClientRect(); + measurementCache.set(element, rect); + } + return rect; +}; + +// Types (kept the same) interface CreateRefCallback { (ref: React.RefObject): void; } @@ -25,256 +54,208 @@ export interface Coordinates { y: number; } -interface DropdownOption { - value: string; - text: string; - disabled: boolean; - selected: boolean; -} - -interface CanvasState { - datePickerInfo: { - coordinates: Coordinates; - selector: string; - } | null; - dropdownInfo: { - coordinates: Coordinates; - selector: string; - options: DropdownOption[]; - } | null; - timePickerInfo: { - coordinates: Coordinates; - selector: string; - } | null; - dateTimeLocalInfo: { - coordinates: Coordinates; - selector: string; - } | null; -} - -type CanvasAction = - | { type: 'SET_DATE_PICKER'; payload: CanvasState['datePickerInfo'] } - | { type: 'SET_DROPDOWN'; payload: CanvasState['dropdownInfo'] } - | { type: 'SET_TIME_PICKER'; payload: CanvasState['timePickerInfo'] } - | { type: 'SET_DATETIME_PICKER'; payload: CanvasState['dateTimeLocalInfo'] }; - -// Helper functions -const throttle = any>(func: T, limit: number): T => { - let inThrottle = false; - return ((...args: Parameters): ReturnType | void => { - if (!inThrottle) { - func.apply(null, args); - inThrottle = true; - setTimeout(() => inThrottle = false, limit); - } - }) as T; +// Batch updates helper +const batchedUpdates = (updates: Array<() => void>) => { + unstable_batchedUpdates(() => { + updates.forEach(update => update()); + }); }; -const createOffscreenCanvas = (width: number, height: number) => { - if (typeof OffscreenCanvas !== 'undefined') { - return new OffscreenCanvas(width, height); - } - const canvas = document.createElement('canvas'); - canvas.width = width; - canvas.height = height; - return canvas; -}; - -// Reducer -const canvasReducer = (state: CanvasState, action: CanvasAction): CanvasState => { - switch (action.type) { - case 'SET_DATE_PICKER': - return { ...state, datePickerInfo: action.payload }; - case 'SET_DROPDOWN': - return { ...state, dropdownInfo: action.payload }; - case 'SET_TIME_PICKER': - return { ...state, timePickerInfo: action.payload }; - case 'SET_DATETIME_PICKER': - return { ...state, dateTimeLocalInfo: action.payload }; - default: - return state; - } -}; - -// Main Component const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { - // Refs const performanceMonitor = useRef(new FrontendPerformanceMonitor()); const canvasRef = useRef(null); - const lastMousePosition = useRef({ x: 0, y: 0 }); - const frameRequest = useRef(); - const renderingContext = useRef(null); - const offscreenCanvas = useRef( - createOffscreenCanvas(width || 900, height || 400) - ); - - // Hooks const { socket } = useSocketStore(); const { setLastAction, lastAction } = useGlobalInfoStore(); const { getText, getList } = useActionContext(); - const getTextRef = useRef(getText); - const getListRef = useRef(getList); + + // Use a single ref object to reduce memory allocations + const refs = useRef({ + getText, + getList, + lastMousePosition: { x: 0, y: 0 }, + frameRequest: 0, + eventQueue: [] as Array<() => void>, + isProcessing: false + }); - // State - const [state, dispatch] = React.useReducer(canvasReducer, { + // Consolidated state using a single reducer + const [state, dispatch] = React.useReducer((state: any, action: any) => { + switch (action.type) { + case 'BATCH_UPDATE': + return { ...state, ...action.payload }; + default: + return state; + } + }, { datePickerInfo: null, dropdownInfo: null, timePickerInfo: null, dateTimeLocalInfo: null }); - // Memoized values - const canvasSize = useMemo(() => ({ - width: width || 900, - height: height || 400 - }), [width, height]); + // Process events in batches + const processEventQueue = useCallback(() => { + if (refs.current.isProcessing || refs.current.eventQueue.length === 0) return; + + refs.current.isProcessing = true; + const events = [...refs.current.eventQueue]; + refs.current.eventQueue = []; - const notifyLastAction = useCallback((action: string) => { - if (lastAction !== action) { - setLastAction(action); + batchedUpdates(events.map(event => () => event())); + + refs.current.isProcessing = false; + + if (refs.current.eventQueue.length > 0) { + requestAnimationFrame(processEventQueue); } - }, [lastAction, setLastAction]); + }, []); - // Socket event handlers - const socketHandlers = useMemo(() => ({ - showDatePicker: (info: CanvasState['datePickerInfo']) => { - dispatch({ type: 'SET_DATE_PICKER', payload: info }); - }, - showDropdown: (info: CanvasState['dropdownInfo']) => { - dispatch({ type: 'SET_DROPDOWN', payload: info }); - }, - showTimePicker: (info: CanvasState['timePickerInfo']) => { - dispatch({ type: 'SET_TIME_PICKER', payload: info }); - }, - showDateTimePicker: (info: CanvasState['dateTimeLocalInfo']) => { - dispatch({ type: 'SET_DATETIME_PICKER', payload: info }); - } - }), []); - - // Event handlers - const handleMouseMove = useCallback( - throttle((coordinates: Coordinates) => { + // Optimized mouse move handler using RAF throttle + const handleMouseMove = useMemo( + () => rafThrottle((coordinates: Coordinates) => { if (!socket) return; - if ( - lastMousePosition.current.x !== coordinates.x || - lastMousePosition.current.y !== coordinates.y - ) { - lastMousePosition.current = coordinates; + const current = refs.current.lastMousePosition; + if (current.x !== coordinates.x || current.y !== coordinates.y) { + refs.current.lastMousePosition = coordinates; socket.emit('input:mousemove', coordinates); - notifyLastAction('move'); + refs.current.eventQueue.push(() => setLastAction('move')); + requestAnimationFrame(processEventQueue); } - }, 16), - [socket, notifyLastAction] + }), + [socket, processEventQueue] ); + // Optimized event handler with better performance characteristics const onMouseEvent = useCallback((event: MouseEvent) => { - performanceMonitor.current.measureEventLatency(event); if (!socket || !canvasRef.current) return; - const rect = canvasRef.current.getBoundingClientRect(); - const clickCoordinates = { + performanceMonitor.current.measureEventLatency(event); + const rect = getBoundingClientRectCached(canvasRef.current); + const coordinates = { x: event.clientX - rect.left, y: event.clientY - rect.top, }; switch (event.type) { case 'mousedown': - if (getTextRef.current) { - console.log('Capturing Text...'); - } else if (getListRef.current) { - console.log('Capturing List...'); - } else { - socket.emit('input:mousedown', clickCoordinates); - } - notifyLastAction('click'); + refs.current.eventQueue.push(() => { + if (refs.current.getText) { + console.log('Capturing Text...'); + } else if (refs.current.getList) { + console.log('Capturing List...'); + } else { + socket.emit('input:mousedown', coordinates); + } + setLastAction('click'); + }); break; case 'mousemove': - handleMouseMove(clickCoordinates); + handleMouseMove(coordinates); break; case 'wheel': - if (frameRequest.current) { - cancelAnimationFrame(frameRequest.current); + if (refs.current.frameRequest) { + cancelAnimationFrame(refs.current.frameRequest); } - frameRequest.current = requestAnimationFrame(() => { + refs.current.frameRequest = requestAnimationFrame(() => { const wheelEvent = event as WheelEvent; socket.emit('input:wheel', { deltaX: Math.round(wheelEvent.deltaX), - deltaY: Math.round(wheelEvent.deltaY), + deltaY: Math.round(wheelEvent.deltaY) }); - notifyLastAction('scroll'); + refs.current.eventQueue.push(() => setLastAction('scroll')); }); break; } - }, [socket, handleMouseMove, notifyLastAction]); - const onKeyboardEvent = useCallback((event: KeyboardEvent) => { - if (!socket) return; + requestAnimationFrame(processEventQueue); + }, [socket, handleMouseMove, processEventQueue]); - switch (event.type) { - case 'keydown': - socket.emit('input:keydown', { - key: event.key, - coordinates: lastMousePosition.current - }); - notifyLastAction(`${event.key} pressed`); - break; + // Optimized keyboard handler + const onKeyboardEvent = useMemo( + () => rafThrottle((event: KeyboardEvent) => { + if (!socket) return; - case 'keyup': - socket.emit('input:keyup', event.key); - break; - } - }, [socket, notifyLastAction]); + refs.current.eventQueue.push(() => { + switch (event.type) { + case 'keydown': + socket.emit('input:keydown', { + key: event.key, + coordinates: refs.current.lastMousePosition + }); + setLastAction(`${event.key} pressed`); + break; + case 'keyup': + socket.emit('input:keyup', event.key); + break; + } + }); + requestAnimationFrame(processEventQueue); + }), + [socket, processEventQueue] + ); - // Effects + // Update refs useEffect(() => { - getTextRef.current = getText; - getListRef.current = getList; + refs.current.getText = getText; + refs.current.getList = getList; }, [getText, getList]); + // Socket event setup with optimized cleanup useEffect(() => { if (!socket) return; - Object.entries(socketHandlers).forEach(([event, handler]) => { + const handlers = { + showDatePicker: (info: any) => dispatch({ type: 'BATCH_UPDATE', payload: { datePickerInfo: info } }), + showDropdown: (info: any) => dispatch({ type: 'BATCH_UPDATE', payload: { dropdownInfo: info } }), + showTimePicker: (info: any) => dispatch({ type: 'BATCH_UPDATE', payload: { timePickerInfo: info } }), + showDateTimePicker: (info: any) => dispatch({ type: 'BATCH_UPDATE', payload: { dateTimeLocalInfo: info } }) + }; + + Object.entries(handlers).forEach(([event, handler]) => { socket.on(event, handler); }); return () => { - Object.keys(socketHandlers).forEach(event => { + Object.keys(handlers).forEach(event => { socket.off(event); }); }; - }, [socket, socketHandlers]); + }, [socket]); useEffect(() => { const monitor = performanceMonitor.current; const intervalId = setInterval(() => { - const report = monitor.getPerformanceReport(); - console.log('Frontend Performance Report:', report); - }, 10000); - + console.log('Frontend Performance Report:', monitor.getPerformanceReport()); + }, 15000); // Increased to 15 seconds + return () => { clearInterval(intervalId); - if (frameRequest.current) { - cancelAnimationFrame(frameRequest.current); + if (refs.current.frameRequest) { + cancelAnimationFrame(refs.current.frameRequest); } + + // Clear measurement cache on unmount + measurementCache = new WeakMap(); // Reset the WeakMap }; }, []); + + // Canvas setup with optimized event binding useEffect(() => { if (!canvasRef.current) return; - renderingContext.current = canvasRef.current.getContext('2d'); onCreateRef(canvasRef); - const canvas = canvasRef.current; - canvas.addEventListener('mousedown', onMouseEvent); - canvas.addEventListener('mousemove', onMouseEvent); - canvas.addEventListener('wheel', onMouseEvent, { passive: true }); - canvas.addEventListener('keydown', onKeyboardEvent); - canvas.addEventListener('keyup', onKeyboardEvent); + + const options = { passive: true }; + canvas.addEventListener('mousedown', onMouseEvent, options); + canvas.addEventListener('mousemove', onMouseEvent, options); + canvas.addEventListener('wheel', onMouseEvent, options); + canvas.addEventListener('keydown', onKeyboardEvent, options); + canvas.addEventListener('keyup', onKeyboardEvent, options); return () => { canvas.removeEventListener('mousedown', onMouseEvent); @@ -285,28 +266,28 @@ const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { }; }, [onMouseEvent, onKeyboardEvent, onCreateRef]); + const memoizedSize = useMemo(() => ({ + width: width || 900, + height: height || 400 + }), [width, height]); + return (
{state.datePickerInfo && ( dispatch({ type: 'SET_DATE_PICKER', payload: null })} - /> - )} - {state.dropdownInfo && ( - dispatch({ type: 'SET_DROPDOWN', payload: null })} + onClose={() => dispatch({ + type: 'BATCH_UPDATE', + payload: { datePickerInfo: null } + })} /> )} {state.timePickerInfo && ( From ba0d84934bf3298188df4d0008bff5195c2c1c01 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 01:46:54 +0530 Subject: [PATCH 07/32] feat: higher fps & latency less than 5ms --- src/components/atoms/canvas.tsx | 419 ++++++++++++++++++-------------- 1 file changed, 240 insertions(+), 179 deletions(-) diff --git a/src/components/atoms/canvas.tsx b/src/components/atoms/canvas.tsx index cd6536a4..2f3cb6c9 100644 --- a/src/components/atoms/canvas.tsx +++ b/src/components/atoms/canvas.tsx @@ -1,84 +1,177 @@ -import React, { useCallback, useEffect, useRef, useMemo } from 'react'; -import { unstable_batchedUpdates } from 'react-dom'; +import React, { useCallback, useEffect, useRef, useMemo, Suspense } from 'react'; import { useSocketStore } from '../../context/socket'; import { useGlobalInfoStore } from "../../context/globalInfo"; import { useActionContext } from '../../context/browserActions'; -import DatePicker from './DatePicker'; -import Dropdown from './Dropdown'; -import TimePicker from './TimePicker'; -import DateTimeLocalPicker from './DateTimeLocalPicker'; import { FrontendPerformanceMonitor } from '../../../perf/performance'; -// Optimized throttle with RAF -const rafThrottle = any>(callback: T) => { - let requestId: number | null = null; - let lastArgs: Parameters; +// Lazy load components that aren't always needed +const DatePicker = React.lazy(() => import('./DatePicker')); +const Dropdown = React.lazy(() => import('./Dropdown')); +const TimePicker = React.lazy(() => import('./TimePicker')); +const DateTimeLocalPicker = React.lazy(() => import('./DateTimeLocalPicker')); - const later = () => { - requestId = null; - callback.apply(null, lastArgs); - }; +// High-performance RAF scheduler +class RAFScheduler { + private queue: Set<() => void> = new Set(); + private isProcessing: boolean = false; + private frameId: number | null = null; - return (...args: Parameters) => { - lastArgs = args; - if (requestId === null) { - requestId = requestAnimationFrame(later); + schedule(callback: () => void): void { + this.queue.add(callback); + if (!this.isProcessing) { + this.process(); } - }; -}; - -// Cache DOM measurements -let measurementCache = new WeakMap(); -const getBoundingClientRectCached = (element: HTMLElement) => { - let rect = measurementCache.get(element); - if (!rect) { - rect = element.getBoundingClientRect(); - measurementCache.set(element, rect); } - return rect; -}; -// Types (kept the same) -interface CreateRefCallback { - (ref: React.RefObject): void; + private process = (): void => { + this.isProcessing = true; + this.frameId = requestAnimationFrame(() => { + const callbacks = Array.from(this.queue); + this.queue.clear(); + + callbacks.forEach(callback => { + try { + callback(); + } catch (error) { + console.error('RAF Scheduler error:', error); + } + }); + + this.isProcessing = false; + this.frameId = null; + + if (this.queue.size > 0) { + this.process(); + } + }); + } + + clear(): void { + this.queue.clear(); + if (this.frameId !== null) { + cancelAnimationFrame(this.frameId); + this.frameId = null; + } + this.isProcessing = false; + } +} + +// Enhanced event debouncer with priority queue +class EventDebouncer { + private highPriorityQueue: Array<() => void> = []; + private lowPriorityQueue: Array<() => void> = []; + private processing: boolean = false; + private scheduler: RAFScheduler; + + constructor(scheduler: RAFScheduler) { + this.scheduler = scheduler; + } + + add(callback: () => void, highPriority: boolean = false): void { + if (highPriority) { + this.highPriorityQueue.push(callback); + } else { + this.lowPriorityQueue.push(callback); + } + + if (!this.processing) { + this.process(); + } + } + + private process(): void { + this.processing = true; + this.scheduler.schedule(() => { + while (this.highPriorityQueue.length > 0) { + const callback = this.highPriorityQueue.shift(); + callback?.(); + } + + if (this.lowPriorityQueue.length > 0) { + const callback = this.lowPriorityQueue.shift(); + callback?.(); + + if (this.lowPriorityQueue.length > 0) { + this.process(); + } + } + + this.processing = false; + }); + } + + clear(): void { + this.highPriorityQueue = []; + this.lowPriorityQueue = []; + this.processing = false; + } +} + +// Optimized measurement cache with LRU +class MeasurementCache { + private cache: Map; + private maxSize: number; + + constructor(maxSize: number = 100) { + this.cache = new Map(); + this.maxSize = maxSize; + } + + get(element: HTMLElement): DOMRect | undefined { + const cached = this.cache.get(element); + if (cached) { + // Refresh the entry + this.cache.delete(element); + this.cache.set(element, cached); + } + return cached; + } + + set(element: HTMLElement, rect: DOMRect): void { + if (this.cache.size >= this.maxSize) { + // Remove oldest entry + const firstKey = this.cache.keys().next().value; + if (firstKey !== undefined) { + this.cache.delete(firstKey); + } + } + this.cache.set(element, rect); + } + + clear(): void { + this.cache.clear(); + } } interface CanvasProps { width: number; height: number; - onCreateRef: CreateRefCallback; + onCreateRef: (ref: React.RefObject) => void; } -export interface Coordinates { - x: number; - y: number; -} - -// Batch updates helper -const batchedUpdates = (updates: Array<() => void>) => { - unstable_batchedUpdates(() => { - updates.forEach(update => update()); - }); -}; - const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { - const performanceMonitor = useRef(new FrontendPerformanceMonitor()); + // Core refs and state const canvasRef = useRef(null); const { socket } = useSocketStore(); - const { setLastAction, lastAction } = useGlobalInfoStore(); + const { setLastAction } = useGlobalInfoStore(); const { getText, getList } = useActionContext(); - - // Use a single ref object to reduce memory allocations + + // Performance optimization instances + const scheduler = useRef(new RAFScheduler()); + const debouncer = useRef(new EventDebouncer(scheduler.current)); + const measurementCache = useRef(new MeasurementCache(50)); + const performanceMonitor = useRef(new FrontendPerformanceMonitor()); + + // Consolidated refs const refs = useRef({ getText, getList, lastMousePosition: { x: 0, y: 0 }, - frameRequest: 0, - eventQueue: [] as Array<() => void>, - isProcessing: false + lastFrameTime: 0, + context: null as CanvasRenderingContext2D | null, }); - // Consolidated state using a single reducer + // Optimized state management const [state, dispatch] = React.useReducer((state: any, action: any) => { switch (action.type) { case 'BATCH_UPDATE': @@ -93,53 +186,32 @@ const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { dateTimeLocalInfo: null }); - // Process events in batches - const processEventQueue = useCallback(() => { - if (refs.current.isProcessing || refs.current.eventQueue.length === 0) return; - - refs.current.isProcessing = true; - const events = [...refs.current.eventQueue]; - refs.current.eventQueue = []; + // Optimized coordinate calculation + const getEventCoordinates = useCallback((event: MouseEvent): { x: number; y: number } => { + if (!canvasRef.current) return { x: 0, y: 0 }; - batchedUpdates(events.map(event => () => event())); - - refs.current.isProcessing = false; - - if (refs.current.eventQueue.length > 0) { - requestAnimationFrame(processEventQueue); + let rect = measurementCache.current.get(canvasRef.current); + if (!rect) { + rect = canvasRef.current.getBoundingClientRect(); + measurementCache.current.set(canvasRef.current, rect); } + + return { + x: event.clientX - rect.left, + y: event.clientY - rect.top + }; }, []); - // Optimized mouse move handler using RAF throttle - const handleMouseMove = useMemo( - () => rafThrottle((coordinates: Coordinates) => { - if (!socket) return; - - const current = refs.current.lastMousePosition; - if (current.x !== coordinates.x || current.y !== coordinates.y) { - refs.current.lastMousePosition = coordinates; - socket.emit('input:mousemove', coordinates); - refs.current.eventQueue.push(() => setLastAction('move')); - requestAnimationFrame(processEventQueue); - } - }), - [socket, processEventQueue] - ); - - // Optimized event handler with better performance characteristics - const onMouseEvent = useCallback((event: MouseEvent) => { + // High-performance mouse handler + const handleMouseEvent = useCallback((event: MouseEvent) => { if (!socket || !canvasRef.current) return; performanceMonitor.current.measureEventLatency(event); - const rect = getBoundingClientRectCached(canvasRef.current); - const coordinates = { - x: event.clientX - rect.left, - y: event.clientY - rect.top, - }; + const coordinates = getEventCoordinates(event); switch (event.type) { case 'mousedown': - refs.current.eventQueue.push(() => { + debouncer.current.add(() => { if (refs.current.getText) { console.log('Capturing Text...'); } else if (refs.current.getList) { @@ -148,62 +220,95 @@ const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { socket.emit('input:mousedown', coordinates); } setLastAction('click'); - }); + }, true); // High priority break; case 'mousemove': - handleMouseMove(coordinates); + if (refs.current.lastMousePosition.x !== coordinates.x || + refs.current.lastMousePosition.y !== coordinates.y) { + debouncer.current.add(() => { + refs.current.lastMousePosition = coordinates; + socket.emit('input:mousemove', coordinates); + setLastAction('move'); + }); + } break; case 'wheel': - if (refs.current.frameRequest) { - cancelAnimationFrame(refs.current.frameRequest); - } - refs.current.frameRequest = requestAnimationFrame(() => { - const wheelEvent = event as WheelEvent; + const wheelEvent = event as WheelEvent; + debouncer.current.add(() => { socket.emit('input:wheel', { deltaX: Math.round(wheelEvent.deltaX), deltaY: Math.round(wheelEvent.deltaY) }); - refs.current.eventQueue.push(() => setLastAction('scroll')); + setLastAction('scroll'); }); break; } - - requestAnimationFrame(processEventQueue); - }, [socket, handleMouseMove, processEventQueue]); + }, [socket, getEventCoordinates]); // Optimized keyboard handler - const onKeyboardEvent = useMemo( - () => rafThrottle((event: KeyboardEvent) => { - if (!socket) return; + const handleKeyboardEvent = useCallback((event: KeyboardEvent) => { + if (!socket) return; - refs.current.eventQueue.push(() => { - switch (event.type) { - case 'keydown': - socket.emit('input:keydown', { - key: event.key, - coordinates: refs.current.lastMousePosition - }); - setLastAction(`${event.key} pressed`); - break; - case 'keyup': - socket.emit('input:keyup', event.key); - break; - } - }); - requestAnimationFrame(processEventQueue); - }), - [socket, processEventQueue] - ); + debouncer.current.add(() => { + switch (event.type) { + case 'keydown': + socket.emit('input:keydown', { + key: event.key, + coordinates: refs.current.lastMousePosition + }); + setLastAction(`${event.key} pressed`); + break; + case 'keyup': + socket.emit('input:keyup', event.key); + break; + } + }, event.type === 'keydown'); // High priority for keydown + }, [socket]); - // Update refs + // Setup and cleanup useEffect(() => { - refs.current.getText = getText; - refs.current.getList = getList; - }, [getText, getList]); + if (!canvasRef.current) return; - // Socket event setup with optimized cleanup + const canvas = canvasRef.current; + refs.current.context = canvas.getContext('2d', { + alpha: false, + desynchronized: true + }); + + onCreateRef(canvasRef); + + const options = { passive: true }; + canvas.addEventListener('mousedown', handleMouseEvent, options); + canvas.addEventListener('mousemove', handleMouseEvent, options); + canvas.addEventListener('wheel', handleMouseEvent, options); + canvas.addEventListener('keydown', handleKeyboardEvent, options); + canvas.addEventListener('keyup', handleKeyboardEvent, options); + + return () => { + canvas.removeEventListener('mousedown', handleMouseEvent); + canvas.removeEventListener('mousemove', handleMouseEvent); + canvas.removeEventListener('wheel', handleMouseEvent); + canvas.removeEventListener('keydown', handleKeyboardEvent); + canvas.removeEventListener('keyup', handleKeyboardEvent); + + scheduler.current.clear(); + debouncer.current.clear(); + measurementCache.current.clear(); + }; + }, [handleMouseEvent, handleKeyboardEvent, onCreateRef]); + + // Performance monitoring + useEffect(() => { + const intervalId = setInterval(() => { + console.log('Performance Report:', performanceMonitor.current.getPerformanceReport()); + }, 20000); // Reduced frequency + + return () => clearInterval(intervalId); + }, []); + + // Socket events useEffect(() => { if (!socket) return; @@ -214,59 +319,13 @@ const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { showDateTimePicker: (info: any) => dispatch({ type: 'BATCH_UPDATE', payload: { dateTimeLocalInfo: info } }) }; - Object.entries(handlers).forEach(([event, handler]) => { - socket.on(event, handler); - }); - + Object.entries(handlers).forEach(([event, handler]) => socket.on(event, handler)); return () => { - Object.keys(handlers).forEach(event => { - socket.off(event); - }); + Object.keys(handlers).forEach(event => socket.off(event)); }; }, [socket]); - useEffect(() => { - const monitor = performanceMonitor.current; - const intervalId = setInterval(() => { - console.log('Frontend Performance Report:', monitor.getPerformanceReport()); - }, 15000); // Increased to 15 seconds - - return () => { - clearInterval(intervalId); - if (refs.current.frameRequest) { - cancelAnimationFrame(refs.current.frameRequest); - } - - // Clear measurement cache on unmount - measurementCache = new WeakMap(); // Reset the WeakMap - }; - }, []); - - - // Canvas setup with optimized event binding - useEffect(() => { - if (!canvasRef.current) return; - - onCreateRef(canvasRef); - const canvas = canvasRef.current; - - const options = { passive: true }; - canvas.addEventListener('mousedown', onMouseEvent, options); - canvas.addEventListener('mousemove', onMouseEvent, options); - canvas.addEventListener('wheel', onMouseEvent, options); - canvas.addEventListener('keydown', onKeyboardEvent, options); - canvas.addEventListener('keyup', onKeyboardEvent, options); - - return () => { - canvas.removeEventListener('mousedown', onMouseEvent); - canvas.removeEventListener('mousemove', onMouseEvent); - canvas.removeEventListener('wheel', onMouseEvent); - canvas.removeEventListener('keydown', onKeyboardEvent); - canvas.removeEventListener('keyup', onKeyboardEvent); - }; - }, [onMouseEvent, onKeyboardEvent, onCreateRef]); - - const memoizedSize = useMemo(() => ({ + const memoizedDimensions = useMemo(() => ({ width: width || 900, height: height || 400 }), [width, height]); @@ -276,10 +335,11 @@ const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { + {state.datePickerInfo && ( { onClose={() => dispatch({ type: 'SET_DATETIME_PICKER', payload: null })} /> )} +
); }); From 2a568ed631038747ca17f6a2647f082dbe50368b Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 02:01:07 +0530 Subject: [PATCH 08/32] chore: cleanup --- src/components/atoms/canvas.tsx | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/components/atoms/canvas.tsx b/src/components/atoms/canvas.tsx index 2f3cb6c9..e7cd8597 100644 --- a/src/components/atoms/canvas.tsx +++ b/src/components/atoms/canvas.tsx @@ -4,13 +4,11 @@ import { useGlobalInfoStore } from "../../context/globalInfo"; import { useActionContext } from '../../context/browserActions'; import { FrontendPerformanceMonitor } from '../../../perf/performance'; -// Lazy load components that aren't always needed const DatePicker = React.lazy(() => import('./DatePicker')); const Dropdown = React.lazy(() => import('./Dropdown')); const TimePicker = React.lazy(() => import('./TimePicker')); const DateTimeLocalPicker = React.lazy(() => import('./DateTimeLocalPicker')); -// High-performance RAF scheduler class RAFScheduler { private queue: Set<() => void> = new Set(); private isProcessing: boolean = false; @@ -56,7 +54,6 @@ class RAFScheduler { } } -// Enhanced event debouncer with priority queue class EventDebouncer { private highPriorityQueue: Array<() => void> = []; private lowPriorityQueue: Array<() => void> = []; @@ -150,19 +147,16 @@ interface CanvasProps { } const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { - // Core refs and state const canvasRef = useRef(null); const { socket } = useSocketStore(); const { setLastAction } = useGlobalInfoStore(); const { getText, getList } = useActionContext(); - // Performance optimization instances const scheduler = useRef(new RAFScheduler()); const debouncer = useRef(new EventDebouncer(scheduler.current)); const measurementCache = useRef(new MeasurementCache(50)); const performanceMonitor = useRef(new FrontendPerformanceMonitor()); - // Consolidated refs const refs = useRef({ getText, getList, @@ -171,7 +165,6 @@ const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { context: null as CanvasRenderingContext2D | null, }); - // Optimized state management const [state, dispatch] = React.useReducer((state: any, action: any) => { switch (action.type) { case 'BATCH_UPDATE': @@ -186,7 +179,6 @@ const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { dateTimeLocalInfo: null }); - // Optimized coordinate calculation const getEventCoordinates = useCallback((event: MouseEvent): { x: number; y: number } => { if (!canvasRef.current) return { x: 0, y: 0 }; @@ -202,7 +194,6 @@ const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { }; }, []); - // High-performance mouse handler const handleMouseEvent = useCallback((event: MouseEvent) => { if (!socket || !canvasRef.current) return; @@ -247,7 +238,6 @@ const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { } }, [socket, getEventCoordinates]); - // Optimized keyboard handler const handleKeyboardEvent = useCallback((event: KeyboardEvent) => { if (!socket) return; @@ -303,12 +293,11 @@ const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { useEffect(() => { const intervalId = setInterval(() => { console.log('Performance Report:', performanceMonitor.current.getPerformanceReport()); - }, 20000); // Reduced frequency + }, 20000); return () => clearInterval(intervalId); }, []); - // Socket events useEffect(() => { if (!socket) return; From b06db2e77839f7ee460ded236ff9fe283e90699d Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 02:01:20 +0530 Subject: [PATCH 09/32] chore: lint --- src/components/atoms/canvas.tsx | 58 ++++++++++++++++----------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/components/atoms/canvas.tsx b/src/components/atoms/canvas.tsx index e7cd8597..d198adb0 100644 --- a/src/components/atoms/canvas.tsx +++ b/src/components/atoms/canvas.tsx @@ -26,7 +26,7 @@ class RAFScheduler { this.frameId = requestAnimationFrame(() => { const callbacks = Array.from(this.queue); this.queue.clear(); - + callbacks.forEach(callback => { try { callback(); @@ -37,7 +37,7 @@ class RAFScheduler { this.isProcessing = false; this.frameId = null; - + if (this.queue.size > 0) { this.process(); } @@ -87,12 +87,12 @@ class EventDebouncer { if (this.lowPriorityQueue.length > 0) { const callback = this.lowPriorityQueue.shift(); callback?.(); - + if (this.lowPriorityQueue.length > 0) { this.process(); } } - + this.processing = false; }); } @@ -282,7 +282,7 @@ const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { canvas.removeEventListener('wheel', handleMouseEvent); canvas.removeEventListener('keydown', handleKeyboardEvent); canvas.removeEventListener('keyup', handleKeyboardEvent); - + scheduler.current.clear(); debouncer.current.clear(); measurementCache.current.clear(); @@ -329,30 +329,30 @@ const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { className="block" /> - {state.datePickerInfo && ( - dispatch({ - type: 'BATCH_UPDATE', - payload: { datePickerInfo: null } - })} - /> - )} - {state.timePickerInfo && ( - dispatch({ type: 'SET_TIME_PICKER', payload: null })} - /> - )} - {state.dateTimeLocalInfo && ( - dispatch({ type: 'SET_DATETIME_PICKER', payload: null })} - /> - )} + {state.datePickerInfo && ( + dispatch({ + type: 'BATCH_UPDATE', + payload: { datePickerInfo: null } + })} + /> + )} + {state.timePickerInfo && ( + dispatch({ type: 'SET_TIME_PICKER', payload: null })} + /> + )} + {state.dateTimeLocalInfo && ( + dispatch({ type: 'SET_DATETIME_PICKER', payload: null })} + /> + )}
); From bc59ae55b2acb836c3c64a04a8e89e6169f078e4 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 10:55:33 +0530 Subject: [PATCH 10/32] chore(deps): install lodash & sharp --- package.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index e89f13de..f8975ea6 100644 --- a/package.json +++ b/package.json @@ -44,9 +44,9 @@ "joi": "^17.6.0", "jsonwebtoken": "^9.0.2", "jwt-decode": "^4.0.0", + "lodash": "^4.17.21", "loglevel": "^1.8.0", "loglevel-plugin-remote": "^0.6.8", - "maxun-core": "^0.0.7", "minio": "^8.0.1", "moment-timezone": "^0.5.45", "node-cron": "^3.0.3", @@ -66,6 +66,7 @@ "react-transition-group": "^4.4.2", "sequelize": "^6.37.3", "sequelize-typescript": "^2.1.6", + "sharp": "^0.33.5", "socket.io": "^4.4.1", "socket.io-client": "^4.4.1", "styled-components": "^5.3.3", @@ -97,6 +98,7 @@ "@types/cookie-parser": "^1.4.7", "@types/express": "^4.17.13", "@types/js-cookie": "^3.0.6", + "@types/lodash": "^4.17.14", "@types/loglevel": "^1.6.3", "@types/node": "22.7.9", "@types/node-cron": "^3.0.11", From 20912a73c632d7c527753bbbcb536377646cf84d Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 10:56:31 +0530 Subject: [PATCH 11/32] fix: missing coordinates interface export --- src/components/atoms/canvas.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/components/atoms/canvas.tsx b/src/components/atoms/canvas.tsx index d198adb0..96933030 100644 --- a/src/components/atoms/canvas.tsx +++ b/src/components/atoms/canvas.tsx @@ -3,7 +3,6 @@ import { useSocketStore } from '../../context/socket'; import { useGlobalInfoStore } from "../../context/globalInfo"; import { useActionContext } from '../../context/browserActions'; import { FrontendPerformanceMonitor } from '../../../perf/performance'; - const DatePicker = React.lazy(() => import('./DatePicker')); const Dropdown = React.lazy(() => import('./Dropdown')); const TimePicker = React.lazy(() => import('./TimePicker')); @@ -146,6 +145,14 @@ interface CanvasProps { onCreateRef: (ref: React.RefObject) => void; } +/** + * Interface for mouse's x,y coordinates + */ +export interface Coordinates { + x: number; + y: number; +}; + const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { const canvasRef = useRef(null); const { socket } = useSocketStore(); From 7fba807525caa2f137f4e098993f52bdca0727fd Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 11:43:02 +0530 Subject: [PATCH 12/32] feat: screenshot queue --- server/src/browser-management/classes/RemoteBrowser.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 2b80b037..dd97a809 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -9,6 +9,8 @@ import { chromium } from 'playwright-extra'; import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import { PlaywrightBlocker } from '@cliqz/adblocker-playwright'; import fetch from 'cross-fetch'; +import { throttle } from 'lodash'; +import sharp from 'sharp'; import logger from '../../logger'; import { InterpreterSettings, RemoteBrowserOptions } from "../../types"; @@ -81,6 +83,10 @@ export class RemoteBrowser { private performanceMonitor: BackendPerformanceMonitor; + private screenshotQueue: Buffer[] = []; + private isProcessingScreenshot = false; + private screencastInterval: NodeJS.Timeout | null = null; + private startPerformanceReporting() { setInterval(() => { const report = this.performanceMonitor.getPerformanceReport(); From b3f17bca4cf317c944f729407da0377f44f5a2d0 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 11:46:06 +0530 Subject: [PATCH 13/32] feat: memory management for screenshot streaming --- .../browser-management/classes/RemoteBrowser.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index dd97a809..ebdb8cac 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -110,6 +110,23 @@ export class RemoteBrowser { this.startPerformanceReporting(); } + private initializeMemoryManagement(): void { + setInterval(() => { + const memoryUsage = process.memoryUsage(); + const heapUsageRatio = memoryUsage.heapUsed / MEMORY_CONFIG.maxHeapSize; + + if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold) { + logger.warn('High memory usage detected, triggering cleanup'); + this.performMemoryCleanup(); + } + + // Clear screenshot queue if it's too large + if (this.screenshotQueue.length > SCREENCAST_CONFIG.maxQueueSize) { + this.screenshotQueue = this.screenshotQueue.slice(-SCREENCAST_CONFIG.maxQueueSize); + } + }, MEMORY_CONFIG.gcInterval); + } + /** * Normalizes URLs to prevent navigation loops while maintaining consistent format */ From 9d0964727adbc773298124d3ea3c094a230f3514 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 11:47:38 +0530 Subject: [PATCH 14/32] feat: perform memory cleanup --- .../classes/RemoteBrowser.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index ebdb8cac..e084e2b3 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -127,6 +127,29 @@ export class RemoteBrowser { }, MEMORY_CONFIG.gcInterval); } + private async performMemoryCleanup(): Promise { + this.screenshotQueue = []; + this.isProcessingScreenshot = false; + + if (global.gc) { + global.gc(); + } + + // Reset CDP session if needed + if (this.client) { + try { + await this.stopScreencast(); + this.client = null; + if (this.currentPage) { + this.client = await this.currentPage.context().newCDPSession(this.currentPage); + await this.startScreencast(); + } + } catch (error) { + logger.error('Error resetting CDP session:', error); + } + } + } + /** * Normalizes URLs to prevent navigation loops while maintaining consistent format */ From d79b2fd80757a93592da9cbb3e9aba2368368bd6 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 11:47:56 +0530 Subject: [PATCH 15/32] feat: memory management config --- server/src/browser-management/classes/RemoteBrowser.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index e084e2b3..d2f83fdf 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -21,6 +21,14 @@ import { getInjectableScript } from 'idcac-playwright'; import { BackendPerformanceMonitor } from '../../../../perf/performance' chromium.use(stealthPlugin()); +// Memory management configuration +const MEMORY_CONFIG = { + gcInterval: 60000, // 1 minute + maxHeapSize: 2048 * 1024 * 1024, // 2GB + heapUsageThreshold: 0.85 // 85% +}; + + /** * This class represents a remote browser instance. From 65c30c0ef4cf4c38bf420f122f1f5253fd8ab807 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 11:54:31 +0530 Subject: [PATCH 16/32] feat: screencast config --- server/src/browser-management/classes/RemoteBrowser.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index d2f83fdf..f64c0613 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -21,13 +21,21 @@ import { getInjectableScript } from 'idcac-playwright'; import { BackendPerformanceMonitor } from '../../../../perf/performance' chromium.use(stealthPlugin()); -// Memory management configuration const MEMORY_CONFIG = { gcInterval: 60000, // 1 minute maxHeapSize: 2048 * 1024 * 1024, // 2GB heapUsageThreshold: 0.85 // 85% }; +const SCREENCAST_CONFIG = { + format: 'jpeg', + quality: 75, + maxWidth: 1280, + maxHeight: 720, + targetFPS: 30, + compressionQuality: 0.8, + maxQueueSize: 2 +}; /** From 60131e12a79f9cec5e08c96f4c16b2b78c24a3a0 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 11:54:48 +0530 Subject: [PATCH 17/32] feat: optimize screenshot --- .../classes/RemoteBrowser.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index f64c0613..53df4900 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -426,6 +426,26 @@ export class RemoteBrowser { } }; + private async optimizeScreenshot(screenshot: Buffer): Promise { + try { + return await sharp(screenshot) + .jpeg({ + quality: Math.round(SCREENCAST_CONFIG.compressionQuality * 100), + progressive: true + }) + .resize({ + width: SCREENCAST_CONFIG.maxWidth, + height: SCREENCAST_CONFIG.maxHeight, + fit: 'inside', + withoutEnlargement: true + }) + .toBuffer(); + } catch (error) { + logger.error('Screenshot optimization failed:', error); + return screenshot; + } + } + /** * Makes and emits a single screenshot to the client side. * @returns {Promise} From c766df6164ce5dcde6b3aa617d8d29c436d7fc7e Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 11:55:15 +0530 Subject: [PATCH 18/32] feat: emit screenshot using screenshot queue --- .../classes/RemoteBrowser.ts | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 53df4900..62958fe1 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -614,11 +614,39 @@ export class RemoteBrowser { * @param payload the screenshot binary data * @returns void */ - private emitScreenshot = (payload: any): void => { - this.performanceMonitor.measureEmitPerformance(() => { - const dataWithMimeType = ('data:image/jpeg;base64,').concat(payload); - this.socket.emit('screencast', dataWithMimeType); - logger.log('debug', `Screenshot emitted`); - }); - }; + private emitScreenshot = throttle(async (payload: Buffer): Promise => { + if (this.isProcessingScreenshot) { + if (this.screenshotQueue.length < SCREENCAST_CONFIG.maxQueueSize) { + this.screenshotQueue.push(payload); + } + return; + } + + this.isProcessingScreenshot = true; + + try { + await this.performanceMonitor.measureEmitPerformance(async () => { + const optimizedScreenshot = await this.optimizeScreenshot(payload); + const base64Data = optimizedScreenshot.toString('base64'); + const dataWithMimeType = `data:image/jpeg;base64,${base64Data}`; + + await new Promise((resolve) => { + this.socket.emit('screencast', dataWithMimeType, () => resolve()); + }); + }); + } catch (error) { + logger.error('Screenshot emission failed:', error); + } finally { + this.isProcessingScreenshot = false; + + // Process next screenshot in queue if any + if (this.screenshotQueue.length > 0) { + const nextScreenshot = this.screenshotQueue.shift(); + if (nextScreenshot) { + this.emitScreenshot(nextScreenshot); + } + } + } + }, 1000 / SCREENCAST_CONFIG.targetFPS); + } From 8c80d5ce6c6c47e49354910d26b8102372b2ef41 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 11:55:45 +0530 Subject: [PATCH 19/32] feat: start/stop screencast --- .../classes/RemoteBrowser.ts | 61 +++++++++++++------ 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 62958fe1..1b3156e6 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -395,7 +395,7 @@ export class RemoteBrowser { return; } this.client.on('Page.screencastFrame', ({ data: base64, sessionId }) => { - this.emitScreenshot(base64) + this.emitScreenshot(Buffer.from(base64, 'base64')) setTimeout(async () => { try { if (!this.client) { @@ -454,7 +454,7 @@ export class RemoteBrowser { try { const screenshot = await this.currentPage?.screenshot(); if (screenshot) { - this.emitScreenshot(screenshot.toString('base64')); + this.emitScreenshot(screenshot); } } catch (e) { const { message } = e as Error; @@ -586,28 +586,51 @@ export class RemoteBrowser { * Should be called only once after the browser is fully initialized. * @returns {Promise} */ - private startScreencast = async (): Promise => { + private async startScreencast(): Promise { if (!this.client) { - logger.log('warn', 'client is not initialized'); + logger.warn('Client is not initialized'); return; } - await this.client.send('Page.startScreencast', { format: 'jpeg', quality: 75 }); - logger.log('info', `Browser started with screencasting a page.`); - }; - /** - * Unsubscribes the current page from the screencast session. - * @returns {Promise} - */ - private stopScreencast = async (): Promise => { - if (!this.client) { - logger.log('error', 'client is not initialized'); - logger.log('error', 'Screencast stop failed'); - } else { - await this.client.send('Page.stopScreencast'); - logger.log('info', `Browser stopped with screencasting.`); + try { + await this.client.send('Page.startScreencast', { + format: SCREENCAST_CONFIG.format, + quality: SCREENCAST_CONFIG.quality + }); + + // Set up screencast frame handler + this.client.on('Page.screencastFrame', async ({ data, sessionId }) => { + try { + const buffer = Buffer.from(data, 'base64'); + await this.emitScreenshot(buffer); + await this.client?.send('Page.screencastFrameAck', { sessionId }); + } catch (error) { + logger.error('Screencast frame processing failed:', error); + } + }); + + logger.info('Screencast started successfully'); + } catch (error) { + logger.error('Failed to start screencast:', error); } - }; + } + + private async stopScreencast(): Promise { + if (!this.client) { + logger.error('Client is not initialized'); + return; + } + + try { + await this.client.send('Page.stopScreencast'); + this.screenshotQueue = []; + this.isProcessingScreenshot = false; + logger.info('Screencast stopped successfully'); + } catch (error) { + logger.error('Failed to stop screencast:', error); + } + } + /** * Helper for emitting the screenshot of browser's active page through websocket. From 6521eac2d7bfdadf66334755be1d74ab065ccc58 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 11:56:10 +0530 Subject: [PATCH 20/32] feat: set screencast type as jpeg | png --- server/src/browser-management/classes/RemoteBrowser.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 1b3156e6..53248aa4 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -27,7 +27,15 @@ const MEMORY_CONFIG = { heapUsageThreshold: 0.85 // 85% }; -const SCREENCAST_CONFIG = { +const SCREENCAST_CONFIG: { + format: "jpeg" | "png"; + quality: number; + maxWidth: number; + maxHeight: number; + targetFPS: number; + compressionQuality: number; + maxQueueSize: number; +} = { format: 'jpeg', quality: 75, maxWidth: 1280, From d25ffe32b8dced7de2e2c70eaf544a714acd66d8 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 19:33:52 +0530 Subject: [PATCH 21/32] feat: change screencast config --- server/src/browser-management/classes/RemoteBrowser.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 53248aa4..27465f0a 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -29,7 +29,6 @@ const MEMORY_CONFIG = { const SCREENCAST_CONFIG: { format: "jpeg" | "png"; - quality: number; maxWidth: number; maxHeight: number; targetFPS: number; @@ -37,9 +36,8 @@ const SCREENCAST_CONFIG: { maxQueueSize: number; } = { format: 'jpeg', - quality: 75, - maxWidth: 1280, - maxHeight: 720, + maxWidth: 900, + maxHeight: 400, targetFPS: 30, compressionQuality: 0.8, maxQueueSize: 2 @@ -603,7 +601,6 @@ export class RemoteBrowser { try { await this.client.send('Page.startScreencast', { format: SCREENCAST_CONFIG.format, - quality: SCREENCAST_CONFIG.quality }); // Set up screencast frame handler From 9cfffd801d036d1b2815d43f708f816d38447127 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 19:34:13 +0530 Subject: [PATCH 22/32] chore: lint --- .../classes/RemoteBrowser.ts | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 27465f0a..fb61cfb6 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -43,7 +43,6 @@ const SCREENCAST_CONFIG: { maxQueueSize: 2 }; - /** * This class represents a remote browser instance. * It is used to allow a variety of interaction with the Playwright's browser instance. @@ -112,9 +111,9 @@ export class RemoteBrowser { private startPerformanceReporting() { setInterval(() => { const report = this.performanceMonitor.getPerformanceReport(); - + console.log('Backend Performance Report:', report); - + }, 5000); } @@ -152,11 +151,11 @@ export class RemoteBrowser { private async performMemoryCleanup(): Promise { this.screenshotQueue = []; this.isProcessingScreenshot = false; - + if (global.gc) { global.gc(); } - + // Reset CDP session if needed if (this.client) { try { @@ -239,7 +238,7 @@ export class RemoteBrowser { 'Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.62 Safari/537.36', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:118.0) Gecko/20100101 Firefox/118.0', ]; - + return userAgents[Math.floor(Math.random() * userAgents.length)]; } @@ -260,7 +259,7 @@ export class RemoteBrowser { "--disable-extensions", "--no-sandbox", "--disable-dev-shm-usage", - ], + ], })); const proxyConfig = await getDecryptedProxyConfig(userId); let proxyOptions: { server: string, username?: string, password?: string } = { server: '' }; @@ -333,11 +332,11 @@ export class RemoteBrowser { this.client = await this.currentPage.context().newCDPSession(this.currentPage); await blocker.disableBlockingInPage(this.currentPage); console.log('Adblocker initialized'); - } catch (error: any) { + } catch (error: any) { console.warn('Failed to initialize adblocker, continuing without it:', error.message); // Still need to set up the CDP session even if blocker fails this.client = await this.currentPage.context().newCDPSession(this.currentPage); - } + } }; /** @@ -657,7 +656,7 @@ export class RemoteBrowser { const optimizedScreenshot = await this.optimizeScreenshot(payload); const base64Data = optimizedScreenshot.toString('base64'); const dataWithMimeType = `data:image/jpeg;base64,${base64Data}`; - + await new Promise((resolve) => { this.socket.emit('screencast', dataWithMimeType, () => resolve()); }); @@ -666,7 +665,7 @@ export class RemoteBrowser { logger.error('Screenshot emission failed:', error); } finally { this.isProcessingScreenshot = false; - + // Process next screenshot in queue if any if (this.screenshotQueue.length > 0) { const nextScreenshot = this.screenshotQueue.shift(); From d33a064dd54b5283cb45f687933083abef861c83 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 19:38:22 +0530 Subject: [PATCH 23/32] feat: set screenshot queue empty while switch off --- .../classes/RemoteBrowser.ts | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index fb61cfb6..b6c0bf5f 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -89,7 +89,7 @@ export class RemoteBrowser { maxConcurrency: 1, maxRepeats: 1, }; - + private lastEmittedUrl: string | null = null; /** @@ -420,16 +420,29 @@ export class RemoteBrowser { * If an interpretation was running it will be stopped. * @returns {Promise} */ - public switchOff = async (): Promise => { - await this.interpreter.stopInterpretation(); - if (this.browser) { - await this.stopScreencast(); - await this.browser.close(); - } else { - logger.log('error', 'Browser wasn\'t initialized'); - logger.log('error', 'Switching off the browser failed'); + public async switchOff(): Promise { + try { + await this.interpreter.stopInterpretation(); + + if (this.screencastInterval) { + clearInterval(this.screencastInterval); + } + + if (this.client) { + await this.stopScreencast(); + } + + if (this.browser) { + await this.browser.close(); + } + + this.screenshotQueue = []; + //this.performanceMonitor.reset(); + + } catch (error) { + logger.error('Error during browser shutdown:', error); } - }; + } private async optimizeScreenshot(screenshot: Buffer): Promise { try { From 4e7ce637e68d9a3f0e56d8bb880ed7628aefa559 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 19:38:37 +0530 Subject: [PATCH 24/32] chore: lint --- .../src/browser-management/classes/RemoteBrowser.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index b6c0bf5f..4419f0b3 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -89,7 +89,7 @@ export class RemoteBrowser { maxConcurrency: 1, maxRepeats: 1, }; - + private lastEmittedUrl: string | null = null; /** @@ -423,22 +423,22 @@ export class RemoteBrowser { public async switchOff(): Promise { try { await this.interpreter.stopInterpretation(); - + if (this.screencastInterval) { clearInterval(this.screencastInterval); } - + if (this.client) { await this.stopScreencast(); } - + if (this.browser) { await this.browser.close(); } - + this.screenshotQueue = []; //this.performanceMonitor.reset(); - + } catch (error) { logger.error('Error during browser shutdown:', error); } From 82e24f9483ba86eca59094c84dd82fff511a2de9 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 19:39:27 +0530 Subject: [PATCH 25/32] feat: disable browser performance monitoring --- server/src/browser-management/classes/RemoteBrowser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 4419f0b3..2e08a506 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -128,7 +128,7 @@ export class RemoteBrowser { this.interpreter = new WorkflowInterpreter(socket); this.generator = new WorkflowGenerator(socket); this.performanceMonitor = new BackendPerformanceMonitor(); - this.startPerformanceReporting(); + //this.startPerformanceReporting(); } private initializeMemoryManagement(): void { From 9b920a111dfd50773683be7bd005582499995383 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 19:41:10 +0530 Subject: [PATCH 26/32] feat: temporarily disable browser performance monitoring --- .../classes/RemoteBrowser.ts | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 2e08a506..a45fc93b 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -102,20 +102,20 @@ export class RemoteBrowser { */ public interpreter: WorkflowInterpreter; - private performanceMonitor: BackendPerformanceMonitor; + //private performanceMonitor: BackendPerformanceMonitor; private screenshotQueue: Buffer[] = []; private isProcessingScreenshot = false; private screencastInterval: NodeJS.Timeout | null = null; - private startPerformanceReporting() { - setInterval(() => { - const report = this.performanceMonitor.getPerformanceReport(); + // private startPerformanceReporting() { + // setInterval(() => { + // const report = this.performanceMonitor.getPerformanceReport(); - console.log('Backend Performance Report:', report); + // console.log('Backend Performance Report:', report); - }, 5000); - } + // }, 5000); + // } /** * Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and @@ -127,7 +127,7 @@ export class RemoteBrowser { this.socket = socket; this.interpreter = new WorkflowInterpreter(socket); this.generator = new WorkflowGenerator(socket); - this.performanceMonitor = new BackendPerformanceMonitor(); + //this.performanceMonitor = new BackendPerformanceMonitor(); //this.startPerformanceReporting(); } @@ -665,7 +665,6 @@ export class RemoteBrowser { this.isProcessingScreenshot = true; try { - await this.performanceMonitor.measureEmitPerformance(async () => { const optimizedScreenshot = await this.optimizeScreenshot(payload); const base64Data = optimizedScreenshot.toString('base64'); const dataWithMimeType = `data:image/jpeg;base64,${base64Data}`; @@ -673,7 +672,6 @@ export class RemoteBrowser { await new Promise((resolve) => { this.socket.emit('screencast', dataWithMimeType, () => resolve()); }); - }); } catch (error) { logger.error('Screenshot emission failed:', error); } finally { From 5ddfc91f8e31d5cdb1defc336246710e9682ccce Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 19:41:31 +0530 Subject: [PATCH 27/32] chore: remove unused import --- server/src/browser-management/classes/RemoteBrowser.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index a45fc93b..36df52cf 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -18,7 +18,6 @@ import { WorkflowGenerator } from "../../workflow-management/classes/Generator"; import { WorkflowInterpreter } from "../../workflow-management/classes/Interpreter"; import { getDecryptedProxyConfig } from '../../routes/proxy'; import { getInjectableScript } from 'idcac-playwright'; -import { BackendPerformanceMonitor } from '../../../../perf/performance' chromium.use(stealthPlugin()); const MEMORY_CONFIG = { From 0497653ca96b6721bf20bd682ca22c3075038a83 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 19:44:04 +0530 Subject: [PATCH 28/32] feat: disable canvas events performance monitoring --- src/components/atoms/canvas.tsx | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/components/atoms/canvas.tsx b/src/components/atoms/canvas.tsx index 96933030..7c216733 100644 --- a/src/components/atoms/canvas.tsx +++ b/src/components/atoms/canvas.tsx @@ -2,7 +2,6 @@ import React, { useCallback, useEffect, useRef, useMemo, Suspense } from 'react' import { useSocketStore } from '../../context/socket'; import { useGlobalInfoStore } from "../../context/globalInfo"; import { useActionContext } from '../../context/browserActions'; -import { FrontendPerformanceMonitor } from '../../../perf/performance'; const DatePicker = React.lazy(() => import('./DatePicker')); const Dropdown = React.lazy(() => import('./Dropdown')); const TimePicker = React.lazy(() => import('./TimePicker')); @@ -162,7 +161,7 @@ const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { const scheduler = useRef(new RAFScheduler()); const debouncer = useRef(new EventDebouncer(scheduler.current)); const measurementCache = useRef(new MeasurementCache(50)); - const performanceMonitor = useRef(new FrontendPerformanceMonitor()); + //const performanceMonitor = useRef(new FrontendPerformanceMonitor()); const refs = useRef({ getText, @@ -204,7 +203,7 @@ const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { const handleMouseEvent = useCallback((event: MouseEvent) => { if (!socket || !canvasRef.current) return; - performanceMonitor.current.measureEventLatency(event); + //performanceMonitor.current.measureEventLatency(event); const coordinates = getEventCoordinates(event); switch (event.type) { @@ -297,13 +296,13 @@ const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { }, [handleMouseEvent, handleKeyboardEvent, onCreateRef]); // Performance monitoring - useEffect(() => { - const intervalId = setInterval(() => { - console.log('Performance Report:', performanceMonitor.current.getPerformanceReport()); - }, 20000); + // useEffect(() => { + // const intervalId = setInterval(() => { + // console.log('Performance Report:', performanceMonitor.current.getPerformanceReport()); + // }, 20000); - return () => clearInterval(intervalId); - }, []); + // return () => clearInterval(intervalId); + // }, []); useEffect(() => { if (!socket) return; From de2ddfd2b8d46555e2191a940578076872985a5d Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 20:18:27 +0530 Subject: [PATCH 29/32] fix: missing dropdown --- src/components/atoms/canvas.tsx | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/components/atoms/canvas.tsx b/src/components/atoms/canvas.tsx index 7c216733..8fd4f791 100644 --- a/src/components/atoms/canvas.tsx +++ b/src/components/atoms/canvas.tsx @@ -155,7 +155,7 @@ export interface Coordinates { const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { const canvasRef = useRef(null); const { socket } = useSocketStore(); - const { setLastAction } = useGlobalInfoStore(); + const { setLastAction, lastAction } = useGlobalInfoStore(); const { getText, getList } = useActionContext(); const scheduler = useRef(new RAFScheduler()); @@ -345,6 +345,17 @@ const Canvas = React.memo(({ width, height, onCreateRef }: CanvasProps) => { })} /> )} + {state.dropdownInfo && ( + dispatch({ + type: 'BATCH_UPDATE', + payload: { dropdownInfo: null } + })} + /> + )} {state.timePickerInfo && ( Date: Mon, 6 Jan 2025 21:10:39 +0530 Subject: [PATCH 30/32] feat: remove throttle wrapper from emit screenshot --- .../classes/RemoteBrowser.ts | 53 +++++++------------ 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 36df52cf..a843fe45 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -18,6 +18,7 @@ import { WorkflowGenerator } from "../../workflow-management/classes/Generator"; import { WorkflowInterpreter } from "../../workflow-management/classes/Interpreter"; import { getDecryptedProxyConfig } from '../../routes/proxy'; import { getInjectableScript } from 'idcac-playwright'; + chromium.use(stealthPlugin()); const MEMORY_CONFIG = { @@ -88,7 +89,7 @@ export class RemoteBrowser { maxConcurrency: 1, maxRepeats: 1, }; - + private lastEmittedUrl: string | null = null; /** @@ -101,20 +102,10 @@ export class RemoteBrowser { */ public interpreter: WorkflowInterpreter; - //private performanceMonitor: BackendPerformanceMonitor; private screenshotQueue: Buffer[] = []; private isProcessingScreenshot = false; - private screencastInterval: NodeJS.Timeout | null = null; - - // private startPerformanceReporting() { - // setInterval(() => { - // const report = this.performanceMonitor.getPerformanceReport(); - - // console.log('Backend Performance Report:', report); - - // }, 5000); - // } + private screencastInterval: NodeJS.Timeout | null = null /** * Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and @@ -126,8 +117,6 @@ export class RemoteBrowser { this.socket = socket; this.interpreter = new WorkflowInterpreter(socket); this.generator = new WorkflowGenerator(socket); - //this.performanceMonitor = new BackendPerformanceMonitor(); - //this.startPerformanceReporting(); } private initializeMemoryManagement(): void { @@ -422,22 +411,22 @@ export class RemoteBrowser { public async switchOff(): Promise { try { await this.interpreter.stopInterpretation(); - + if (this.screencastInterval) { clearInterval(this.screencastInterval); } - + if (this.client) { await this.stopScreencast(); } - + if (this.browser) { await this.browser.close(); } - + this.screenshotQueue = []; //this.performanceMonitor.reset(); - + } catch (error) { logger.error('Error during browser shutdown:', error); } @@ -653,37 +642,35 @@ export class RemoteBrowser { * @param payload the screenshot binary data * @returns void */ - private emitScreenshot = throttle(async (payload: Buffer): Promise => { + private emitScreenshot = async (payload: Buffer): Promise => { if (this.isProcessingScreenshot) { if (this.screenshotQueue.length < SCREENCAST_CONFIG.maxQueueSize) { this.screenshotQueue.push(payload); } return; } - + this.isProcessingScreenshot = true; - + try { - const optimizedScreenshot = await this.optimizeScreenshot(payload); - const base64Data = optimizedScreenshot.toString('base64'); - const dataWithMimeType = `data:image/jpeg;base64,${base64Data}`; - - await new Promise((resolve) => { - this.socket.emit('screencast', dataWithMimeType, () => resolve()); - }); + const optimizedScreenshot = await this.optimizeScreenshot(payload); + const base64Data = optimizedScreenshot.toString('base64'); + const dataWithMimeType = `data:image/jpeg;base64,${base64Data}`; + + this.socket.emit('screencast', dataWithMimeType); + logger.debug('Screenshot emitted'); } catch (error) { logger.error('Screenshot emission failed:', error); } finally { this.isProcessingScreenshot = false; - - // Process next screenshot in queue if any + if (this.screenshotQueue.length > 0) { const nextScreenshot = this.screenshotQueue.shift(); if (nextScreenshot) { - this.emitScreenshot(nextScreenshot); + setTimeout(() => this.emitScreenshot(nextScreenshot), 1000 / SCREENCAST_CONFIG.targetFPS); } } } - }, 1000 / SCREENCAST_CONFIG.targetFPS); + }; } From 434409ead3e58754ca3c08c6e7d2efd9cd153501 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 21:11:11 +0530 Subject: [PATCH 31/32] chore: lint --- .../classes/RemoteBrowser.ts | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index a843fe45..8ff4f601 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -89,7 +89,7 @@ export class RemoteBrowser { maxConcurrency: 1, maxRepeats: 1, }; - + private lastEmittedUrl: string | null = null; /** @@ -411,22 +411,22 @@ export class RemoteBrowser { public async switchOff(): Promise { try { await this.interpreter.stopInterpretation(); - + if (this.screencastInterval) { clearInterval(this.screencastInterval); } - + if (this.client) { await this.stopScreencast(); } - + if (this.browser) { await this.browser.close(); } - + this.screenshotQueue = []; //this.performanceMonitor.reset(); - + } catch (error) { logger.error('Error during browser shutdown:', error); } @@ -649,21 +649,21 @@ export class RemoteBrowser { } return; } - + this.isProcessingScreenshot = true; - + try { const optimizedScreenshot = await this.optimizeScreenshot(payload); const base64Data = optimizedScreenshot.toString('base64'); const dataWithMimeType = `data:image/jpeg;base64,${base64Data}`; - + this.socket.emit('screencast', dataWithMimeType); logger.debug('Screenshot emitted'); } catch (error) { logger.error('Screenshot emission failed:', error); } finally { this.isProcessingScreenshot = false; - + if (this.screenshotQueue.length > 0) { const nextScreenshot = this.screenshotQueue.shift(); if (nextScreenshot) { From 2d79591a7682fd0e2d967a07ae672bba851d5432 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Mon, 6 Jan 2025 21:24:27 +0530 Subject: [PATCH 32/32] chore(deps): use maxun-core 0.0.7 --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index f8975ea6..b8ca18b9 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "lodash": "^4.17.21", "loglevel": "^1.8.0", "loglevel-plugin-remote": "^0.6.8", + "maxun-core": "^0.0.7", "minio": "^8.0.1", "moment-timezone": "^0.5.45", "node-cron": "^3.0.3",