From a5ea68655b1a4eb9e9ba4f417d1369613454ed95 Mon Sep 17 00:00:00 2001 From: Rohit Date: Thu, 12 Jun 2025 14:07:18 +0530 Subject: [PATCH 1/4] feat: throttle highlighter, request animation frame --- src/components/browser/BrowserWindow.tsx | 31 +++++++++++++++++------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/components/browser/BrowserWindow.tsx b/src/components/browser/BrowserWindow.tsx index 8ce6e0f9..a0065e7b 100644 --- a/src/components/browser/BrowserWindow.tsx +++ b/src/components/browser/BrowserWindow.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useContext, useEffect, useState } from 'react'; +import React, { useCallback, useContext, useEffect, useRef, useState } from 'react'; import { useSocketStore } from '../../context/socket'; import { Button } from '@mui/material'; import Canvas from "../recorder/canvas"; @@ -84,6 +84,8 @@ export const BrowserWindow = () => { const [fields, setFields] = useState>({}); const [paginationSelector, setPaginationSelector] = useState(''); + const highlighterUpdateRef = useRef(0); + const { socket } = useSocketStore(); const { notify, currentTextActionId, currentListActionId } = useGlobalInfoStore(); const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext(); @@ -103,12 +105,12 @@ export const BrowserWindow = () => { useEffect(() => { if (listSelector) { - window.sessionStorage.setItem('recordingListSelector', listSelector); + sessionStorage.setItem('recordingListSelector', listSelector); } }, [listSelector]); useEffect(() => { - const storedListSelector = window.sessionStorage.getItem('recordingListSelector'); + const storedListSelector = sessionStorage.getItem('recordingListSelector'); // Only restore state if it exists in sessionStorage if (storedListSelector && !listSelector) { @@ -172,6 +174,12 @@ export const BrowserWindow = () => { }, [screenShot, canvasRef, socket, screencastHandler]); const highlighterHandler = useCallback((data: { rect: DOMRect, selector: string, elementInfo: ElementInfo | null, childSelectors?: string[] }) => { + const now = performance.now(); + if (now - highlighterUpdateRef.current < 16) { + return; + } + highlighterUpdateRef.current = now; + // Map the incoming DOMRect from browser coordinates to canvas coordinates const mappedRect = new DOMRect( data.rect.x, @@ -573,17 +581,22 @@ export const BrowserWindow = () => { }; const drawImage = (image: string, canvas: HTMLCanvasElement): void => { - const ctx = canvas.getContext('2d'); + if (!ctx) return; const img = new Image(); - - img.src = image; img.onload = () => { - URL.revokeObjectURL(img.src); - ctx?.drawImage(img, 0, 0, canvas.width, canvas.height); + requestAnimationFrame(() => { + ctx.drawImage(img, 0, 0, canvas.width, canvas.height); + }); + if (image.startsWith('blob:')) { + URL.revokeObjectURL(image); + } }; - + img.onerror = () => { + console.warn('Failed to load image'); + }; + img.src = image; }; const modalStyle = { From 50575d90e7deb61dad2d9f48dfcf608e7ba7dd65 Mon Sep 17 00:00:00 2001 From: Rohit Date: Thu, 12 Jun 2025 14:14:55 +0530 Subject: [PATCH 2/4] feat: optimize canvas perf --- src/components/recorder/canvas.tsx | 252 ++++++++++++++++------------- 1 file changed, 138 insertions(+), 114 deletions(-) diff --git a/src/components/recorder/canvas.tsx b/src/components/recorder/canvas.tsx index 52d29b00..c8c7502c 100644 --- a/src/components/recorder/canvas.tsx +++ b/src/components/recorder/canvas.tsx @@ -29,13 +29,17 @@ export interface Coordinates { const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { const canvasRef = useRef(null); + const contextRef = useRef(null); + const imageDataRef = useRef(null); + const animationFrameRef = useRef(null); + const { socket } = useSocketStore(); const { setLastAction, lastAction } = useGlobalInfoStore(); const { getText, getList } = useActionContext(); const getTextRef = useRef(getText); const getListRef = useRef(getList); - const MOUSE_MOVE_THROTTLE = 16; // ~60fps + const MOUSE_MOVE_THROTTLE = 8; const lastMouseMoveTime = useRef(0); const [datePickerInfo, setDatePickerInfo] = React.useState<{ @@ -72,6 +76,22 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { const lastMousePosition = useRef({ x: 0, y: 0 }); + useEffect(() => { + if (canvasRef.current && !contextRef.current) { + const ctx = canvasRef.current.getContext('2d', { + alpha: false, + desynchronized: true, + willReadFrequently: false + }); + + if (ctx) { + contextRef.current = ctx; + + imageDataRef.current = ctx.createImageData(width, height); + } + } + }, [width, height]); + useEffect(() => { getTextRef.current = getText; getListRef.current = getList; @@ -79,115 +99,100 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { useEffect(() => { if (socket) { - socket.on('showDatePicker', (info: { coordinates: Coordinates, selector: string }) => { + const handleDatePicker = (info: { coordinates: Coordinates, selector: string }) => { const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates); - setDatePickerInfo({ - ...info, - coordinates: canvasCoords - }); - }); + setDatePickerInfo({ ...info, coordinates: canvasCoords }); + }; - socket.on('showDropdown', (info: { + const handleDropdown = (info: { coordinates: Coordinates, selector: string, - options: Array<{ - value: string; - text: string; - disabled: boolean; - selected: boolean; - }>; + options: Array<{ value: string; text: string; disabled: boolean; selected: boolean; }>; }) => { const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates); - setDropdownInfo({ - ...info, - coordinates: canvasCoords - }); - }); + setDropdownInfo({ ...info, coordinates: canvasCoords }); + }; - socket.on('showTimePicker', (info: { coordinates: Coordinates, selector: string }) => { + const handleTimePicker = (info: { coordinates: Coordinates, selector: string }) => { const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates); - setTimePickerInfo({ - ...info, - coordinates: canvasCoords - }); - }); + setTimePickerInfo({ ...info, coordinates: canvasCoords }); + }; - socket.on('showDateTimePicker', (info: { coordinates: Coordinates, selector: string }) => { + const handleDateTimePicker = (info: { coordinates: Coordinates, selector: string }) => { const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates); - setDateTimeLocalInfo({ - ...info, - coordinates: canvasCoords - }); - }); + setDateTimeLocalInfo({ ...info, coordinates: canvasCoords }); + }; + + socket.on('showDatePicker', handleDatePicker); + socket.on('showDropdown', handleDropdown); + socket.on('showTimePicker', handleTimePicker); + socket.on('showDateTimePicker', handleDateTimePicker); return () => { - socket.off('showDatePicker'); - socket.off('showDropdown'); - socket.off('showTimePicker'); - socket.off('showDateTimePicker'); + socket.off('showDatePicker', handleDatePicker); + socket.off('showDropdown', handleDropdown); + socket.off('showTimePicker', handleTimePicker); + socket.off('showDateTimePicker', handleDateTimePicker); }; } }, [socket]); const onMouseEvent = useCallback((event: MouseEvent) => { - if (socket && canvasRef.current) { - const rect = canvasRef.current.getBoundingClientRect(); - const clickCoordinates = { - x: event.clientX - rect.left, // Use relative x coordinate - y: event.clientY - rect.top, // Use relative y coordinate - }; + if (!socket || !canvasRef.current) return; - const browserCoordinates = coordinateMapper.mapCanvasToBrowser(clickCoordinates); + const rect = canvasRef.current.getBoundingClientRect(); + const clickCoordinates = { + x: event.clientX - rect.left, + y: event.clientY - rect.top, + }; - switch (event.type) { - case 'mousedown': - if (getTextRef.current === true) { - console.log('Capturing Text...'); - } else if (getListRef.current === true) { - console.log('Capturing List...'); - } else { - socket.emit('input:mousedown', browserCoordinates); - } - notifyLastAction('click'); - break; - case 'mousemove': { - const now = performance.now(); - if (now - lastMouseMoveTime.current < MOUSE_MOVE_THROTTLE) { - return; - } - lastMouseMoveTime.current = now; - - const dx = Math.abs(lastMousePosition.current.x - clickCoordinates.x); - const dy = Math.abs(lastMousePosition.current.y - clickCoordinates.y); - if (dx > 1 || dy > 1) { - lastMousePosition.current = { - x: clickCoordinates.x, - y: clickCoordinates.y, - }; - socket.emit('input:mousemove', browserCoordinates); - notifyLastAction('move'); - } - break; + const browserCoordinates = coordinateMapper.mapCanvasToBrowser(clickCoordinates); + + switch (event.type) { + case 'mousedown': + if (getTextRef.current === true) { + console.log('Capturing Text...'); + } else if (getListRef.current === true) { + console.log('Capturing List...'); + } else { + socket.emit('input:mousedown', browserCoordinates); } + notifyLastAction('click'); + break; - // Optimize wheel events - case 'wheel': { - const wheelEvent = event as WheelEvent; - const deltaX = Math.round(wheelEvent.deltaX / 10) * 10; - const deltaY = Math.round(wheelEvent.deltaY / 10) * 10; - - if (Math.abs(deltaX) > 5 || Math.abs(deltaY) > 5) { - socket.emit('input:wheel', { deltaX, deltaY }); - notifyLastAction('scroll'); - } - break; + case 'mousemove': { + const now = performance.now(); + if (now - lastMouseMoveTime.current < MOUSE_MOVE_THROTTLE) { + return; } - default: - console.log('Default mouseEvent registered'); - return; + lastMouseMoveTime.current = now; + + const dx = Math.abs(lastMousePosition.current.x - clickCoordinates.x); + const dy = Math.abs(lastMousePosition.current.y - clickCoordinates.y); + + if (dx > 0.5 || dy > 0.5) { + lastMousePosition.current = clickCoordinates; + socket.emit('input:mousemove', browserCoordinates); + notifyLastAction('move'); + } + break; } + + case 'wheel': { + const wheelEvent = event as WheelEvent; + const deltaX = Math.round(wheelEvent.deltaX / 5) * 5; + const deltaY = Math.round(wheelEvent.deltaY / 5) * 5; + + if (Math.abs(deltaX) > 2 || Math.abs(deltaY) > 2) { + socket.emit('input:wheel', { deltaX, deltaY }); + notifyLastAction('scroll'); + } + break; + } + default: + return; } - }, [socket]); + }, [socket, notifyLastAction]); const onKeyboardEvent = useCallback((event: KeyboardEvent) => { if (socket) { @@ -206,47 +211,66 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { return; } } - }, [socket]); + }, [socket, notifyLastAction]); useEffect(() => { - if (canvasRef.current) { - onCreateRef(canvasRef); - canvasRef.current.addEventListener('mousedown', onMouseEvent); - canvasRef.current.addEventListener('mousemove', onMouseEvent); - canvasRef.current.addEventListener('wheel', onMouseEvent, { passive: true }); - canvasRef.current.addEventListener('keydown', onKeyboardEvent); - canvasRef.current.addEventListener('keyup', onKeyboardEvent); + const canvas = canvasRef.current; + if (!canvas) return; - return () => { - if (canvasRef.current) { - canvasRef.current.removeEventListener('mousedown', onMouseEvent); - canvasRef.current.removeEventListener('mousemove', onMouseEvent); - canvasRef.current.removeEventListener('wheel', onMouseEvent); - canvasRef.current.removeEventListener('keydown', onKeyboardEvent); - canvasRef.current.removeEventListener('keyup', onKeyboardEvent); - } + onCreateRef(canvasRef); - }; - } else { - console.log('Canvas not initialized'); - } + const options = { passive: true }; + + canvas.addEventListener('mousedown', onMouseEvent, options); + canvas.addEventListener('mousemove', onMouseEvent, options); + canvas.addEventListener('wheel', onMouseEvent, options); + canvas.addEventListener('keydown', onKeyboardEvent); + canvas.addEventListener('keyup', onKeyboardEvent); - }, [onMouseEvent]); + return () => { + canvas.removeEventListener('mousedown', onMouseEvent); + canvas.removeEventListener('mousemove', onMouseEvent); + canvas.removeEventListener('wheel', onMouseEvent); + canvas.removeEventListener('keydown', onKeyboardEvent); + canvas.removeEventListener('keyup', onKeyboardEvent); + }; + }, [onMouseEvent, onKeyboardEvent, onCreateRef]); + + useEffect(() => { + return () => { + if (animationFrameRef.current) { + cancelAnimationFrame(animationFrameRef.current); + } + }; + }, []); + + const containerStyle = React.useMemo(() => ({ + borderRadius: '0px 0px 5px 5px', + overflow: 'hidden', + backgroundColor: 'white', + contain: 'layout style paint', + isolation: 'isolate' as React.CSSProperties['isolation'] + }), []); + + const canvasStyle = React.useMemo(() => ({ + display: 'block', + imageRendering: 'crisp-edges' as const, + willChange: 'contents', + backfaceVisibility: 'hidden' as const, + transform: 'translateZ(0)', + maxWidth: '100%', + maxHeight: '100%' + }), []); return ( -
+
{datePickerInfo && ( Date: Thu, 12 Jun 2025 14:15:57 +0530 Subject: [PATCH 3/4] feat: memoize highlighter rect calc --- src/components/recorder/Highlighter.tsx | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/components/recorder/Highlighter.tsx b/src/components/recorder/Highlighter.tsx index a81c9b37..fd4763e7 100644 --- a/src/components/recorder/Highlighter.tsx +++ b/src/components/recorder/Highlighter.tsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useMemo } from 'react'; import styled from "styled-components"; import { coordinateMapper } from '../../helpers/coordinateMapper'; @@ -14,16 +14,15 @@ const HighlighterComponent = ({ unmodifiedRect, displayedSelector = '', width, h if (!unmodifiedRect) { return null; } else { - const mappedRect = coordinateMapper.mapBrowserRectToCanvas(unmodifiedRect); - - const rect = { - top: mappedRect.top + canvasRect.top + window.scrollY, - left: mappedRect.left + canvasRect.left + window.scrollX, - right: mappedRect.right + canvasRect.left, - bottom: mappedRect.bottom + canvasRect.top, - width: mappedRect.width, - height: mappedRect.height, - }; + const rect = useMemo(() => { + const mappedRect = coordinateMapper.mapBrowserRectToCanvas(unmodifiedRect); + return { + top: mappedRect.top + canvasRect.top + window.scrollY, + left: mappedRect.left + canvasRect.left + window.scrollX, + width: mappedRect.width, + height: mappedRect.height, + }; + }, [unmodifiedRect, canvasRect.top, canvasRect.left]); return ( From 7633c4730d4121cfb8109e7370fe430d56f7e6c7 Mon Sep 17 00:00:00 2001 From: Rohit Date: Thu, 12 Jun 2025 14:30:43 +0530 Subject: [PATCH 4/4] feat: optimize remote browser perf --- .../classes/RemoteBrowser.ts | 69 +++++++++++++------ 1 file changed, 49 insertions(+), 20 deletions(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index ddfa9ab5..d04dcefe 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -25,6 +25,13 @@ const MEMORY_CONFIG = { heapUsageThreshold: 0.7 // 70% (reduced threshold to react earlier) }; +const DEFAULT_VIEWPORT = { + width: 1280, + height: 720, + deviceScaleFactor: 1, + mobile: false +}; + const SCREENCAST_CONFIG: { format: "jpeg" | "png"; maxWidth: number; @@ -32,13 +39,17 @@ const SCREENCAST_CONFIG: { targetFPS: number; compressionQuality: number; maxQueueSize: number; + skipFrameThreshold: number, + enableAdaptiveQuality: boolean, } = { - format: 'png', - maxWidth: 1280, - maxHeight: 720, - targetFPS: 15, - compressionQuality: 0.95, - maxQueueSize: 1 + format: 'jpeg', + maxWidth: DEFAULT_VIEWPORT.width, + maxHeight: DEFAULT_VIEWPORT.height, + targetFPS: 30, + compressionQuality: 0.8, + maxQueueSize: 2, + skipFrameThreshold: 100, + enableAdaptiveQuality: true, }; /** @@ -125,6 +136,18 @@ export class RemoteBrowser { this.generator = new WorkflowGenerator(socket, poolId); } + private cleanupMemory(): void { + if (this.screenshotQueue.length > 10) { + this.screenshotQueue = this.screenshotQueue.slice(-3); // Keep only last 3 + } + } + + private setupMemoryCleanup(): void { + setInterval(() => { + this.cleanupMemory(); + }, 30000); // Every 30 seconds + } + private initializeMemoryManagement(): void { setInterval(() => { const memoryUsage = process.memoryUsage(); @@ -412,6 +435,7 @@ export class RemoteBrowser { } } + this.setupMemoryCleanup(); // this.initializeMemoryManagement(); }; @@ -1399,7 +1423,7 @@ export class RemoteBrowser { */ private emitScreenshot = async (payload: Buffer, viewportSize?: { width: number, height: number }): Promise => { if (this.screenshotQueue.length > SCREENCAST_CONFIG.maxQueueSize) { - this.screenshotQueue = this.screenshotQueue.slice(-SCREENCAST_CONFIG.maxQueueSize); + this.screenshotQueue = this.screenshotQueue.slice(-1); } if (this.isProcessingScreenshot) { @@ -1414,7 +1438,7 @@ export class RemoteBrowser { try { const optimizationPromise = this.optimizeScreenshot(payload); const timeoutPromise = new Promise((resolve) => { - setTimeout(() => resolve(payload), 150); + setTimeout(() => resolve(payload), 100); }); const optimizedScreenshot = await Promise.race([optimizationPromise, timeoutPromise]); @@ -1423,10 +1447,12 @@ export class RemoteBrowser { payload = null as any; - this.socket.emit('screencast', { + setImmediate(async () => { + this.socket.emit('screencast', { image: dataWithMimeType, userId: this.userId, viewport: viewportSize || await this.currentPage?.viewportSize() || null + }); }); } catch (error) { logger.error('Screenshot emission failed:', error); @@ -1434,24 +1460,27 @@ export class RemoteBrowser { const base64Data = payload.toString('base64'); const dataWithMimeType = `data:image/png;base64,${base64Data}`; - this.socket.emit('screencast', { - image: dataWithMimeType, - userId: this.userId, - viewport: viewportSize || await this.currentPage?.viewportSize() || null + setImmediate(async () => { + this.socket.emit('screencast', { + image: dataWithMimeType, + userId: this.userId, + viewport: viewportSize || await this.currentPage?.viewportSize() || null + }); }); } catch (e) { logger.error('Fallback screenshot emission also failed:', e); } } finally { this.isProcessingScreenshot = false; - + if (this.screenshotQueue.length > 0) { - const nextScreenshot = this.screenshotQueue.shift(); - if (nextScreenshot) { - setTimeout(() => { - this.emitScreenshot(nextScreenshot); - }, 1000 / SCREENCAST_CONFIG.targetFPS); - } + const nextScreenshot = this.screenshotQueue.shift(); + if (nextScreenshot) { + const delay = this.screenshotQueue.length > 0 ? 16 : 33; + setTimeout(() => { + this.emitScreenshot(nextScreenshot); + }, delay); + } } } };