Merge pull request #640 from getmaxun/browser-perf

feat: browser performance optimization

server/src/browser-management/classes/RemoteBrowser.ts

@@ -25,6 +25,13 @@ const MEMORY_CONFIG = {
     heapUsageThreshold: 0.7 // 70% (reduced threshold to react earlier)
 };
 
+const DEFAULT_VIEWPORT = {
+  width: 1280,
+  height: 720, 
+  deviceScaleFactor: 1,
+  mobile: false
+};
+
 const SCREENCAST_CONFIG: {
     format: "jpeg" | "png";
     maxWidth: number;
@@ -32,13 +39,17 @@ const SCREENCAST_CONFIG: {
     targetFPS: number;
     compressionQuality: number;
     maxQueueSize: number;
+    skipFrameThreshold: number, 
+    enableAdaptiveQuality: boolean, 
 } = {
-    format: 'png', 
+    format: 'jpeg', 
-    maxWidth: 1280,
+    maxWidth: DEFAULT_VIEWPORT.width,
-    maxHeight: 720,
+    maxHeight: DEFAULT_VIEWPORT.height,
-    targetFPS: 15, 
+    targetFPS: 30, 
-    compressionQuality: 0.95, 
+    compressionQuality: 0.8, 
-    maxQueueSize: 1 
+    maxQueueSize: 2,
+    skipFrameThreshold: 100, 
+    enableAdaptiveQuality: true,  
 };
 
 /**
@@ -125,6 +136,18 @@ export class RemoteBrowser {
         this.generator = new WorkflowGenerator(socket, poolId);
     }
 
+    private cleanupMemory(): void {
+      if (this.screenshotQueue.length > 10) {
+          this.screenshotQueue = this.screenshotQueue.slice(-3); // Keep only last 3
+      }
+    }
+
+    private setupMemoryCleanup(): void {
+      setInterval(() => {
+          this.cleanupMemory();
+      }, 30000); // Every 30 seconds
+    }
+
     private initializeMemoryManagement(): void {
         setInterval(() => {
             const memoryUsage = process.memoryUsage();
@@ -412,6 +435,7 @@ export class RemoteBrowser {
             }
         }
 
+        this.setupMemoryCleanup();
         // this.initializeMemoryManagement();
     };
 
@@ -1399,7 +1423,7 @@ export class RemoteBrowser {
      */
     private emitScreenshot = async (payload: Buffer, viewportSize?: { width: number, height: number }): Promise<void> => {
         if (this.screenshotQueue.length > SCREENCAST_CONFIG.maxQueueSize) {
-            this.screenshotQueue = this.screenshotQueue.slice(-SCREENCAST_CONFIG.maxQueueSize);
+          this.screenshotQueue = this.screenshotQueue.slice(-1);
         }
         
         if (this.isProcessingScreenshot) {
@@ -1414,7 +1438,7 @@ export class RemoteBrowser {
         try {
             const optimizationPromise = this.optimizeScreenshot(payload);
             const timeoutPromise = new Promise<Buffer>((resolve) => {
-                setTimeout(() => resolve(payload), 150);
+                setTimeout(() => resolve(payload), 100);
             });
             
             const optimizedScreenshot = await Promise.race([optimizationPromise, timeoutPromise]);
@@ -1423,10 +1447,12 @@ export class RemoteBrowser {
             
             payload = null as any;
             
-            this.socket.emit('screencast', {
+            setImmediate(async () => {
+              this.socket.emit('screencast', {
                 image: dataWithMimeType,
                 userId: this.userId,
                 viewport: viewportSize || await this.currentPage?.viewportSize() || null
+              });
             });
         } catch (error) {
             logger.error('Screenshot emission failed:', error);
@@ -1434,24 +1460,27 @@ export class RemoteBrowser {
                 const base64Data = payload.toString('base64');
                 const dataWithMimeType = `data:image/png;base64,${base64Data}`;
                 
-                this.socket.emit('screencast', {
+                setImmediate(async () => {
-                    image: dataWithMimeType,
+                  this.socket.emit('screencast', {
-                    userId: this.userId,
+                      image: dataWithMimeType,
-                    viewport: viewportSize || await this.currentPage?.viewportSize() || null
+                      userId: this.userId,
+                      viewport: viewportSize || await this.currentPage?.viewportSize() || null
+                  });
                 });
             } catch (e) {
                 logger.error('Fallback screenshot emission also failed:', e);
             }
         } finally {
             this.isProcessingScreenshot = false;
-            
+        
             if (this.screenshotQueue.length > 0) {
-                const nextScreenshot = this.screenshotQueue.shift();  
+              const nextScreenshot = this.screenshotQueue.shift();
-                if (nextScreenshot) {
+              if (nextScreenshot) {
-                    setTimeout(() => {
+                const delay = this.screenshotQueue.length > 0 ? 16 : 33;
-                        this.emitScreenshot(nextScreenshot);
+                setTimeout(() => {
-                    }, 1000 / SCREENCAST_CONFIG.targetFPS);
+                  this.emitScreenshot(nextScreenshot);
-                }
+                }, delay);
+              }
             }
         }
     };

src/components/browser/BrowserWindow.tsx

@@ -1,4 +1,4 @@
-import React, { useCallback, useContext, useEffect, useState } from 'react';
+import React, { useCallback, useContext, useEffect, useRef, useState } from 'react';
 import { useSocketStore } from '../../context/socket';
 import { Button } from '@mui/material';
 import Canvas from "../recorder/canvas";
@@ -84,6 +84,8 @@ export const BrowserWindow = () => {
     const [fields, setFields] = useState<Record<string, TextStep>>({});
     const [paginationSelector, setPaginationSelector] = useState<string>('');
 
+    const highlighterUpdateRef = useRef<number>(0);
+
     const { socket } = useSocketStore();
     const { notify, currentTextActionId, currentListActionId } = useGlobalInfoStore();
     const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext();
@@ -103,12 +105,12 @@ export const BrowserWindow = () => {
 
     useEffect(() => {
         if (listSelector) {
-          window.sessionStorage.setItem('recordingListSelector', listSelector);
+          sessionStorage.setItem('recordingListSelector', listSelector);
         }
     }, [listSelector]);
 
     useEffect(() => {
-        const storedListSelector = window.sessionStorage.getItem('recordingListSelector');
+        const storedListSelector = sessionStorage.getItem('recordingListSelector');
         
         // Only restore state if it exists in sessionStorage
         if (storedListSelector && !listSelector) {
@@ -172,6 +174,12 @@ export const BrowserWindow = () => {
     }, [screenShot, canvasRef, socket, screencastHandler]);
 
     const highlighterHandler = useCallback((data: { rect: DOMRect, selector: string, elementInfo: ElementInfo | null, childSelectors?: string[] }) => {
+        const now = performance.now();
+        if (now - highlighterUpdateRef.current < 16) {
+            return;
+        }
+        highlighterUpdateRef.current = now;
+        
         // Map the incoming DOMRect from browser coordinates to canvas coordinates
         const mappedRect = new DOMRect(
             data.rect.x,
@@ -573,17 +581,22 @@ export const BrowserWindow = () => {
 };
 
 const drawImage = (image: string, canvas: HTMLCanvasElement): void => {
-
     const ctx = canvas.getContext('2d');
+    if (!ctx) return;
 
     const img = new Image();
-
-    img.src = image;
     img.onload = () => {
-        URL.revokeObjectURL(img.src);
+        requestAnimationFrame(() => {
-        ctx?.drawImage(img, 0, 0, canvas.width, canvas.height);
+            ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
+        });
+        if (image.startsWith('blob:')) {
+            URL.revokeObjectURL(image);
+        }
     };
-
+    img.onerror = () => {
+        console.warn('Failed to load image');
+    };
+    img.src = image;
 };
 
 const modalStyle = {

src/components/recorder/Highlighter.tsx

@@ -1,4 +1,4 @@
-import React from 'react';
+import React, { useMemo } from 'react';
 import styled from "styled-components";
 import { coordinateMapper } from '../../helpers/coordinateMapper';
 
@@ -14,16 +14,15 @@ const HighlighterComponent = ({ unmodifiedRect, displayedSelector = '', width, h
   if (!unmodifiedRect) {
     return null;
   } else {
-    const mappedRect = coordinateMapper.mapBrowserRectToCanvas(unmodifiedRect);
+    const rect = useMemo(() => {
-
+      const mappedRect = coordinateMapper.mapBrowserRectToCanvas(unmodifiedRect);
-    const rect = {
+      return {
-      top: mappedRect.top + canvasRect.top + window.scrollY,
+        top: mappedRect.top + canvasRect.top + window.scrollY,
-      left: mappedRect.left + canvasRect.left + window.scrollX,
+        left: mappedRect.left + canvasRect.left + window.scrollX,
-      right: mappedRect.right + canvasRect.left,
+        width: mappedRect.width,
-      bottom: mappedRect.bottom + canvasRect.top,
+        height: mappedRect.height,
-      width: mappedRect.width,
+      };
-      height: mappedRect.height,
+    }, [unmodifiedRect, canvasRect.top, canvasRect.left]);
-    };
 
 
     return (

src/components/recorder/canvas.tsx

@@ -29,13 +29,17 @@ export interface Coordinates {
 const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
 
     const canvasRef = useRef<HTMLCanvasElement>(null);
+    const contextRef = useRef<CanvasRenderingContext2D | null>(null);
+    const imageDataRef = useRef<ImageData | null>(null);
+    const animationFrameRef = useRef<number | null>(null);
+
     const { socket } = useSocketStore();
     const { setLastAction, lastAction } = useGlobalInfoStore();
     const { getText, getList } = useActionContext();
     const getTextRef = useRef(getText);
     const getListRef = useRef(getList);
 
-    const MOUSE_MOVE_THROTTLE = 16; // ~60fps
+    const MOUSE_MOVE_THROTTLE = 8;
     const lastMouseMoveTime = useRef(0);
 
     const [datePickerInfo, setDatePickerInfo] = React.useState<{
@@ -72,6 +76,22 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
 
     const lastMousePosition = useRef<Coordinates>({ x: 0, y: 0 });
 
+    useEffect(() => {
+        if (canvasRef.current && !contextRef.current) {
+            const ctx = canvasRef.current.getContext('2d', {
+                alpha: false,           
+                desynchronized: true,   
+                willReadFrequently: false 
+            });
+            
+            if (ctx) {                
+                contextRef.current = ctx;
+                
+                imageDataRef.current = ctx.createImageData(width, height);
+            }
+        }
+    }, [width, height]);
+
     useEffect(() => {
         getTextRef.current = getText;
         getListRef.current = getList;
@@ -79,115 +99,100 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
 
     useEffect(() => {
         if (socket) {
-            socket.on('showDatePicker', (info: { coordinates: Coordinates, selector: string }) => {
+            const handleDatePicker = (info: { coordinates: Coordinates, selector: string }) => {
                 const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates);
-                setDatePickerInfo({
+                setDatePickerInfo({ ...info, coordinates: canvasCoords });
-                    ...info,
+            };
-                    coordinates: canvasCoords
-                });
-            });
 
-            socket.on('showDropdown', (info: {
+            const handleDropdown = (info: {
                 coordinates: Coordinates,
                 selector: string,
-                options: Array<{
+                options: Array<{ value: string; text: string; disabled: boolean; selected: boolean; }>;
-                    value: string;
-                    text: string;
-                    disabled: boolean;
-                    selected: boolean;
-                }>;
             }) => {
                 const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates);
-                setDropdownInfo({
+                setDropdownInfo({ ...info, coordinates: canvasCoords });
-                    ...info,
+            };
-                    coordinates: canvasCoords
-                });
-            });
 
-            socket.on('showTimePicker', (info: { coordinates: Coordinates, selector: string }) => {
+            const handleTimePicker = (info: { coordinates: Coordinates, selector: string }) => {
                 const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates);
-                setTimePickerInfo({
+                setTimePickerInfo({ ...info, coordinates: canvasCoords });
-                    ...info,
+            };
-                    coordinates: canvasCoords
-                });
-            });
 
-            socket.on('showDateTimePicker', (info: { coordinates: Coordinates, selector: string }) => {
+            const handleDateTimePicker = (info: { coordinates: Coordinates, selector: string }) => {
                 const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates);
-                setDateTimeLocalInfo({
+                setDateTimeLocalInfo({ ...info, coordinates: canvasCoords });
-                    ...info,
+            };
-                    coordinates: canvasCoords
+
-                });
+            socket.on('showDatePicker', handleDatePicker);
-            });
+            socket.on('showDropdown', handleDropdown);
+            socket.on('showTimePicker', handleTimePicker);
+            socket.on('showDateTimePicker', handleDateTimePicker);
 
             return () => {
-                socket.off('showDatePicker');
+                socket.off('showDatePicker', handleDatePicker);
-                socket.off('showDropdown');
+                socket.off('showDropdown', handleDropdown);
-                socket.off('showTimePicker');
+                socket.off('showTimePicker', handleTimePicker);
-                socket.off('showDateTimePicker');
+                socket.off('showDateTimePicker', handleDateTimePicker);
             };
         }
     }, [socket]);
 
     const onMouseEvent = useCallback((event: MouseEvent) => {
-        if (socket && canvasRef.current) {
+        if (!socket || !canvasRef.current) return;
-            const rect = canvasRef.current.getBoundingClientRect();
-            const clickCoordinates = {
-                x: event.clientX - rect.left, // Use relative x coordinate
-                y: event.clientY - rect.top, // Use relative y coordinate
-            };
 
-            const browserCoordinates = coordinateMapper.mapCanvasToBrowser(clickCoordinates);
+        const rect = canvasRef.current.getBoundingClientRect();
+        const clickCoordinates = {
+            x: event.clientX - rect.left,
+            y: event.clientY - rect.top,
+        };
 
-            switch (event.type) {
+        const browserCoordinates = coordinateMapper.mapCanvasToBrowser(clickCoordinates);
-                case 'mousedown':
+
-                    if (getTextRef.current === true) {
+        switch (event.type) {
-                        console.log('Capturing Text...');
+            case 'mousedown':
-                    } else if (getListRef.current === true) {
+                if (getTextRef.current === true) {
-                        console.log('Capturing List...');
+                    console.log('Capturing Text...');
-                    } else {
+                } else if (getListRef.current === true) {
-                        socket.emit('input:mousedown', browserCoordinates);
+                    console.log('Capturing List...');
-                    }
+                } else {
-                    notifyLastAction('click');
+                    socket.emit('input:mousedown', browserCoordinates);
-                    break;
-                case 'mousemove': {
-                    const now = performance.now();
-                    if (now - lastMouseMoveTime.current < MOUSE_MOVE_THROTTLE) {
-                        return; 
-                    }
-                    lastMouseMoveTime.current = now;
-                    
-                    const dx = Math.abs(lastMousePosition.current.x - clickCoordinates.x);
-                    const dy = Math.abs(lastMousePosition.current.y - clickCoordinates.y);
-                    if (dx > 1 || dy > 1) {
-                        lastMousePosition.current = {
-                            x: clickCoordinates.x,
-                            y: clickCoordinates.y,
-                        };
-                        socket.emit('input:mousemove', browserCoordinates);
-                        notifyLastAction('move');
-                    }
-                    break;
                 }
+                notifyLastAction('click');
+                break;
                 
-                // Optimize wheel events
+            case 'mousemove': {
-                case 'wheel': {
+                const now = performance.now();
-                    const wheelEvent = event as WheelEvent;
+                if (now - lastMouseMoveTime.current < MOUSE_MOVE_THROTTLE) {
-                    const deltaX = Math.round(wheelEvent.deltaX / 10) * 10;
+                    return; 
-                    const deltaY = Math.round(wheelEvent.deltaY / 10) * 10;
-                    
-                    if (Math.abs(deltaX) > 5 || Math.abs(deltaY) > 5) {
-                        socket.emit('input:wheel', { deltaX, deltaY });
-                        notifyLastAction('scroll');
-                    }
-                    break;
                 }
-                default:
+                lastMouseMoveTime.current = now;
-                    console.log('Default mouseEvent registered');
+                
-                    return;
+                const dx = Math.abs(lastMousePosition.current.x - clickCoordinates.x);
+                const dy = Math.abs(lastMousePosition.current.y - clickCoordinates.y);
+                
+                if (dx > 0.5 || dy > 0.5) {
+                    lastMousePosition.current = clickCoordinates;
+                    socket.emit('input:mousemove', browserCoordinates);
+                    notifyLastAction('move');
+                }
+                break;
             }
+            
+            case 'wheel': {
+                const wheelEvent = event as WheelEvent;
+                const deltaX = Math.round(wheelEvent.deltaX / 5) * 5;
+                const deltaY = Math.round(wheelEvent.deltaY / 5) * 5;
+                
+                if (Math.abs(deltaX) > 2 || Math.abs(deltaY) > 2) {
+                    socket.emit('input:wheel', { deltaX, deltaY });
+                    notifyLastAction('scroll');
+                }
+                break;
+            }
+            default:
+                return;
         }
-    }, [socket]);
+    }, [socket, notifyLastAction]);
 
     const onKeyboardEvent = useCallback((event: KeyboardEvent) => {
         if (socket) {
@@ -206,47 +211,66 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
                     return;
             }
         }
-    }, [socket]);
+    }, [socket, notifyLastAction]);
 
 
     useEffect(() => {
-        if (canvasRef.current) {
+        const canvas = canvasRef.current;
-            onCreateRef(canvasRef);
+        if (!canvas) return;
-            canvasRef.current.addEventListener('mousedown', onMouseEvent);
-            canvasRef.current.addEventListener('mousemove', onMouseEvent);
-            canvasRef.current.addEventListener('wheel', onMouseEvent, { passive: true });
-            canvasRef.current.addEventListener('keydown', onKeyboardEvent);
-            canvasRef.current.addEventListener('keyup', onKeyboardEvent);
 
-            return () => {
+        onCreateRef(canvasRef);
-                if (canvasRef.current) {
-                    canvasRef.current.removeEventListener('mousedown', onMouseEvent);
-                    canvasRef.current.removeEventListener('mousemove', onMouseEvent);
-                    canvasRef.current.removeEventListener('wheel', onMouseEvent);
-                    canvasRef.current.removeEventListener('keydown', onKeyboardEvent);
-                    canvasRef.current.removeEventListener('keyup', onKeyboardEvent);
-                }
 
-            };
+        const options = { passive: true };
-        } else {
+        
-            console.log('Canvas not initialized');
+        canvas.addEventListener('mousedown', onMouseEvent, options);
-        }
+        canvas.addEventListener('mousemove', onMouseEvent, options);
+        canvas.addEventListener('wheel', onMouseEvent, options);
+        canvas.addEventListener('keydown', onKeyboardEvent);
+        canvas.addEventListener('keyup', onKeyboardEvent);
 
-    }, [onMouseEvent]);
+        return () => {
+            canvas.removeEventListener('mousedown', onMouseEvent);
+            canvas.removeEventListener('mousemove', onMouseEvent);
+            canvas.removeEventListener('wheel', onMouseEvent);
+            canvas.removeEventListener('keydown', onKeyboardEvent);
+            canvas.removeEventListener('keyup', onKeyboardEvent);
+        };
+    }, [onMouseEvent, onKeyboardEvent, onCreateRef]);
+
+    useEffect(() => {
+        return () => {
+            if (animationFrameRef.current) {
+                cancelAnimationFrame(animationFrameRef.current);
+            }
+        };
+    }, []);
+
+    const containerStyle = React.useMemo<React.CSSProperties>(() => ({
+        borderRadius: '0px 0px 5px 5px',
+        overflow: 'hidden',
+        backgroundColor: 'white',
+        contain: 'layout style paint', 
+        isolation: 'isolate' as React.CSSProperties['isolation'] 
+    }), []);
+
+    const canvasStyle = React.useMemo(() => ({
+        display: 'block',
+        imageRendering: 'crisp-edges' as const,
+        willChange: 'contents', 
+        backfaceVisibility: 'hidden' as const, 
+        transform: 'translateZ(0)', 
+        maxWidth: '100%',
+        maxHeight: '100%'
+    }), []);
 
     return (
-        <div style={{ borderRadius: '0px 0px 5px 5px', overflow: 'hidden', backgroundColor: 'white' }}>
+        <div style={containerStyle}>
             <canvas
                 tabIndex={0}
                 ref={canvasRef}
                 height={height}
                 width={width}
-                style={{ 
+                style={canvasStyle}
-                    display: 'block',
-                    imageRendering: 'crisp-edges', 
-                    willChange: 'transform', 
-                    transform: 'translateZ(0)' 
-                }}
             />
             {datePickerInfo && (
                 <DatePicker