Merge pull request #640 from getmaxun/browser-perf

feat: browser performance optimization
2025-06-12 21:26:42 +05:30
parent 4b077a660b 7633c4730d
commit 154243b1de
4 changed files with 219 additions and 154 deletions
--- a/server/src/browser-management/classes/RemoteBrowser.ts
+++ b/server/src/browser-management/classes/RemoteBrowser.ts
@@ -25,6 +25,13 @@ const MEMORY_CONFIG = {
    heapUsageThreshold: 0.7 // 70% (reduced threshold to react earlier)
 };

+const DEFAULT_VIEWPORT = {
+  width: 1280,
+  height: 720, 
+  deviceScaleFactor: 1,
+  mobile: false
+};
+
 const SCREENCAST_CONFIG: {
    format: "jpeg" | "png";
    maxWidth: number;
@@ -32,13 +39,17 @@ const SCREENCAST_CONFIG: {
    targetFPS: number;
    compressionQuality: number;
    maxQueueSize: number;
+    skipFrameThreshold: number, 
+    enableAdaptiveQuality: boolean, 
 } = {
-    format: 'png', 
-    maxWidth: 1280,
-    maxHeight: 720,
-    targetFPS: 15, 
-    compressionQuality: 0.95, 
-    maxQueueSize: 1 
+    format: 'jpeg', 
+    maxWidth: DEFAULT_VIEWPORT.width,
+    maxHeight: DEFAULT_VIEWPORT.height,
+    targetFPS: 30, 
+    compressionQuality: 0.8, 
+    maxQueueSize: 2,
+    skipFrameThreshold: 100, 
+    enableAdaptiveQuality: true,  
 };

 /**
@@ -125,6 +136,18 @@ export class RemoteBrowser {
        this.generator = new WorkflowGenerator(socket, poolId);
    }

+    private cleanupMemory(): void {
+      if (this.screenshotQueue.length > 10) {
+          this.screenshotQueue = this.screenshotQueue.slice(-3); // Keep only last 3
+      }
+    }
+
+    private setupMemoryCleanup(): void {
+      setInterval(() => {
+          this.cleanupMemory();
+      }, 30000); // Every 30 seconds
+    }
+
    private initializeMemoryManagement(): void {
        setInterval(() => {
            const memoryUsage = process.memoryUsage();
@@ -412,6 +435,7 @@ export class RemoteBrowser {
            }
        }

+        this.setupMemoryCleanup();
        // this.initializeMemoryManagement();
    };

@@ -1399,7 +1423,7 @@ export class RemoteBrowser {
     */
    private emitScreenshot = async (payload: Buffer, viewportSize?: { width: number, height: number }): Promise<void> => {
        if (this.screenshotQueue.length > SCREENCAST_CONFIG.maxQueueSize) {
-            this.screenshotQueue = this.screenshotQueue.slice(-SCREENCAST_CONFIG.maxQueueSize);
+          this.screenshotQueue = this.screenshotQueue.slice(-1);
        }
        
        if (this.isProcessingScreenshot) {
@@ -1414,7 +1438,7 @@ export class RemoteBrowser {
        try {
            const optimizationPromise = this.optimizeScreenshot(payload);
            const timeoutPromise = new Promise<Buffer>((resolve) => {
-                setTimeout(() => resolve(payload), 150);
+                setTimeout(() => resolve(payload), 100);
            });
            
            const optimizedScreenshot = await Promise.race([optimizationPromise, timeoutPromise]);
@@ -1423,10 +1447,12 @@ export class RemoteBrowser {
            
            payload = null as any;
            
-            this.socket.emit('screencast', {
+            setImmediate(async () => {
+              this.socket.emit('screencast', {
                image: dataWithMimeType,
                userId: this.userId,
                viewport: viewportSize || await this.currentPage?.viewportSize() || null
+              });
            });
        } catch (error) {
            logger.error('Screenshot emission failed:', error);
@@ -1434,24 +1460,27 @@ export class RemoteBrowser {
                const base64Data = payload.toString('base64');
                const dataWithMimeType = `data:image/png;base64,${base64Data}`;
                
-                this.socket.emit('screencast', {
-                    image: dataWithMimeType,
-                    userId: this.userId,
-                    viewport: viewportSize || await this.currentPage?.viewportSize() || null
+                setImmediate(async () => {
+                  this.socket.emit('screencast', {
+                      image: dataWithMimeType,
+                      userId: this.userId,
+                      viewport: viewportSize || await this.currentPage?.viewportSize() || null
+                  });
                });
            } catch (e) {
                logger.error('Fallback screenshot emission also failed:', e);
            }
        } finally {
            this.isProcessingScreenshot = false;
-            
+        
            if (this.screenshotQueue.length > 0) {
-                const nextScreenshot = this.screenshotQueue.shift();  
-                if (nextScreenshot) {
-                    setTimeout(() => {
-                        this.emitScreenshot(nextScreenshot);
-                    }, 1000 / SCREENCAST_CONFIG.targetFPS);
-                }
+              const nextScreenshot = this.screenshotQueue.shift();
+              if (nextScreenshot) {
+                const delay = this.screenshotQueue.length > 0 ? 16 : 33;
+                setTimeout(() => {
+                  this.emitScreenshot(nextScreenshot);
+                }, delay);
+              }
            }
        }
    };
--- a/src/components/browser/BrowserWindow.tsx
+++ b/src/components/browser/BrowserWindow.tsx
@@ -1,4 +1,4 @@
-import React, { useCallback, useContext, useEffect, useState } from 'react';
+import React, { useCallback, useContext, useEffect, useRef, useState } from 'react';
 import { useSocketStore } from '../../context/socket';
 import { Button } from '@mui/material';
 import Canvas from "../recorder/canvas";
@@ -84,6 +84,8 @@ export const BrowserWindow = () => {
    const [fields, setFields] = useState<Record<string, TextStep>>({});
    const [paginationSelector, setPaginationSelector] = useState<string>('');

+    const highlighterUpdateRef = useRef<number>(0);
+
    const { socket } = useSocketStore();
    const { notify, currentTextActionId, currentListActionId } = useGlobalInfoStore();
    const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext();
@@ -103,12 +105,12 @@ export const BrowserWindow = () => {

    useEffect(() => {
        if (listSelector) {
-          window.sessionStorage.setItem('recordingListSelector', listSelector);
+          sessionStorage.setItem('recordingListSelector', listSelector);
        }
    }, [listSelector]);

    useEffect(() => {
-        const storedListSelector = window.sessionStorage.getItem('recordingListSelector');
+        const storedListSelector = sessionStorage.getItem('recordingListSelector');
        
        // Only restore state if it exists in sessionStorage
        if (storedListSelector && !listSelector) {
@@ -172,6 +174,12 @@ export const BrowserWindow = () => {
    }, [screenShot, canvasRef, socket, screencastHandler]);

    const highlighterHandler = useCallback((data: { rect: DOMRect, selector: string, elementInfo: ElementInfo | null, childSelectors?: string[] }) => {
+        const now = performance.now();
+        if (now - highlighterUpdateRef.current < 16) {
+            return;
+        }
+        highlighterUpdateRef.current = now;
+        
        // Map the incoming DOMRect from browser coordinates to canvas coordinates
        const mappedRect = new DOMRect(
            data.rect.x,
@@ -573,17 +581,22 @@ export const BrowserWindow = () => {
 };

 const drawImage = (image: string, canvas: HTMLCanvasElement): void => {
-
    const ctx = canvas.getContext('2d');
+    if (!ctx) return;

    const img = new Image();
-
-    img.src = image;
    img.onload = () => {
-        URL.revokeObjectURL(img.src);
-        ctx?.drawImage(img, 0, 0, canvas.width, canvas.height);
+        requestAnimationFrame(() => {
+            ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
+        });
+        if (image.startsWith('blob:')) {
+            URL.revokeObjectURL(image);
+        }
    };
-
+    img.onerror = () => {
+        console.warn('Failed to load image');
+    };
+    img.src = image;
 };

 const modalStyle = {
--- a/src/components/recorder/Highlighter.tsx
+++ b/src/components/recorder/Highlighter.tsx
@@ -1,4 +1,4 @@
-import React from 'react';
+import React, { useMemo } from 'react';
 import styled from "styled-components";
 import { coordinateMapper } from '../../helpers/coordinateMapper';

@@ -14,16 +14,15 @@ const HighlighterComponent = ({ unmodifiedRect, displayedSelector = '', width, h
  if (!unmodifiedRect) {
    return null;
  } else {
-    const mappedRect = coordinateMapper.mapBrowserRectToCanvas(unmodifiedRect);
-
-    const rect = {
-      top: mappedRect.top + canvasRect.top + window.scrollY,
-      left: mappedRect.left + canvasRect.left + window.scrollX,
-      right: mappedRect.right + canvasRect.left,
-      bottom: mappedRect.bottom + canvasRect.top,
-      width: mappedRect.width,
-      height: mappedRect.height,
-    };
+    const rect = useMemo(() => {
+      const mappedRect = coordinateMapper.mapBrowserRectToCanvas(unmodifiedRect);
+      return {
+        top: mappedRect.top + canvasRect.top + window.scrollY,
+        left: mappedRect.left + canvasRect.left + window.scrollX,
+        width: mappedRect.width,
+        height: mappedRect.height,
+      };
+    }, [unmodifiedRect, canvasRect.top, canvasRect.left]);


    return (
--- a/src/components/recorder/canvas.tsx
+++ b/src/components/recorder/canvas.tsx
@@ -29,13 +29,17 @@ export interface Coordinates {
 const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {

    const canvasRef = useRef<HTMLCanvasElement>(null);
+    const contextRef = useRef<CanvasRenderingContext2D | null>(null);
+    const imageDataRef = useRef<ImageData | null>(null);
+    const animationFrameRef = useRef<number | null>(null);
+
    const { socket } = useSocketStore();
    const { setLastAction, lastAction } = useGlobalInfoStore();
    const { getText, getList } = useActionContext();
    const getTextRef = useRef(getText);
    const getListRef = useRef(getList);

-    const MOUSE_MOVE_THROTTLE = 16; // ~60fps
+    const MOUSE_MOVE_THROTTLE = 8;
    const lastMouseMoveTime = useRef(0);

    const [datePickerInfo, setDatePickerInfo] = React.useState<{
@@ -72,6 +76,22 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {

    const lastMousePosition = useRef<Coordinates>({ x: 0, y: 0 });

+    useEffect(() => {
+        if (canvasRef.current && !contextRef.current) {
+            const ctx = canvasRef.current.getContext('2d', {
+                alpha: false,           
+                desynchronized: true,   
+                willReadFrequently: false 
+            });
+            
+            if (ctx) {                
+                contextRef.current = ctx;
+                
+                imageDataRef.current = ctx.createImageData(width, height);
+            }
+        }
+    }, [width, height]);
+
    useEffect(() => {
        getTextRef.current = getText;
        getListRef.current = getList;
@@ -79,115 +99,100 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {

    useEffect(() => {
        if (socket) {
-            socket.on('showDatePicker', (info: { coordinates: Coordinates, selector: string }) => {
+            const handleDatePicker = (info: { coordinates: Coordinates, selector: string }) => {
                const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates);
-                setDatePickerInfo({
-                    ...info,
-                    coordinates: canvasCoords
-                });
-            });
+                setDatePickerInfo({ ...info, coordinates: canvasCoords });
+            };

-            socket.on('showDropdown', (info: {
+            const handleDropdown = (info: {
                coordinates: Coordinates,
                selector: string,
-                options: Array<{
-                    value: string;
-                    text: string;
-                    disabled: boolean;
-                    selected: boolean;
-                }>;
+                options: Array<{ value: string; text: string; disabled: boolean; selected: boolean; }>;
            }) => {
                const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates);
-                setDropdownInfo({
-                    ...info,
-                    coordinates: canvasCoords
-                });
-            });
+                setDropdownInfo({ ...info, coordinates: canvasCoords });
+            };

-            socket.on('showTimePicker', (info: { coordinates: Coordinates, selector: string }) => {
+            const handleTimePicker = (info: { coordinates: Coordinates, selector: string }) => {
                const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates);
-                setTimePickerInfo({
-                    ...info,
-                    coordinates: canvasCoords
-                });
-            });
+                setTimePickerInfo({ ...info, coordinates: canvasCoords });
+            };

-            socket.on('showDateTimePicker', (info: { coordinates: Coordinates, selector: string }) => {
+            const handleDateTimePicker = (info: { coordinates: Coordinates, selector: string }) => {
                const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates);
-                setDateTimeLocalInfo({
-                    ...info,
-                    coordinates: canvasCoords
-                });
-            });
+                setDateTimeLocalInfo({ ...info, coordinates: canvasCoords });
+            };
+
+            socket.on('showDatePicker', handleDatePicker);
+            socket.on('showDropdown', handleDropdown);
+            socket.on('showTimePicker', handleTimePicker);
+            socket.on('showDateTimePicker', handleDateTimePicker);

            return () => {
-                socket.off('showDatePicker');
-                socket.off('showDropdown');
-                socket.off('showTimePicker');
-                socket.off('showDateTimePicker');
+                socket.off('showDatePicker', handleDatePicker);
+                socket.off('showDropdown', handleDropdown);
+                socket.off('showTimePicker', handleTimePicker);
+                socket.off('showDateTimePicker', handleDateTimePicker);
            };
        }
    }, [socket]);

    const onMouseEvent = useCallback((event: MouseEvent) => {
-        if (socket && canvasRef.current) {
-            const rect = canvasRef.current.getBoundingClientRect();
-            const clickCoordinates = {
-                x: event.clientX - rect.left, // Use relative x coordinate
-                y: event.clientY - rect.top, // Use relative y coordinate
-            };
+        if (!socket || !canvasRef.current) return;

-            const browserCoordinates = coordinateMapper.mapCanvasToBrowser(clickCoordinates);
+        const rect = canvasRef.current.getBoundingClientRect();
+        const clickCoordinates = {
+            x: event.clientX - rect.left,
+            y: event.clientY - rect.top,
+        };

-            switch (event.type) {
-                case 'mousedown':
-                    if (getTextRef.current === true) {
-                        console.log('Capturing Text...');
-                    } else if (getListRef.current === true) {
-                        console.log('Capturing List...');
-                    } else {
-                        socket.emit('input:mousedown', browserCoordinates);
-                    }
-                    notifyLastAction('click');
-                    break;
-                case 'mousemove': {
-                    const now = performance.now();
-                    if (now - lastMouseMoveTime.current < MOUSE_MOVE_THROTTLE) {
-                        return; 
-                    }
-                    lastMouseMoveTime.current = now;
-                    
-                    const dx = Math.abs(lastMousePosition.current.x - clickCoordinates.x);
-                    const dy = Math.abs(lastMousePosition.current.y - clickCoordinates.y);
-                    if (dx > 1 || dy > 1) {
-                        lastMousePosition.current = {
-                            x: clickCoordinates.x,
-                            y: clickCoordinates.y,
-                        };
-                        socket.emit('input:mousemove', browserCoordinates);
-                        notifyLastAction('move');
-                    }
-                    break;
+        const browserCoordinates = coordinateMapper.mapCanvasToBrowser(clickCoordinates);
+
+        switch (event.type) {
+            case 'mousedown':
+                if (getTextRef.current === true) {
+                    console.log('Capturing Text...');
+                } else if (getListRef.current === true) {
+                    console.log('Capturing List...');
+                } else {
+                    socket.emit('input:mousedown', browserCoordinates);
                }
+                notifyLastAction('click');
+                break;
                
-                // Optimize wheel events
-                case 'wheel': {
-                    const wheelEvent = event as WheelEvent;
-                    const deltaX = Math.round(wheelEvent.deltaX / 10) * 10;
-                    const deltaY = Math.round(wheelEvent.deltaY / 10) * 10;
-                    
-                    if (Math.abs(deltaX) > 5 || Math.abs(deltaY) > 5) {
-                        socket.emit('input:wheel', { deltaX, deltaY });
-                        notifyLastAction('scroll');
-                    }
-                    break;
+            case 'mousemove': {
+                const now = performance.now();
+                if (now - lastMouseMoveTime.current < MOUSE_MOVE_THROTTLE) {
+                    return; 
                }
-                default:
-                    console.log('Default mouseEvent registered');
-                    return;
+                lastMouseMoveTime.current = now;
+                
+                const dx = Math.abs(lastMousePosition.current.x - clickCoordinates.x);
+                const dy = Math.abs(lastMousePosition.current.y - clickCoordinates.y);
+                
+                if (dx > 0.5 || dy > 0.5) {
+                    lastMousePosition.current = clickCoordinates;
+                    socket.emit('input:mousemove', browserCoordinates);
+                    notifyLastAction('move');
+                }
+                break;
            }
+            
+            case 'wheel': {
+                const wheelEvent = event as WheelEvent;
+                const deltaX = Math.round(wheelEvent.deltaX / 5) * 5;
+                const deltaY = Math.round(wheelEvent.deltaY / 5) * 5;
+                
+                if (Math.abs(deltaX) > 2 || Math.abs(deltaY) > 2) {
+                    socket.emit('input:wheel', { deltaX, deltaY });
+                    notifyLastAction('scroll');
+                }
+                break;
+            }
+            default:
+                return;
        }
-    }, [socket]);
+    }, [socket, notifyLastAction]);

    const onKeyboardEvent = useCallback((event: KeyboardEvent) => {
        if (socket) {
@@ -206,47 +211,66 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
                    return;
            }
        }
-    }, [socket]);
+    }, [socket, notifyLastAction]);


    useEffect(() => {
-        if (canvasRef.current) {
-            onCreateRef(canvasRef);
-            canvasRef.current.addEventListener('mousedown', onMouseEvent);
-            canvasRef.current.addEventListener('mousemove', onMouseEvent);
-            canvasRef.current.addEventListener('wheel', onMouseEvent, { passive: true });
-            canvasRef.current.addEventListener('keydown', onKeyboardEvent);
-            canvasRef.current.addEventListener('keyup', onKeyboardEvent);
+        const canvas = canvasRef.current;
+        if (!canvas) return;

-            return () => {
-                if (canvasRef.current) {
-                    canvasRef.current.removeEventListener('mousedown', onMouseEvent);
-                    canvasRef.current.removeEventListener('mousemove', onMouseEvent);
-                    canvasRef.current.removeEventListener('wheel', onMouseEvent);
-                    canvasRef.current.removeEventListener('keydown', onKeyboardEvent);
-                    canvasRef.current.removeEventListener('keyup', onKeyboardEvent);
-                }
+        onCreateRef(canvasRef);

-            };
-        } else {
-            console.log('Canvas not initialized');
-        }
+        const options = { passive: true };
+        
+        canvas.addEventListener('mousedown', onMouseEvent, options);
+        canvas.addEventListener('mousemove', onMouseEvent, options);
+        canvas.addEventListener('wheel', onMouseEvent, options);
+        canvas.addEventListener('keydown', onKeyboardEvent);
+        canvas.addEventListener('keyup', onKeyboardEvent);

-    }, [onMouseEvent]);
+        return () => {
+            canvas.removeEventListener('mousedown', onMouseEvent);
+            canvas.removeEventListener('mousemove', onMouseEvent);
+            canvas.removeEventListener('wheel', onMouseEvent);
+            canvas.removeEventListener('keydown', onKeyboardEvent);
+            canvas.removeEventListener('keyup', onKeyboardEvent);
+        };
+    }, [onMouseEvent, onKeyboardEvent, onCreateRef]);
+
+    useEffect(() => {
+        return () => {
+            if (animationFrameRef.current) {
+                cancelAnimationFrame(animationFrameRef.current);
+            }
+        };
+    }, []);
+
+    const containerStyle = React.useMemo<React.CSSProperties>(() => ({
+        borderRadius: '0px 0px 5px 5px',
+        overflow: 'hidden',
+        backgroundColor: 'white',
+        contain: 'layout style paint', 
+        isolation: 'isolate' as React.CSSProperties['isolation'] 
+    }), []);
+
+    const canvasStyle = React.useMemo(() => ({
+        display: 'block',
+        imageRendering: 'crisp-edges' as const,
+        willChange: 'contents', 
+        backfaceVisibility: 'hidden' as const, 
+        transform: 'translateZ(0)', 
+        maxWidth: '100%',
+        maxHeight: '100%'
+    }), []);

    return (
-        <div style={{ borderRadius: '0px 0px 5px 5px', overflow: 'hidden', backgroundColor: 'white' }}>
+        <div style={containerStyle}>
            <canvas
                tabIndex={0}
                ref={canvasRef}
                height={height}
                width={width}
-                style={{ 
-                    display: 'block',
-                    imageRendering: 'crisp-edges', 
-                    willChange: 'transform', 
-                    transform: 'translateZ(0)' 
-                }}
+                style={canvasStyle}
            />
            {datePickerInfo && (
                <DatePicker