Merge pull request #860 from getmaxun/fast-loader

feat: faster remote browser loading

server/src/browser-management/classes/RemoteBrowser.ts

@@ -201,6 +201,11 @@ export class RemoteBrowser {
     private networkRequestTimeout: NodeJS.Timeout | null = null;
     private pendingNetworkRequests: string[] = [];
     private readonly NETWORK_QUIET_PERIOD = 8000;
+    private readonly INITIAL_LOAD_QUIET_PERIOD = 3000;
+    private networkWaitStartTime: number = 0;
+    private progressInterval: NodeJS.Timeout | null = null;
+    private hasShownInitialLoader: boolean = false;
+    private isInitialLoadInProgress: boolean = false;
 
     /**
      * Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and
@@ -432,17 +437,19 @@ export class RemoteBrowser {
       if (!this.currentPage) return;
 
       this.currentPage.on("domcontentloaded", async () => {
-        logger.info("DOM content loaded - triggering snapshot");
+        if (!this.isInitialLoadInProgress) {
-        await this.makeAndEmitDOMSnapshot();
+          logger.info("DOM content loaded - triggering snapshot");
+          await this.makeAndEmitDOMSnapshot();
+        }
       });
 
       this.currentPage.on("response", async (response) => {
         const url = response.url();
-        if (
+        const isDocumentRequest = response.request().resourceType() === "document";
-          response.request().resourceType() === "document" ||
+
-          url.includes("api/") ||
+        if (!this.hasShownInitialLoader && isDocumentRequest && !url.includes("about:blank")) {
-          url.includes("ajax")
+          this.hasShownInitialLoader = true;
-        ) {
+          this.isInitialLoadInProgress = true;
           this.pendingNetworkRequests.push(url);
 
           if (this.networkRequestTimeout) {
@@ -450,24 +457,54 @@ export class RemoteBrowser {
             this.networkRequestTimeout = null;
           }
 
+          if (this.progressInterval) {
+            clearInterval(this.progressInterval);
+            this.progressInterval = null;
+          }
+
+          this.networkWaitStartTime = Date.now();
+          this.progressInterval = setInterval(() => {
+            const elapsed = Date.now() - this.networkWaitStartTime;
+            const navigationProgress = Math.min((elapsed / this.INITIAL_LOAD_QUIET_PERIOD) * 40, 35);
+            const totalProgress = 60 + navigationProgress;
+            this.emitLoadingProgress(totalProgress, this.pendingNetworkRequests.length);
+          }, 500);
+
           logger.debug(
-            `Network request received: ${url}. Total pending: ${this.pendingNetworkRequests.length}`
+            `Initial load network request received: ${url}. Using ${this.INITIAL_LOAD_QUIET_PERIOD}ms quiet period`
           );
 
           this.networkRequestTimeout = setTimeout(async () => {
             logger.info(
-              `Network quiet period reached. Processing ${this.pendingNetworkRequests.length} requests`
+              `Initial load network quiet period reached (${this.INITIAL_LOAD_QUIET_PERIOD}ms)`
             );
 
+            if (this.progressInterval) {
+              clearInterval(this.progressInterval);
+              this.progressInterval = null;
+            }
+
+            this.emitLoadingProgress(100, this.pendingNetworkRequests.length);
+
             this.pendingNetworkRequests = [];
             this.networkRequestTimeout = null;
+            this.isInitialLoadInProgress = false;
 
             await this.makeAndEmitDOMSnapshot();
-          }, this.NETWORK_QUIET_PERIOD);
+          }, this.INITIAL_LOAD_QUIET_PERIOD);
         }
       });
     }
 
+    private emitLoadingProgress(progress: number, pendingRequests: number): void {
+      this.socket.emit("domLoadingProgress", {
+        progress: Math.round(progress),
+        pendingRequests,
+        userId: this.userId,
+        timestamp: Date.now(),
+      });
+    }
+
     private async setupPageEventListeners(page: Page) {
         page.on('framenavigated', async (frame) => {
             if (frame === page.mainFrame()) {
@@ -522,6 +559,12 @@ export class RemoteBrowser {
         let retryCount = 0;
         let success = false;
 
+        this.socket.emit("dom-snapshot-loading", {
+          userId: this.userId,
+          timestamp: Date.now(),
+        });
+        this.emitLoadingProgress(0, 0);
+
         while (!success && retryCount < MAX_RETRIES) {
             try {
                 this.browser = <Browser>(await chromium.launch({
@@ -546,6 +589,8 @@ export class RemoteBrowser {
                     throw new Error('Browser failed to launch or is not connected');
                 }
 
+                this.emitLoadingProgress(20, 0);
+
                 const proxyConfig = await getDecryptedProxyConfig(userId);
                 let proxyOptions: { server: string, username?: string, password?: string } = { server: '' };
                 
@@ -623,6 +668,8 @@ export class RemoteBrowser {
                 
                 this.currentPage = await this.context.newPage();
 
+                this.emitLoadingProgress(40, 0);
+
                 await this.setupPageEventListeners(this.currentPage);
     
                 const viewportSize = await this.currentPage.viewportSize();
@@ -646,6 +693,8 @@ export class RemoteBrowser {
                     this.client = await this.currentPage.context().newCDPSession(this.currentPage);
                 }
 
+                this.emitLoadingProgress(60, 0);
+
                 success = true;
                 logger.log('debug', `Browser initialized successfully for user ${userId}`);
             } catch (error: any) {
@@ -1521,9 +1570,6 @@ export class RemoteBrowser {
         this.isDOMStreamingActive = true;
         logger.info("DOM streaming started successfully");
 
-        // Initial DOM snapshot
-        await this.makeAndEmitDOMSnapshot();
-
         this.setupScrollEventListener();
         this.setupPageChangeListeners();
       } catch (error) {

src/components/browser/BrowserContent.tsx

@@ -13,7 +13,7 @@ import {
 export const BrowserContent = () => {
   const { socket } = useSocketStore();
 
-  const [tabs, setTabs] = useState<string[]>(["current"]);
+  const [tabs, setTabs] = useState<string[]>(["Loading..."]);
   const [tabIndex, setTabIndex] = React.useState(0);
   const [showOutputData, setShowOutputData] = useState(false);
   const { browserWidth } = useBrowserDimensionsStore();
@@ -125,7 +125,7 @@ export const BrowserContent = () => {
   useEffect(() => {
     getCurrentTabs()
       .then((response) => {
-        if (response) {
+        if (response && response.length > 0) {
           setTabs(response);
         }
       })

src/components/browser/BrowserWindow.tsx

@@ -1,8 +1,6 @@
 import React, { useCallback, useContext, useEffect, useRef, useState } from 'react';
 import { useSocketStore } from '../../context/socket';
 import { Button } from '@mui/material';
-import Canvas from "../recorder/Canvas";
-import { Highlighter } from "../recorder/Highlighter";
 import { GenericModal } from '../ui/GenericModal';
 import { useActionContext } from '../../context/browserActions';
 import { useBrowserSteps, TextStep, ListStep } from '../../context/browserSteps';
@@ -38,12 +36,6 @@ interface AttributeOption {
     value: string;
 }
 
-interface ScreencastData {
-    image: string;
-    userId: string;
-    viewport?: ViewportInfo | null;
-}
-
 interface ViewportInfo {
     width: number;
     height: number;
@@ -146,8 +138,6 @@ const getAttributeOptions = (tagName: string, elementInfo: ElementInfo | null):
 export const BrowserWindow = () => {
     const { t } = useTranslation();
     const { browserWidth, browserHeight } = useBrowserDimensionsStore();
-    const [canvasRef, setCanvasReference] = useState<React.RefObject<HTMLCanvasElement> | undefined>(undefined);
-    const [screenShot, setScreenShot] = useState<string>("");
     const [highlighterData, setHighlighterData] = useState<{
         rect: DOMRect;
         selector: string;
@@ -1303,17 +1293,6 @@ export const BrowserWindow = () => {
     }, []); 
 
     const onMouseMove = (e: MouseEvent) => {
-        if (canvasRef && canvasRef.current && highlighterData) {
-            const canvasRect = canvasRef.current.getBoundingClientRect();
-            if (
-                e.pageX < canvasRect.left
-                || e.pageX > canvasRect.right
-                || e.pageY < canvasRect.top
-                || e.pageY > canvasRect.bottom
-            ) {
-                setHighlighterData(null);
-            }
-        }
     };
 
     const resetListState = useCallback(() => {
@@ -1331,35 +1310,15 @@ export const BrowserWindow = () => {
         }
     }, [getList, resetListState]);
 
-    const screencastHandler = useCallback((data: string | ScreencastData) => {
-        if (typeof data === 'string') {
-            setScreenShot(data);
-        } else if (data && typeof data === 'object' && 'image' in data) {
-            if (!data.userId || data.userId === user?.id) {
-                setScreenShot(data.image);
-                
-                if (data.viewport) {
-                    setViewportInfo(data.viewport);
-                }
-            }
-        }
-    }, [user?.id]);
-
     useEffect(() => {
         if (socket) {
-            socket.on("screencast", screencastHandler);
             socket.on("domcast", rrwebSnapshotHandler);
             socket.on("dom-mode-enabled", domModeHandler);
             socket.on("dom-mode-error", domModeErrorHandler);
         }
 
-        if (canvasRef?.current && !isDOMMode && screenShot) {
-            drawImage(screenShot, canvasRef.current);
-        }
-
         return () => {
             if (socket) {
-                socket.off("screencast", screencastHandler);
                 socket.off("domcast", rrwebSnapshotHandler);
                 socket.off("dom-mode-enabled", domModeHandler);
                 socket.off("dom-mode-error", domModeErrorHandler);
@@ -1367,10 +1326,6 @@ export const BrowserWindow = () => {
         };
     }, [
         socket,
-        screenShot,
-        canvasRef,
-        isDOMMode,
-        screencastHandler,
         rrwebSnapshotHandler,
         domModeHandler,
         domModeErrorHandler,
@@ -1847,24 +1802,7 @@ export const BrowserWindow = () => {
 
     const handleClick = (e: React.MouseEvent<HTMLDivElement>) => {
       if (highlighterData) {
-        let shouldProcessClick = false;
+        const shouldProcessClick = true;
-
-        if (!isDOMMode && canvasRef?.current) {
-          const canvasRect = canvasRef.current.getBoundingClientRect();
-          const clickX = e.clientX - canvasRect.left;
-          const clickY = e.clientY - canvasRect.top;
-          const highlightRect = highlighterData.rect;
-          const mappedRect =
-            coordinateMapper.mapBrowserRectToCanvas(highlightRect);
-
-          shouldProcessClick =
-            clickX >= mappedRect.left &&
-            clickX <= mappedRect.right &&
-            clickY >= mappedRect.top &&
-            clickY <= mappedRect.bottom;
-        } else {
-          shouldProcessClick = true;
-        }
 
         if (shouldProcessClick) {
           const options = getAttributeOptions(
@@ -2209,17 +2147,7 @@ export const BrowserWindow = () => {
             !showAttributeModal &&
             highlighterData?.rect != null && (
               <>
-                {!isDOMMode && canvasRef?.current && (
+                {highlighterData && (
-                  <Highlighter
-                    unmodifiedRect={highlighterData?.rect}
-                    displayedSelector={highlighterData?.selector}
-                    width={dimensions.width}
-                    height={dimensions.height}
-                    canvasRect={canvasRef.current.getBoundingClientRect()}
-                  />
-                )}
-
-                {isDOMMode && highlighterData && (
                   <div
                   id="dom-highlight-overlay"
                   style={{
@@ -2355,31 +2283,27 @@ export const BrowserWindow = () => {
             borderRadius: "0px 0px 5px 5px",
           }}
         >
-          {isDOMMode ? (
+          {currentSnapshot ? (
             <>
-              {currentSnapshot ? (
+              <DOMBrowserRenderer
-                <DOMBrowserRenderer
+                width={dimensions.width}
-                  width={dimensions.width}
+                height={dimensions.height}
-                  height={dimensions.height}
+                snapshot={currentSnapshot}
-                  snapshot={currentSnapshot}
+                getList={getList}
-                  getList={getList}
+                getText={getText}
-                  getText={getText}
+                listSelector={listSelector}
-                  listSelector={listSelector}
+                cachedChildSelectors={cachedChildSelectors}
-                  cachedChildSelectors={cachedChildSelectors}
+                paginationMode={paginationMode}
-                  paginationMode={paginationMode}
+                paginationType={paginationType}
-                  paginationType={paginationType}
+                limitMode={limitMode}
-                  limitMode={limitMode}
+                isCachingChildSelectors={isCachingChildSelectors}
-                  isCachingChildSelectors={isCachingChildSelectors}
+                onHighlight={domHighlighterHandler}
-                  onHighlight={domHighlighterHandler}
+                onElementSelect={handleDOMElementSelection}
-                  onElementSelect={handleDOMElementSelection}
+                onShowDatePicker={handleShowDatePicker}
-                  onShowDatePicker={handleShowDatePicker}
+                onShowDropdown={handleShowDropdown}
-                  onShowDropdown={handleShowDropdown}
+                onShowTimePicker={handleShowTimePicker}
-                  onShowTimePicker={handleShowTimePicker}
+                onShowDateTimePicker={handleShowDateTimePicker}
-                  onShowDateTimePicker={handleShowDateTimePicker}
+              />
-                />
-              ) : (
-                <DOMLoadingIndicator />
-              )}
 
               {/* --- Loading overlay --- */}
               {isCachingChildSelectors && (
@@ -2492,11 +2416,7 @@ export const BrowserWindow = () => {
               )}
             </>
           ) : (
-            <Canvas
+            <DOMLoadingIndicator />
-              onCreateRef={setCanvasReference}
-              width={dimensions.width}
-              height={dimensions.height}
-            />
           )}
         </div>
       </div>
@@ -2591,26 +2511,6 @@ const DOMLoadingIndicator: React.FC = () => {
   );
 };
 
-
-const drawImage = (image: string, canvas: HTMLCanvasElement): void => {
-    const ctx = canvas.getContext('2d');
-    if (!ctx) return;
-
-    const img = new Image();
-    img.onload = () => {
-        requestAnimationFrame(() => {
-            ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
-        });
-        if (image.startsWith('blob:')) {
-            URL.revokeObjectURL(image);
-        }
-    };
-    img.onerror = () => {
-        console.warn('Failed to load image');
-    };
-    img.src = image;
-};
-
 const modalStyle = {
     top: '50%',
     left: '50%',

src/pages/PageWrapper.tsx

@@ -133,6 +133,10 @@ export const PageWrapper = () => {
                   path="/register"
                   element={<Register />}
                 />
+                <Route
+                  path="/recording-setup"
+                  element={<div />}
+                />
                 <Route path="*" element={<NotFoundPage />} />
               </Routes>
             </Box>

src/pages/RecordingPage.tsx

@@ -43,7 +43,7 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
 
   const { setId, socket } = useSocketStore();
   const { setWidth } = useBrowserDimensionsStore();
-  const { browserId, setBrowserId, recordingId, recordingUrl, setRecordingUrl, setRecordingName, setRetrainRobotId } = useGlobalInfoStore();
+  const { browserId, setBrowserId, recordingId, recordingUrl, setRecordingUrl, setRecordingName, setRetrainRobotId, setIsDOMMode } = useGlobalInfoStore();
 
   const handleShowOutputData = useCallback(() => {
     setShowOutputData(true);
@@ -77,6 +77,8 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
   useEffect(() => {
     let isCancelled = false;
     const handleRecording = async () => {
+      setIsDOMMode(true);
+
       const storedUrl = window.sessionStorage.getItem('recordingUrl');
       if (storedUrl && !recordingUrl) {
         setRecordingUrl(storedUrl);
@@ -137,9 +139,12 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
       if (browserId === 'new-recording') {
         socket?.emit('new-recording');
       }
+      if (recordingUrl && socket) {
+        socket.emit('input:url', recordingUrl);
+      }
       setIsLoaded(true);
     }
-  }, [socket, browserId, recordingName, recordingId, isLoaded]);
+  }, [socket, browserId, recordingName, recordingId, recordingUrl, isLoaded]);
 
   useEffect(() => {
     socket?.on('loaded', handleLoaded);
@@ -153,26 +158,20 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
     <ActionProvider>
       <BrowserStepsProvider>
         <div id="browser-recorder">
-          {isLoaded ? (
+          <Grid container direction="row" style={{ flexGrow: 1, height: '100%' }}>
-            <>
+            <Grid item xs={12} md={9} lg={9} style={{ height: '100%', overflow: 'hidden', position: 'relative' }}>
-              <Grid container direction="row" style={{ flexGrow: 1, height: '100%' }}>
+              <div style={{ height: '100%', overflow: 'auto' }}>
-                <Grid item xs={12} md={9} lg={9} style={{ height: '100%', overflow: 'hidden', position: 'relative' }}>
+                <BrowserContent />
-                  <div style={{ height: '100%', overflow: 'auto' }}>
+                <InterpretationLog isOpen={showOutputData} setIsOpen={setShowOutputData} />
-                    <BrowserContent />
+              </div>
-                    <InterpretationLog isOpen={showOutputData} setIsOpen={setShowOutputData} />
+            </Grid>
-                  </div>
+            <Grid item xs={12} md={3} lg={3} style={{ height: '100%', overflow: 'hidden' }}>
-                </Grid>
+              <div className="right-side-panel" style={{ height: '100%' }}>
-                <Grid item xs={12} md={3} lg={3} style={{ height: '100%', overflow: 'hidden' }}>
+                <RightSidePanel onFinishCapture={handleShowOutputData} />
-                  <div className="right-side-panel" style={{ height: '100%' }}>
+                <BrowserRecordingSave />
-                    <RightSidePanel onFinishCapture={handleShowOutputData} />
+              </div>
-                    <BrowserRecordingSave />
+            </Grid>
-                  </div>
+          </Grid>
-                </Grid>
-              </Grid>
-            </>
-          ) : (
-            <Loader text={t('recording_page.loader.browser_startup', { url: recordingUrl })} />
-          )}
         </div>
       </BrowserStepsProvider>
     </ActionProvider>