Merge pull request #860 from getmaxun/fast-loader

feat: faster remote browser loading
2025-11-12 21:50:14 +05:30
parent 653a332524 fcb3a12aca
commit 1e07aa5dbb
5 changed files with 112 additions and 163 deletions
--- a/server/src/browser-management/classes/RemoteBrowser.ts
+++ b/server/src/browser-management/classes/RemoteBrowser.ts
@@ -201,6 +201,11 @@ export class RemoteBrowser {
    private networkRequestTimeout: NodeJS.Timeout | null = null;
    private pendingNetworkRequests: string[] = [];
    private readonly NETWORK_QUIET_PERIOD = 8000;
+    private readonly INITIAL_LOAD_QUIET_PERIOD = 3000;
+    private networkWaitStartTime: number = 0;
+    private progressInterval: NodeJS.Timeout | null = null;
+    private hasShownInitialLoader: boolean = false;
+    private isInitialLoadInProgress: boolean = false;

    /**
     * Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and
@@ -432,17 +437,19 @@ export class RemoteBrowser {
      if (!this.currentPage) return;

      this.currentPage.on("domcontentloaded", async () => {
-        logger.info("DOM content loaded - triggering snapshot");
-        await this.makeAndEmitDOMSnapshot();
+        if (!this.isInitialLoadInProgress) {
+          logger.info("DOM content loaded - triggering snapshot");
+          await this.makeAndEmitDOMSnapshot();
+        }
      });

      this.currentPage.on("response", async (response) => {
        const url = response.url();
-        if (
-          response.request().resourceType() === "document" ||
-          url.includes("api/") ||
-          url.includes("ajax")
-        ) {
+        const isDocumentRequest = response.request().resourceType() === "document";
+
+        if (!this.hasShownInitialLoader && isDocumentRequest && !url.includes("about:blank")) {
+          this.hasShownInitialLoader = true;
+          this.isInitialLoadInProgress = true;
          this.pendingNetworkRequests.push(url);

          if (this.networkRequestTimeout) {
@@ -450,24 +457,54 @@ export class RemoteBrowser {
            this.networkRequestTimeout = null;
          }

+          if (this.progressInterval) {
+            clearInterval(this.progressInterval);
+            this.progressInterval = null;
+          }
+
+          this.networkWaitStartTime = Date.now();
+          this.progressInterval = setInterval(() => {
+            const elapsed = Date.now() - this.networkWaitStartTime;
+            const navigationProgress = Math.min((elapsed / this.INITIAL_LOAD_QUIET_PERIOD) * 40, 35);
+            const totalProgress = 60 + navigationProgress;
+            this.emitLoadingProgress(totalProgress, this.pendingNetworkRequests.length);
+          }, 500);
+
          logger.debug(
-            `Network request received: ${url}. Total pending: ${this.pendingNetworkRequests.length}`
+            `Initial load network request received: ${url}. Using ${this.INITIAL_LOAD_QUIET_PERIOD}ms quiet period`
          );

          this.networkRequestTimeout = setTimeout(async () => {
            logger.info(
-              `Network quiet period reached. Processing ${this.pendingNetworkRequests.length} requests`
+              `Initial load network quiet period reached (${this.INITIAL_LOAD_QUIET_PERIOD}ms)`
            );

+            if (this.progressInterval) {
+              clearInterval(this.progressInterval);
+              this.progressInterval = null;
+            }
+
+            this.emitLoadingProgress(100, this.pendingNetworkRequests.length);
+
            this.pendingNetworkRequests = [];
            this.networkRequestTimeout = null;
+            this.isInitialLoadInProgress = false;

            await this.makeAndEmitDOMSnapshot();
-          }, this.NETWORK_QUIET_PERIOD);
+          }, this.INITIAL_LOAD_QUIET_PERIOD);
        }
      });
    }

+    private emitLoadingProgress(progress: number, pendingRequests: number): void {
+      this.socket.emit("domLoadingProgress", {
+        progress: Math.round(progress),
+        pendingRequests,
+        userId: this.userId,
+        timestamp: Date.now(),
+      });
+    }
+
    private async setupPageEventListeners(page: Page) {
        page.on('framenavigated', async (frame) => {
            if (frame === page.mainFrame()) {
@@ -521,7 +558,13 @@ export class RemoteBrowser {
        const MAX_RETRIES = 3;
        let retryCount = 0;
        let success = false;
-    
+
+        this.socket.emit("dom-snapshot-loading", {
+          userId: this.userId,
+          timestamp: Date.now(),
+        });
+        this.emitLoadingProgress(0, 0);
+
        while (!success && retryCount < MAX_RETRIES) {
            try {
                this.browser = <Browser>(await chromium.launch({
@@ -545,7 +588,9 @@ export class RemoteBrowser {
                if (!this.browser || this.browser.isConnected() === false) {
                    throw new Error('Browser failed to launch or is not connected');
                }
-                
+
+                this.emitLoadingProgress(20, 0);
+
                const proxyConfig = await getDecryptedProxyConfig(userId);
                let proxyOptions: { server: string, username?: string, password?: string } = { server: '' };
                
@@ -623,6 +668,8 @@ export class RemoteBrowser {
                
                this.currentPage = await this.context.newPage();

+                this.emitLoadingProgress(40, 0);
+
                await this.setupPageEventListeners(this.currentPage);
    
                const viewportSize = await this.currentPage.viewportSize();
@@ -645,7 +692,9 @@ export class RemoteBrowser {
                    // Still need to set up the CDP session even if blocker fails
                    this.client = await this.currentPage.context().newCDPSession(this.currentPage);
                }
-                
+
+                this.emitLoadingProgress(60, 0);
+
                success = true;
                logger.log('debug', `Browser initialized successfully for user ${userId}`);
            } catch (error: any) {
@@ -1521,9 +1570,6 @@ export class RemoteBrowser {
        this.isDOMStreamingActive = true;
        logger.info("DOM streaming started successfully");

-        // Initial DOM snapshot
-        await this.makeAndEmitDOMSnapshot();
-
        this.setupScrollEventListener();
        this.setupPageChangeListeners();
      } catch (error) {
--- a/src/components/browser/BrowserContent.tsx
+++ b/src/components/browser/BrowserContent.tsx
@@ -13,7 +13,7 @@ import {
 export const BrowserContent = () => {
  const { socket } = useSocketStore();

-  const [tabs, setTabs] = useState<string[]>(["current"]);
+  const [tabs, setTabs] = useState<string[]>(["Loading..."]);
  const [tabIndex, setTabIndex] = React.useState(0);
  const [showOutputData, setShowOutputData] = useState(false);
  const { browserWidth } = useBrowserDimensionsStore();
@@ -125,7 +125,7 @@ export const BrowserContent = () => {
  useEffect(() => {
    getCurrentTabs()
      .then((response) => {
-        if (response) {
+        if (response && response.length > 0) {
          setTabs(response);
        }
      })
--- a/src/components/browser/BrowserWindow.tsx
+++ b/src/components/browser/BrowserWindow.tsx
@@ -1,8 +1,6 @@
 import React, { useCallback, useContext, useEffect, useRef, useState } from 'react';
 import { useSocketStore } from '../../context/socket';
 import { Button } from '@mui/material';
-import Canvas from "../recorder/Canvas";
-import { Highlighter } from "../recorder/Highlighter";
 import { GenericModal } from '../ui/GenericModal';
 import { useActionContext } from '../../context/browserActions';
 import { useBrowserSteps, TextStep, ListStep } from '../../context/browserSteps';
@@ -38,12 +36,6 @@ interface AttributeOption {
    value: string;
 }

-interface ScreencastData {
-    image: string;
-    userId: string;
-    viewport?: ViewportInfo | null;
-}
-
 interface ViewportInfo {
    width: number;
    height: number;
@@ -146,8 +138,6 @@ const getAttributeOptions = (tagName: string, elementInfo: ElementInfo | null):
 export const BrowserWindow = () => {
    const { t } = useTranslation();
    const { browserWidth, browserHeight } = useBrowserDimensionsStore();
-    const [canvasRef, setCanvasReference] = useState<React.RefObject<HTMLCanvasElement> | undefined>(undefined);
-    const [screenShot, setScreenShot] = useState<string>("");
    const [highlighterData, setHighlighterData] = useState<{
        rect: DOMRect;
        selector: string;
@@ -1303,17 +1293,6 @@ export const BrowserWindow = () => {
    }, []); 

    const onMouseMove = (e: MouseEvent) => {
-        if (canvasRef && canvasRef.current && highlighterData) {
-            const canvasRect = canvasRef.current.getBoundingClientRect();
-            if (
-                e.pageX < canvasRect.left
-                || e.pageX > canvasRect.right
-                || e.pageY < canvasRect.top
-                || e.pageY > canvasRect.bottom
-            ) {
-                setHighlighterData(null);
-            }
-        }
    };

    const resetListState = useCallback(() => {
@@ -1331,35 +1310,15 @@ export const BrowserWindow = () => {
        }
    }, [getList, resetListState]);

-    const screencastHandler = useCallback((data: string | ScreencastData) => {
-        if (typeof data === 'string') {
-            setScreenShot(data);
-        } else if (data && typeof data === 'object' && 'image' in data) {
-            if (!data.userId || data.userId === user?.id) {
-                setScreenShot(data.image);
-                
-                if (data.viewport) {
-                    setViewportInfo(data.viewport);
-                }
-            }
-        }
-    }, [user?.id]);
-
    useEffect(() => {
        if (socket) {
-            socket.on("screencast", screencastHandler);
            socket.on("domcast", rrwebSnapshotHandler);
            socket.on("dom-mode-enabled", domModeHandler);
            socket.on("dom-mode-error", domModeErrorHandler);
        }

-        if (canvasRef?.current && !isDOMMode && screenShot) {
-            drawImage(screenShot, canvasRef.current);
-        }
-
        return () => {
            if (socket) {
-                socket.off("screencast", screencastHandler);
                socket.off("domcast", rrwebSnapshotHandler);
                socket.off("dom-mode-enabled", domModeHandler);
                socket.off("dom-mode-error", domModeErrorHandler);
@@ -1367,10 +1326,6 @@ export const BrowserWindow = () => {
        };
    }, [
        socket,
-        screenShot,
-        canvasRef,
-        isDOMMode,
-        screencastHandler,
        rrwebSnapshotHandler,
        domModeHandler,
        domModeErrorHandler,
@@ -1847,24 +1802,7 @@ export const BrowserWindow = () => {

    const handleClick = (e: React.MouseEvent<HTMLDivElement>) => {
      if (highlighterData) {
-        let shouldProcessClick = false;
-
-        if (!isDOMMode && canvasRef?.current) {
-          const canvasRect = canvasRef.current.getBoundingClientRect();
-          const clickX = e.clientX - canvasRect.left;
-          const clickY = e.clientY - canvasRect.top;
-          const highlightRect = highlighterData.rect;
-          const mappedRect =
-            coordinateMapper.mapBrowserRectToCanvas(highlightRect);
-
-          shouldProcessClick =
-            clickX >= mappedRect.left &&
-            clickX <= mappedRect.right &&
-            clickY >= mappedRect.top &&
-            clickY <= mappedRect.bottom;
-        } else {
-          shouldProcessClick = true;
-        }
+        const shouldProcessClick = true;

        if (shouldProcessClick) {
          const options = getAttributeOptions(
@@ -2209,17 +2147,7 @@ export const BrowserWindow = () => {
            !showAttributeModal &&
            highlighterData?.rect != null && (
              <>
-                {!isDOMMode && canvasRef?.current && (
-                  <Highlighter
-                    unmodifiedRect={highlighterData?.rect}
-                    displayedSelector={highlighterData?.selector}
-                    width={dimensions.width}
-                    height={dimensions.height}
-                    canvasRect={canvasRef.current.getBoundingClientRect()}
-                  />
-                )}
-
-                {isDOMMode && highlighterData && (
+                {highlighterData && (
                  <div
                  id="dom-highlight-overlay"
                  style={{
@@ -2355,31 +2283,27 @@ export const BrowserWindow = () => {
            borderRadius: "0px 0px 5px 5px",
          }}
        >
-          {isDOMMode ? (
+          {currentSnapshot ? (
            <>
-              {currentSnapshot ? (
-                <DOMBrowserRenderer
-                  width={dimensions.width}
-                  height={dimensions.height}
-                  snapshot={currentSnapshot}
-                  getList={getList}
-                  getText={getText}
-                  listSelector={listSelector}
-                  cachedChildSelectors={cachedChildSelectors}
-                  paginationMode={paginationMode}
-                  paginationType={paginationType}
-                  limitMode={limitMode}
-                  isCachingChildSelectors={isCachingChildSelectors}
-                  onHighlight={domHighlighterHandler}
-                  onElementSelect={handleDOMElementSelection}
-                  onShowDatePicker={handleShowDatePicker}
-                  onShowDropdown={handleShowDropdown}
-                  onShowTimePicker={handleShowTimePicker}
-                  onShowDateTimePicker={handleShowDateTimePicker}
-                />
-              ) : (
-                <DOMLoadingIndicator />
-              )}
+              <DOMBrowserRenderer
+                width={dimensions.width}
+                height={dimensions.height}
+                snapshot={currentSnapshot}
+                getList={getList}
+                getText={getText}
+                listSelector={listSelector}
+                cachedChildSelectors={cachedChildSelectors}
+                paginationMode={paginationMode}
+                paginationType={paginationType}
+                limitMode={limitMode}
+                isCachingChildSelectors={isCachingChildSelectors}
+                onHighlight={domHighlighterHandler}
+                onElementSelect={handleDOMElementSelection}
+                onShowDatePicker={handleShowDatePicker}
+                onShowDropdown={handleShowDropdown}
+                onShowTimePicker={handleShowTimePicker}
+                onShowDateTimePicker={handleShowDateTimePicker}
+              />

              {/* --- Loading overlay --- */}
              {isCachingChildSelectors && (
@@ -2492,11 +2416,7 @@ export const BrowserWindow = () => {
              )}
            </>
          ) : (
-            <Canvas
-              onCreateRef={setCanvasReference}
-              width={dimensions.width}
-              height={dimensions.height}
-            />
+            <DOMLoadingIndicator />
          )}
        </div>
      </div>
@@ -2591,26 +2511,6 @@ const DOMLoadingIndicator: React.FC = () => {
  );
 };

-
-const drawImage = (image: string, canvas: HTMLCanvasElement): void => {
-    const ctx = canvas.getContext('2d');
-    if (!ctx) return;
-
-    const img = new Image();
-    img.onload = () => {
-        requestAnimationFrame(() => {
-            ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
-        });
-        if (image.startsWith('blob:')) {
-            URL.revokeObjectURL(image);
-        }
-    };
-    img.onerror = () => {
-        console.warn('Failed to load image');
-    };
-    img.src = image;
-};
-
 const modalStyle = {
    top: '50%',
    left: '50%',
--- a/src/pages/PageWrapper.tsx
+++ b/src/pages/PageWrapper.tsx
@@ -133,6 +133,10 @@ export const PageWrapper = () => {
                  path="/register"
                  element={<Register />}
                />
+                <Route
+                  path="/recording-setup"
+                  element={<div />}
+                />
                <Route path="*" element={<NotFoundPage />} />
              </Routes>
            </Box>
--- a/src/pages/RecordingPage.tsx
+++ b/src/pages/RecordingPage.tsx
@@ -43,7 +43,7 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {

  const { setId, socket } = useSocketStore();
  const { setWidth } = useBrowserDimensionsStore();
-  const { browserId, setBrowserId, recordingId, recordingUrl, setRecordingUrl, setRecordingName, setRetrainRobotId } = useGlobalInfoStore();
+  const { browserId, setBrowserId, recordingId, recordingUrl, setRecordingUrl, setRecordingName, setRetrainRobotId, setIsDOMMode } = useGlobalInfoStore();

  const handleShowOutputData = useCallback(() => {
    setShowOutputData(true);
@@ -77,6 +77,8 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
  useEffect(() => {
    let isCancelled = false;
    const handleRecording = async () => {
+      setIsDOMMode(true);
+
      const storedUrl = window.sessionStorage.getItem('recordingUrl');
      if (storedUrl && !recordingUrl) {
        setRecordingUrl(storedUrl);
@@ -137,9 +139,12 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
      if (browserId === 'new-recording') {
        socket?.emit('new-recording');
      }
+      if (recordingUrl && socket) {
+        socket.emit('input:url', recordingUrl);
+      }
      setIsLoaded(true);
    }
-  }, [socket, browserId, recordingName, recordingId, isLoaded]);
+  }, [socket, browserId, recordingName, recordingId, recordingUrl, isLoaded]);

  useEffect(() => {
    socket?.on('loaded', handleLoaded);
@@ -153,26 +158,20 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
    <ActionProvider>
      <BrowserStepsProvider>
        <div id="browser-recorder">
-          {isLoaded ? (
-            <>
-              <Grid container direction="row" style={{ flexGrow: 1, height: '100%' }}>
-                <Grid item xs={12} md={9} lg={9} style={{ height: '100%', overflow: 'hidden', position: 'relative' }}>
-                  <div style={{ height: '100%', overflow: 'auto' }}>
-                    <BrowserContent />
-                    <InterpretationLog isOpen={showOutputData} setIsOpen={setShowOutputData} />
-                  </div>
-                </Grid>
-                <Grid item xs={12} md={3} lg={3} style={{ height: '100%', overflow: 'hidden' }}>
-                  <div className="right-side-panel" style={{ height: '100%' }}>
-                    <RightSidePanel onFinishCapture={handleShowOutputData} />
-                    <BrowserRecordingSave />
-                  </div>
-                </Grid>
-              </Grid>
-            </>
-          ) : (
-            <Loader text={t('recording_page.loader.browser_startup', { url: recordingUrl })} />
-          )}
+          <Grid container direction="row" style={{ flexGrow: 1, height: '100%' }}>
+            <Grid item xs={12} md={9} lg={9} style={{ height: '100%', overflow: 'hidden', position: 'relative' }}>
+              <div style={{ height: '100%', overflow: 'auto' }}>
+                <BrowserContent />
+                <InterpretationLog isOpen={showOutputData} setIsOpen={setShowOutputData} />
+              </div>
+            </Grid>
+            <Grid item xs={12} md={3} lg={3} style={{ height: '100%', overflow: 'hidden' }}>
+              <div className="right-side-panel" style={{ height: '100%' }}>
+                <RightSidePanel onFinishCapture={handleShowOutputData} />
+                <BrowserRecordingSave />
+              </div>
+            </Grid>
+          </Grid>
        </div>
      </BrowserStepsProvider>
    </ActionProvider>