From d7830ee6efe964bd500f2bbe8432a133b45671e4 Mon Sep 17 00:00:00 2001 From: Rohit Rajan Date: Fri, 9 Jan 2026 16:39:02 +0530 Subject: [PATCH 1/2] feat: add rrweb live recorder server side --- .gitignore | 1 + package.json | 2 +- .../classes/RemoteBrowser.ts | 563 ++++++------------ .../classes/bundle-rrweb.js | 10 - .../classes/rrweb-bundle.js | 1 - .../browser-management/classes/rrweb-entry.js | 2 - server/src/browser-management/controller.ts | 5 - .../src/browser-management/inputHandlers.ts | 239 +------- .../workflow-management/classes/Generator.ts | 91 +-- 9 files changed, 187 insertions(+), 727 deletions(-) delete mode 100644 server/src/browser-management/classes/bundle-rrweb.js delete mode 100644 server/src/browser-management/classes/rrweb-bundle.js delete mode 100644 server/src/browser-management/classes/rrweb-entry.js diff --git a/.gitignore b/.gitignore index 6b2b3f98..367f117b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # dependencies /node_modules +/browser/node_modules # misc .DS_Store diff --git a/package.json b/package.json index 3b26d722..f6611a3b 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ "react-dom": "^18.0.0", "react-i18next": "^15.1.3", "react-router-dom": "^6.26.1", - "rrweb-snapshot": "^2.0.0-alpha.4", + "rrweb": "^2.0.0-alpha.4", "sequelize": "^6.37.3", "sequelize-typescript": "^2.1.6", "sharp": "^0.33.5", diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 41a59176..eed60616 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -8,6 +8,7 @@ import { Socket } from "socket.io"; import { PlaywrightBlocker } from '@cliqz/adblocker-playwright'; import fetch from 'cross-fetch'; import logger from '../../logger'; +import { readFileSync } from "fs"; import { InterpreterSettings } from "../../types"; import { WorkflowGenerator } from "../../workflow-management/classes/Generator"; import { WorkflowInterpreter } from "../../workflow-management/classes/Interpreter"; @@ -19,30 +20,17 @@ import { connectToRemoteBrowser } from '../browserConnection'; declare global { interface Window { - rrwebSnapshot?: any; + rrweb?: any; + isRecording?: boolean; + emitEventToBackend?: (event: any) => Promise; } } -interface RRWebSnapshot { - type: number; - childNodes?: RRWebSnapshot[]; - tagName?: string; - attributes?: Record; - textContent?: string; - id: number; - [key: string]: any; -} - -interface ProcessedSnapshot { - snapshot: RRWebSnapshot; - baseUrl: string; -} - -const MEMORY_CONFIG = { - gcInterval: 20000, // Check memory more frequently (20s instead of 60s) - maxHeapSize: 1536 * 1024 * 1024, // 1.5GB - heapUsageThreshold: 0.7 // 70% (reduced threshold to react earlier) -}; +// const MEMORY_CONFIG = { +// gcInterval: 20000, +// maxHeapSize: 1536 * 1024 * 1024, +// heapUsageThreshold: 0.7 +// }; /** * This class represents a remote browser instance. @@ -110,22 +98,11 @@ export class RemoteBrowser { public interpreter: WorkflowInterpreter; public isDOMStreamingActive: boolean = false; - private domUpdateInterval: NodeJS.Timeout | null = null; - private lastScrollPosition = { x: 0, y: 0 }; - private scrollThreshold = 200; // pixels - private snapshotDebounceTimeout: NodeJS.Timeout | null = null; + private scrollThreshold = 200; - private networkRequestTimeout: NodeJS.Timeout | null = null; - private pendingNetworkRequests: string[] = []; - private readonly INITIAL_LOAD_QUIET_PERIOD = 3000; - private networkWaitStartTime: number = 0; - private progressInterval: NodeJS.Timeout | null = null; - private hasShownInitialLoader: boolean = false; - private isInitialLoadInProgress: boolean = false; - - private memoryCleanupInterval: NodeJS.Timeout | null = null; - private memoryManagementInterval: NodeJS.Timeout | null = null; + // private memoryCleanupInterval: NodeJS.Timeout | null = null; + // private memoryManagementInterval: NodeJS.Timeout | null = null; /** * Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and @@ -140,64 +117,53 @@ export class RemoteBrowser { this.generator = new WorkflowGenerator(socket, poolId); } - private async processRRWebSnapshot( - snapshot: RRWebSnapshot - ): Promise { - const baseUrl = this.currentPage?.url() || ""; + // private initializeMemoryManagement(): void { + // this.memoryManagementInterval = setInterval(() => { + // const memoryUsage = process.memoryUsage(); + // const heapUsageRatio = memoryUsage.heapUsed / MEMORY_CONFIG.maxHeapSize; - return { - snapshot, - baseUrl - }; - } + // if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.2) { + // logger.warn( + // "Critical memory pressure detected, triggering emergency cleanup" + // ); + // this.performMemoryCleanup(); + // } else if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold) { + // logger.warn("High memory usage detected, triggering cleanup"); - private initializeMemoryManagement(): void { - this.memoryManagementInterval = setInterval(() => { - const memoryUsage = process.memoryUsage(); - const heapUsageRatio = memoryUsage.heapUsed / MEMORY_CONFIG.maxHeapSize; + // if ( + // global.gc && + // heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.1 + // ) { + // global.gc(); + // } + // } + // }, MEMORY_CONFIG.gcInterval); + // } - if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.2) { - logger.warn( - "Critical memory pressure detected, triggering emergency cleanup" - ); - this.performMemoryCleanup(); - } else if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold) { - logger.warn("High memory usage detected, triggering cleanup"); + // private async performMemoryCleanup(): Promise { + // if (global.gc) { + // try { + // global.gc(); + // logger.info("Garbage collection requested"); + // } catch (error) { + // logger.error("Error during garbage collection:", error); + // } + // } - if ( - global.gc && - heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.1 - ) { - global.gc(); - } - } - }, MEMORY_CONFIG.gcInterval); - } + // if (this.currentPage) { + // try { + // await new Promise((resolve) => setTimeout(resolve, 500)); + // logger.info("CDP session reset completed"); + // } catch (error) { + // logger.error("Error resetting CDP session:", error); + // } + // } - private async performMemoryCleanup(): Promise { - if (global.gc) { - try { - global.gc(); - logger.info("Garbage collection requested"); - } catch (error) { - logger.error("Error during garbage collection:", error); - } - } - - if (this.currentPage) { - try { - await new Promise((resolve) => setTimeout(resolve, 500)); - logger.info("CDP session reset completed"); - } catch (error) { - logger.error("Error resetting CDP session:", error); - } - } - - this.socket.emit("memory-cleanup", { - userId: this.userId, - timestamp: Date.now(), - }); - } + // this.socket.emit("memory-cleanup", { + // userId: this.userId, + // timestamp: Date.now(), + // }); + // } /** * Normalizes URLs to prevent navigation loops while maintaining consistent format @@ -205,9 +171,7 @@ export class RemoteBrowser { private normalizeUrl(url: string): string { try { const parsedUrl = new URL(url); - // Remove trailing slashes except for root path parsedUrl.pathname = parsedUrl.pathname.replace(/\/+$/, '') || '/'; - // Ensure consistent protocol handling parsedUrl.protocol = parsedUrl.protocol.toLowerCase(); return parsedUrl.toString(); } catch { @@ -260,14 +224,6 @@ export class RemoteBrowser { if (scrollDelta > this.scrollThreshold) { this.lastScrollPosition = { x: scrollInfo.x, y: scrollInfo.y }; - - if (this.snapshotDebounceTimeout) { - clearTimeout(this.snapshotDebounceTimeout); - } - - this.snapshotDebounceTimeout = setTimeout(async () => { - await this.makeAndEmitDOMSnapshot(); - }, 300); } } catch (error) { logger.error("Error handling scroll event:", error); @@ -276,79 +232,6 @@ export class RemoteBrowser { ); } - private setupPageChangeListeners(): void { - if (!this.currentPage) return; - - try { - if (!this.currentPage.isClosed()) { - this.currentPage.removeAllListeners("domcontentloaded"); - this.currentPage.removeAllListeners("response"); - } - } catch (error: any) { - logger.warn(`Error removing page change listeners: ${error.message}`); - } - - this.currentPage.on("domcontentloaded", async () => { - if (!this.isInitialLoadInProgress) { - logger.info("DOM content loaded - triggering snapshot"); - await this.makeAndEmitDOMSnapshot(); - } - }); - - this.currentPage.on("response", async (response) => { - const url = response.url(); - const isDocumentRequest = response.request().resourceType() === "document"; - - if (!this.hasShownInitialLoader && isDocumentRequest && !url.includes("about:blank")) { - this.hasShownInitialLoader = true; - this.isInitialLoadInProgress = true; - this.pendingNetworkRequests.push(url); - - if (this.networkRequestTimeout) { - clearTimeout(this.networkRequestTimeout); - this.networkRequestTimeout = null; - } - - if (this.progressInterval) { - clearInterval(this.progressInterval); - this.progressInterval = null; - } - - this.networkWaitStartTime = Date.now(); - - this.progressInterval = setInterval(() => { - const elapsed = Date.now() - this.networkWaitStartTime; - const navigationProgress = Math.min((elapsed / this.INITIAL_LOAD_QUIET_PERIOD) * 40, 35); - const totalProgress = 60 + navigationProgress; - this.emitLoadingProgress(totalProgress, this.pendingNetworkRequests.length); - }, 500); - - logger.debug( - `Initial load network request received: ${url}. Using ${this.INITIAL_LOAD_QUIET_PERIOD}ms quiet period` - ); - - this.networkRequestTimeout = setTimeout(async () => { - logger.info( - `Initial load network quiet period reached (${this.INITIAL_LOAD_QUIET_PERIOD}ms)` - ); - - if (this.progressInterval) { - clearInterval(this.progressInterval); - this.progressInterval = null; - } - - this.emitLoadingProgress(100, this.pendingNetworkRequests.length); - - this.pendingNetworkRequests = []; - this.networkRequestTimeout = null; - this.isInitialLoadInProgress = false; - - await this.makeAndEmitDOMSnapshot(); - }, this.INITIAL_LOAD_QUIET_PERIOD); - } - }); - } - private emitLoadingProgress(progress: number, pendingRequests: number): void { this.socket.emit("domLoadingProgress", { progress: Math.round(progress), @@ -368,16 +251,27 @@ export class RemoteBrowser { } page.on('framenavigated', async (frame) => { - if (frame === page.mainFrame()) { - const currentUrl = page.url(); - if (this.shouldEmitUrlChange(currentUrl)) { - this.lastEmittedUrl = currentUrl; - this.socket.emit('urlChanged', {url: currentUrl, userId: this.userId}); - } + if (frame === page.mainFrame()) { + const currentUrl = page.url(); + if (this.shouldEmitUrlChange(currentUrl)) { + this.lastEmittedUrl = currentUrl; + this.socket.emit('urlChanged', { url: currentUrl, userId: this.userId }); } + + await page.evaluate(() => { + if (window.rrweb && window.isRecording) { + window.isRecording = false; + } + }); + + await page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => { + logger.warn('[rrweb] Network idle timeout on navigation, proceeding with rrweb initialization'); + }); + + await this.initializeRRWebRecording(page); + } }); - // Handle page load events with retry mechanism page.on('load', async () => { const injectScript = async (): Promise => { try { @@ -401,6 +295,96 @@ export class RemoteBrowser { }); } + /** + * Initialize rrweb recording for real-time DOM streaming + * This replaces the snapshot-based approach with live event streaming + */ + private async initializeRRWebRecording(page: Page): Promise { + try { + const rrwebJsPath = require.resolve('rrweb/dist/rrweb.min.js'); + const rrwebScriptContent = readFileSync(rrwebJsPath, 'utf8'); + + await page.context().addInitScript(rrwebScriptContent); + + await page.evaluate((scriptContent) => { + if (typeof window.rrweb === 'undefined') { + try { + (0, eval)(scriptContent); + } catch (e) { + console.error('[rrweb] eval failed:', e); + } + } + }, rrwebScriptContent); + + const rrwebLoaded = await page.evaluate(() => typeof window.rrweb !== 'undefined'); + if (rrwebLoaded) { + logger.debug('[rrweb] Script injected successfully'); + } else { + logger.warn('[rrweb] Script injection failed - window.rrweb not found'); + } + + const isAlreadyExposed = await page.evaluate(() => { + return typeof window.emitEventToBackend === 'function'; + }); + + if (!isAlreadyExposed) { + let hasEmittedFullSnapshot = false; + await page.exposeFunction('emitEventToBackend', (event: any) => { + this.socket.emit('rrweb-event', event); + + if (event.type === 2 && !hasEmittedFullSnapshot) { + hasEmittedFullSnapshot = true; + this.emitLoadingProgress(100, 0); + logger.debug(`[rrweb] Full snapshot sent, loading progress at 100%`); + } + }); + } + + const rrwebStatus = await page.evaluate(() => { + if (!window.rrweb) { + console.error('[rrweb] window.rrweb is not defined!'); + return { success: false, error: 'window.rrweb is not defined' }; + } + + if (window.isRecording) { + return { success: false, error: 'already recording' }; + } + + window.isRecording = true; + + try { + const recordHandle = window.rrweb.record({ + emit(event: any) { + if (window.emitEventToBackend) { + window.emitEventToBackend(event).catch(() => { }); + } + }, + maskAllInputs: false, + recordCanvas: true, + input: true + }); + + (window as any).rrwebRecordHandle = recordHandle; + + return { success: true }; + } catch (error: any) { + console.error('[rrweb] Failed to start recording:', error); + return { success: false, error: error.message }; + } + }); + + if (rrwebStatus.success) { + this.isDOMStreamingActive = true; + this.emitLoadingProgress(80, 0); + this.setupScrollEventListener(); + } else { + logger.error(`Failed to initialize rrweb recording: ${rrwebStatus.error}`); + } + } catch (error: any) { + logger.error(`Failed to initialize rrweb recording: ${error.message}`); + } + } + private getUserAgent() { const userAgents = [ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.140 Safari/537.36', @@ -432,7 +416,6 @@ export class RemoteBrowser { } } catch (error: any) { logger.error(`Enhanced fingerprinting failed: ${error.message}`); - // Don't throw - fallback to basic functionality } } @@ -540,14 +523,18 @@ export class RemoteBrowser { patchedGetter.toString();` ); - await this.context.addInitScript({ path: './server/src/browser-management/classes/rrweb-bundle.js' }); - this.currentPage = await this.context.newPage(); this.emitLoadingProgress(40, 0); await this.setupPageEventListeners(this.currentPage); + await this.currentPage.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => { + logger.warn('[rrweb] Network idle timeout, proceeding with rrweb initialization'); + }); + + await this.initializeRRWebRecording(this.currentPage); + try { const blocker = await PlaywrightBlocker.fromLists(fetch, ['https://easylist.to/easylist/easylist.txt']); await blocker.enableBlockingInPage(this.currentPage as any); @@ -556,12 +543,9 @@ export class RemoteBrowser { console.log('Adblocker initialized'); } catch (error: any) { console.warn('Failed to initialize adblocker, continuing without it:', error.message); - // Still need to set up the CDP session even if blocker fails this.client = await this.currentPage.context().newCDPSession(this.currentPage); } - this.emitLoadingProgress(60, 0); - success = true; logger.log('debug', `Browser initialized successfully for user ${userId}`); } catch (error: any) { @@ -683,13 +667,6 @@ export class RemoteBrowser { await this.captureDirectScreenshot(settings); }); - this.socket.on("rerender", async () => { - logger.debug( - `General rerender event received, checking if for user ${this.userId}` - ); - await this.makeAndEmitDOMSnapshot(); - }); - this.socket.on( "changeTab", async (tabIndex) => await this.changeTab(tabIndex) @@ -719,157 +696,6 @@ export class RemoteBrowser { }; /** - * Subscribe to DOM streaming - simplified version following screenshot pattern - */ - public async subscribeToDOM(): Promise { - if (!this.client) { - logger.warn("DOM streaming requires scraping browser with CDP client"); - return; - } - - try { - this.isDOMStreamingActive = true; - logger.info("DOM streaming started successfully"); - - this.setupScrollEventListener(); - this.setupPageChangeListeners(); - } catch (error) { - logger.error("Failed to start DOM streaming:", error); - this.isDOMStreamingActive = false; - } - } - - /** - * CDP-based DOM snapshot creation using captured network resources - */ - public async makeAndEmitDOMSnapshot(): Promise { - if (!this.currentPage || !this.isDOMStreamingActive) { - return; - } - - try { - // Check if page is still valid and not closed - if (this.currentPage.isClosed()) { - logger.debug("Skipping DOM snapshot - page is closed"); - return; - } - - // Double-check page state after network wait - if (this.currentPage.isClosed()) { - logger.debug("Skipping DOM snapshot - page closed during network wait"); - return; - } - - // Get current scroll position - const currentScrollInfo = await this.currentPage.evaluate(() => ({ - x: window.scrollX, - y: window.scrollY, - maxX: Math.max( - 0, - document.documentElement.scrollWidth - window.innerWidth - ), - maxY: Math.max( - 0, - document.documentElement.scrollHeight - window.innerHeight - ), - documentHeight: document.documentElement.scrollHeight, - })); - - logger.info( - `Creating rrweb snapshot at scroll position: ${currentScrollInfo.y}/${currentScrollInfo.maxY}` - ); - - // Update our tracked scroll position - this.lastScrollPosition = { - x: currentScrollInfo.x, - y: currentScrollInfo.y, - }; - - // Final check before snapshot - if (this.currentPage.isClosed()) { - logger.debug("Skipping DOM snapshot - page closed before snapshot"); - return; - } - - // Capture snapshot using rrweb - const rawSnapshot = await this.currentPage.evaluate(() => { - if (typeof window.rrwebSnapshot === "undefined") { - throw new Error("rrweb-snapshot library not available"); - } - - return window.rrwebSnapshot.snapshot(document, { - inlineImages: false, - collectFonts: true, - }); - }); - - // Process the snapshot to proxy resources - const processedSnapshot = await this.processRRWebSnapshot(rawSnapshot); - - // Add scroll position information - const enhancedSnapshot = { - ...processedSnapshot, - scrollPosition: currentScrollInfo, - captureTime: Date.now(), - }; - - // Emit the processed snapshot - this.emitRRWebSnapshot(enhancedSnapshot); - } catch (error) { - // Handle navigation context destruction gracefully - if ( - error instanceof Error && - (error.message.includes("Execution context was destroyed") || - error.message.includes("most likely because of a navigation") || - error.message.includes("Target closed")) - ) { - logger.debug("DOM snapshot skipped due to page navigation or closure"); - return; - } - - logger.error("Failed to create rrweb snapshot:", error); - this.socket.emit("dom-mode-error", { - userId: this.userId, - message: "Failed to create rrweb snapshot", - error: error instanceof Error ? error.message : String(error), - timestamp: Date.now(), - }); - } - } - - /** - * Emit DOM snapshot to client - following screenshot pattern - */ - private emitRRWebSnapshot(processedSnapshot: ProcessedSnapshot): void { - this.socket.emit("domcast", { - snapshotData: processedSnapshot, - userId: this.userId, - timestamp: Date.now(), - }); - } - - /** - * Stop DOM streaming - following dom snapshot pattern - */ - private async stopDOM(): Promise { - this.isDOMStreamingActive = false; - - if (this.domUpdateInterval) { - clearInterval(this.domUpdateInterval); - this.domUpdateInterval = null; - } - - if (this.networkRequestTimeout) { - clearTimeout(this.networkRequestTimeout); - this.networkRequestTimeout = null; - } - - this.pendingNetworkRequests = []; - - logger.info("DOM streaming stopped successfully"); - } - - /**rrweb-bundle * Terminates the dom snapshot session and closes the remote browser. * If an interpretation was running it will be stopped. * @returns {Promise} @@ -877,35 +703,15 @@ export class RemoteBrowser { public async switchOff(): Promise { this.isDOMStreamingActive = false; - if (this.domUpdateInterval) { - clearInterval(this.domUpdateInterval); - this.domUpdateInterval = null; - } + // if (this.memoryCleanupInterval) { + // clearInterval(this.memoryCleanupInterval); + // this.memoryCleanupInterval = null; + // } - if (this.memoryCleanupInterval) { - clearInterval(this.memoryCleanupInterval); - this.memoryCleanupInterval = null; - } - - if (this.memoryManagementInterval) { - clearInterval(this.memoryManagementInterval); - this.memoryManagementInterval = null; - } - - if (this.progressInterval) { - clearInterval(this.progressInterval); - this.progressInterval = null; - } - - if (this.snapshotDebounceTimeout) { - clearTimeout(this.snapshotDebounceTimeout); - this.snapshotDebounceTimeout = null; - } - - if (this.networkRequestTimeout) { - clearTimeout(this.networkRequestTimeout); - this.networkRequestTimeout = null; - } + // if (this.memoryManagementInterval) { + // clearInterval(this.memoryManagementInterval); + // this.memoryManagementInterval = null; + // } this.removeAllSocketListeners(); @@ -923,7 +729,6 @@ export class RemoteBrowser { logger.warn(`Error removing page listeners: ${error.message}`); } - // Clean up Generator listeners to prevent memory leaks if (this.generator) { try { this.generator.cleanup(); @@ -933,20 +738,12 @@ export class RemoteBrowser { } } - // Stop interpretation with individual error handling (also calls clearState which removes pausing listeners) try { await this.interpreter.stopInterpretation(); } catch (error) { logger.error("Error stopping interpretation during shutdown:", error); } - // Stop DOM streaming with individual error handling - try { - await this.stopDOM(); - } catch (error) { - logger.error("Error stopping DOM during shutdown:", error); - } - try { if (this.client && this.currentPage && !this.currentPage.isClosed()) { const detachPromise = this.client.detach(); @@ -1081,7 +878,6 @@ export class RemoteBrowser { private changeTab = async (tabIndex: number): Promise => { const page = this.currentPage?.context().pages()[tabIndex]; if (page) { - await this.stopDOM(); this.currentPage = page; await this.setupPageEventListeners(this.currentPage); @@ -1093,10 +889,6 @@ export class RemoteBrowser { url: this.currentPage.url(), userId: this.userId }); - if (this.isDOMStreamingActive) { - await this.makeAndEmitDOMSnapshot(); - await this.subscribeToDOM(); - } } else { logger.log('error', `${tabIndex} index out of range of pages`) } @@ -1118,8 +910,7 @@ export class RemoteBrowser { this.currentPage = newPage; if (this.currentPage) { await this.setupPageEventListeners(this.currentPage); - - await this.subscribeToDOM(); + logger.debug('Using rrweb live recording for new page'); } else { logger.log('error', 'Could not get a new page, returned undefined'); } diff --git a/server/src/browser-management/classes/bundle-rrweb.js b/server/src/browser-management/classes/bundle-rrweb.js deleted file mode 100644 index c2fe8b8a..00000000 --- a/server/src/browser-management/classes/bundle-rrweb.js +++ /dev/null @@ -1,10 +0,0 @@ -const esbuild = require('esbuild'); - -esbuild.build({ - entryPoints: ['rrweb-entry.js'], - bundle: true, - minify: true, - outfile: 'rrweb-bundle.js', - format: 'iife', // so that rrwebSnapshot is available on window - globalName: 'rrwebSnapshotBundle' -}).catch(() => process.exit(1)); diff --git a/server/src/browser-management/classes/rrweb-bundle.js b/server/src/browser-management/classes/rrweb-bundle.js deleted file mode 100644 index 58532e0a..00000000 --- a/server/src/browser-management/classes/rrweb-bundle.js +++ /dev/null @@ -1 +0,0 @@ -"use strict";var rrwebSnapshotBundle=(()=>{var g;(function(e){e[e.Document=0]="Document",e[e.DocumentType=1]="DocumentType",e[e.Element=2]="Element",e[e.Text=3]="Text",e[e.CDATA=4]="CDATA",e[e.Comment=5]="Comment"})(g||(g={}));function ue(e){return e.nodeType===e.ELEMENT_NODE}function fe(e){var t=e?.host;return t?.shadowRoot===e}function Q(e){return Object.prototype.toString.call(e)==="[object ShadowRoot]"}function de(e){return e.includes(" background-clip: text;")&&!e.includes(" -webkit-background-clip: text;")&&(e=e.replace(" background-clip: text;"," -webkit-background-clip: text; background-clip: text;")),e}function Z(e){try{var t=e.rules||e.cssRules;return t?de(Array.from(t).map(me).join("")):null}catch{return null}}function me(e){var t=e.cssText;if(pe(e))try{t=Z(e.styleSheet)||t}catch{}return t}function pe(e){return"styleSheet"in e}var he=function(){function e(){this.idNodeMap=new Map,this.nodeMetaMap=new WeakMap}return e.prototype.getId=function(t){var r;if(!t)return-1;var a=(r=this.getMeta(t))===null||r===void 0?void 0:r.id;return a??-1},e.prototype.getNode=function(t){return this.idNodeMap.get(t)||null},e.prototype.getIds=function(){return Array.from(this.idNodeMap.keys())},e.prototype.getMeta=function(t){return this.nodeMetaMap.get(t)||null},e.prototype.removeNodeFromMap=function(t){var r=this,a=this.getId(t);this.idNodeMap.delete(a),t.childNodes&&t.childNodes.forEach(function(i){return r.removeNodeFromMap(i)})},e.prototype.has=function(t){return this.idNodeMap.has(t)},e.prototype.hasNode=function(t){return this.nodeMetaMap.has(t)},e.prototype.add=function(t,r){var a=r.id;this.idNodeMap.set(a,t),this.nodeMetaMap.set(t,r)},e.prototype.replace=function(t,r){var a=this.getNode(t);if(a){var i=this.nodeMetaMap.get(a);i&&this.nodeMetaMap.set(r,i)}this.idNodeMap.set(t,r)},e.prototype.reset=function(){this.idNodeMap=new Map,this.nodeMetaMap=new WeakMap},e}();function ve(e){var t=e.maskInputOptions,r=e.tagName,a=e.type,i=e.value,n=e.maskInputFn,l=i||"";return(t[r.toLowerCase()]||t[a])&&(n?l=n(l):l="*".repeat(l.length)),l}var oe="__rrweb_original__";function ge(e){var t=e.getContext("2d");if(!t)return!0;for(var r=50,a=0;a-1?t=e.split("/").slice(0,3).join("/"):t=e.split("/")[0],t=t.split("?")[0],t}var H,le,Te=/url\((?:(')([^']*)'|(")(.*?)"|([^)]*))\)/gm,Ie=/^(?!www\.|(?:http|ftp)s?:\/\/|[A-Za-z]:\\|\/\/|#).*/,Ee=/^(data:)([^,]*),(.*)/i;function $(e,t){return(e||"").replace(Te,function(r,a,i,n,l,u){var o=i||l||u,f=a||n||"";if(!o)return r;if(!Ie.test(o)||Ee.test(o))return"url(".concat(f).concat(o).concat(f,")");if(o[0]==="/")return"url(".concat(f).concat(we(t)+o).concat(f,")");var c=t.split("/"),p=o.split("/");c.pop();for(var C=0,k=p;C=t.length);){var n=a(Le);if(n.slice(-1)===",")n=j(e,n.substring(0,n.length-1)),i.push(n);else{var l="";n=j(e,n);for(var u=!1;;){var o=t.charAt(r);if(o===""){i.push((n+l).trim());break}else if(u)o===")"&&(u=!1);else if(o===","){r+=1,i.push((n+l).trim());break}else o==="("&&(u=!0);l+=o,r+=1}}}return i.join(", ")}function j(e,t){if(!t||t.trim()==="")return t;var r=e.createElement("a");return r.href=t,r.href}function Me(e){return!!(e.tagName==="svg"||e.ownerSVGElement)}function ee(){var e=document.createElement("a");return e.href="",e.href}function Oe(e,t,r,a){return r==="src"||r==="href"&&a&&!(t==="use"&&a[0]==="#")||r==="xlink:href"&&a&&a[0]!=="#"||r==="background"&&a&&(t==="table"||t==="td"||t==="th")?j(e,a):r==="srcset"&&a?Ne(e,a):r==="style"&&a?$(a,ee()):t==="object"&&r==="data"&&a?j(e,a):a}function Re(e,t,r){if(typeof t=="string"){if(e.classList.contains(t))return!0}else for(var a=e.classList.length;a--;){var i=e.classList[a];if(t.test(i))return!0}return r?e.matches(r):!1}function J(e,t,r){if(!e)return!1;if(e.nodeType!==e.ELEMENT_NODE)return r?J(e.parentNode,t,r):!1;for(var a=e.classList.length;a--;){var i=e.classList[a];if(t.test(i))return!0}return r?J(e.parentNode,t,r):!1}function Ae(e,t,r){var a=e.nodeType===e.ELEMENT_NODE?e:e.parentElement;if(a===null)return!1;if(typeof t=="string"){if(a.classList.contains(t)||a.closest(".".concat(t)))return!0}else if(J(a,t,!0))return!0;return!!(r&&(a.matches(r)||a.closest(r)))}function De(e,t,r){var a=e.contentWindow;if(a){var i=!1,n;try{n=a.document.readyState}catch{return}if(n!=="complete"){var l=setTimeout(function(){i||(t(),i=!0)},r);e.addEventListener("load",function(){clearTimeout(l),i=!0,t()});return}var u="about:blank";if(a.location.href!==u||e.src===u||e.src==="")return setTimeout(t,0),e.addEventListener("load",t);e.addEventListener("load",t)}}function Fe(e,t,r){var a=!1,i;try{i=e.sheet}catch{return}if(!i){var n=setTimeout(function(){a||(t(),a=!0)},r);e.addEventListener("load",function(){clearTimeout(n),a=!0,t()})}}function Ue(e,t){var r=t.doc,a=t.mirror,i=t.blockClass,n=t.blockSelector,l=t.maskTextClass,u=t.maskTextSelector,o=t.inlineStylesheet,f=t.maskInputOptions,c=f===void 0?{}:f,p=t.maskTextFn,C=t.maskInputFn,k=t.dataURLOptions,S=k===void 0?{}:k,T=t.inlineImages,I=t.recordCanvas,E=t.keepIframeSrcFn,m=t.newlyAddedElement,s=m===void 0?!1:m,y=We(r,a);switch(e.nodeType){case e.DOCUMENT_NODE:return e.compatMode!=="CSS1Compat"?{type:g.Document,childNodes:[],compatMode:e.compatMode}:{type:g.Document,childNodes:[]};case e.DOCUMENT_TYPE_NODE:return{type:g.DocumentType,name:e.name,publicId:e.publicId,systemId:e.systemId,rootId:y};case e.ELEMENT_NODE:return _e(e,{doc:r,blockClass:i,blockSelector:n,inlineStylesheet:o,maskInputOptions:c,maskInputFn:C,dataURLOptions:S,inlineImages:T,recordCanvas:I,keepIframeSrcFn:E,newlyAddedElement:s,rootId:y});case e.TEXT_NODE:return Pe(e,{maskTextClass:l,maskTextSelector:u,maskTextFn:p,rootId:y});case e.CDATA_SECTION_NODE:return{type:g.CDATA,textContent:"",rootId:y};case e.COMMENT_NODE:return{type:g.Comment,textContent:e.textContent||"",rootId:y};default:return!1}}function We(e,t){if(t.hasNode(e)){var r=t.getId(e);return r===1?void 0:r}}function Pe(e,t){var r,a=t.maskTextClass,i=t.maskTextSelector,n=t.maskTextFn,l=t.rootId,u=e.parentNode&&e.parentNode.tagName,o=e.textContent,f=u==="STYLE"?!0:void 0,c=u==="SCRIPT"?!0:void 0;if(f&&o){try{e.nextSibling||e.previousSibling||!((r=e.parentNode.sheet)===null||r===void 0)&&r.cssRules&&(o=Ce(e.parentNode.sheet))}catch(p){console.warn("Cannot get CSS styles from text's parentNode. Error: ".concat(p),e)}o=$(o,ee())}return c&&(o=""),!f&&!c&&o&&Ae(e,a,i)&&(o=n?n(o):o.replace(/[\S]/g,"*")),{type:g.Text,textContent:o||"",isStyle:f,rootId:l}}function _e(e,t){for(var r=t.doc,a=t.blockClass,i=t.blockSelector,n=t.inlineStylesheet,l=t.maskInputOptions,u=l===void 0?{}:l,o=t.maskInputFn,f=t.dataURLOptions,c=f===void 0?{}:f,p=t.inlineImages,C=t.recordCanvas,k=t.keepIframeSrcFn,S=t.newlyAddedElement,T=S===void 0?!1:S,I=t.rootId,E=Re(e,a,i),m=be(e),s={},y=e.attributes.length,A=0;A { - logger.log('debug', 'Handling mousedown event emitted from client'); - await handleWrapper(handleMousedown, userId, coordinates); -} - -/** - * A mousedown event handler. - * Reproduces the click on the remote browser instance - * and generates pair data for the recorded workflow. - * @param activeBrowser - the active remote browser {@link RemoteBrowser} - * @param page - the active page of the remote browser - * @param x - the x coordinate of the mousedown event - * @param y - the y coordinate of the mousedown event - * @category BrowserManagement - */ -const handleMousedown = async (activeBrowser: RemoteBrowser, page: Page, { x, y }: Coordinates) => { - try { - if (page.isClosed()) { - logger.log("debug", `Ignoring mousedown event: page is closed`); - return; - } - - const generator = activeBrowser.generator; - await generator.onClick({ x, y }, page); - const previousUrl = page.url(); - const tabsBeforeClick = page.context().pages().length; - await page.mouse.click(x, y); - // try if the click caused a navigation to a new url - try { - await page.waitForNavigation({ timeout: 2000 }); - const currentUrl = page.url(); - if (currentUrl !== previousUrl) { - generator.notifyUrlChange(currentUrl); - } - } catch (e) { - const { message } = e as Error; - } //ignore possible timeouts - - // check if any new page was opened by the click - const tabsAfterClick = page.context().pages().length; - const numOfNewPages = tabsAfterClick - tabsBeforeClick; - if (numOfNewPages > 0) { - for (let i = 1; i <= numOfNewPages; i++) { - const newPage = page.context().pages()[tabsAfterClick - i]; - if (newPage) { - generator.notifyOnNewTab(newPage, tabsAfterClick - i); - } - } - } - logger.log("debug", `Clicked on position x:${x}, y:${y}`); - } catch (e) { - const { message } = e as Error; - logger.log("warn", `Error handling mousedown event: ${message}`); - } -}; - -/** - * A wrapper function for handling the wheel event. - * @param socket The socket connection - * @param scrollDeltas - the scroll deltas of the wheel event - * @category HelperFunctions - */ -const onWheel = async (scrollDeltas: ScrollDeltas, userId: string) => { - logger.log('debug', 'Handling scroll event emitted from client'); - await handleWrapper(handleWheel, userId, scrollDeltas); -}; - -/** - * A wheel event handler. - * Reproduces the wheel event on the remote browser instance. - * Scroll is not generated for the workflow pair. This is because - * Playwright scrolls elements into focus on any action. - * @param activeBrowser - the active remote browser {@link RemoteBrowser} - * @param page - the active page of the remote browser - * @param deltaX - the delta x of the wheel event - * @param deltaY - the delta y of the wheel event - * @category BrowserManagement - */ -const handleWheel = async (activeBrowser: RemoteBrowser, page: Page, { deltaX, deltaY }: ScrollDeltas) => { - try { - if (page.isClosed()) { - logger.log("debug", `Ignoring wheel event: page is closed`); - return; - } - - await page.mouse.wheel(deltaX, deltaY).catch(error => { - logger.log('warn', `Wheel event failed: ${error.message}`); - }); - logger.log('debug', `Scrolled horizontally ${deltaX} pixels and vertically ${deltaY} pixels`); - } catch (e) { - const { message } = e as Error; - logger.log('warn', `Error handling wheel event: ${message}`); - } -}; - -/** - * A wrapper function for handling the mousemove event. - * @param socket The socket connection - * @param coordinates - the coordinates of the mousemove event - * @category HelperFunctions - */ -const onMousemove = async (coordinates: Coordinates, userId: string) => { - logger.log('debug', 'Handling mousemove event emitted from client'); - await handleWrapper(handleMousemove, userId, coordinates); -} - -/** - * A mousemove event handler. - * Reproduces the mousemove event on the remote browser instance - * and generates data for the client's highlighter. - * Mousemove is also not reflected in the workflow. - * @param activeBrowser - the active remote browser {@link RemoteBrowser} - * @param page - the active page of the remote browser - * @param x - the x coordinate of the mousemove event - * @param y - the y coordinate of the mousemove event - * @category BrowserManagement - */ -const handleMousemove = async (activeBrowser: RemoteBrowser, page: Page, { x, y }: Coordinates) => { - try { - if (page.isClosed()) { - logger.log("debug", `Ignoring mousemove event: page is closed`); - return; - } - - const generator = activeBrowser.generator; - await page.mouse.move(x, y); - // throttle(async () => { - // if (!page.isClosed()) { - // await generator.generateDataForHighlighter(page, { x, y }); - // } - // }, 100)(); - logger.log("debug", `Moved over position x:${x}, y:${y}`); - } catch (e) { - const { message } = e as Error; - logger.log("error", message); - } -} - -/** - * A wrapper function for handling the keydown event. - * @param socket The socket connection - * @param keyboardInput - the keyboard input of the keydown event - * @category HelperFunctions - */ -const onKeydown = async (keyboardInput: KeyboardInput, userId: string) => { - logger.log('debug', 'Handling keydown event emitted from client'); - await handleWrapper(handleKeydown, userId, keyboardInput); -} - -/** - * A keydown event handler. - * Reproduces the keydown event on the remote browser instance - * and generates the workflow pair data. - * @param activeBrowser - the active remote browser {@link RemoteBrowser} - * @param page - the active page of the remote browser - * @param key - the pressed key - * @param coordinates - the coordinates, where the keydown event happened - * @category BrowserManagement - */ -const handleKeydown = async (activeBrowser: RemoteBrowser, page: Page, { key, coordinates }: KeyboardInput) => { - try { - if (page.isClosed()) { - logger.log("debug", `Ignoring keydown event: page is closed`); - return; - } - - const generator = activeBrowser.generator; - await page.keyboard.down(key); - await generator.onKeyboardInput(key, coordinates, page); - logger.log("debug", `Key ${key} pressed`); - } catch (e) { - const { message } = e as Error; - logger.log("warn", `Error handling keydown event: ${message}`); - } -}; - /** * Handles the date selection event. * @param activeBrowser - the active remote browser {@link RemoteBrowser} @@ -493,7 +311,7 @@ const handleChangeUrl = async (activeBrowser: RemoteBrowser, page: Page, url: st try { await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 }); - await page.waitForTimeout(2000); + await page.waitForTimeout(500); logger.log("debug", `Went to ${url}`); } catch (e) { const { message } = e as Error; @@ -689,8 +507,7 @@ const handleClickAction = async ( } } - await new Promise((resolve) => setTimeout(resolve, 2000)); - await activeBrowser.makeAndEmitDOMSnapshot(); + await new Promise((resolve) => setTimeout(resolve, 300)); } catch (e) { const { message } = e as Error; logger.log( @@ -778,47 +595,6 @@ const onDOMKeyboardAction = async ( await handleWrapper(handleKeyboardAction, userId, data); }; -/** - * Handles the workflow pair event. - * @param activeBrowser - the active remote browser {@link RemoteBrowser} - * @param page - the active page of the remote browser - * @param data - the data of the workflow pair event - * @category BrowserManagement - */ -const handleWorkflowPair = async ( - activeBrowser: RemoteBrowser, - page: Page, - data: { pair: WhereWhatPair; userId: string } -) => { - try { - if (page.isClosed()) { - logger.log("debug", `Ignoring workflow pair event: page is closed`); - return; - } - - const generator = activeBrowser.generator; - await generator.onDOMWorkflowPair(page, data); - logger.log("debug", `Workflow pair processed from frontend`); - } catch (e) { - const { message } = e as Error; - logger.log("warn", `Error handling workflow pair event: ${message}`); - } -}; - -/** - * A wrapper function for handling the workflow pair event. - * @param socket The socket connection - * @param data - the data of the workflow pair event - * @category HelperFunctions - */ -const onDOMWorkflowPair = async ( - data: { pair: WhereWhatPair; userId: string }, - userId: string -) => { - logger.log("debug", "Handling workflow pair event emitted from client"); - await handleWrapper(handleWorkflowPair, userId, data); -}; - /** * Handles the remove action event. * This is called when a user discards a capture action (list or text) that was already emitted to the backend. @@ -1026,11 +802,6 @@ const onTestPaginationScroll = async ( * @category BrowserManagement */ const registerInputHandlers = (socket: Socket, userId: string) => { - // Register handlers with the socket - socket.on("input:mousedown", (data) => onMousedown(data, userId)); - socket.on("input:wheel", (data) => onWheel(data, userId)); - socket.on("input:mousemove", (data) => onMousemove(data, userId)); - socket.on("input:keydown", (data) => onKeydown(data, userId)); socket.on("input:keyup", (data) => onKeyup(data, userId)); socket.on("input:url", (data) => onChangeUrl(data, userId)); socket.on("input:refresh", () => onRefresh(userId)); @@ -1045,7 +816,6 @@ const registerInputHandlers = (socket: Socket, userId: string) => { socket.on("dom:click", (data) => onDOMClickAction(data, userId)); socket.on("dom:keypress", (data) => onDOMKeyboardAction(data, userId)); - socket.on("dom:addpair", (data) => onDOMWorkflowPair(data, userId)); socket.on("testPaginationScroll", (data) => onTestPaginationScroll(data, userId, socket)); }; @@ -1058,10 +828,6 @@ const registerInputHandlers = (socket: Socket, userId: string) => { */ const removeInputHandlers = (socket: Socket) => { try { - socket.removeAllListeners("input:mousedown"); - socket.removeAllListeners("input:wheel"); - socket.removeAllListeners("input:mousemove"); - socket.removeAllListeners("input:keydown"); socket.removeAllListeners("input:keyup"); socket.removeAllListeners("input:url"); socket.removeAllListeners("input:refresh"); @@ -1075,7 +841,6 @@ const removeInputHandlers = (socket: Socket) => { socket.removeAllListeners("dom:input"); socket.removeAllListeners("dom:click"); socket.removeAllListeners("dom:keypress"); - socket.removeAllListeners("dom:addpair"); socket.removeAllListeners("removeAction"); socket.removeAllListeners("testPaginationScroll"); } catch (error: any) { diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 63a466d0..25ecf9f7 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -198,7 +198,6 @@ export class WorkflowGenerator { private async getSelectorsForSchema(page: Page, schema: Record): Promise { const selectors = Object.values(schema).map((field) => field.selector); - // Verify if the selectors are present and actionable on the current page const actionableSelectors: string[] = []; for (const selector of selectors) { const isActionable = await page.isVisible(selector).catch(() => false); @@ -235,7 +234,6 @@ export class WorkflowGenerator { private addPairToWorkflowAndNotifyClient = async (pair: WhereWhatPair, page: Page) => { let matched = false; - // Check for scrapeSchema actions and enhance the where condition if (pair.what[0].action === 'scrapeSchema') { const schema = pair.what[0]?.args?.[0]; if (schema) { @@ -244,7 +242,6 @@ export class WorkflowGenerator { } } - // Validate if the pair is already in the workflow if (pair.where.selectors && pair.where.selectors[0]) { const match = selectorAlreadyInWorkflow(pair.where.selectors[0], this.workflowRecord.workflow); if (match) { @@ -260,7 +257,6 @@ export class WorkflowGenerator { } } - // Handle cases where the where condition isn't already present if (!matched) { const handled = await this.handleOverShadowing(pair, page, this.generatedData.lastIndex || 0); if (!handled) { @@ -282,7 +278,6 @@ export class WorkflowGenerator { } } - // Emit the updated workflow to the client this.socket.emit('workflow', this.workflowRecord); logger.log('info', `Workflow emitted`); }; @@ -367,7 +362,6 @@ export class WorkflowGenerator { await this.addPairToWorkflowAndNotifyClient(pair, page); }; - // Handles click events on the DOM, generating a pair for the click action public onDOMClickAction = async (page: Page, data: { selector: string, url: string, @@ -388,7 +382,6 @@ export class WorkflowGenerator { }], }; - // Handle special input elements with cursor positioning if (elementInfo && coordinates && (elementInfo.tagName === 'INPUT' || elementInfo.tagName === 'TEXTAREA')) { pair.what[0] = { @@ -403,7 +396,6 @@ export class WorkflowGenerator { await this.addPairToWorkflowAndNotifyClient(pair, page); }; - // Handles keyboard actions on the DOM, generating a pair for the key press action public onDOMKeyboardAction = async (page: Page, data: { selector: string, key: string, @@ -430,7 +422,6 @@ export class WorkflowGenerator { await this.addPairToWorkflowAndNotifyClient(pair, page); }; - // Handles navigation events on the DOM, generating a pair for the navigation action public onDOMNavigation = async (page: Page, data: { url: string, currentUrl: string, @@ -450,13 +441,6 @@ export class WorkflowGenerator { await this.addPairToWorkflowAndNotifyClient(pair, page); }; - // Handles workflow pair events on the DOM - public onDOMWorkflowPair = async (page: Page, data: { pair: WhereWhatPair, userId: string }) => { - const { pair } = data; - - await this.addPairToWorkflowAndNotifyClient(pair, page); - }; - /** * Generates a pair for the click event. * @param coordinates The coordinates of the click event. @@ -471,25 +455,22 @@ export class WorkflowGenerator { const elementInfo = await getElementInformation(page, coordinates, '', false); console.log("Element info: ", elementInfo); - // Check if clicked element is a select dropdown const isDropdown = elementInfo?.tagName === 'SELECT'; if (isDropdown && elementInfo.innerHTML) { - // Parse options from innerHTML const options = elementInfo.innerHTML .split(' { const valueMatch = optionHtml.match(/value="([^"]*)"/); const disabledMatch = optionHtml.includes('disabled="disabled"'); const selectedMatch = optionHtml.includes('selected="selected"'); - // Extract text content between > and const textMatch = optionHtml.match(/>([^<]*)} - */ - public onKeyboardInput = async (key: string, coordinates: Coordinates, page: Page) => { - let where: WhereWhatPair["where"] = { url: this.getBestUrl(page.url()) }; - const selector = await this.generateSelector(page, coordinates, ActionType.Keydown); - - const elementInfo = await getElementInformation(page, coordinates, '', false); - const inputType = elementInfo?.attributes?.type || "text"; - - if (selector) { - where.selectors = [selector]; - } - const pair: WhereWhatPair = { - where, - what: [{ - action: 'press', - args: [selector, encrypt(key), inputType], - }], - } - if (selector) { - this.generatedData.lastUsedSelector = selector; - this.generatedData.lastAction = 'press'; - } - await this.addPairToWorkflowAndNotifyClient(pair, page); - }; - /** * Returns tag name and text content for the specified selector * used in customAction for decision modal @@ -940,7 +885,7 @@ export class WorkflowGenerator { return pair; }) - .filter((pair) => pair !== null) as WhereWhatPair[]; // Remove null entries + .filter((pair) => pair !== null) as WhereWhatPair[]; if (actionWasRemoved) { logger.log("info", `Action with actionId ${actionId} removed from workflow`); @@ -996,24 +941,6 @@ export class WorkflowGenerator { } } - /** - * Returns the currently generated workflow without all the generated flag actions. - * @param workflow The workflow for removing the generated flag actions from. - * @private - * @returns {WorkflowFile} - */ - private removeAllGeneratedFlags = (workflow: WorkflowFile): WorkflowFile => { - for (let i = 0; i < workflow.workflow.length; i++) { - if ( - workflow.workflow[i].what[0] && - workflow.workflow[i].what[0].action === 'flag' && - workflow.workflow[i].what[0].args?.includes('generated')) { - workflow.workflow[i].what.splice(0, 1); - } - } - return workflow; - }; - /** * Adds generated flag actions to the workflow's pairs' what conditions. * @param workflow The workflow for adding the generated flag actions from. @@ -1127,7 +1054,6 @@ export class WorkflowGenerator { : await getSelectors(page, coordinates); if (this.paginationMode && selectorBasedOnCustomAction) { - // Chain selectors in specific priority order const selectors = selectorBasedOnCustomAction; const selectorChain = [ selectors?.iframeSelector?.full, @@ -1324,17 +1250,15 @@ export class WorkflowGenerator { if (this.workflowRecord.workflow[index].where.selectors?.includes(selector)) { break; } else { - // add new selector to the where part of the overshadowing pair this.workflowRecord.workflow[index].where.selectors?.push(selector); } } } - // push the action automatically to the first/the closest rule which would be overShadowed + this.workflowRecord.workflow[index].what = this.workflowRecord.workflow[index].what.concat(pair.what); return true; } else { - // notify client about overshadowing a further rule return false; } } @@ -1352,8 +1276,7 @@ export class WorkflowGenerator { const parsedUrl = new URL(url); const protocol = parsedUrl.protocol === 'https:' || parsedUrl.protocol === 'http:' ? `${parsedUrl.protocol}//` : parsedUrl.protocol; const regex = new RegExp(/(?=.*[A-Z])/g) - // remove all params with uppercase letters, they are most likely dynamically generated - // also escapes all regex characters from the params + const search = parsedUrl.search .replace(/[.*+?^${}()|[\]\\]/g, '\\$&') .split('&').map((param, index) => { @@ -1380,8 +1303,6 @@ export class WorkflowGenerator { * @param workflow The workflow to be checked. */ private checkWorkflowForParams = (workflow: WorkflowFile): string[] | null => { - // for now the where condition cannot have any params, so we're checking only what part of the pair - // where only the args part of what condition can have a parameter for (const pair of workflow.workflow) { for (const condition of pair.what) { if (condition.args) { From dddac2d9665bb9db480a33839e62bf899d8a79d1 Mon Sep 17 00:00:00 2001 From: Rohit Rajan Date: Fri, 9 Jan 2026 17:59:18 +0530 Subject: [PATCH 2/2] feat: add replayer func client side --- .../recorder => legacy/src}/Canvas.tsx | 0 .../recorder => legacy/src}/Highlighter.tsx | 0 .../src}/coordinateMapper.ts | 0 {src/helpers => legacy/src}/inputHelpers.ts | 0 .../classes/RemoteBrowser.ts | 1 - src/components/browser/BrowserWindow.tsx | 340 +++-------- src/components/pickers/DatePicker.tsx | 6 +- .../pickers/DateTimeLocalPicker.tsx | 6 +- src/components/pickers/Dropdown.tsx | 6 +- src/components/pickers/TimePicker.tsx | 6 +- .../recorder/DOMBrowserRenderer.tsx | 551 +++++++----------- src/components/recorder/RightSidePanel.tsx | 41 +- src/context/globalInfo.tsx | 60 +- src/helpers/capturedElementHighlighter.ts | 7 +- src/pages/RecordingPage.tsx | 16 - 15 files changed, 308 insertions(+), 732 deletions(-) rename {src/components/recorder => legacy/src}/Canvas.tsx (100%) rename {src/components/recorder => legacy/src}/Highlighter.tsx (100%) rename {src/helpers => legacy/src}/coordinateMapper.ts (100%) rename {src/helpers => legacy/src}/inputHelpers.ts (100%) diff --git a/src/components/recorder/Canvas.tsx b/legacy/src/Canvas.tsx similarity index 100% rename from src/components/recorder/Canvas.tsx rename to legacy/src/Canvas.tsx diff --git a/src/components/recorder/Highlighter.tsx b/legacy/src/Highlighter.tsx similarity index 100% rename from src/components/recorder/Highlighter.tsx rename to legacy/src/Highlighter.tsx diff --git a/src/helpers/coordinateMapper.ts b/legacy/src/coordinateMapper.ts similarity index 100% rename from src/helpers/coordinateMapper.ts rename to legacy/src/coordinateMapper.ts diff --git a/src/helpers/inputHelpers.ts b/legacy/src/inputHelpers.ts similarity index 100% rename from src/helpers/inputHelpers.ts rename to legacy/src/inputHelpers.ts diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index eed60616..a6aa1ccf 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -640,7 +640,6 @@ export class RemoteBrowser { private removeAllSocketListeners(): void { try { this.socket.removeAllListeners('captureDirectScreenshot'); - this.socket.removeAllListeners('rerender'); this.socket.removeAllListeners('settings'); this.socket.removeAllListeners('changeTab'); this.socket.removeAllListeners('addTab'); diff --git a/src/components/browser/BrowserWindow.tsx b/src/components/browser/BrowserWindow.tsx index af9d61a9..013d5a94 100644 --- a/src/components/browser/BrowserWindow.tsx +++ b/src/components/browser/BrowserWindow.tsx @@ -7,7 +7,6 @@ import { useBrowserSteps, TextStep, ListStep } from '../../context/browserSteps' import { useGlobalInfoStore } from '../../context/globalInfo'; import { useTranslation } from 'react-i18next'; import { AuthContext } from '../../context/auth'; -import { coordinateMapper } from '../../helpers/coordinateMapper'; import { useBrowserDimensionsStore } from '../../context/browserDimensions'; import { clientSelectorGenerator, ElementFingerprint } from "../../helpers/clientSelectorGenerator"; import { capturedElementHighlighter } from "../../helpers/capturedElementHighlighter"; @@ -36,79 +35,6 @@ interface AttributeOption { value: string; } -interface ViewportInfo { - width: number; - height: number; -} - -interface RRWebSnapshot { - type: number; - childNodes?: RRWebSnapshot[]; - tagName?: string; - attributes?: Record; - textContent: string; - id: number; - [key: string]: any; -} - -interface ProcessedSnapshot { - snapshot: RRWebSnapshot; - resources: { - stylesheets: Array<{ - href: string; - content: string; - media?: string; - }>; - images: Array<{ - src: string; - dataUrl: string; - alt?: string; - }>; - fonts: Array<{ - url: string; - dataUrl: string; - format?: string; - }>; - scripts: Array<{ - src: string; - content: string; - type?: string; - }>; - media: Array<{ - src: string; - dataUrl: string; - type: string; - }>; - }; - baseUrl: string; - viewport: { width: number; height: number }; - timestamp: number; - processingStats: { - totalReplacements: number; - discoveredResources: { - images: number; - stylesheets: number; - scripts: number; - fonts: number; - media: number; - }; - cachedResources: { - stylesheets: number; - images: number; - fonts: number; - scripts: number; - media: number; - }; - totalCacheSize: number; - }; -} - -interface RRWebDOMCastData { - snapshotData: ProcessedSnapshot; - userId: string; - timestamp: number; -} - const getAttributeOptions = (tagName: string, elementInfo: ElementInfo | null): AttributeOption[] => { if (!elementInfo) return []; switch (tagName.toLowerCase()) { @@ -154,8 +80,6 @@ export const BrowserWindow = () => { const [attributeOptions, setAttributeOptions] = useState([]); const [selectedElement, setSelectedElement] = useState<{ selector: string, info: ElementInfo | null } | null>(null); const [currentListId, setCurrentListId] = useState(null); - const [viewportInfo, setViewportInfo] = useState({ width: browserWidth, height: browserHeight }); - const [isLoading, setIsLoading] = useState(false); const [cachedChildSelectors, setCachedChildSelectors] = useState([]); const [processingGroupCoordinates, setProcessingGroupCoordinates] = useState>([]); const [listSelector, setListSelector] = useState(null); @@ -177,7 +101,7 @@ export const BrowserWindow = () => { const [manuallyAddedFieldIds, setManuallyAddedFieldIds] = useState>(new Set()); const { socket } = useSocketStore(); - const { notify, currentTextActionId, currentListActionId, updateDOMMode, isDOMMode, currentSnapshot } = useGlobalInfoStore(); + const { notify, currentTextActionId, currentListActionId, updateDOMMode, isDOMMode } = useGlobalInfoStore(); const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext(); const { addTextStep, addListStep, browserSteps } = useBrowserSteps(); @@ -258,27 +182,11 @@ export const BrowserWindow = () => { [] ); - const rrwebSnapshotHandler = useCallback( - (data: RRWebDOMCastData) => { - if (!data.userId || data.userId === user?.id) { - if (data.snapshotData && data.snapshotData.snapshot) { - updateDOMMode(true, data.snapshotData); - socket?.emit("dom-mode-enabled"); - setIsLoading(false); - } else { - setIsLoading(false); - } - } - }, - [user?.id, socket, updateDOMMode] - ); - const domModeHandler = useCallback( (data: any) => { if (!data.userId || data.userId === user?.id) { updateDOMMode(true); socket?.emit("dom-mode-enabled"); - setIsLoading(false); } }, [user?.id, socket, updateDOMMode] @@ -288,18 +196,21 @@ export const BrowserWindow = () => { (data: any) => { if (!data.userId || data.userId === user?.id) { updateDOMMode(false); - setIsLoading(false); + + if (data.error) { + notify("error", data.error); + } } }, - [user?.id, updateDOMMode] + [user?.id, updateDOMMode, notify] ); useEffect(() => { - if (isDOMMode) { + if (isDOMMode) { clientSelectorGenerator.setGetList(getList); clientSelectorGenerator.setListSelector(listSelector || ""); clientSelectorGenerator.setPaginationMode(paginationMode); - } + } }, [isDOMMode, getList, listSelector, paginationMode]); const createFieldsFromChildSelectors = useCallback( @@ -698,7 +609,7 @@ export const BrowserWindow = () => { return finalFields; }, - [currentSnapshot] + [] ); const removeParentChildDuplicates = ( @@ -798,7 +709,7 @@ export const BrowserWindow = () => { clientSelectorGenerator.setListSelector(listSelector); - if (currentSnapshot && cachedListSelector !== listSelector) { + if (cachedListSelector !== listSelector) { setCachedChildSelectors([]); setIsCachingChildSelectors(true); setCachedListSelector(listSelector); @@ -878,7 +789,6 @@ export const BrowserWindow = () => { listSelector, socket, getList, - currentSnapshot, cachedListSelector, pendingNotification, notify, @@ -989,10 +899,6 @@ export const BrowserWindow = () => { } }, [browserSteps, getText, getList, listSelector, currentTextActionId, currentListActionId, isDOMMode, manuallyAddedFieldIds]); - useEffect(() => { - coordinateMapper.updateDimensions(dimensions.width, dimensions.height, viewportInfo.width, viewportInfo.height); - }, [viewportInfo, dimensions.width, dimensions.height]); - useEffect(() => { if (listSelector) { sessionStorage.setItem('recordingListSelector', listSelector); @@ -1027,21 +933,18 @@ export const BrowserWindow = () => { useEffect(() => { if (socket) { - socket.on("domcast", rrwebSnapshotHandler); socket.on("dom-mode-enabled", domModeHandler); socket.on("dom-mode-error", domModeErrorHandler); } return () => { if (socket) { - socket.off("domcast", rrwebSnapshotHandler); socket.off("dom-mode-enabled", domModeHandler); socket.off("dom-mode-error", domModeErrorHandler); } }; }, [ socket, - rrwebSnapshotHandler, domModeHandler, domModeErrorHandler, ]); @@ -1066,14 +969,15 @@ export const BrowserWindow = () => { isDOMMode?: boolean; }) => { if (paginationMode && paginationSelector) { - return; + return; } + if (!getText && !getList) { setHighlighterData(null); return; } - if (!isDOMMode || !currentSnapshot) { + if (!isDOMMode) { return; } @@ -1187,7 +1091,6 @@ export const BrowserWindow = () => { }, [ isDOMMode, - currentSnapshot, getText, getList, socket, @@ -1200,105 +1103,6 @@ export const BrowserWindow = () => { ] ); - const highlighterHandler = useCallback((data: { rect: DOMRect, selector: string, elementInfo: ElementInfo | null, childSelectors?: string[], isDOMMode?: boolean; }) => { - if (paginationMode && paginationSelector) { - return; - } - if (isDOMMode || data.isDOMMode) { - domHighlighterHandler(data); - return; - } - - const now = performance.now(); - if (now - highlighterUpdateRef.current < 16) { - return; - } - highlighterUpdateRef.current = now; - - const mappedRect = new DOMRect( - data.rect.x, - data.rect.y, - data.rect.width, - data.rect.height - ); - - const mappedData = { - ...data, - rect: mappedRect - }; - - if (getList === true) { - if (listSelector) { - socket?.emit('listSelector', { selector: listSelector }); - const hasValidChildSelectors = Array.isArray(mappedData.childSelectors) && mappedData.childSelectors.length > 0; - - if (limitMode) { - setHighlighterData(null); - } else if (paginationMode) { - if (paginationType !== '' && !['none', 'scrollDown', 'scrollUp'].includes(paginationType)) { - setHighlighterData(mappedData); - } else { - setHighlighterData(null); - } - } else if (mappedData.childSelectors && mappedData.childSelectors.includes(mappedData.selector)) { - setHighlighterData(mappedData); - } else if (mappedData.elementInfo?.isIframeContent && mappedData.childSelectors) { - const isIframeChild = mappedData.childSelectors.some(childSelector => - mappedData.selector.includes(':>>') && - childSelector.split(':>>').some(part => - mappedData.selector.includes(part.trim()) - ) - ); - setHighlighterData(isIframeChild ? mappedData : null); - } else if (mappedData.selector.includes(':>>') && hasValidChildSelectors) { - const selectorParts = mappedData.selector.split(':>>').map(part => part.trim()); - const isValidMixedSelector = selectorParts.some(part => - mappedData.childSelectors!.some(childSelector => - childSelector.includes(part) - ) - ); - setHighlighterData(isValidMixedSelector ? mappedData : null); - } else if (mappedData.elementInfo?.isShadowRoot && mappedData.childSelectors) { - const isShadowChild = mappedData.childSelectors.some(childSelector => - mappedData.selector.includes('>>') && - childSelector.split('>>').some(part => - mappedData.selector.includes(part.trim()) - ) - ); - setHighlighterData(isShadowChild ? mappedData : null); - } else if (mappedData.selector.includes('>>') && hasValidChildSelectors) { - const selectorParts = mappedData.selector.split('>>').map(part => part.trim()); - const isValidMixedSelector = selectorParts.some(part => - mappedData.childSelectors!.some(childSelector => - childSelector.includes(part) - ) - ); - setHighlighterData(isValidMixedSelector ? mappedData : null); - } else { - setHighlighterData(null); - } - } else { - setHighlighterData(mappedData); - } - } else { - setHighlighterData(mappedData); - } - }, [getList, socket, listSelector, paginationMode, paginationType, limitMode]); - - useEffect(() => { - document.addEventListener("mousemove", onMouseMove, false); - if (socket) { - socket.off("highlighter", highlighterHandler); - socket.on("highlighter", highlighterHandler); - } - return () => { - document.removeEventListener("mousemove", onMouseMove); - if (socket) { - socket.off("highlighter", highlighterHandler); - } - }; - }, [socket, highlighterHandler, getList, listSelector]); - useEffect(() => { if (socket && listSelector) { socket.emit('setGetList', { getList: true }); @@ -1533,7 +1337,6 @@ export const BrowserWindow = () => { ] ); - const handleClick = (e: React.MouseEvent) => { if (highlighterData) { const shouldProcessClick = true; @@ -1720,49 +1523,50 @@ export const BrowserWindow = () => { default: data = selectedElement.info?.innerText || ''; } - { - if (getText === true) { - addTextStep('', data, { + + if (getText === true) { + addTextStep('', data, { + selector: selectedElement.selector, + tag: selectedElement.info?.tagName, + isShadow: highlighterData?.isShadow || selectedElement.info?.isShadowRoot, + attribute: attribute + }, currentTextActionId || `text-${crypto.randomUUID()}`); + } + + if (getList === true && listSelector && currentListId) { + const newField: TextStep = { + id: Date.now(), + type: 'text', + label: `Label ${Object.keys(fields).length + 1}`, + data: data, + selectorObj: { selector: selectedElement.selector, tag: selectedElement.info?.tagName, - isShadow: highlighterData?.isShadow || selectedElement.info?.isShadowRoot, + isShadow: highlighterData?.isShadow || highlighterData?.elementInfo?.isShadowRoot, attribute: attribute - }, currentTextActionId || `text-${crypto.randomUUID()}`); - } - if (getList === true && listSelector && currentListId) { - const newField: TextStep = { - id: Date.now(), - type: 'text', - label: `Label ${Object.keys(fields).length + 1}`, - data: data, - selectorObj: { - selector: selectedElement.selector, - tag: selectedElement.info?.tagName, - isShadow: highlighterData?.isShadow || highlighterData?.elementInfo?.isShadowRoot, - attribute: attribute - } - }; - - const updatedFields = { - ...fields, - [newField.id]: newField - }; - - setFields(updatedFields); - - if (listSelector) { - addListStep( - listSelector, - updatedFields, - currentListId, - currentListActionId || `list-${crypto.randomUUID()}`, - { type: "", selector: paginationSelector, isShadow: highlighterData?.isShadow }, - undefined, - highlighterData?.isShadow - ); } + }; + + const updatedFields = { + ...fields, + [newField.id]: newField + }; + + setFields(updatedFields); + + if (listSelector) { + addListStep( + listSelector, + updatedFields, + currentListId, + currentListActionId || `list-${crypto.randomUUID()}`, + { type: "", selector: paginationSelector, isShadow: highlighterData?.isShadow }, + undefined, + highlighterData?.isShadow + ); } } + } setShowAttributeModal(false); @@ -1817,7 +1621,6 @@ export const BrowserWindow = () => { style={{ width: browserWidth }} id="browser-window" > - {/* Attribute selection modal */} {(getText === true || getList === true) && ( { /> )} - {/* Main content area */}
- {/* Add CSS for the spinner animation */} + style={{ + position: "relative", + width: "100%", + height: dimensions.height, + overflow: "hidden", + borderRadius: "0px 0px 5px 5px", + }} + > - - - - `; - - if (!iframeDoc) { - throw new Error("Cannot access iframe document"); - } - - // Write the skeleton into the iframe - iframeDoc.open(); - iframeDoc.write(skeleton); - iframeDoc.close(); - - const mirror = createMirror(); - - try { - rebuild(snapshotData.snapshot, { - doc: iframeDoc, - mirror: mirror, - hackCss: false, - cache: { stylesWithHoverClass: new Map() }, - afterAppend: (node) => { - if (node.nodeType === Node.TEXT_NODE && node.textContent) { - const text = node.textContent.trim(); - - if ( - text.startsWith("<") && - text.includes(">") && - text.length > 50 - ) { - if (node.parentNode) { - node.parentNode.removeChild(node); - } - } - } - }, - }); - } catch (rebuildError) { - console.error("rrweb rebuild failed:", rebuildError); - throw new Error(`rrweb rebuild failed: ${rebuildError}`); - } - - setIsRendered(true); - setupIframeInteractions(iframeDoc); - } catch (error) { - console.error("Error rendering rrweb snapshot:", error); - } - }, - [setupIframeInteractions, isInCaptureMode, isCachingChildSelectors] - ); - - useEffect(() => { - if (snapshot && iframeRef.current) { - renderRRWebSnapshot(snapshot); - } - }, [snapshot]); - - useEffect(() => { - if (isRendered && iframeRef.current) { - const iframeDoc = iframeRef.current.contentDocument; - if (iframeDoc) { - setupIframeInteractions(iframeDoc); - } - } - }, [getText, getList, listSelector, isRendered, setupIframeInteractions]); - useEffect(() => { return () => { - if (iframeRef.current) { - const iframeDoc = iframeRef.current.contentDocument; - if (iframeDoc) { - const handlers = (iframeDoc as any)._domRendererHandlers; - if (handlers) { - Object.entries(handlers).forEach(([event, handler]) => { - const options: boolean | AddEventListenerOptions = ['wheel', 'touchstart', 'touchmove'].includes(event) - ? { passive: false } - : false; - iframeDoc.removeEventListener( - event, - handler as EventListener, - options - ); - }); - } - } + if (replayerRef.current) { + replayerRef.current.pause(); + replayerRef.current = null; } }; }, []); + /** + * Listen for rrweb events from backend and add to replayer + */ + useEffect(() => { + if (!socket) { + console.warn('No socket available, skipping event listener setup'); + return; + } + + const handleRRWebEvent = (event: any) => { + if (!replayerRef.current && event.type === 2) { + const container = document.getElementById('mirror-container'); + if (!container) { + console.warn('Container #mirror-container not found'); + return; + } + + const replayer = new Replayer([], { + root: container, + liveMode: true, + mouseTail: false + }); + + replayer.startLive(); + replayer.addEvent(event); + + replayerRef.current = replayer; + + setTimeout(() => { + const replayerWrapper = container.querySelector('.replayer-wrapper'); + const replayerIframe = replayerWrapper?.querySelector('iframe') as HTMLIFrameElement; + + if (replayerIframe) { + replayerIframe.style.width = '100%'; + replayerIframe.style.height = '100%'; + replayerIframe.style.border = 'none'; + replayerIframe.style.position = 'absolute'; + replayerIframe.style.top = '0'; + replayerIframe.style.left = '0'; + replayerIframe.style.backgroundColor = '#ffffff'; + replayerIframe.style.display = 'block'; + replayerIframe.style.pointerEvents = 'auto'; + + replayerIframe.id = 'dom-browser-iframe'; + + replayerIframeRef.current = replayerIframe; + + try { + const iframeDoc = replayerIframe.contentDocument; + if (iframeDoc) { + setupIframeInteractions(iframeDoc); + } + } catch (err) { + console.warn('Error accessing iframe:', err); + } + + replayer.on('fullsnapshot-rebuilded', () => { + const iframe = replayerIframeRef.current; + if (iframe && iframe.contentDocument) { + setupIframeInteractions(iframe.contentDocument); + + iframe.style.pointerEvents = 'auto'; + const wrapper = container.querySelector('.replayer-wrapper') as HTMLElement; + if(wrapper) wrapper.style.pointerEvents = 'auto'; + + setIsRendered(true); + } + }); + + } else { + console.warn('Could not find iframe in replayer-wrapper'); + } + }, 150); + } else if (replayerRef.current) { + replayerRef.current.addEvent(event); + } + }; + + socket.on('rrweb-event', handleRRWebEvent); + socket.emit('request-refresh'); + + return () => { + socket.off('rrweb-event', handleRRWebEvent); + }; + }, [socket, setupIframeInteractions]); + + useEffect(() => { + const iframe = replayerIframeRef.current; + if (iframe && iframe.contentDocument) { + setupIframeInteractions(iframe.contentDocument); + } + }, [setupIframeInteractions]); + return (
-