vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add rrweb live recorder server side

Browse Source
This commit is contained in:
Rohit Rajan
2026-01-09 16:39:02 +05:30
parent 670fcfe186
commit d7830ee6ef
9 changed files with 187 additions and 727 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1,5 +1,6 @@
# dependencies
/node_modules
/browser/node_modules
# misc
.DS_Store

2
package.json
View File

@@ -67,7 +67,7 @@
"react-dom": "^18.0.0",
"react-i18next": "^15.1.3",
"react-router-dom": "^6.26.1",
"rrweb-snapshot": "^2.0.0-alpha.4",
"rrweb": "^2.0.0-alpha.4",
"sequelize": "^6.37.3",
"sequelize-typescript": "^2.1.6",
"sharp": "^0.33.5",

563
server/src/browser-management/classes/RemoteBrowser.ts
View File

@@ -8,6 +8,7 @@ import { Socket } from "socket.io";
import { PlaywrightBlocker } from '@cliqz/adblocker-playwright';
import fetch from 'cross-fetch';
import logger from '../../logger';
import { readFileSync } from "fs";
import { InterpreterSettings } from "../../types";
import { WorkflowGenerator } from "../../workflow-management/classes/Generator";
import { WorkflowInterpreter } from "../../workflow-management/classes/Interpreter";
@@ -19,30 +20,17 @@ import { connectToRemoteBrowser } from '../browserConnection';
declare global {
interface Window {
rrwebSnapshot?: any;
rrweb?: any;
isRecording?: boolean;
emitEventToBackend?: (event: any) => Promise<void>;
}
}
interface RRWebSnapshot {
type: number;
childNodes?: RRWebSnapshot[];
tagName?: string;
attributes?: Record<string, string>;
textContent?: string;
id: number;
[key: string]: any;
}
interface ProcessedSnapshot {
snapshot: RRWebSnapshot;
baseUrl: string;
}
const MEMORY_CONFIG = {
gcInterval: 20000, // Check memory more frequently (20s instead of 60s)
maxHeapSize: 1536 * 1024 * 1024, // 1.5GB
heapUsageThreshold: 0.7 // 70% (reduced threshold to react earlier)
};
// const MEMORY_CONFIG = {
// gcInterval: 20000,
// maxHeapSize: 1536 * 1024 * 1024,
// heapUsageThreshold: 0.7
// };
/**
* This class represents a remote browser instance.
@@ -110,22 +98,11 @@ export class RemoteBrowser {
public interpreter: WorkflowInterpreter;
public isDOMStreamingActive: boolean = false;
private domUpdateInterval: NodeJS.Timeout | null = null;
private lastScrollPosition = { x: 0, y: 0 };
private scrollThreshold = 200; // pixels
private snapshotDebounceTimeout: NodeJS.Timeout | null = null;
private scrollThreshold = 200;
private networkRequestTimeout: NodeJS.Timeout | null = null;
private pendingNetworkRequests: string[] = [];
private readonly INITIAL_LOAD_QUIET_PERIOD = 3000;
private networkWaitStartTime: number = 0;
private progressInterval: NodeJS.Timeout | null = null;
private hasShownInitialLoader: boolean = false;
private isInitialLoadInProgress: boolean = false;
private memoryCleanupInterval: NodeJS.Timeout | null = null;
private memoryManagementInterval: NodeJS.Timeout | null = null;
// private memoryCleanupInterval: NodeJS.Timeout | null = null;
// private memoryManagementInterval: NodeJS.Timeout | null = null;
/**
* Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and
@@ -140,64 +117,53 @@ export class RemoteBrowser {
this.generator = new WorkflowGenerator(socket, poolId);
}
private async processRRWebSnapshot(
snapshot: RRWebSnapshot
): Promise<ProcessedSnapshot> {
const baseUrl = this.currentPage?.url() || "";
// private initializeMemoryManagement(): void {
// this.memoryManagementInterval = setInterval(() => {
// const memoryUsage = process.memoryUsage();
// const heapUsageRatio = memoryUsage.heapUsed / MEMORY_CONFIG.maxHeapSize;
return {
snapshot,
baseUrl
};
}
// if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.2) {
// logger.warn(
// "Critical memory pressure detected, triggering emergency cleanup"
// );
// this.performMemoryCleanup();
// } else if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold) {
// logger.warn("High memory usage detected, triggering cleanup");
private initializeMemoryManagement(): void {
this.memoryManagementInterval = setInterval(() => {
const memoryUsage = process.memoryUsage();
const heapUsageRatio = memoryUsage.heapUsed / MEMORY_CONFIG.maxHeapSize;
// if (
// global.gc &&
// heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.1
// ) {
// global.gc();
// }
// }
// }, MEMORY_CONFIG.gcInterval);
// }
if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.2) {
logger.warn(
"Critical memory pressure detected, triggering emergency cleanup"
);
this.performMemoryCleanup();
} else if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold) {
logger.warn("High memory usage detected, triggering cleanup");
// private async performMemoryCleanup(): Promise<void> {
// if (global.gc) {
// try {
// global.gc();
// logger.info("Garbage collection requested");
// } catch (error) {
// logger.error("Error during garbage collection:", error);
// }
// }
if (
global.gc &&
heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.1
) {
global.gc();
}
}
}, MEMORY_CONFIG.gcInterval);
}
// if (this.currentPage) {
// try {
// await new Promise((resolve) => setTimeout(resolve, 500));
// logger.info("CDP session reset completed");
// } catch (error) {
// logger.error("Error resetting CDP session:", error);
// }
// }
private async performMemoryCleanup(): Promise<void> {
if (global.gc) {
try {
global.gc();
logger.info("Garbage collection requested");
} catch (error) {
logger.error("Error during garbage collection:", error);
}
}
if (this.currentPage) {
try {
await new Promise((resolve) => setTimeout(resolve, 500));
logger.info("CDP session reset completed");
} catch (error) {
logger.error("Error resetting CDP session:", error);
}
}
this.socket.emit("memory-cleanup", {
userId: this.userId,
timestamp: Date.now(),
});
}
// this.socket.emit("memory-cleanup", {
// userId: this.userId,
// timestamp: Date.now(),
// });
// }
/**
* Normalizes URLs to prevent navigation loops while maintaining consistent format
@@ -205,9 +171,7 @@ export class RemoteBrowser {
private normalizeUrl(url: string): string {
try {
const parsedUrl = new URL(url);
// Remove trailing slashes except for root path
parsedUrl.pathname = parsedUrl.pathname.replace(/\/+$/, '') || '/';
// Ensure consistent protocol handling
parsedUrl.protocol = parsedUrl.protocol.toLowerCase();
return parsedUrl.toString();
} catch {
@@ -260,14 +224,6 @@ export class RemoteBrowser {
if (scrollDelta > this.scrollThreshold) {
this.lastScrollPosition = { x: scrollInfo.x, y: scrollInfo.y };
if (this.snapshotDebounceTimeout) {
clearTimeout(this.snapshotDebounceTimeout);
}
this.snapshotDebounceTimeout = setTimeout(async () => {
await this.makeAndEmitDOMSnapshot();
}, 300);
}
} catch (error) {
logger.error("Error handling scroll event:", error);
@@ -276,79 +232,6 @@ export class RemoteBrowser {
);
}
private setupPageChangeListeners(): void {
if (!this.currentPage) return;
try {
if (!this.currentPage.isClosed()) {
this.currentPage.removeAllListeners("domcontentloaded");
this.currentPage.removeAllListeners("response");
}
} catch (error: any) {
logger.warn(`Error removing page change listeners: ${error.message}`);
}
this.currentPage.on("domcontentloaded", async () => {
if (!this.isInitialLoadInProgress) {
logger.info("DOM content loaded - triggering snapshot");
await this.makeAndEmitDOMSnapshot();
}
});
this.currentPage.on("response", async (response) => {
const url = response.url();
const isDocumentRequest = response.request().resourceType() === "document";
if (!this.hasShownInitialLoader && isDocumentRequest && !url.includes("about:blank")) {
this.hasShownInitialLoader = true;
this.isInitialLoadInProgress = true;
this.pendingNetworkRequests.push(url);
if (this.networkRequestTimeout) {
clearTimeout(this.networkRequestTimeout);
this.networkRequestTimeout = null;
}
if (this.progressInterval) {
clearInterval(this.progressInterval);
this.progressInterval = null;
}
this.networkWaitStartTime = Date.now();
this.progressInterval = setInterval(() => {
const elapsed = Date.now() - this.networkWaitStartTime;
const navigationProgress = Math.min((elapsed / this.INITIAL_LOAD_QUIET_PERIOD) * 40, 35);
const totalProgress = 60 + navigationProgress;
this.emitLoadingProgress(totalProgress, this.pendingNetworkRequests.length);
}, 500);
logger.debug(
`Initial load network request received: ${url}. Using ${this.INITIAL_LOAD_QUIET_PERIOD}ms quiet period`
);
this.networkRequestTimeout = setTimeout(async () => {
logger.info(
`Initial load network quiet period reached (${this.INITIAL_LOAD_QUIET_PERIOD}ms)`
);
if (this.progressInterval) {
clearInterval(this.progressInterval);
this.progressInterval = null;
}
this.emitLoadingProgress(100, this.pendingNetworkRequests.length);
this.pendingNetworkRequests = [];
this.networkRequestTimeout = null;
this.isInitialLoadInProgress = false;
await this.makeAndEmitDOMSnapshot();
}, this.INITIAL_LOAD_QUIET_PERIOD);
}
});
}
private emitLoadingProgress(progress: number, pendingRequests: number): void {
this.socket.emit("domLoadingProgress", {
progress: Math.round(progress),
@@ -368,16 +251,27 @@ export class RemoteBrowser {
}
page.on('framenavigated', async (frame) => {
if (frame === page.mainFrame()) {
const currentUrl = page.url();
if (this.shouldEmitUrlChange(currentUrl)) {
this.lastEmittedUrl = currentUrl;
this.socket.emit('urlChanged', {url: currentUrl, userId: this.userId});
}
if (frame === page.mainFrame()) {
const currentUrl = page.url();
if (this.shouldEmitUrlChange(currentUrl)) {
this.lastEmittedUrl = currentUrl;
this.socket.emit('urlChanged', { url: currentUrl, userId: this.userId });
}
await page.evaluate(() => {
if (window.rrweb && window.isRecording) {
window.isRecording = false;
}
});
await page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {
logger.warn('[rrweb] Network idle timeout on navigation, proceeding with rrweb initialization');
});
await this.initializeRRWebRecording(page);
}
});
// Handle page load events with retry mechanism
page.on('load', async () => {
const injectScript = async (): Promise<boolean> => {
try {
@@ -401,6 +295,96 @@ export class RemoteBrowser {
});
}
/**
* Initialize rrweb recording for real-time DOM streaming
* This replaces the snapshot-based approach with live event streaming
*/
private async initializeRRWebRecording(page: Page): Promise<void> {
try {
const rrwebJsPath = require.resolve('rrweb/dist/rrweb.min.js');
const rrwebScriptContent = readFileSync(rrwebJsPath, 'utf8');
await page.context().addInitScript(rrwebScriptContent);
await page.evaluate((scriptContent) => {
if (typeof window.rrweb === 'undefined') {
try {
(0, eval)(scriptContent);
} catch (e) {
console.error('[rrweb] eval failed:', e);
}
}
}, rrwebScriptContent);
const rrwebLoaded = await page.evaluate(() => typeof window.rrweb !== 'undefined');
if (rrwebLoaded) {
logger.debug('[rrweb] Script injected successfully');
} else {
logger.warn('[rrweb] Script injection failed - window.rrweb not found');
}
const isAlreadyExposed = await page.evaluate(() => {
return typeof window.emitEventToBackend === 'function';
});
if (!isAlreadyExposed) {
let hasEmittedFullSnapshot = false;
await page.exposeFunction('emitEventToBackend', (event: any) => {
this.socket.emit('rrweb-event', event);
if (event.type === 2 && !hasEmittedFullSnapshot) {
hasEmittedFullSnapshot = true;
this.emitLoadingProgress(100, 0);
logger.debug(`[rrweb] Full snapshot sent, loading progress at 100%`);
}
});
}
const rrwebStatus = await page.evaluate(() => {
if (!window.rrweb) {
console.error('[rrweb] window.rrweb is not defined!');
return { success: false, error: 'window.rrweb is not defined' };
}
if (window.isRecording) {
return { success: false, error: 'already recording' };
}
window.isRecording = true;
try {
const recordHandle = window.rrweb.record({
emit(event: any) {
if (window.emitEventToBackend) {
window.emitEventToBackend(event).catch(() => { });
}
},
maskAllInputs: false,
recordCanvas: true,
input: true
});
(window as any).rrwebRecordHandle = recordHandle;
return { success: true };
} catch (error: any) {
console.error('[rrweb] Failed to start recording:', error);
return { success: false, error: error.message };
}
});
if (rrwebStatus.success) {
this.isDOMStreamingActive = true;
this.emitLoadingProgress(80, 0);
this.setupScrollEventListener();
} else {
logger.error(`Failed to initialize rrweb recording: ${rrwebStatus.error}`);
}
} catch (error: any) {
logger.error(`Failed to initialize rrweb recording: ${error.message}`);
}
}
private getUserAgent() {
const userAgents = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.140 Safari/537.36',
@@ -432,7 +416,6 @@ export class RemoteBrowser {
}
} catch (error: any) {
logger.error(`Enhanced fingerprinting failed: ${error.message}`);
// Don't throw - fallback to basic functionality
}
}
@@ -540,14 +523,18 @@ export class RemoteBrowser {
patchedGetter.toString();`
);
await this.context.addInitScript({ path: './server/src/browser-management/classes/rrweb-bundle.js' });
this.currentPage = await this.context.newPage();
this.emitLoadingProgress(40, 0);
await this.setupPageEventListeners(this.currentPage);
await this.currentPage.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {
logger.warn('[rrweb] Network idle timeout, proceeding with rrweb initialization');
});
await this.initializeRRWebRecording(this.currentPage);
try {
const blocker = await PlaywrightBlocker.fromLists(fetch, ['https://easylist.to/easylist/easylist.txt']);
await blocker.enableBlockingInPage(this.currentPage as any);
@@ -556,12 +543,9 @@ export class RemoteBrowser {
console.log('Adblocker initialized');
} catch (error: any) {
console.warn('Failed to initialize adblocker, continuing without it:', error.message);
// Still need to set up the CDP session even if blocker fails
this.client = await this.currentPage.context().newCDPSession(this.currentPage);
}
this.emitLoadingProgress(60, 0);
success = true;
logger.log('debug', `Browser initialized successfully for user ${userId}`);
} catch (error: any) {
@@ -683,13 +667,6 @@ export class RemoteBrowser {
await this.captureDirectScreenshot(settings);
});
this.socket.on("rerender", async () => {
logger.debug(
`General rerender event received, checking if for user ${this.userId}`
);
await this.makeAndEmitDOMSnapshot();
});
this.socket.on(
"changeTab",
async (tabIndex) => await this.changeTab(tabIndex)
@@ -719,157 +696,6 @@ export class RemoteBrowser {
};
/**
* Subscribe to DOM streaming - simplified version following screenshot pattern
*/
public async subscribeToDOM(): Promise<void> {
if (!this.client) {
logger.warn("DOM streaming requires scraping browser with CDP client");
return;
}
try {
this.isDOMStreamingActive = true;
logger.info("DOM streaming started successfully");
this.setupScrollEventListener();
this.setupPageChangeListeners();
} catch (error) {
logger.error("Failed to start DOM streaming:", error);
this.isDOMStreamingActive = false;
}
}
/**
* CDP-based DOM snapshot creation using captured network resources
*/
public async makeAndEmitDOMSnapshot(): Promise<void> {
if (!this.currentPage || !this.isDOMStreamingActive) {
return;
}
try {
// Check if page is still valid and not closed
if (this.currentPage.isClosed()) {
logger.debug("Skipping DOM snapshot - page is closed");
return;
}
// Double-check page state after network wait
if (this.currentPage.isClosed()) {
logger.debug("Skipping DOM snapshot - page closed during network wait");
return;
}
// Get current scroll position
const currentScrollInfo = await this.currentPage.evaluate(() => ({
x: window.scrollX,
y: window.scrollY,
maxX: Math.max(
0,
document.documentElement.scrollWidth - window.innerWidth
),
maxY: Math.max(
0,
document.documentElement.scrollHeight - window.innerHeight
),
documentHeight: document.documentElement.scrollHeight,
}));
logger.info(
`Creating rrweb snapshot at scroll position: ${currentScrollInfo.y}/${currentScrollInfo.maxY}`
);
// Update our tracked scroll position
this.lastScrollPosition = {
x: currentScrollInfo.x,
y: currentScrollInfo.y,
};
// Final check before snapshot
if (this.currentPage.isClosed()) {
logger.debug("Skipping DOM snapshot - page closed before snapshot");
return;
}
// Capture snapshot using rrweb
const rawSnapshot = await this.currentPage.evaluate(() => {
if (typeof window.rrwebSnapshot === "undefined") {
throw new Error("rrweb-snapshot library not available");
}
return window.rrwebSnapshot.snapshot(document, {
inlineImages: false,
collectFonts: true,
});
});
// Process the snapshot to proxy resources
const processedSnapshot = await this.processRRWebSnapshot(rawSnapshot);
// Add scroll position information
const enhancedSnapshot = {
...processedSnapshot,
scrollPosition: currentScrollInfo,
captureTime: Date.now(),
};
// Emit the processed snapshot
this.emitRRWebSnapshot(enhancedSnapshot);
} catch (error) {
// Handle navigation context destruction gracefully
if (
error instanceof Error &&
(error.message.includes("Execution context was destroyed") ||
error.message.includes("most likely because of a navigation") ||
error.message.includes("Target closed"))
) {
logger.debug("DOM snapshot skipped due to page navigation or closure");
return;
}
logger.error("Failed to create rrweb snapshot:", error);
this.socket.emit("dom-mode-error", {
userId: this.userId,
message: "Failed to create rrweb snapshot",
error: error instanceof Error ? error.message : String(error),
timestamp: Date.now(),
});
}
}
/**
* Emit DOM snapshot to client - following screenshot pattern
*/
private emitRRWebSnapshot(processedSnapshot: ProcessedSnapshot): void {
this.socket.emit("domcast", {
snapshotData: processedSnapshot,
userId: this.userId,
timestamp: Date.now(),
});
}
/**
* Stop DOM streaming - following dom snapshot pattern
*/
private async stopDOM(): Promise<void> {
this.isDOMStreamingActive = false;
if (this.domUpdateInterval) {
clearInterval(this.domUpdateInterval);
this.domUpdateInterval = null;
}
if (this.networkRequestTimeout) {
clearTimeout(this.networkRequestTimeout);
this.networkRequestTimeout = null;
}
this.pendingNetworkRequests = [];
logger.info("DOM streaming stopped successfully");
}
/**rrweb-bundle
* Terminates the dom snapshot session and closes the remote browser.
* If an interpretation was running it will be stopped.
* @returns {Promise<void>}
@@ -877,35 +703,15 @@ export class RemoteBrowser {
public async switchOff(): Promise<void> {
this.isDOMStreamingActive = false;
if (this.domUpdateInterval) {
clearInterval(this.domUpdateInterval);
this.domUpdateInterval = null;
}
// if (this.memoryCleanupInterval) {
// clearInterval(this.memoryCleanupInterval);
// this.memoryCleanupInterval = null;
// }
if (this.memoryCleanupInterval) {
clearInterval(this.memoryCleanupInterval);
this.memoryCleanupInterval = null;
}
if (this.memoryManagementInterval) {
clearInterval(this.memoryManagementInterval);
this.memoryManagementInterval = null;
}
if (this.progressInterval) {
clearInterval(this.progressInterval);
this.progressInterval = null;
}
if (this.snapshotDebounceTimeout) {
clearTimeout(this.snapshotDebounceTimeout);
this.snapshotDebounceTimeout = null;
}
if (this.networkRequestTimeout) {
clearTimeout(this.networkRequestTimeout);
this.networkRequestTimeout = null;
}
// if (this.memoryManagementInterval) {
// clearInterval(this.memoryManagementInterval);
// this.memoryManagementInterval = null;
// }
this.removeAllSocketListeners();
@@ -923,7 +729,6 @@ export class RemoteBrowser {
logger.warn(`Error removing page listeners: ${error.message}`);
}
// Clean up Generator listeners to prevent memory leaks
if (this.generator) {
try {
this.generator.cleanup();
@@ -933,20 +738,12 @@ export class RemoteBrowser {
}
}
// Stop interpretation with individual error handling (also calls clearState which removes pausing listeners)
try {
await this.interpreter.stopInterpretation();
} catch (error) {
logger.error("Error stopping interpretation during shutdown:", error);
}
// Stop DOM streaming with individual error handling
try {
await this.stopDOM();
} catch (error) {
logger.error("Error stopping DOM during shutdown:", error);
}
try {
if (this.client && this.currentPage && !this.currentPage.isClosed()) {
const detachPromise = this.client.detach();
@@ -1081,7 +878,6 @@ export class RemoteBrowser {
private changeTab = async (tabIndex: number): Promise<void> => {
const page = this.currentPage?.context().pages()[tabIndex];
if (page) {
await this.stopDOM();
this.currentPage = page;
await this.setupPageEventListeners(this.currentPage);
@@ -1093,10 +889,6 @@ export class RemoteBrowser {
url: this.currentPage.url(),
userId: this.userId
});
if (this.isDOMStreamingActive) {
await this.makeAndEmitDOMSnapshot();
await this.subscribeToDOM();
}
} else {
logger.log('error', `${tabIndex} index out of range of pages`)
}
@@ -1118,8 +910,7 @@ export class RemoteBrowser {
this.currentPage = newPage;
if (this.currentPage) {
await this.setupPageEventListeners(this.currentPage);
await this.subscribeToDOM();
logger.debug('Using rrweb live recording for new page');
} else {
logger.log('error', 'Could not get a new page, returned undefined');
}

10
server/src/browser-management/classes/bundle-rrweb.js
View File

@@ -1,10 +0,0 @@
const esbuild = require('esbuild');
esbuild.build({
entryPoints: ['rrweb-entry.js'],
bundle: true,
minify: true,
outfile: 'rrweb-bundle.js',
format: 'iife', // so that rrwebSnapshot is available on window
globalName: 'rrwebSnapshotBundle'
}).catch(() => process.exit(1));

1
server/src/browser-management/classes/rrweb-bundle.js
View File

File diff suppressed because one or more lines are too long

2
server/src/browser-management/classes/rrweb-entry.js
View File

@@ -1,2 +0,0 @@
import { snapshot } from 'rrweb-snapshot';
window.rrwebSnapshot = { snapshot };

5
server/src/browser-management/controller.ts
View File

@@ -31,10 +31,6 @@ export const initializeRemoteBrowserForRecording = (userId: string, mode: string
if (activeId) {
const remoteBrowser = browserPool.getRemoteBrowser(activeId);
remoteBrowser?.updateSocket(socket);
if (remoteBrowser?.isDOMStreamingActive) {
remoteBrowser?.makeAndEmitDOMSnapshot();
}
} else {
const browserSession = new RemoteBrowser(socket, userId, id);
browserSession.interpreter.subscribeToPausing();
@@ -43,7 +39,6 @@ export const initializeRemoteBrowserForRecording = (userId: string, mode: string
await browserSession.initialize(userId);
await browserSession.registerEditorEvents();
await browserSession.subscribeToDOM();
logger.info('DOM streaming started for remote browser in recording mode');
browserPool.addRemoteBrowser(id, browserSession, userId, false, "recording");

239
server/src/browser-management/inputHandlers.ts
View File

@@ -102,188 +102,6 @@ const handleGenerateAction =
}
}
/**
* A wrapper function for handling mousedown event.
* @param socket The socket connection
* @param coordinates - coordinates of the mouse click
* @category HelperFunctions
*/
const onMousedown = async (coordinates: Coordinates, userId: string) => {
logger.log('debug', 'Handling mousedown event emitted from client');
await handleWrapper(handleMousedown, userId, coordinates);
}
/**
* A mousedown event handler.
* Reproduces the click on the remote browser instance
* and generates pair data for the recorded workflow.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param x - the x coordinate of the mousedown event
* @param y - the y coordinate of the mousedown event
* @category BrowserManagement
*/
const handleMousedown = async (activeBrowser: RemoteBrowser, page: Page, { x, y }: Coordinates) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring mousedown event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await generator.onClick({ x, y }, page);
const previousUrl = page.url();
const tabsBeforeClick = page.context().pages().length;
await page.mouse.click(x, y);
// try if the click caused a navigation to a new url
try {
await page.waitForNavigation({ timeout: 2000 });
const currentUrl = page.url();
if (currentUrl !== previousUrl) {
generator.notifyUrlChange(currentUrl);
}
} catch (e) {
const { message } = e as Error;
} //ignore possible timeouts
// check if any new page was opened by the click
const tabsAfterClick = page.context().pages().length;
const numOfNewPages = tabsAfterClick - tabsBeforeClick;
if (numOfNewPages > 0) {
for (let i = 1; i <= numOfNewPages; i++) {
const newPage = page.context().pages()[tabsAfterClick - i];
if (newPage) {
generator.notifyOnNewTab(newPage, tabsAfterClick - i);
}
}
}
logger.log("debug", `Clicked on position x:${x}, y:${y}`);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling mousedown event: ${message}`);
}
};
/**
* A wrapper function for handling the wheel event.
* @param socket The socket connection
* @param scrollDeltas - the scroll deltas of the wheel event
* @category HelperFunctions
*/
const onWheel = async (scrollDeltas: ScrollDeltas, userId: string) => {
logger.log('debug', 'Handling scroll event emitted from client');
await handleWrapper(handleWheel, userId, scrollDeltas);
};
/**
* A wheel event handler.
* Reproduces the wheel event on the remote browser instance.
* Scroll is not generated for the workflow pair. This is because
* Playwright scrolls elements into focus on any action.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param deltaX - the delta x of the wheel event
* @param deltaY - the delta y of the wheel event
* @category BrowserManagement
*/
const handleWheel = async (activeBrowser: RemoteBrowser, page: Page, { deltaX, deltaY }: ScrollDeltas) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring wheel event: page is closed`);
return;
}
await page.mouse.wheel(deltaX, deltaY).catch(error => {
logger.log('warn', `Wheel event failed: ${error.message}`);
});
logger.log('debug', `Scrolled horizontally ${deltaX} pixels and vertically ${deltaY} pixels`);
} catch (e) {
const { message } = e as Error;
logger.log('warn', `Error handling wheel event: ${message}`);
}
};
/**
* A wrapper function for handling the mousemove event.
* @param socket The socket connection
* @param coordinates - the coordinates of the mousemove event
* @category HelperFunctions
*/
const onMousemove = async (coordinates: Coordinates, userId: string) => {
logger.log('debug', 'Handling mousemove event emitted from client');
await handleWrapper(handleMousemove, userId, coordinates);
}
/**
* A mousemove event handler.
* Reproduces the mousemove event on the remote browser instance
* and generates data for the client's highlighter.
* Mousemove is also not reflected in the workflow.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param x - the x coordinate of the mousemove event
* @param y - the y coordinate of the mousemove event
* @category BrowserManagement
*/
const handleMousemove = async (activeBrowser: RemoteBrowser, page: Page, { x, y }: Coordinates) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring mousemove event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await page.mouse.move(x, y);
// throttle(async () => {
// if (!page.isClosed()) {
// await generator.generateDataForHighlighter(page, { x, y });
// }
// }, 100)();
logger.log("debug", `Moved over position x:${x}, y:${y}`);
} catch (e) {
const { message } = e as Error;
logger.log("error", message);
}
}
/**
* A wrapper function for handling the keydown event.
* @param socket The socket connection
* @param keyboardInput - the keyboard input of the keydown event
* @category HelperFunctions
*/
const onKeydown = async (keyboardInput: KeyboardInput, userId: string) => {
logger.log('debug', 'Handling keydown event emitted from client');
await handleWrapper(handleKeydown, userId, keyboardInput);
}
/**
* A keydown event handler.
* Reproduces the keydown event on the remote browser instance
* and generates the workflow pair data.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param key - the pressed key
* @param coordinates - the coordinates, where the keydown event happened
* @category BrowserManagement
*/
const handleKeydown = async (activeBrowser: RemoteBrowser, page: Page, { key, coordinates }: KeyboardInput) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring keydown event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await page.keyboard.down(key);
await generator.onKeyboardInput(key, coordinates, page);
logger.log("debug", `Key ${key} pressed`);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling keydown event: ${message}`);
}
};
/**
* Handles the date selection event.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
@@ -493,7 +311,7 @@ const handleChangeUrl = async (activeBrowser: RemoteBrowser, page: Page, url: st
try {
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
await page.waitForTimeout(2000);
await page.waitForTimeout(500);
logger.log("debug", `Went to ${url}`);
} catch (e) {
const { message } = e as Error;
@@ -689,8 +507,7 @@ const handleClickAction = async (
}
}
await new Promise((resolve) => setTimeout(resolve, 2000));
await activeBrowser.makeAndEmitDOMSnapshot();
await new Promise((resolve) => setTimeout(resolve, 300));
} catch (e) {
const { message } = e as Error;
logger.log(
@@ -778,47 +595,6 @@ const onDOMKeyboardAction = async (
await handleWrapper(handleKeyboardAction, userId, data);
};
/**
* Handles the workflow pair event.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param data - the data of the workflow pair event
* @category BrowserManagement
*/
const handleWorkflowPair = async (
activeBrowser: RemoteBrowser,
page: Page,
data: { pair: WhereWhatPair; userId: string }
) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring workflow pair event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await generator.onDOMWorkflowPair(page, data);
logger.log("debug", `Workflow pair processed from frontend`);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling workflow pair event: ${message}`);
}
};
/**
* A wrapper function for handling the workflow pair event.
* @param socket The socket connection
* @param data - the data of the workflow pair event
* @category HelperFunctions
*/
const onDOMWorkflowPair = async (
data: { pair: WhereWhatPair; userId: string },
userId: string
) => {
logger.log("debug", "Handling workflow pair event emitted from client");
await handleWrapper(handleWorkflowPair, userId, data);
};
/**
* Handles the remove action event.
* This is called when a user discards a capture action (list or text) that was already emitted to the backend.
@@ -1026,11 +802,6 @@ const onTestPaginationScroll = async (
* @category BrowserManagement
*/
const registerInputHandlers = (socket: Socket, userId: string) => {
// Register handlers with the socket
socket.on("input:mousedown", (data) => onMousedown(data, userId));
socket.on("input:wheel", (data) => onWheel(data, userId));
socket.on("input:mousemove", (data) => onMousemove(data, userId));
socket.on("input:keydown", (data) => onKeydown(data, userId));
socket.on("input:keyup", (data) => onKeyup(data, userId));
socket.on("input:url", (data) => onChangeUrl(data, userId));
socket.on("input:refresh", () => onRefresh(userId));
@@ -1045,7 +816,6 @@ const registerInputHandlers = (socket: Socket, userId: string) => {
socket.on("dom:click", (data) => onDOMClickAction(data, userId));
socket.on("dom:keypress", (data) => onDOMKeyboardAction(data, userId));
socket.on("dom:addpair", (data) => onDOMWorkflowPair(data, userId));
socket.on("testPaginationScroll", (data) => onTestPaginationScroll(data, userId, socket));
};
@@ -1058,10 +828,6 @@ const registerInputHandlers = (socket: Socket, userId: string) => {
*/
const removeInputHandlers = (socket: Socket) => {
try {
socket.removeAllListeners("input:mousedown");
socket.removeAllListeners("input:wheel");
socket.removeAllListeners("input:mousemove");
socket.removeAllListeners("input:keydown");
socket.removeAllListeners("input:keyup");
socket.removeAllListeners("input:url");
socket.removeAllListeners("input:refresh");
@@ -1075,7 +841,6 @@ const removeInputHandlers = (socket: Socket) => {
socket.removeAllListeners("dom:input");
socket.removeAllListeners("dom:click");
socket.removeAllListeners("dom:keypress");
socket.removeAllListeners("dom:addpair");
socket.removeAllListeners("removeAction");
socket.removeAllListeners("testPaginationScroll");
} catch (error: any) {

91
server/src/workflow-management/classes/Generator.ts
View File

@@ -198,7 +198,6 @@ export class WorkflowGenerator {
private async getSelectorsForSchema(page: Page, schema: Record<string, { selector: string }>): Promise<string[]> {
const selectors = Object.values(schema).map((field) => field.selector);
// Verify if the selectors are present and actionable on the current page
const actionableSelectors: string[] = [];
for (const selector of selectors) {
const isActionable = await page.isVisible(selector).catch(() => false);
@@ -235,7 +234,6 @@ export class WorkflowGenerator {
private addPairToWorkflowAndNotifyClient = async (pair: WhereWhatPair, page: Page) => {
let matched = false;
// Check for scrapeSchema actions and enhance the where condition
if (pair.what[0].action === 'scrapeSchema') {
const schema = pair.what[0]?.args?.[0];
if (schema) {
@@ -244,7 +242,6 @@ export class WorkflowGenerator {
}
}
// Validate if the pair is already in the workflow
if (pair.where.selectors && pair.where.selectors[0]) {
const match = selectorAlreadyInWorkflow(pair.where.selectors[0], this.workflowRecord.workflow);
if (match) {
@@ -260,7 +257,6 @@ export class WorkflowGenerator {
}
}
// Handle cases where the where condition isn't already present
if (!matched) {
const handled = await this.handleOverShadowing(pair, page, this.generatedData.lastIndex || 0);
if (!handled) {
@@ -282,7 +278,6 @@ export class WorkflowGenerator {
}
}
// Emit the updated workflow to the client
this.socket.emit('workflow', this.workflowRecord);
logger.log('info', `Workflow emitted`);
};
@@ -367,7 +362,6 @@ export class WorkflowGenerator {
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
// Handles click events on the DOM, generating a pair for the click action
public onDOMClickAction = async (page: Page, data: {
selector: string,
url: string,
@@ -388,7 +382,6 @@ export class WorkflowGenerator {
}],
};
// Handle special input elements with cursor positioning
if (elementInfo && coordinates &&
(elementInfo.tagName === 'INPUT' || elementInfo.tagName === 'TEXTAREA')) {
pair.what[0] = {
@@ -403,7 +396,6 @@ export class WorkflowGenerator {
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
// Handles keyboard actions on the DOM, generating a pair for the key press action
public onDOMKeyboardAction = async (page: Page, data: {
selector: string,
key: string,
@@ -430,7 +422,6 @@ export class WorkflowGenerator {
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
// Handles navigation events on the DOM, generating a pair for the navigation action
public onDOMNavigation = async (page: Page, data: {
url: string,
currentUrl: string,
@@ -450,13 +441,6 @@ export class WorkflowGenerator {
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
// Handles workflow pair events on the DOM
public onDOMWorkflowPair = async (page: Page, data: { pair: WhereWhatPair, userId: string }) => {
const { pair } = data;
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
/**
* Generates a pair for the click event.
* @param coordinates The coordinates of the click event.
@@ -471,25 +455,22 @@ export class WorkflowGenerator {
const elementInfo = await getElementInformation(page, coordinates, '', false);
console.log("Element info: ", elementInfo);
// Check if clicked element is a select dropdown
const isDropdown = elementInfo?.tagName === 'SELECT';
if (isDropdown && elementInfo.innerHTML) {
// Parse options from innerHTML
const options = elementInfo.innerHTML
.split('<option')
.slice(1) // Remove first empty element
.slice(1)
.map(optionHtml => {
const valueMatch = optionHtml.match(/value="([^"]*)"/);
const disabledMatch = optionHtml.includes('disabled="disabled"');
const selectedMatch = optionHtml.includes('selected="selected"');
// Extract text content between > and </option>
const textMatch = optionHtml.match(/>([^<]*)</);
const text = textMatch
? textMatch[1]
.replace(/\n/g, '') // Remove all newlines
.replace(/\s+/g, ' ') // Replace multiple spaces with single space
.replace(/\n/g, '')
.replace(/\s+/g, ' ')
.trim()
: '';
@@ -501,7 +482,6 @@ export class WorkflowGenerator {
};
});
// Notify client to show dropdown overlay
this.socket.emit('showDropdown', {
coordinates,
selector,
@@ -510,11 +490,9 @@ export class WorkflowGenerator {
return;
}
// Check if clicked element is a date input
const isDateInput = elementInfo?.tagName === 'INPUT' && elementInfo?.attributes?.type === 'date';
if (isDateInput) {
// Notify client to show datepicker overlay
this.socket.emit('showDatePicker', {
coordinates,
selector
@@ -640,8 +618,6 @@ export class WorkflowGenerator {
}
}
//const element = await getElementMouseIsOver(page, coordinates);
//logger.log('debug', `Element: ${JSON.stringify(element, null, 2)}`);
if (selector) {
where.selectors = [selector];
}
@@ -679,37 +655,6 @@ export class WorkflowGenerator {
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
/**
* Generates a pair for the keypress event.
* @param key The key to be pressed.
* @param coordinates The coordinates of the keypress event.
* @param page The page to use for obtaining the needed data.
* @returns {Promise<void>}
*/
public onKeyboardInput = async (key: string, coordinates: Coordinates, page: Page) => {
let where: WhereWhatPair["where"] = { url: this.getBestUrl(page.url()) };
const selector = await this.generateSelector(page, coordinates, ActionType.Keydown);
const elementInfo = await getElementInformation(page, coordinates, '', false);
const inputType = elementInfo?.attributes?.type || "text";
if (selector) {
where.selectors = [selector];
}
const pair: WhereWhatPair = {
where,
what: [{
action: 'press',
args: [selector, encrypt(key), inputType],
}],
}
if (selector) {
this.generatedData.lastUsedSelector = selector;
this.generatedData.lastAction = 'press';
}
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
/**
* Returns tag name and text content for the specified selector
* used in customAction for decision modal
@@ -940,7 +885,7 @@ export class WorkflowGenerator {
return pair;
})
.filter((pair) => pair !== null) as WhereWhatPair[]; // Remove null entries
.filter((pair) => pair !== null) as WhereWhatPair[];
if (actionWasRemoved) {
logger.log("info", `Action with actionId ${actionId} removed from workflow`);
@@ -996,24 +941,6 @@ export class WorkflowGenerator {
}
}
/**
* Returns the currently generated workflow without all the generated flag actions.
* @param workflow The workflow for removing the generated flag actions from.
* @private
* @returns {WorkflowFile}
*/
private removeAllGeneratedFlags = (workflow: WorkflowFile): WorkflowFile => {
for (let i = 0; i < workflow.workflow.length; i++) {
if (
workflow.workflow[i].what[0] &&
workflow.workflow[i].what[0].action === 'flag' &&
workflow.workflow[i].what[0].args?.includes('generated')) {
workflow.workflow[i].what.splice(0, 1);
}
}
return workflow;
};
/**
* Adds generated flag actions to the workflow's pairs' what conditions.
* @param workflow The workflow for adding the generated flag actions from.
@@ -1127,7 +1054,6 @@ export class WorkflowGenerator {
: await getSelectors(page, coordinates);
if (this.paginationMode && selectorBasedOnCustomAction) {
// Chain selectors in specific priority order
const selectors = selectorBasedOnCustomAction;
const selectorChain = [
selectors?.iframeSelector?.full,
@@ -1324,17 +1250,15 @@ export class WorkflowGenerator {
if (this.workflowRecord.workflow[index].where.selectors?.includes(selector)) {
break;
} else {
// add new selector to the where part of the overshadowing pair
this.workflowRecord.workflow[index].where.selectors?.push(selector);
}
}
}
// push the action automatically to the first/the closest rule which would be overShadowed
this.workflowRecord.workflow[index].what =
this.workflowRecord.workflow[index].what.concat(pair.what);
return true;
} else {
// notify client about overshadowing a further rule
return false;
}
}
@@ -1352,8 +1276,7 @@ export class WorkflowGenerator {
const parsedUrl = new URL(url);
const protocol = parsedUrl.protocol === 'https:' || parsedUrl.protocol === 'http:' ? `${parsedUrl.protocol}//` : parsedUrl.protocol;
const regex = new RegExp(/(?=.*[A-Z])/g)
// remove all params with uppercase letters, they are most likely dynamically generated
// also escapes all regex characters from the params
const search = parsedUrl.search
.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
.split('&').map((param, index) => {
@@ -1380,8 +1303,6 @@ export class WorkflowGenerator {
* @param workflow The workflow to be checked.
*/
private checkWorkflowForParams = (workflow: WorkflowFile): string[] | null => {
// for now the where condition cannot have any params, so we're checking only what part of the pair
// where only the args part of what condition can have a parameter
for (const pair of workflow.workflow) {
for (const condition of pair.what) {
if (condition.args) {