vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #942 from getmaxun/fix-recorder

Browse Source 
fix: recorder flow
This commit is contained in:
Karishma Shukla
2026-01-19 13:02:15 +05:30
committed by GitHub
parent f3cb9273ec dddac2d966
commit 5157c6c6d2
23 changed files with 495 additions and 1459 deletions
Download Patch File Download Diff File Expand all files Collapse all files

564
server/src/browser-management/classes/RemoteBrowser.ts
View File

@@ -8,6 +8,7 @@ import { Socket } from "socket.io";
import { PlaywrightBlocker } from '@cliqz/adblocker-playwright';
import fetch from 'cross-fetch';
import logger from '../../logger';
import { readFileSync } from "fs";
import { InterpreterSettings } from "../../types";
import { WorkflowGenerator } from "../../workflow-management/classes/Generator";
import { WorkflowInterpreter } from "../../workflow-management/classes/Interpreter";
@@ -19,30 +20,17 @@ import { connectToRemoteBrowser } from '../browserConnection';
declare global {
interface Window {
rrwebSnapshot?: any;
rrweb?: any;
isRecording?: boolean;
emitEventToBackend?: (event: any) => Promise<void>;
}
}
interface RRWebSnapshot {
type: number;
childNodes?: RRWebSnapshot[];
tagName?: string;
attributes?: Record<string, string>;
textContent?: string;
id: number;
[key: string]: any;
}
interface ProcessedSnapshot {
snapshot: RRWebSnapshot;
baseUrl: string;
}
const MEMORY_CONFIG = {
gcInterval: 20000, // Check memory more frequently (20s instead of 60s)
maxHeapSize: 1536 * 1024 * 1024, // 1.5GB
heapUsageThreshold: 0.7 // 70% (reduced threshold to react earlier)
};
// const MEMORY_CONFIG = {
// gcInterval: 20000,
// maxHeapSize: 1536 * 1024 * 1024,
// heapUsageThreshold: 0.7
// };
/**
* This class represents a remote browser instance.
@@ -110,22 +98,11 @@ export class RemoteBrowser {
public interpreter: WorkflowInterpreter;
public isDOMStreamingActive: boolean = false;
private domUpdateInterval: NodeJS.Timeout | null = null;
private lastScrollPosition = { x: 0, y: 0 };
private scrollThreshold = 200; // pixels
private snapshotDebounceTimeout: NodeJS.Timeout | null = null;
private scrollThreshold = 200;
private networkRequestTimeout: NodeJS.Timeout | null = null;
private pendingNetworkRequests: string[] = [];
private readonly INITIAL_LOAD_QUIET_PERIOD = 3000;
private networkWaitStartTime: number = 0;
private progressInterval: NodeJS.Timeout | null = null;
private hasShownInitialLoader: boolean = false;
private isInitialLoadInProgress: boolean = false;
private memoryCleanupInterval: NodeJS.Timeout | null = null;
private memoryManagementInterval: NodeJS.Timeout | null = null;
// private memoryCleanupInterval: NodeJS.Timeout | null = null;
// private memoryManagementInterval: NodeJS.Timeout | null = null;
/**
* Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and
@@ -140,64 +117,53 @@ export class RemoteBrowser {
this.generator = new WorkflowGenerator(socket, poolId);
}
private async processRRWebSnapshot(
snapshot: RRWebSnapshot
): Promise<ProcessedSnapshot> {
const baseUrl = this.currentPage?.url() || "";
// private initializeMemoryManagement(): void {
// this.memoryManagementInterval = setInterval(() => {
// const memoryUsage = process.memoryUsage();
// const heapUsageRatio = memoryUsage.heapUsed / MEMORY_CONFIG.maxHeapSize;
return {
snapshot,
baseUrl
};
}
// if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.2) {
// logger.warn(
// "Critical memory pressure detected, triggering emergency cleanup"
// );
// this.performMemoryCleanup();
// } else if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold) {
// logger.warn("High memory usage detected, triggering cleanup");
private initializeMemoryManagement(): void {
this.memoryManagementInterval = setInterval(() => {
const memoryUsage = process.memoryUsage();
const heapUsageRatio = memoryUsage.heapUsed / MEMORY_CONFIG.maxHeapSize;
// if (
// global.gc &&
// heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.1
// ) {
// global.gc();
// }
// }
// }, MEMORY_CONFIG.gcInterval);
// }
if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.2) {
logger.warn(
"Critical memory pressure detected, triggering emergency cleanup"
);
this.performMemoryCleanup();
} else if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold) {
logger.warn("High memory usage detected, triggering cleanup");
// private async performMemoryCleanup(): Promise<void> {
// if (global.gc) {
// try {
// global.gc();
// logger.info("Garbage collection requested");
// } catch (error) {
// logger.error("Error during garbage collection:", error);
// }
// }
if (
global.gc &&
heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.1
) {
global.gc();
}
}
}, MEMORY_CONFIG.gcInterval);
}
// if (this.currentPage) {
// try {
// await new Promise((resolve) => setTimeout(resolve, 500));
// logger.info("CDP session reset completed");
// } catch (error) {
// logger.error("Error resetting CDP session:", error);
// }
// }
private async performMemoryCleanup(): Promise<void> {
if (global.gc) {
try {
global.gc();
logger.info("Garbage collection requested");
} catch (error) {
logger.error("Error during garbage collection:", error);
}
}
if (this.currentPage) {
try {
await new Promise((resolve) => setTimeout(resolve, 500));
logger.info("CDP session reset completed");
} catch (error) {
logger.error("Error resetting CDP session:", error);
}
}
this.socket.emit("memory-cleanup", {
userId: this.userId,
timestamp: Date.now(),
});
}
// this.socket.emit("memory-cleanup", {
// userId: this.userId,
// timestamp: Date.now(),
// });
// }
/**
* Normalizes URLs to prevent navigation loops while maintaining consistent format
@@ -205,9 +171,7 @@ export class RemoteBrowser {
private normalizeUrl(url: string): string {
try {
const parsedUrl = new URL(url);
// Remove trailing slashes except for root path
parsedUrl.pathname = parsedUrl.pathname.replace(/\/+$/, '') || '/';
// Ensure consistent protocol handling
parsedUrl.protocol = parsedUrl.protocol.toLowerCase();
return parsedUrl.toString();
} catch {
@@ -260,14 +224,6 @@ export class RemoteBrowser {
if (scrollDelta > this.scrollThreshold) {
this.lastScrollPosition = { x: scrollInfo.x, y: scrollInfo.y };
if (this.snapshotDebounceTimeout) {
clearTimeout(this.snapshotDebounceTimeout);
}
this.snapshotDebounceTimeout = setTimeout(async () => {
await this.makeAndEmitDOMSnapshot();
}, 300);
}
} catch (error) {
logger.error("Error handling scroll event:", error);
@@ -276,79 +232,6 @@ export class RemoteBrowser {
);
}
private setupPageChangeListeners(): void {
if (!this.currentPage) return;
try {
if (!this.currentPage.isClosed()) {
this.currentPage.removeAllListeners("domcontentloaded");
this.currentPage.removeAllListeners("response");
}
} catch (error: any) {
logger.warn(`Error removing page change listeners: ${error.message}`);
}
this.currentPage.on("domcontentloaded", async () => {
if (!this.isInitialLoadInProgress) {
logger.info("DOM content loaded - triggering snapshot");
await this.makeAndEmitDOMSnapshot();
}
});
this.currentPage.on("response", async (response) => {
const url = response.url();
const isDocumentRequest = response.request().resourceType() === "document";
if (!this.hasShownInitialLoader && isDocumentRequest && !url.includes("about:blank")) {
this.hasShownInitialLoader = true;
this.isInitialLoadInProgress = true;
this.pendingNetworkRequests.push(url);
if (this.networkRequestTimeout) {
clearTimeout(this.networkRequestTimeout);
this.networkRequestTimeout = null;
}
if (this.progressInterval) {
clearInterval(this.progressInterval);
this.progressInterval = null;
}
this.networkWaitStartTime = Date.now();
this.progressInterval = setInterval(() => {
const elapsed = Date.now() - this.networkWaitStartTime;
const navigationProgress = Math.min((elapsed / this.INITIAL_LOAD_QUIET_PERIOD) * 40, 35);
const totalProgress = 60 + navigationProgress;
this.emitLoadingProgress(totalProgress, this.pendingNetworkRequests.length);
}, 500);
logger.debug(
`Initial load network request received: ${url}. Using ${this.INITIAL_LOAD_QUIET_PERIOD}ms quiet period`
);
this.networkRequestTimeout = setTimeout(async () => {
logger.info(
`Initial load network quiet period reached (${this.INITIAL_LOAD_QUIET_PERIOD}ms)`
);
if (this.progressInterval) {
clearInterval(this.progressInterval);
this.progressInterval = null;
}
this.emitLoadingProgress(100, this.pendingNetworkRequests.length);
this.pendingNetworkRequests = [];
this.networkRequestTimeout = null;
this.isInitialLoadInProgress = false;
await this.makeAndEmitDOMSnapshot();
}, this.INITIAL_LOAD_QUIET_PERIOD);
}
});
}
private emitLoadingProgress(progress: number, pendingRequests: number): void {
this.socket.emit("domLoadingProgress", {
progress: Math.round(progress),
@@ -368,16 +251,27 @@ export class RemoteBrowser {
}
page.on('framenavigated', async (frame) => {
if (frame === page.mainFrame()) {
const currentUrl = page.url();
if (this.shouldEmitUrlChange(currentUrl)) {
this.lastEmittedUrl = currentUrl;
this.socket.emit('urlChanged', {url: currentUrl, userId: this.userId});
}
if (frame === page.mainFrame()) {
const currentUrl = page.url();
if (this.shouldEmitUrlChange(currentUrl)) {
this.lastEmittedUrl = currentUrl;
this.socket.emit('urlChanged', { url: currentUrl, userId: this.userId });
}
await page.evaluate(() => {
if (window.rrweb && window.isRecording) {
window.isRecording = false;
}
});
await page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {
logger.warn('[rrweb] Network idle timeout on navigation, proceeding with rrweb initialization');
});
await this.initializeRRWebRecording(page);
}
});
// Handle page load events with retry mechanism
page.on('load', async () => {
const injectScript = async (): Promise<boolean> => {
try {
@@ -401,6 +295,96 @@ export class RemoteBrowser {
});
}
/**
* Initialize rrweb recording for real-time DOM streaming
* This replaces the snapshot-based approach with live event streaming
*/
private async initializeRRWebRecording(page: Page): Promise<void> {
try {
const rrwebJsPath = require.resolve('rrweb/dist/rrweb.min.js');
const rrwebScriptContent = readFileSync(rrwebJsPath, 'utf8');
await page.context().addInitScript(rrwebScriptContent);
await page.evaluate((scriptContent) => {
if (typeof window.rrweb === 'undefined') {
try {
(0, eval)(scriptContent);
} catch (e) {
console.error('[rrweb] eval failed:', e);
}
}
}, rrwebScriptContent);
const rrwebLoaded = await page.evaluate(() => typeof window.rrweb !== 'undefined');
if (rrwebLoaded) {
logger.debug('[rrweb] Script injected successfully');
} else {
logger.warn('[rrweb] Script injection failed - window.rrweb not found');
}
const isAlreadyExposed = await page.evaluate(() => {
return typeof window.emitEventToBackend === 'function';
});
if (!isAlreadyExposed) {
let hasEmittedFullSnapshot = false;
await page.exposeFunction('emitEventToBackend', (event: any) => {
this.socket.emit('rrweb-event', event);
if (event.type === 2 && !hasEmittedFullSnapshot) {
hasEmittedFullSnapshot = true;
this.emitLoadingProgress(100, 0);
logger.debug(`[rrweb] Full snapshot sent, loading progress at 100%`);
}
});
}
const rrwebStatus = await page.evaluate(() => {
if (!window.rrweb) {
console.error('[rrweb] window.rrweb is not defined!');
return { success: false, error: 'window.rrweb is not defined' };
}
if (window.isRecording) {
return { success: false, error: 'already recording' };
}
window.isRecording = true;
try {
const recordHandle = window.rrweb.record({
emit(event: any) {
if (window.emitEventToBackend) {
window.emitEventToBackend(event).catch(() => { });
}
},
maskAllInputs: false,
recordCanvas: true,
input: true
});
(window as any).rrwebRecordHandle = recordHandle;
return { success: true };
} catch (error: any) {
console.error('[rrweb] Failed to start recording:', error);
return { success: false, error: error.message };
}
});
if (rrwebStatus.success) {
this.isDOMStreamingActive = true;
this.emitLoadingProgress(80, 0);
this.setupScrollEventListener();
} else {
logger.error(`Failed to initialize rrweb recording: ${rrwebStatus.error}`);
}
} catch (error: any) {
logger.error(`Failed to initialize rrweb recording: ${error.message}`);
}
}
private getUserAgent() {
const userAgents = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.140 Safari/537.36',
@@ -432,7 +416,6 @@ export class RemoteBrowser {
}
} catch (error: any) {
logger.error(`Enhanced fingerprinting failed: ${error.message}`);
// Don't throw - fallback to basic functionality
}
}
@@ -540,14 +523,18 @@ export class RemoteBrowser {
patchedGetter.toString();`
);
await this.context.addInitScript({ path: './server/src/browser-management/classes/rrweb-bundle.js' });
this.currentPage = await this.context.newPage();
this.emitLoadingProgress(40, 0);
await this.setupPageEventListeners(this.currentPage);
await this.currentPage.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {
logger.warn('[rrweb] Network idle timeout, proceeding with rrweb initialization');
});
await this.initializeRRWebRecording(this.currentPage);
try {
const blocker = await PlaywrightBlocker.fromLists(fetch, ['https://easylist.to/easylist/easylist.txt']);
await blocker.enableBlockingInPage(this.currentPage as any);
@@ -556,12 +543,9 @@ export class RemoteBrowser {
console.log('Adblocker initialized');
} catch (error: any) {
console.warn('Failed to initialize adblocker, continuing without it:', error.message);
// Still need to set up the CDP session even if blocker fails
this.client = await this.currentPage.context().newCDPSession(this.currentPage);
}
this.emitLoadingProgress(60, 0);
success = true;
logger.log('debug', `Browser initialized successfully for user ${userId}`);
} catch (error: any) {
@@ -656,7 +640,6 @@ export class RemoteBrowser {
private removeAllSocketListeners(): void {
try {
this.socket.removeAllListeners('captureDirectScreenshot');
this.socket.removeAllListeners('rerender');
this.socket.removeAllListeners('settings');
this.socket.removeAllListeners('changeTab');
this.socket.removeAllListeners('addTab');
@@ -683,13 +666,6 @@ export class RemoteBrowser {
await this.captureDirectScreenshot(settings);
});
this.socket.on("rerender", async () => {
logger.debug(
`General rerender event received, checking if for user ${this.userId}`
);
await this.makeAndEmitDOMSnapshot();
});
this.socket.on(
"changeTab",
async (tabIndex) => await this.changeTab(tabIndex)
@@ -719,157 +695,6 @@ export class RemoteBrowser {
};
/**
* Subscribe to DOM streaming - simplified version following screenshot pattern
*/
public async subscribeToDOM(): Promise<void> {
if (!this.client) {
logger.warn("DOM streaming requires scraping browser with CDP client");
return;
}
try {
this.isDOMStreamingActive = true;
logger.info("DOM streaming started successfully");
this.setupScrollEventListener();
this.setupPageChangeListeners();
} catch (error) {
logger.error("Failed to start DOM streaming:", error);
this.isDOMStreamingActive = false;
}
}
/**
* CDP-based DOM snapshot creation using captured network resources
*/
public async makeAndEmitDOMSnapshot(): Promise<void> {
if (!this.currentPage || !this.isDOMStreamingActive) {
return;
}
try {
// Check if page is still valid and not closed
if (this.currentPage.isClosed()) {
logger.debug("Skipping DOM snapshot - page is closed");
return;
}
// Double-check page state after network wait
if (this.currentPage.isClosed()) {
logger.debug("Skipping DOM snapshot - page closed during network wait");
return;
}
// Get current scroll position
const currentScrollInfo = await this.currentPage.evaluate(() => ({
x: window.scrollX,
y: window.scrollY,
maxX: Math.max(
0,
document.documentElement.scrollWidth - window.innerWidth
),
maxY: Math.max(
0,
document.documentElement.scrollHeight - window.innerHeight
),
documentHeight: document.documentElement.scrollHeight,
}));
logger.info(
`Creating rrweb snapshot at scroll position: ${currentScrollInfo.y}/${currentScrollInfo.maxY}`
);
// Update our tracked scroll position
this.lastScrollPosition = {
x: currentScrollInfo.x,
y: currentScrollInfo.y,
};
// Final check before snapshot
if (this.currentPage.isClosed()) {
logger.debug("Skipping DOM snapshot - page closed before snapshot");
return;
}
// Capture snapshot using rrweb
const rawSnapshot = await this.currentPage.evaluate(() => {
if (typeof window.rrwebSnapshot === "undefined") {
throw new Error("rrweb-snapshot library not available");
}
return window.rrwebSnapshot.snapshot(document, {
inlineImages: false,
collectFonts: true,
});
});
// Process the snapshot to proxy resources
const processedSnapshot = await this.processRRWebSnapshot(rawSnapshot);
// Add scroll position information
const enhancedSnapshot = {
...processedSnapshot,
scrollPosition: currentScrollInfo,
captureTime: Date.now(),
};
// Emit the processed snapshot
this.emitRRWebSnapshot(enhancedSnapshot);
} catch (error) {
// Handle navigation context destruction gracefully
if (
error instanceof Error &&
(error.message.includes("Execution context was destroyed") ||
error.message.includes("most likely because of a navigation") ||
error.message.includes("Target closed"))
) {
logger.debug("DOM snapshot skipped due to page navigation or closure");
return;
}
logger.error("Failed to create rrweb snapshot:", error);
this.socket.emit("dom-mode-error", {
userId: this.userId,
message: "Failed to create rrweb snapshot",
error: error instanceof Error ? error.message : String(error),
timestamp: Date.now(),
});
}
}
/**
* Emit DOM snapshot to client - following screenshot pattern
*/
private emitRRWebSnapshot(processedSnapshot: ProcessedSnapshot): void {
this.socket.emit("domcast", {
snapshotData: processedSnapshot,
userId: this.userId,
timestamp: Date.now(),
});
}
/**
* Stop DOM streaming - following dom snapshot pattern
*/
private async stopDOM(): Promise<void> {
this.isDOMStreamingActive = false;
if (this.domUpdateInterval) {
clearInterval(this.domUpdateInterval);
this.domUpdateInterval = null;
}
if (this.networkRequestTimeout) {
clearTimeout(this.networkRequestTimeout);
this.networkRequestTimeout = null;
}
this.pendingNetworkRequests = [];
logger.info("DOM streaming stopped successfully");
}
/**rrweb-bundle
* Terminates the dom snapshot session and closes the remote browser.
* If an interpretation was running it will be stopped.
* @returns {Promise<void>}
@@ -877,35 +702,15 @@ export class RemoteBrowser {
public async switchOff(): Promise<void> {
this.isDOMStreamingActive = false;
if (this.domUpdateInterval) {
clearInterval(this.domUpdateInterval);
this.domUpdateInterval = null;
}
// if (this.memoryCleanupInterval) {
// clearInterval(this.memoryCleanupInterval);
// this.memoryCleanupInterval = null;
// }
if (this.memoryCleanupInterval) {
clearInterval(this.memoryCleanupInterval);
this.memoryCleanupInterval = null;
}
if (this.memoryManagementInterval) {
clearInterval(this.memoryManagementInterval);
this.memoryManagementInterval = null;
}
if (this.progressInterval) {
clearInterval(this.progressInterval);
this.progressInterval = null;
}
if (this.snapshotDebounceTimeout) {
clearTimeout(this.snapshotDebounceTimeout);
this.snapshotDebounceTimeout = null;
}
if (this.networkRequestTimeout) {
clearTimeout(this.networkRequestTimeout);
this.networkRequestTimeout = null;
}
// if (this.memoryManagementInterval) {
// clearInterval(this.memoryManagementInterval);
// this.memoryManagementInterval = null;
// }
this.removeAllSocketListeners();
@@ -923,7 +728,6 @@ export class RemoteBrowser {
logger.warn(`Error removing page listeners: ${error.message}`);
}
// Clean up Generator listeners to prevent memory leaks
if (this.generator) {
try {
this.generator.cleanup();
@@ -933,20 +737,12 @@ export class RemoteBrowser {
}
}
// Stop interpretation with individual error handling (also calls clearState which removes pausing listeners)
try {
await this.interpreter.stopInterpretation();
} catch (error) {
logger.error("Error stopping interpretation during shutdown:", error);
}
// Stop DOM streaming with individual error handling
try {
await this.stopDOM();
} catch (error) {
logger.error("Error stopping DOM during shutdown:", error);
}
try {
if (this.client && this.currentPage && !this.currentPage.isClosed()) {
const detachPromise = this.client.detach();
@@ -1081,7 +877,6 @@ export class RemoteBrowser {
private changeTab = async (tabIndex: number): Promise<void> => {
const page = this.currentPage?.context().pages()[tabIndex];
if (page) {
await this.stopDOM();
this.currentPage = page;
await this.setupPageEventListeners(this.currentPage);
@@ -1093,10 +888,6 @@ export class RemoteBrowser {
url: this.currentPage.url(),
userId: this.userId
});
if (this.isDOMStreamingActive) {
await this.makeAndEmitDOMSnapshot();
await this.subscribeToDOM();
}
} else {
logger.log('error', `${tabIndex} index out of range of pages`)
}
@@ -1118,8 +909,7 @@ export class RemoteBrowser {
this.currentPage = newPage;
if (this.currentPage) {
await this.setupPageEventListeners(this.currentPage);
await this.subscribeToDOM();
logger.debug('Using rrweb live recording for new page');
} else {
logger.log('error', 'Could not get a new page, returned undefined');
}

10
server/src/browser-management/classes/bundle-rrweb.js
View File

@@ -1,10 +0,0 @@
const esbuild = require('esbuild');
esbuild.build({
entryPoints: ['rrweb-entry.js'],
bundle: true,
minify: true,
outfile: 'rrweb-bundle.js',
format: 'iife', // so that rrwebSnapshot is available on window
globalName: 'rrwebSnapshotBundle'
}).catch(() => process.exit(1));

1
server/src/browser-management/classes/rrweb-bundle.js
View File

File diff suppressed because one or more lines are too long

2
server/src/browser-management/classes/rrweb-entry.js
View File

@@ -1,2 +0,0 @@
import { snapshot } from 'rrweb-snapshot';
window.rrwebSnapshot = { snapshot };

5
server/src/browser-management/controller.ts
View File

@@ -31,10 +31,6 @@ export const initializeRemoteBrowserForRecording = (userId: string, mode: string
if (activeId) {
const remoteBrowser = browserPool.getRemoteBrowser(activeId);
remoteBrowser?.updateSocket(socket);
if (remoteBrowser?.isDOMStreamingActive) {
remoteBrowser?.makeAndEmitDOMSnapshot();
}
} else {
const browserSession = new RemoteBrowser(socket, userId, id);
browserSession.interpreter.subscribeToPausing();
@@ -43,7 +39,6 @@ export const initializeRemoteBrowserForRecording = (userId: string, mode: string
await browserSession.initialize(userId);
await browserSession.registerEditorEvents();
await browserSession.subscribeToDOM();
logger.info('DOM streaming started for remote browser in recording mode');
browserPool.addRemoteBrowser(id, browserSession, userId, false, "recording");

239
server/src/browser-management/inputHandlers.ts
View File

@@ -102,188 +102,6 @@ const handleGenerateAction =
}
}
/**
* A wrapper function for handling mousedown event.
* @param socket The socket connection
* @param coordinates - coordinates of the mouse click
* @category HelperFunctions
*/
const onMousedown = async (coordinates: Coordinates, userId: string) => {
logger.log('debug', 'Handling mousedown event emitted from client');
await handleWrapper(handleMousedown, userId, coordinates);
}
/**
* A mousedown event handler.
* Reproduces the click on the remote browser instance
* and generates pair data for the recorded workflow.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param x - the x coordinate of the mousedown event
* @param y - the y coordinate of the mousedown event
* @category BrowserManagement
*/
const handleMousedown = async (activeBrowser: RemoteBrowser, page: Page, { x, y }: Coordinates) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring mousedown event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await generator.onClick({ x, y }, page);
const previousUrl = page.url();
const tabsBeforeClick = page.context().pages().length;
await page.mouse.click(x, y);
// try if the click caused a navigation to a new url
try {
await page.waitForNavigation({ timeout: 2000 });
const currentUrl = page.url();
if (currentUrl !== previousUrl) {
generator.notifyUrlChange(currentUrl);
}
} catch (e) {
const { message } = e as Error;
} //ignore possible timeouts
// check if any new page was opened by the click
const tabsAfterClick = page.context().pages().length;
const numOfNewPages = tabsAfterClick - tabsBeforeClick;
if (numOfNewPages > 0) {
for (let i = 1; i <= numOfNewPages; i++) {
const newPage = page.context().pages()[tabsAfterClick - i];
if (newPage) {
generator.notifyOnNewTab(newPage, tabsAfterClick - i);
}
}
}
logger.log("debug", `Clicked on position x:${x}, y:${y}`);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling mousedown event: ${message}`);
}
};
/**
* A wrapper function for handling the wheel event.
* @param socket The socket connection
* @param scrollDeltas - the scroll deltas of the wheel event
* @category HelperFunctions
*/
const onWheel = async (scrollDeltas: ScrollDeltas, userId: string) => {
logger.log('debug', 'Handling scroll event emitted from client');
await handleWrapper(handleWheel, userId, scrollDeltas);
};
/**
* A wheel event handler.
* Reproduces the wheel event on the remote browser instance.
* Scroll is not generated for the workflow pair. This is because
* Playwright scrolls elements into focus on any action.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param deltaX - the delta x of the wheel event
* @param deltaY - the delta y of the wheel event
* @category BrowserManagement
*/
const handleWheel = async (activeBrowser: RemoteBrowser, page: Page, { deltaX, deltaY }: ScrollDeltas) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring wheel event: page is closed`);
return;
}
await page.mouse.wheel(deltaX, deltaY).catch(error => {
logger.log('warn', `Wheel event failed: ${error.message}`);
});
logger.log('debug', `Scrolled horizontally ${deltaX} pixels and vertically ${deltaY} pixels`);
} catch (e) {
const { message } = e as Error;
logger.log('warn', `Error handling wheel event: ${message}`);
}
};
/**
* A wrapper function for handling the mousemove event.
* @param socket The socket connection
* @param coordinates - the coordinates of the mousemove event
* @category HelperFunctions
*/
const onMousemove = async (coordinates: Coordinates, userId: string) => {
logger.log('debug', 'Handling mousemove event emitted from client');
await handleWrapper(handleMousemove, userId, coordinates);
}
/**
* A mousemove event handler.
* Reproduces the mousemove event on the remote browser instance
* and generates data for the client's highlighter.
* Mousemove is also not reflected in the workflow.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param x - the x coordinate of the mousemove event
* @param y - the y coordinate of the mousemove event
* @category BrowserManagement
*/
const handleMousemove = async (activeBrowser: RemoteBrowser, page: Page, { x, y }: Coordinates) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring mousemove event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await page.mouse.move(x, y);
// throttle(async () => {
// if (!page.isClosed()) {
// await generator.generateDataForHighlighter(page, { x, y });
// }
// }, 100)();
logger.log("debug", `Moved over position x:${x}, y:${y}`);
} catch (e) {
const { message } = e as Error;
logger.log("error", message);
}
}
/**
* A wrapper function for handling the keydown event.
* @param socket The socket connection
* @param keyboardInput - the keyboard input of the keydown event
* @category HelperFunctions
*/
const onKeydown = async (keyboardInput: KeyboardInput, userId: string) => {
logger.log('debug', 'Handling keydown event emitted from client');
await handleWrapper(handleKeydown, userId, keyboardInput);
}
/**
* A keydown event handler.
* Reproduces the keydown event on the remote browser instance
* and generates the workflow pair data.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param key - the pressed key
* @param coordinates - the coordinates, where the keydown event happened
* @category BrowserManagement
*/
const handleKeydown = async (activeBrowser: RemoteBrowser, page: Page, { key, coordinates }: KeyboardInput) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring keydown event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await page.keyboard.down(key);
await generator.onKeyboardInput(key, coordinates, page);
logger.log("debug", `Key ${key} pressed`);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling keydown event: ${message}`);
}
};
/**
* Handles the date selection event.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
@@ -493,7 +311,7 @@ const handleChangeUrl = async (activeBrowser: RemoteBrowser, page: Page, url: st
try {
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
await page.waitForTimeout(2000);
await page.waitForTimeout(500);
logger.log("debug", `Went to ${url}`);
} catch (e) {
const { message } = e as Error;
@@ -689,8 +507,7 @@ const handleClickAction = async (
}
}
await new Promise((resolve) => setTimeout(resolve, 2000));
await activeBrowser.makeAndEmitDOMSnapshot();
await new Promise((resolve) => setTimeout(resolve, 300));
} catch (e) {
const { message } = e as Error;
logger.log(
@@ -778,47 +595,6 @@ const onDOMKeyboardAction = async (
await handleWrapper(handleKeyboardAction, userId, data);
};
/**
* Handles the workflow pair event.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param data - the data of the workflow pair event
* @category BrowserManagement
*/
const handleWorkflowPair = async (
activeBrowser: RemoteBrowser,
page: Page,
data: { pair: WhereWhatPair; userId: string }
) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring workflow pair event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await generator.onDOMWorkflowPair(page, data);
logger.log("debug", `Workflow pair processed from frontend`);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling workflow pair event: ${message}`);
}
};
/**
* A wrapper function for handling the workflow pair event.
* @param socket The socket connection
* @param data - the data of the workflow pair event
* @category HelperFunctions
*/
const onDOMWorkflowPair = async (
data: { pair: WhereWhatPair; userId: string },
userId: string
) => {
logger.log("debug", "Handling workflow pair event emitted from client");
await handleWrapper(handleWorkflowPair, userId, data);
};
/**
* Handles the remove action event.
* This is called when a user discards a capture action (list or text) that was already emitted to the backend.
@@ -1026,11 +802,6 @@ const onTestPaginationScroll = async (
* @category BrowserManagement
*/
const registerInputHandlers = (socket: Socket, userId: string) => {
// Register handlers with the socket
socket.on("input:mousedown", (data) => onMousedown(data, userId));
socket.on("input:wheel", (data) => onWheel(data, userId));
socket.on("input:mousemove", (data) => onMousemove(data, userId));
socket.on("input:keydown", (data) => onKeydown(data, userId));
socket.on("input:keyup", (data) => onKeyup(data, userId));
socket.on("input:url", (data) => onChangeUrl(data, userId));
socket.on("input:refresh", () => onRefresh(userId));
@@ -1045,7 +816,6 @@ const registerInputHandlers = (socket: Socket, userId: string) => {
socket.on("dom:click", (data) => onDOMClickAction(data, userId));
socket.on("dom:keypress", (data) => onDOMKeyboardAction(data, userId));
socket.on("dom:addpair", (data) => onDOMWorkflowPair(data, userId));
socket.on("testPaginationScroll", (data) => onTestPaginationScroll(data, userId, socket));
};
@@ -1058,10 +828,6 @@ const registerInputHandlers = (socket: Socket, userId: string) => {
*/
const removeInputHandlers = (socket: Socket) => {
try {
socket.removeAllListeners("input:mousedown");
socket.removeAllListeners("input:wheel");
socket.removeAllListeners("input:mousemove");
socket.removeAllListeners("input:keydown");
socket.removeAllListeners("input:keyup");
socket.removeAllListeners("input:url");
socket.removeAllListeners("input:refresh");
@@ -1075,7 +841,6 @@ const removeInputHandlers = (socket: Socket) => {
socket.removeAllListeners("dom:input");
socket.removeAllListeners("dom:click");
socket.removeAllListeners("dom:keypress");
socket.removeAllListeners("dom:addpair");
socket.removeAllListeners("removeAction");
socket.removeAllListeners("testPaginationScroll");
} catch (error: any) {

91
server/src/workflow-management/classes/Generator.ts
View File

@@ -198,7 +198,6 @@ export class WorkflowGenerator {
private async getSelectorsForSchema(page: Page, schema: Record<string, { selector: string }>): Promise<string[]> {
const selectors = Object.values(schema).map((field) => field.selector);
// Verify if the selectors are present and actionable on the current page
const actionableSelectors: string[] = [];
for (const selector of selectors) {
const isActionable = await page.isVisible(selector).catch(() => false);
@@ -235,7 +234,6 @@ export class WorkflowGenerator {
private addPairToWorkflowAndNotifyClient = async (pair: WhereWhatPair, page: Page) => {
let matched = false;
// Check for scrapeSchema actions and enhance the where condition
if (pair.what[0].action === 'scrapeSchema') {
const schema = pair.what[0]?.args?.[0];
if (schema) {
@@ -244,7 +242,6 @@ export class WorkflowGenerator {
}
}
// Validate if the pair is already in the workflow
if (pair.where.selectors && pair.where.selectors[0]) {
const match = selectorAlreadyInWorkflow(pair.where.selectors[0], this.workflowRecord.workflow);
if (match) {
@@ -260,7 +257,6 @@ export class WorkflowGenerator {
}
}
// Handle cases where the where condition isn't already present
if (!matched) {
const handled = await this.handleOverShadowing(pair, page, this.generatedData.lastIndex || 0);
if (!handled) {
@@ -282,7 +278,6 @@ export class WorkflowGenerator {
}
}
// Emit the updated workflow to the client
this.socket.emit('workflow', this.workflowRecord);
logger.log('info', `Workflow emitted`);
};
@@ -367,7 +362,6 @@ export class WorkflowGenerator {
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
// Handles click events on the DOM, generating a pair for the click action
public onDOMClickAction = async (page: Page, data: {
selector: string,
url: string,
@@ -388,7 +382,6 @@ export class WorkflowGenerator {
}],
};
// Handle special input elements with cursor positioning
if (elementInfo && coordinates &&
(elementInfo.tagName === 'INPUT' || elementInfo.tagName === 'TEXTAREA')) {
pair.what[0] = {
@@ -403,7 +396,6 @@ export class WorkflowGenerator {
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
// Handles keyboard actions on the DOM, generating a pair for the key press action
public onDOMKeyboardAction = async (page: Page, data: {
selector: string,
key: string,
@@ -430,7 +422,6 @@ export class WorkflowGenerator {
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
// Handles navigation events on the DOM, generating a pair for the navigation action
public onDOMNavigation = async (page: Page, data: {
url: string,
currentUrl: string,
@@ -450,13 +441,6 @@ export class WorkflowGenerator {
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
// Handles workflow pair events on the DOM
public onDOMWorkflowPair = async (page: Page, data: { pair: WhereWhatPair, userId: string }) => {
const { pair } = data;
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
/**
* Generates a pair for the click event.
* @param coordinates The coordinates of the click event.
@@ -471,25 +455,22 @@ export class WorkflowGenerator {
const elementInfo = await getElementInformation(page, coordinates, '', false);
console.log("Element info: ", elementInfo);
// Check if clicked element is a select dropdown
const isDropdown = elementInfo?.tagName === 'SELECT';
if (isDropdown && elementInfo.innerHTML) {
// Parse options from innerHTML
const options = elementInfo.innerHTML
.split('<option')
.slice(1) // Remove first empty element
.slice(1)
.map(optionHtml => {
const valueMatch = optionHtml.match(/value="([^"]*)"/);
const disabledMatch = optionHtml.includes('disabled="disabled"');
const selectedMatch = optionHtml.includes('selected="selected"');
// Extract text content between > and </option>
const textMatch = optionHtml.match(/>([^<]*)</);
const text = textMatch
? textMatch[1]
.replace(/\n/g, '') // Remove all newlines
.replace(/\s+/g, ' ') // Replace multiple spaces with single space
.replace(/\n/g, '')
.replace(/\s+/g, ' ')
.trim()
: '';
@@ -501,7 +482,6 @@ export class WorkflowGenerator {
};
});
// Notify client to show dropdown overlay
this.socket.emit('showDropdown', {
coordinates,
selector,
@@ -510,11 +490,9 @@ export class WorkflowGenerator {
return;
}
// Check if clicked element is a date input
const isDateInput = elementInfo?.tagName === 'INPUT' && elementInfo?.attributes?.type === 'date';
if (isDateInput) {
// Notify client to show datepicker overlay
this.socket.emit('showDatePicker', {
coordinates,
selector
@@ -640,8 +618,6 @@ export class WorkflowGenerator {
}
}
//const element = await getElementMouseIsOver(page, coordinates);
//logger.log('debug', `Element: ${JSON.stringify(element, null, 2)}`);
if (selector) {
where.selectors = [selector];
}
@@ -679,37 +655,6 @@ export class WorkflowGenerator {
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
/**
* Generates a pair for the keypress event.
* @param key The key to be pressed.
* @param coordinates The coordinates of the keypress event.
* @param page The page to use for obtaining the needed data.
* @returns {Promise<void>}
*/
public onKeyboardInput = async (key: string, coordinates: Coordinates, page: Page) => {
let where: WhereWhatPair["where"] = { url: this.getBestUrl(page.url()) };
const selector = await this.generateSelector(page, coordinates, ActionType.Keydown);
const elementInfo = await getElementInformation(page, coordinates, '', false);
const inputType = elementInfo?.attributes?.type || "text";
if (selector) {
where.selectors = [selector];
}
const pair: WhereWhatPair = {
where,
what: [{
action: 'press',
args: [selector, encrypt(key), inputType],
}],
}
if (selector) {
this.generatedData.lastUsedSelector = selector;
this.generatedData.lastAction = 'press';
}
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
/**
* Returns tag name and text content for the specified selector
* used in customAction for decision modal
@@ -940,7 +885,7 @@ export class WorkflowGenerator {
return pair;
})
.filter((pair) => pair !== null) as WhereWhatPair[]; // Remove null entries
.filter((pair) => pair !== null) as WhereWhatPair[];
if (actionWasRemoved) {
logger.log("info", `Action with actionId ${actionId} removed from workflow`);
@@ -996,24 +941,6 @@ export class WorkflowGenerator {
}
}
/**
* Returns the currently generated workflow without all the generated flag actions.
* @param workflow The workflow for removing the generated flag actions from.
* @private
* @returns {WorkflowFile}
*/
private removeAllGeneratedFlags = (workflow: WorkflowFile): WorkflowFile => {
for (let i = 0; i < workflow.workflow.length; i++) {
if (
workflow.workflow[i].what[0] &&
workflow.workflow[i].what[0].action === 'flag' &&
workflow.workflow[i].what[0].args?.includes('generated')) {
workflow.workflow[i].what.splice(0, 1);
}
}
return workflow;
};
/**
* Adds generated flag actions to the workflow's pairs' what conditions.
* @param workflow The workflow for adding the generated flag actions from.
@@ -1127,7 +1054,6 @@ export class WorkflowGenerator {
: await getSelectors(page, coordinates);
if (this.paginationMode && selectorBasedOnCustomAction) {
// Chain selectors in specific priority order
const selectors = selectorBasedOnCustomAction;
const selectorChain = [
selectors?.iframeSelector?.full,
@@ -1324,17 +1250,15 @@ export class WorkflowGenerator {
if (this.workflowRecord.workflow[index].where.selectors?.includes(selector)) {
break;
} else {
// add new selector to the where part of the overshadowing pair
this.workflowRecord.workflow[index].where.selectors?.push(selector);
}
}
}
// push the action automatically to the first/the closest rule which would be overShadowed
this.workflowRecord.workflow[index].what =
this.workflowRecord.workflow[index].what.concat(pair.what);
return true;
} else {
// notify client about overshadowing a further rule
return false;
}
}
@@ -1352,8 +1276,7 @@ export class WorkflowGenerator {
const parsedUrl = new URL(url);
const protocol = parsedUrl.protocol === 'https:' || parsedUrl.protocol === 'http:' ? `${parsedUrl.protocol}//` : parsedUrl.protocol;
const regex = new RegExp(/(?=.*[A-Z])/g)
// remove all params with uppercase letters, they are most likely dynamically generated
// also escapes all regex characters from the params
const search = parsedUrl.search
.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
.split('&').map((param, index) => {
@@ -1380,8 +1303,6 @@ export class WorkflowGenerator {
* @param workflow The workflow to be checked.
*/
private checkWorkflowForParams = (workflow: WorkflowFile): string[] | null => {
// for now the where condition cannot have any params, so we're checking only what part of the pair
// where only the args part of what condition can have a parameter
for (const pair of workflow.workflow) {
for (const condition of pair.what) {
if (condition.args) {