Merge pull request #942 from getmaxun/fix-recorder

fix: recorder flow
This commit is contained in:
Karishma Shukla
2026-01-19 13:02:15 +05:30
committed by GitHub
23 changed files with 495 additions and 1459 deletions

1
.gitignore vendored
View File

@@ -1,5 +1,6 @@
# dependencies # dependencies
/node_modules /node_modules
/browser/node_modules
# misc # misc
.DS_Store .DS_Store

View File

@@ -67,7 +67,7 @@
"react-dom": "^18.0.0", "react-dom": "^18.0.0",
"react-i18next": "^15.1.3", "react-i18next": "^15.1.3",
"react-router-dom": "^6.26.1", "react-router-dom": "^6.26.1",
"rrweb-snapshot": "^2.0.0-alpha.4", "rrweb": "^2.0.0-alpha.4",
"sequelize": "^6.37.3", "sequelize": "^6.37.3",
"sequelize-typescript": "^2.1.6", "sequelize-typescript": "^2.1.6",
"sharp": "^0.33.5", "sharp": "^0.33.5",

View File

@@ -8,6 +8,7 @@ import { Socket } from "socket.io";
import { PlaywrightBlocker } from '@cliqz/adblocker-playwright'; import { PlaywrightBlocker } from '@cliqz/adblocker-playwright';
import fetch from 'cross-fetch'; import fetch from 'cross-fetch';
import logger from '../../logger'; import logger from '../../logger';
import { readFileSync } from "fs";
import { InterpreterSettings } from "../../types"; import { InterpreterSettings } from "../../types";
import { WorkflowGenerator } from "../../workflow-management/classes/Generator"; import { WorkflowGenerator } from "../../workflow-management/classes/Generator";
import { WorkflowInterpreter } from "../../workflow-management/classes/Interpreter"; import { WorkflowInterpreter } from "../../workflow-management/classes/Interpreter";
@@ -19,30 +20,17 @@ import { connectToRemoteBrowser } from '../browserConnection';
declare global { declare global {
interface Window { interface Window {
rrwebSnapshot?: any; rrweb?: any;
isRecording?: boolean;
emitEventToBackend?: (event: any) => Promise<void>;
} }
} }
interface RRWebSnapshot { // const MEMORY_CONFIG = {
type: number; // gcInterval: 20000,
childNodes?: RRWebSnapshot[]; // maxHeapSize: 1536 * 1024 * 1024,
tagName?: string; // heapUsageThreshold: 0.7
attributes?: Record<string, string>; // };
textContent?: string;
id: number;
[key: string]: any;
}
interface ProcessedSnapshot {
snapshot: RRWebSnapshot;
baseUrl: string;
}
const MEMORY_CONFIG = {
gcInterval: 20000, // Check memory more frequently (20s instead of 60s)
maxHeapSize: 1536 * 1024 * 1024, // 1.5GB
heapUsageThreshold: 0.7 // 70% (reduced threshold to react earlier)
};
/** /**
* This class represents a remote browser instance. * This class represents a remote browser instance.
@@ -110,22 +98,11 @@ export class RemoteBrowser {
public interpreter: WorkflowInterpreter; public interpreter: WorkflowInterpreter;
public isDOMStreamingActive: boolean = false; public isDOMStreamingActive: boolean = false;
private domUpdateInterval: NodeJS.Timeout | null = null;
private lastScrollPosition = { x: 0, y: 0 }; private lastScrollPosition = { x: 0, y: 0 };
private scrollThreshold = 200; // pixels private scrollThreshold = 200;
private snapshotDebounceTimeout: NodeJS.Timeout | null = null;
private networkRequestTimeout: NodeJS.Timeout | null = null; // private memoryCleanupInterval: NodeJS.Timeout | null = null;
private pendingNetworkRequests: string[] = []; // private memoryManagementInterval: NodeJS.Timeout | null = null;
private readonly INITIAL_LOAD_QUIET_PERIOD = 3000;
private networkWaitStartTime: number = 0;
private progressInterval: NodeJS.Timeout | null = null;
private hasShownInitialLoader: boolean = false;
private isInitialLoadInProgress: boolean = false;
private memoryCleanupInterval: NodeJS.Timeout | null = null;
private memoryManagementInterval: NodeJS.Timeout | null = null;
/** /**
* Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and * Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and
@@ -140,64 +117,53 @@ export class RemoteBrowser {
this.generator = new WorkflowGenerator(socket, poolId); this.generator = new WorkflowGenerator(socket, poolId);
} }
private async processRRWebSnapshot( // private initializeMemoryManagement(): void {
snapshot: RRWebSnapshot // this.memoryManagementInterval = setInterval(() => {
): Promise<ProcessedSnapshot> { // const memoryUsage = process.memoryUsage();
const baseUrl = this.currentPage?.url() || ""; // const heapUsageRatio = memoryUsage.heapUsed / MEMORY_CONFIG.maxHeapSize;
return { // if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.2) {
snapshot, // logger.warn(
baseUrl // "Critical memory pressure detected, triggering emergency cleanup"
}; // );
} // this.performMemoryCleanup();
// } else if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold) {
// logger.warn("High memory usage detected, triggering cleanup");
private initializeMemoryManagement(): void { // if (
this.memoryManagementInterval = setInterval(() => { // global.gc &&
const memoryUsage = process.memoryUsage(); // heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.1
const heapUsageRatio = memoryUsage.heapUsed / MEMORY_CONFIG.maxHeapSize; // ) {
// global.gc();
// }
// }
// }, MEMORY_CONFIG.gcInterval);
// }
if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.2) { // private async performMemoryCleanup(): Promise<void> {
logger.warn( // if (global.gc) {
"Critical memory pressure detected, triggering emergency cleanup" // try {
); // global.gc();
this.performMemoryCleanup(); // logger.info("Garbage collection requested");
} else if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold) { // } catch (error) {
logger.warn("High memory usage detected, triggering cleanup"); // logger.error("Error during garbage collection:", error);
// }
// }
if ( // if (this.currentPage) {
global.gc && // try {
heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.1 // await new Promise((resolve) => setTimeout(resolve, 500));
) { // logger.info("CDP session reset completed");
global.gc(); // } catch (error) {
} // logger.error("Error resetting CDP session:", error);
} // }
}, MEMORY_CONFIG.gcInterval); // }
}
private async performMemoryCleanup(): Promise<void> { // this.socket.emit("memory-cleanup", {
if (global.gc) { // userId: this.userId,
try { // timestamp: Date.now(),
global.gc(); // });
logger.info("Garbage collection requested"); // }
} catch (error) {
logger.error("Error during garbage collection:", error);
}
}
if (this.currentPage) {
try {
await new Promise((resolve) => setTimeout(resolve, 500));
logger.info("CDP session reset completed");
} catch (error) {
logger.error("Error resetting CDP session:", error);
}
}
this.socket.emit("memory-cleanup", {
userId: this.userId,
timestamp: Date.now(),
});
}
/** /**
* Normalizes URLs to prevent navigation loops while maintaining consistent format * Normalizes URLs to prevent navigation loops while maintaining consistent format
@@ -205,9 +171,7 @@ export class RemoteBrowser {
private normalizeUrl(url: string): string { private normalizeUrl(url: string): string {
try { try {
const parsedUrl = new URL(url); const parsedUrl = new URL(url);
// Remove trailing slashes except for root path
parsedUrl.pathname = parsedUrl.pathname.replace(/\/+$/, '') || '/'; parsedUrl.pathname = parsedUrl.pathname.replace(/\/+$/, '') || '/';
// Ensure consistent protocol handling
parsedUrl.protocol = parsedUrl.protocol.toLowerCase(); parsedUrl.protocol = parsedUrl.protocol.toLowerCase();
return parsedUrl.toString(); return parsedUrl.toString();
} catch { } catch {
@@ -260,14 +224,6 @@ export class RemoteBrowser {
if (scrollDelta > this.scrollThreshold) { if (scrollDelta > this.scrollThreshold) {
this.lastScrollPosition = { x: scrollInfo.x, y: scrollInfo.y }; this.lastScrollPosition = { x: scrollInfo.x, y: scrollInfo.y };
if (this.snapshotDebounceTimeout) {
clearTimeout(this.snapshotDebounceTimeout);
}
this.snapshotDebounceTimeout = setTimeout(async () => {
await this.makeAndEmitDOMSnapshot();
}, 300);
} }
} catch (error) { } catch (error) {
logger.error("Error handling scroll event:", error); logger.error("Error handling scroll event:", error);
@@ -276,79 +232,6 @@ export class RemoteBrowser {
); );
} }
private setupPageChangeListeners(): void {
if (!this.currentPage) return;
try {
if (!this.currentPage.isClosed()) {
this.currentPage.removeAllListeners("domcontentloaded");
this.currentPage.removeAllListeners("response");
}
} catch (error: any) {
logger.warn(`Error removing page change listeners: ${error.message}`);
}
this.currentPage.on("domcontentloaded", async () => {
if (!this.isInitialLoadInProgress) {
logger.info("DOM content loaded - triggering snapshot");
await this.makeAndEmitDOMSnapshot();
}
});
this.currentPage.on("response", async (response) => {
const url = response.url();
const isDocumentRequest = response.request().resourceType() === "document";
if (!this.hasShownInitialLoader && isDocumentRequest && !url.includes("about:blank")) {
this.hasShownInitialLoader = true;
this.isInitialLoadInProgress = true;
this.pendingNetworkRequests.push(url);
if (this.networkRequestTimeout) {
clearTimeout(this.networkRequestTimeout);
this.networkRequestTimeout = null;
}
if (this.progressInterval) {
clearInterval(this.progressInterval);
this.progressInterval = null;
}
this.networkWaitStartTime = Date.now();
this.progressInterval = setInterval(() => {
const elapsed = Date.now() - this.networkWaitStartTime;
const navigationProgress = Math.min((elapsed / this.INITIAL_LOAD_QUIET_PERIOD) * 40, 35);
const totalProgress = 60 + navigationProgress;
this.emitLoadingProgress(totalProgress, this.pendingNetworkRequests.length);
}, 500);
logger.debug(
`Initial load network request received: ${url}. Using ${this.INITIAL_LOAD_QUIET_PERIOD}ms quiet period`
);
this.networkRequestTimeout = setTimeout(async () => {
logger.info(
`Initial load network quiet period reached (${this.INITIAL_LOAD_QUIET_PERIOD}ms)`
);
if (this.progressInterval) {
clearInterval(this.progressInterval);
this.progressInterval = null;
}
this.emitLoadingProgress(100, this.pendingNetworkRequests.length);
this.pendingNetworkRequests = [];
this.networkRequestTimeout = null;
this.isInitialLoadInProgress = false;
await this.makeAndEmitDOMSnapshot();
}, this.INITIAL_LOAD_QUIET_PERIOD);
}
});
}
private emitLoadingProgress(progress: number, pendingRequests: number): void { private emitLoadingProgress(progress: number, pendingRequests: number): void {
this.socket.emit("domLoadingProgress", { this.socket.emit("domLoadingProgress", {
progress: Math.round(progress), progress: Math.round(progress),
@@ -368,16 +251,27 @@ export class RemoteBrowser {
} }
page.on('framenavigated', async (frame) => { page.on('framenavigated', async (frame) => {
if (frame === page.mainFrame()) { if (frame === page.mainFrame()) {
const currentUrl = page.url(); const currentUrl = page.url();
if (this.shouldEmitUrlChange(currentUrl)) { if (this.shouldEmitUrlChange(currentUrl)) {
this.lastEmittedUrl = currentUrl; this.lastEmittedUrl = currentUrl;
this.socket.emit('urlChanged', {url: currentUrl, userId: this.userId}); this.socket.emit('urlChanged', { url: currentUrl, userId: this.userId });
}
} }
await page.evaluate(() => {
if (window.rrweb && window.isRecording) {
window.isRecording = false;
}
});
await page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {
logger.warn('[rrweb] Network idle timeout on navigation, proceeding with rrweb initialization');
});
await this.initializeRRWebRecording(page);
}
}); });
// Handle page load events with retry mechanism
page.on('load', async () => { page.on('load', async () => {
const injectScript = async (): Promise<boolean> => { const injectScript = async (): Promise<boolean> => {
try { try {
@@ -401,6 +295,96 @@ export class RemoteBrowser {
}); });
} }
/**
* Initialize rrweb recording for real-time DOM streaming
* This replaces the snapshot-based approach with live event streaming
*/
private async initializeRRWebRecording(page: Page): Promise<void> {
try {
const rrwebJsPath = require.resolve('rrweb/dist/rrweb.min.js');
const rrwebScriptContent = readFileSync(rrwebJsPath, 'utf8');
await page.context().addInitScript(rrwebScriptContent);
await page.evaluate((scriptContent) => {
if (typeof window.rrweb === 'undefined') {
try {
(0, eval)(scriptContent);
} catch (e) {
console.error('[rrweb] eval failed:', e);
}
}
}, rrwebScriptContent);
const rrwebLoaded = await page.evaluate(() => typeof window.rrweb !== 'undefined');
if (rrwebLoaded) {
logger.debug('[rrweb] Script injected successfully');
} else {
logger.warn('[rrweb] Script injection failed - window.rrweb not found');
}
const isAlreadyExposed = await page.evaluate(() => {
return typeof window.emitEventToBackend === 'function';
});
if (!isAlreadyExposed) {
let hasEmittedFullSnapshot = false;
await page.exposeFunction('emitEventToBackend', (event: any) => {
this.socket.emit('rrweb-event', event);
if (event.type === 2 && !hasEmittedFullSnapshot) {
hasEmittedFullSnapshot = true;
this.emitLoadingProgress(100, 0);
logger.debug(`[rrweb] Full snapshot sent, loading progress at 100%`);
}
});
}
const rrwebStatus = await page.evaluate(() => {
if (!window.rrweb) {
console.error('[rrweb] window.rrweb is not defined!');
return { success: false, error: 'window.rrweb is not defined' };
}
if (window.isRecording) {
return { success: false, error: 'already recording' };
}
window.isRecording = true;
try {
const recordHandle = window.rrweb.record({
emit(event: any) {
if (window.emitEventToBackend) {
window.emitEventToBackend(event).catch(() => { });
}
},
maskAllInputs: false,
recordCanvas: true,
input: true
});
(window as any).rrwebRecordHandle = recordHandle;
return { success: true };
} catch (error: any) {
console.error('[rrweb] Failed to start recording:', error);
return { success: false, error: error.message };
}
});
if (rrwebStatus.success) {
this.isDOMStreamingActive = true;
this.emitLoadingProgress(80, 0);
this.setupScrollEventListener();
} else {
logger.error(`Failed to initialize rrweb recording: ${rrwebStatus.error}`);
}
} catch (error: any) {
logger.error(`Failed to initialize rrweb recording: ${error.message}`);
}
}
private getUserAgent() { private getUserAgent() {
const userAgents = [ const userAgents = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.140 Safari/537.36', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.140 Safari/537.36',
@@ -432,7 +416,6 @@ export class RemoteBrowser {
} }
} catch (error: any) { } catch (error: any) {
logger.error(`Enhanced fingerprinting failed: ${error.message}`); logger.error(`Enhanced fingerprinting failed: ${error.message}`);
// Don't throw - fallback to basic functionality
} }
} }
@@ -540,14 +523,18 @@ export class RemoteBrowser {
patchedGetter.toString();` patchedGetter.toString();`
); );
await this.context.addInitScript({ path: './server/src/browser-management/classes/rrweb-bundle.js' });
this.currentPage = await this.context.newPage(); this.currentPage = await this.context.newPage();
this.emitLoadingProgress(40, 0); this.emitLoadingProgress(40, 0);
await this.setupPageEventListeners(this.currentPage); await this.setupPageEventListeners(this.currentPage);
await this.currentPage.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {
logger.warn('[rrweb] Network idle timeout, proceeding with rrweb initialization');
});
await this.initializeRRWebRecording(this.currentPage);
try { try {
const blocker = await PlaywrightBlocker.fromLists(fetch, ['https://easylist.to/easylist/easylist.txt']); const blocker = await PlaywrightBlocker.fromLists(fetch, ['https://easylist.to/easylist/easylist.txt']);
await blocker.enableBlockingInPage(this.currentPage as any); await blocker.enableBlockingInPage(this.currentPage as any);
@@ -556,12 +543,9 @@ export class RemoteBrowser {
console.log('Adblocker initialized'); console.log('Adblocker initialized');
} catch (error: any) { } catch (error: any) {
console.warn('Failed to initialize adblocker, continuing without it:', error.message); console.warn('Failed to initialize adblocker, continuing without it:', error.message);
// Still need to set up the CDP session even if blocker fails
this.client = await this.currentPage.context().newCDPSession(this.currentPage); this.client = await this.currentPage.context().newCDPSession(this.currentPage);
} }
this.emitLoadingProgress(60, 0);
success = true; success = true;
logger.log('debug', `Browser initialized successfully for user ${userId}`); logger.log('debug', `Browser initialized successfully for user ${userId}`);
} catch (error: any) { } catch (error: any) {
@@ -656,7 +640,6 @@ export class RemoteBrowser {
private removeAllSocketListeners(): void { private removeAllSocketListeners(): void {
try { try {
this.socket.removeAllListeners('captureDirectScreenshot'); this.socket.removeAllListeners('captureDirectScreenshot');
this.socket.removeAllListeners('rerender');
this.socket.removeAllListeners('settings'); this.socket.removeAllListeners('settings');
this.socket.removeAllListeners('changeTab'); this.socket.removeAllListeners('changeTab');
this.socket.removeAllListeners('addTab'); this.socket.removeAllListeners('addTab');
@@ -683,13 +666,6 @@ export class RemoteBrowser {
await this.captureDirectScreenshot(settings); await this.captureDirectScreenshot(settings);
}); });
this.socket.on("rerender", async () => {
logger.debug(
`General rerender event received, checking if for user ${this.userId}`
);
await this.makeAndEmitDOMSnapshot();
});
this.socket.on( this.socket.on(
"changeTab", "changeTab",
async (tabIndex) => await this.changeTab(tabIndex) async (tabIndex) => await this.changeTab(tabIndex)
@@ -719,157 +695,6 @@ export class RemoteBrowser {
}; };
/** /**
* Subscribe to DOM streaming - simplified version following screenshot pattern
*/
public async subscribeToDOM(): Promise<void> {
if (!this.client) {
logger.warn("DOM streaming requires scraping browser with CDP client");
return;
}
try {
this.isDOMStreamingActive = true;
logger.info("DOM streaming started successfully");
this.setupScrollEventListener();
this.setupPageChangeListeners();
} catch (error) {
logger.error("Failed to start DOM streaming:", error);
this.isDOMStreamingActive = false;
}
}
/**
* CDP-based DOM snapshot creation using captured network resources
*/
public async makeAndEmitDOMSnapshot(): Promise<void> {
if (!this.currentPage || !this.isDOMStreamingActive) {
return;
}
try {
// Check if page is still valid and not closed
if (this.currentPage.isClosed()) {
logger.debug("Skipping DOM snapshot - page is closed");
return;
}
// Double-check page state after network wait
if (this.currentPage.isClosed()) {
logger.debug("Skipping DOM snapshot - page closed during network wait");
return;
}
// Get current scroll position
const currentScrollInfo = await this.currentPage.evaluate(() => ({
x: window.scrollX,
y: window.scrollY,
maxX: Math.max(
0,
document.documentElement.scrollWidth - window.innerWidth
),
maxY: Math.max(
0,
document.documentElement.scrollHeight - window.innerHeight
),
documentHeight: document.documentElement.scrollHeight,
}));
logger.info(
`Creating rrweb snapshot at scroll position: ${currentScrollInfo.y}/${currentScrollInfo.maxY}`
);
// Update our tracked scroll position
this.lastScrollPosition = {
x: currentScrollInfo.x,
y: currentScrollInfo.y,
};
// Final check before snapshot
if (this.currentPage.isClosed()) {
logger.debug("Skipping DOM snapshot - page closed before snapshot");
return;
}
// Capture snapshot using rrweb
const rawSnapshot = await this.currentPage.evaluate(() => {
if (typeof window.rrwebSnapshot === "undefined") {
throw new Error("rrweb-snapshot library not available");
}
return window.rrwebSnapshot.snapshot(document, {
inlineImages: false,
collectFonts: true,
});
});
// Process the snapshot to proxy resources
const processedSnapshot = await this.processRRWebSnapshot(rawSnapshot);
// Add scroll position information
const enhancedSnapshot = {
...processedSnapshot,
scrollPosition: currentScrollInfo,
captureTime: Date.now(),
};
// Emit the processed snapshot
this.emitRRWebSnapshot(enhancedSnapshot);
} catch (error) {
// Handle navigation context destruction gracefully
if (
error instanceof Error &&
(error.message.includes("Execution context was destroyed") ||
error.message.includes("most likely because of a navigation") ||
error.message.includes("Target closed"))
) {
logger.debug("DOM snapshot skipped due to page navigation or closure");
return;
}
logger.error("Failed to create rrweb snapshot:", error);
this.socket.emit("dom-mode-error", {
userId: this.userId,
message: "Failed to create rrweb snapshot",
error: error instanceof Error ? error.message : String(error),
timestamp: Date.now(),
});
}
}
/**
* Emit DOM snapshot to client - following screenshot pattern
*/
private emitRRWebSnapshot(processedSnapshot: ProcessedSnapshot): void {
this.socket.emit("domcast", {
snapshotData: processedSnapshot,
userId: this.userId,
timestamp: Date.now(),
});
}
/**
* Stop DOM streaming - following dom snapshot pattern
*/
private async stopDOM(): Promise<void> {
this.isDOMStreamingActive = false;
if (this.domUpdateInterval) {
clearInterval(this.domUpdateInterval);
this.domUpdateInterval = null;
}
if (this.networkRequestTimeout) {
clearTimeout(this.networkRequestTimeout);
this.networkRequestTimeout = null;
}
this.pendingNetworkRequests = [];
logger.info("DOM streaming stopped successfully");
}
/**rrweb-bundle
* Terminates the dom snapshot session and closes the remote browser. * Terminates the dom snapshot session and closes the remote browser.
* If an interpretation was running it will be stopped. * If an interpretation was running it will be stopped.
* @returns {Promise<void>} * @returns {Promise<void>}
@@ -877,35 +702,15 @@ export class RemoteBrowser {
public async switchOff(): Promise<void> { public async switchOff(): Promise<void> {
this.isDOMStreamingActive = false; this.isDOMStreamingActive = false;
if (this.domUpdateInterval) { // if (this.memoryCleanupInterval) {
clearInterval(this.domUpdateInterval); // clearInterval(this.memoryCleanupInterval);
this.domUpdateInterval = null; // this.memoryCleanupInterval = null;
} // }
if (this.memoryCleanupInterval) { // if (this.memoryManagementInterval) {
clearInterval(this.memoryCleanupInterval); // clearInterval(this.memoryManagementInterval);
this.memoryCleanupInterval = null; // this.memoryManagementInterval = null;
} // }
if (this.memoryManagementInterval) {
clearInterval(this.memoryManagementInterval);
this.memoryManagementInterval = null;
}
if (this.progressInterval) {
clearInterval(this.progressInterval);
this.progressInterval = null;
}
if (this.snapshotDebounceTimeout) {
clearTimeout(this.snapshotDebounceTimeout);
this.snapshotDebounceTimeout = null;
}
if (this.networkRequestTimeout) {
clearTimeout(this.networkRequestTimeout);
this.networkRequestTimeout = null;
}
this.removeAllSocketListeners(); this.removeAllSocketListeners();
@@ -923,7 +728,6 @@ export class RemoteBrowser {
logger.warn(`Error removing page listeners: ${error.message}`); logger.warn(`Error removing page listeners: ${error.message}`);
} }
// Clean up Generator listeners to prevent memory leaks
if (this.generator) { if (this.generator) {
try { try {
this.generator.cleanup(); this.generator.cleanup();
@@ -933,20 +737,12 @@ export class RemoteBrowser {
} }
} }
// Stop interpretation with individual error handling (also calls clearState which removes pausing listeners)
try { try {
await this.interpreter.stopInterpretation(); await this.interpreter.stopInterpretation();
} catch (error) { } catch (error) {
logger.error("Error stopping interpretation during shutdown:", error); logger.error("Error stopping interpretation during shutdown:", error);
} }
// Stop DOM streaming with individual error handling
try {
await this.stopDOM();
} catch (error) {
logger.error("Error stopping DOM during shutdown:", error);
}
try { try {
if (this.client && this.currentPage && !this.currentPage.isClosed()) { if (this.client && this.currentPage && !this.currentPage.isClosed()) {
const detachPromise = this.client.detach(); const detachPromise = this.client.detach();
@@ -1081,7 +877,6 @@ export class RemoteBrowser {
private changeTab = async (tabIndex: number): Promise<void> => { private changeTab = async (tabIndex: number): Promise<void> => {
const page = this.currentPage?.context().pages()[tabIndex]; const page = this.currentPage?.context().pages()[tabIndex];
if (page) { if (page) {
await this.stopDOM();
this.currentPage = page; this.currentPage = page;
await this.setupPageEventListeners(this.currentPage); await this.setupPageEventListeners(this.currentPage);
@@ -1093,10 +888,6 @@ export class RemoteBrowser {
url: this.currentPage.url(), url: this.currentPage.url(),
userId: this.userId userId: this.userId
}); });
if (this.isDOMStreamingActive) {
await this.makeAndEmitDOMSnapshot();
await this.subscribeToDOM();
}
} else { } else {
logger.log('error', `${tabIndex} index out of range of pages`) logger.log('error', `${tabIndex} index out of range of pages`)
} }
@@ -1118,8 +909,7 @@ export class RemoteBrowser {
this.currentPage = newPage; this.currentPage = newPage;
if (this.currentPage) { if (this.currentPage) {
await this.setupPageEventListeners(this.currentPage); await this.setupPageEventListeners(this.currentPage);
logger.debug('Using rrweb live recording for new page');
await this.subscribeToDOM();
} else { } else {
logger.log('error', 'Could not get a new page, returned undefined'); logger.log('error', 'Could not get a new page, returned undefined');
} }

View File

@@ -1,10 +0,0 @@
const esbuild = require('esbuild');
esbuild.build({
entryPoints: ['rrweb-entry.js'],
bundle: true,
minify: true,
outfile: 'rrweb-bundle.js',
format: 'iife', // so that rrwebSnapshot is available on window
globalName: 'rrwebSnapshotBundle'
}).catch(() => process.exit(1));

File diff suppressed because one or more lines are too long

View File

@@ -1,2 +0,0 @@
import { snapshot } from 'rrweb-snapshot';
window.rrwebSnapshot = { snapshot };

View File

@@ -31,10 +31,6 @@ export const initializeRemoteBrowserForRecording = (userId: string, mode: string
if (activeId) { if (activeId) {
const remoteBrowser = browserPool.getRemoteBrowser(activeId); const remoteBrowser = browserPool.getRemoteBrowser(activeId);
remoteBrowser?.updateSocket(socket); remoteBrowser?.updateSocket(socket);
if (remoteBrowser?.isDOMStreamingActive) {
remoteBrowser?.makeAndEmitDOMSnapshot();
}
} else { } else {
const browserSession = new RemoteBrowser(socket, userId, id); const browserSession = new RemoteBrowser(socket, userId, id);
browserSession.interpreter.subscribeToPausing(); browserSession.interpreter.subscribeToPausing();
@@ -43,7 +39,6 @@ export const initializeRemoteBrowserForRecording = (userId: string, mode: string
await browserSession.initialize(userId); await browserSession.initialize(userId);
await browserSession.registerEditorEvents(); await browserSession.registerEditorEvents();
await browserSession.subscribeToDOM();
logger.info('DOM streaming started for remote browser in recording mode'); logger.info('DOM streaming started for remote browser in recording mode');
browserPool.addRemoteBrowser(id, browserSession, userId, false, "recording"); browserPool.addRemoteBrowser(id, browserSession, userId, false, "recording");

View File

@@ -102,188 +102,6 @@ const handleGenerateAction =
} }
} }
/**
* A wrapper function for handling mousedown event.
* @param socket The socket connection
* @param coordinates - coordinates of the mouse click
* @category HelperFunctions
*/
const onMousedown = async (coordinates: Coordinates, userId: string) => {
logger.log('debug', 'Handling mousedown event emitted from client');
await handleWrapper(handleMousedown, userId, coordinates);
}
/**
* A mousedown event handler.
* Reproduces the click on the remote browser instance
* and generates pair data for the recorded workflow.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param x - the x coordinate of the mousedown event
* @param y - the y coordinate of the mousedown event
* @category BrowserManagement
*/
const handleMousedown = async (activeBrowser: RemoteBrowser, page: Page, { x, y }: Coordinates) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring mousedown event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await generator.onClick({ x, y }, page);
const previousUrl = page.url();
const tabsBeforeClick = page.context().pages().length;
await page.mouse.click(x, y);
// try if the click caused a navigation to a new url
try {
await page.waitForNavigation({ timeout: 2000 });
const currentUrl = page.url();
if (currentUrl !== previousUrl) {
generator.notifyUrlChange(currentUrl);
}
} catch (e) {
const { message } = e as Error;
} //ignore possible timeouts
// check if any new page was opened by the click
const tabsAfterClick = page.context().pages().length;
const numOfNewPages = tabsAfterClick - tabsBeforeClick;
if (numOfNewPages > 0) {
for (let i = 1; i <= numOfNewPages; i++) {
const newPage = page.context().pages()[tabsAfterClick - i];
if (newPage) {
generator.notifyOnNewTab(newPage, tabsAfterClick - i);
}
}
}
logger.log("debug", `Clicked on position x:${x}, y:${y}`);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling mousedown event: ${message}`);
}
};
/**
* A wrapper function for handling the wheel event.
* @param socket The socket connection
* @param scrollDeltas - the scroll deltas of the wheel event
* @category HelperFunctions
*/
const onWheel = async (scrollDeltas: ScrollDeltas, userId: string) => {
logger.log('debug', 'Handling scroll event emitted from client');
await handleWrapper(handleWheel, userId, scrollDeltas);
};
/**
* A wheel event handler.
* Reproduces the wheel event on the remote browser instance.
* Scroll is not generated for the workflow pair. This is because
* Playwright scrolls elements into focus on any action.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param deltaX - the delta x of the wheel event
* @param deltaY - the delta y of the wheel event
* @category BrowserManagement
*/
const handleWheel = async (activeBrowser: RemoteBrowser, page: Page, { deltaX, deltaY }: ScrollDeltas) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring wheel event: page is closed`);
return;
}
await page.mouse.wheel(deltaX, deltaY).catch(error => {
logger.log('warn', `Wheel event failed: ${error.message}`);
});
logger.log('debug', `Scrolled horizontally ${deltaX} pixels and vertically ${deltaY} pixels`);
} catch (e) {
const { message } = e as Error;
logger.log('warn', `Error handling wheel event: ${message}`);
}
};
/**
* A wrapper function for handling the mousemove event.
* @param socket The socket connection
* @param coordinates - the coordinates of the mousemove event
* @category HelperFunctions
*/
const onMousemove = async (coordinates: Coordinates, userId: string) => {
logger.log('debug', 'Handling mousemove event emitted from client');
await handleWrapper(handleMousemove, userId, coordinates);
}
/**
* A mousemove event handler.
* Reproduces the mousemove event on the remote browser instance
* and generates data for the client's highlighter.
* Mousemove is also not reflected in the workflow.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param x - the x coordinate of the mousemove event
* @param y - the y coordinate of the mousemove event
* @category BrowserManagement
*/
const handleMousemove = async (activeBrowser: RemoteBrowser, page: Page, { x, y }: Coordinates) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring mousemove event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await page.mouse.move(x, y);
// throttle(async () => {
// if (!page.isClosed()) {
// await generator.generateDataForHighlighter(page, { x, y });
// }
// }, 100)();
logger.log("debug", `Moved over position x:${x}, y:${y}`);
} catch (e) {
const { message } = e as Error;
logger.log("error", message);
}
}
/**
* A wrapper function for handling the keydown event.
* @param socket The socket connection
* @param keyboardInput - the keyboard input of the keydown event
* @category HelperFunctions
*/
const onKeydown = async (keyboardInput: KeyboardInput, userId: string) => {
logger.log('debug', 'Handling keydown event emitted from client');
await handleWrapper(handleKeydown, userId, keyboardInput);
}
/**
* A keydown event handler.
* Reproduces the keydown event on the remote browser instance
* and generates the workflow pair data.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param key - the pressed key
* @param coordinates - the coordinates, where the keydown event happened
* @category BrowserManagement
*/
const handleKeydown = async (activeBrowser: RemoteBrowser, page: Page, { key, coordinates }: KeyboardInput) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring keydown event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await page.keyboard.down(key);
await generator.onKeyboardInput(key, coordinates, page);
logger.log("debug", `Key ${key} pressed`);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling keydown event: ${message}`);
}
};
/** /**
* Handles the date selection event. * Handles the date selection event.
* @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param activeBrowser - the active remote browser {@link RemoteBrowser}
@@ -493,7 +311,7 @@ const handleChangeUrl = async (activeBrowser: RemoteBrowser, page: Page, url: st
try { try {
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 }); await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
await page.waitForTimeout(2000); await page.waitForTimeout(500);
logger.log("debug", `Went to ${url}`); logger.log("debug", `Went to ${url}`);
} catch (e) { } catch (e) {
const { message } = e as Error; const { message } = e as Error;
@@ -689,8 +507,7 @@ const handleClickAction = async (
} }
} }
await new Promise((resolve) => setTimeout(resolve, 2000)); await new Promise((resolve) => setTimeout(resolve, 300));
await activeBrowser.makeAndEmitDOMSnapshot();
} catch (e) { } catch (e) {
const { message } = e as Error; const { message } = e as Error;
logger.log( logger.log(
@@ -778,47 +595,6 @@ const onDOMKeyboardAction = async (
await handleWrapper(handleKeyboardAction, userId, data); await handleWrapper(handleKeyboardAction, userId, data);
}; };
/**
* Handles the workflow pair event.
* @param activeBrowser - the active remote browser {@link RemoteBrowser}
* @param page - the active page of the remote browser
* @param data - the data of the workflow pair event
* @category BrowserManagement
*/
const handleWorkflowPair = async (
activeBrowser: RemoteBrowser,
page: Page,
data: { pair: WhereWhatPair; userId: string }
) => {
try {
if (page.isClosed()) {
logger.log("debug", `Ignoring workflow pair event: page is closed`);
return;
}
const generator = activeBrowser.generator;
await generator.onDOMWorkflowPair(page, data);
logger.log("debug", `Workflow pair processed from frontend`);
} catch (e) {
const { message } = e as Error;
logger.log("warn", `Error handling workflow pair event: ${message}`);
}
};
/**
* A wrapper function for handling the workflow pair event.
* @param socket The socket connection
* @param data - the data of the workflow pair event
* @category HelperFunctions
*/
const onDOMWorkflowPair = async (
data: { pair: WhereWhatPair; userId: string },
userId: string
) => {
logger.log("debug", "Handling workflow pair event emitted from client");
await handleWrapper(handleWorkflowPair, userId, data);
};
/** /**
* Handles the remove action event. * Handles the remove action event.
* This is called when a user discards a capture action (list or text) that was already emitted to the backend. * This is called when a user discards a capture action (list or text) that was already emitted to the backend.
@@ -1026,11 +802,6 @@ const onTestPaginationScroll = async (
* @category BrowserManagement * @category BrowserManagement
*/ */
const registerInputHandlers = (socket: Socket, userId: string) => { const registerInputHandlers = (socket: Socket, userId: string) => {
// Register handlers with the socket
socket.on("input:mousedown", (data) => onMousedown(data, userId));
socket.on("input:wheel", (data) => onWheel(data, userId));
socket.on("input:mousemove", (data) => onMousemove(data, userId));
socket.on("input:keydown", (data) => onKeydown(data, userId));
socket.on("input:keyup", (data) => onKeyup(data, userId)); socket.on("input:keyup", (data) => onKeyup(data, userId));
socket.on("input:url", (data) => onChangeUrl(data, userId)); socket.on("input:url", (data) => onChangeUrl(data, userId));
socket.on("input:refresh", () => onRefresh(userId)); socket.on("input:refresh", () => onRefresh(userId));
@@ -1045,7 +816,6 @@ const registerInputHandlers = (socket: Socket, userId: string) => {
socket.on("dom:click", (data) => onDOMClickAction(data, userId)); socket.on("dom:click", (data) => onDOMClickAction(data, userId));
socket.on("dom:keypress", (data) => onDOMKeyboardAction(data, userId)); socket.on("dom:keypress", (data) => onDOMKeyboardAction(data, userId));
socket.on("dom:addpair", (data) => onDOMWorkflowPair(data, userId));
socket.on("testPaginationScroll", (data) => onTestPaginationScroll(data, userId, socket)); socket.on("testPaginationScroll", (data) => onTestPaginationScroll(data, userId, socket));
}; };
@@ -1058,10 +828,6 @@ const registerInputHandlers = (socket: Socket, userId: string) => {
*/ */
const removeInputHandlers = (socket: Socket) => { const removeInputHandlers = (socket: Socket) => {
try { try {
socket.removeAllListeners("input:mousedown");
socket.removeAllListeners("input:wheel");
socket.removeAllListeners("input:mousemove");
socket.removeAllListeners("input:keydown");
socket.removeAllListeners("input:keyup"); socket.removeAllListeners("input:keyup");
socket.removeAllListeners("input:url"); socket.removeAllListeners("input:url");
socket.removeAllListeners("input:refresh"); socket.removeAllListeners("input:refresh");
@@ -1075,7 +841,6 @@ const removeInputHandlers = (socket: Socket) => {
socket.removeAllListeners("dom:input"); socket.removeAllListeners("dom:input");
socket.removeAllListeners("dom:click"); socket.removeAllListeners("dom:click");
socket.removeAllListeners("dom:keypress"); socket.removeAllListeners("dom:keypress");
socket.removeAllListeners("dom:addpair");
socket.removeAllListeners("removeAction"); socket.removeAllListeners("removeAction");
socket.removeAllListeners("testPaginationScroll"); socket.removeAllListeners("testPaginationScroll");
} catch (error: any) { } catch (error: any) {

View File

@@ -198,7 +198,6 @@ export class WorkflowGenerator {
private async getSelectorsForSchema(page: Page, schema: Record<string, { selector: string }>): Promise<string[]> { private async getSelectorsForSchema(page: Page, schema: Record<string, { selector: string }>): Promise<string[]> {
const selectors = Object.values(schema).map((field) => field.selector); const selectors = Object.values(schema).map((field) => field.selector);
// Verify if the selectors are present and actionable on the current page
const actionableSelectors: string[] = []; const actionableSelectors: string[] = [];
for (const selector of selectors) { for (const selector of selectors) {
const isActionable = await page.isVisible(selector).catch(() => false); const isActionable = await page.isVisible(selector).catch(() => false);
@@ -235,7 +234,6 @@ export class WorkflowGenerator {
private addPairToWorkflowAndNotifyClient = async (pair: WhereWhatPair, page: Page) => { private addPairToWorkflowAndNotifyClient = async (pair: WhereWhatPair, page: Page) => {
let matched = false; let matched = false;
// Check for scrapeSchema actions and enhance the where condition
if (pair.what[0].action === 'scrapeSchema') { if (pair.what[0].action === 'scrapeSchema') {
const schema = pair.what[0]?.args?.[0]; const schema = pair.what[0]?.args?.[0];
if (schema) { if (schema) {
@@ -244,7 +242,6 @@ export class WorkflowGenerator {
} }
} }
// Validate if the pair is already in the workflow
if (pair.where.selectors && pair.where.selectors[0]) { if (pair.where.selectors && pair.where.selectors[0]) {
const match = selectorAlreadyInWorkflow(pair.where.selectors[0], this.workflowRecord.workflow); const match = selectorAlreadyInWorkflow(pair.where.selectors[0], this.workflowRecord.workflow);
if (match) { if (match) {
@@ -260,7 +257,6 @@ export class WorkflowGenerator {
} }
} }
// Handle cases where the where condition isn't already present
if (!matched) { if (!matched) {
const handled = await this.handleOverShadowing(pair, page, this.generatedData.lastIndex || 0); const handled = await this.handleOverShadowing(pair, page, this.generatedData.lastIndex || 0);
if (!handled) { if (!handled) {
@@ -282,7 +278,6 @@ export class WorkflowGenerator {
} }
} }
// Emit the updated workflow to the client
this.socket.emit('workflow', this.workflowRecord); this.socket.emit('workflow', this.workflowRecord);
logger.log('info', `Workflow emitted`); logger.log('info', `Workflow emitted`);
}; };
@@ -367,7 +362,6 @@ export class WorkflowGenerator {
await this.addPairToWorkflowAndNotifyClient(pair, page); await this.addPairToWorkflowAndNotifyClient(pair, page);
}; };
// Handles click events on the DOM, generating a pair for the click action
public onDOMClickAction = async (page: Page, data: { public onDOMClickAction = async (page: Page, data: {
selector: string, selector: string,
url: string, url: string,
@@ -388,7 +382,6 @@ export class WorkflowGenerator {
}], }],
}; };
// Handle special input elements with cursor positioning
if (elementInfo && coordinates && if (elementInfo && coordinates &&
(elementInfo.tagName === 'INPUT' || elementInfo.tagName === 'TEXTAREA')) { (elementInfo.tagName === 'INPUT' || elementInfo.tagName === 'TEXTAREA')) {
pair.what[0] = { pair.what[0] = {
@@ -403,7 +396,6 @@ export class WorkflowGenerator {
await this.addPairToWorkflowAndNotifyClient(pair, page); await this.addPairToWorkflowAndNotifyClient(pair, page);
}; };
// Handles keyboard actions on the DOM, generating a pair for the key press action
public onDOMKeyboardAction = async (page: Page, data: { public onDOMKeyboardAction = async (page: Page, data: {
selector: string, selector: string,
key: string, key: string,
@@ -430,7 +422,6 @@ export class WorkflowGenerator {
await this.addPairToWorkflowAndNotifyClient(pair, page); await this.addPairToWorkflowAndNotifyClient(pair, page);
}; };
// Handles navigation events on the DOM, generating a pair for the navigation action
public onDOMNavigation = async (page: Page, data: { public onDOMNavigation = async (page: Page, data: {
url: string, url: string,
currentUrl: string, currentUrl: string,
@@ -450,13 +441,6 @@ export class WorkflowGenerator {
await this.addPairToWorkflowAndNotifyClient(pair, page); await this.addPairToWorkflowAndNotifyClient(pair, page);
}; };
// Handles workflow pair events on the DOM
public onDOMWorkflowPair = async (page: Page, data: { pair: WhereWhatPair, userId: string }) => {
const { pair } = data;
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
/** /**
* Generates a pair for the click event. * Generates a pair for the click event.
* @param coordinates The coordinates of the click event. * @param coordinates The coordinates of the click event.
@@ -471,25 +455,22 @@ export class WorkflowGenerator {
const elementInfo = await getElementInformation(page, coordinates, '', false); const elementInfo = await getElementInformation(page, coordinates, '', false);
console.log("Element info: ", elementInfo); console.log("Element info: ", elementInfo);
// Check if clicked element is a select dropdown
const isDropdown = elementInfo?.tagName === 'SELECT'; const isDropdown = elementInfo?.tagName === 'SELECT';
if (isDropdown && elementInfo.innerHTML) { if (isDropdown && elementInfo.innerHTML) {
// Parse options from innerHTML
const options = elementInfo.innerHTML const options = elementInfo.innerHTML
.split('<option') .split('<option')
.slice(1) // Remove first empty element .slice(1)
.map(optionHtml => { .map(optionHtml => {
const valueMatch = optionHtml.match(/value="([^"]*)"/); const valueMatch = optionHtml.match(/value="([^"]*)"/);
const disabledMatch = optionHtml.includes('disabled="disabled"'); const disabledMatch = optionHtml.includes('disabled="disabled"');
const selectedMatch = optionHtml.includes('selected="selected"'); const selectedMatch = optionHtml.includes('selected="selected"');
// Extract text content between > and </option>
const textMatch = optionHtml.match(/>([^<]*)</); const textMatch = optionHtml.match(/>([^<]*)</);
const text = textMatch const text = textMatch
? textMatch[1] ? textMatch[1]
.replace(/\n/g, '') // Remove all newlines .replace(/\n/g, '')
.replace(/\s+/g, ' ') // Replace multiple spaces with single space .replace(/\s+/g, ' ')
.trim() .trim()
: ''; : '';
@@ -501,7 +482,6 @@ export class WorkflowGenerator {
}; };
}); });
// Notify client to show dropdown overlay
this.socket.emit('showDropdown', { this.socket.emit('showDropdown', {
coordinates, coordinates,
selector, selector,
@@ -510,11 +490,9 @@ export class WorkflowGenerator {
return; return;
} }
// Check if clicked element is a date input
const isDateInput = elementInfo?.tagName === 'INPUT' && elementInfo?.attributes?.type === 'date'; const isDateInput = elementInfo?.tagName === 'INPUT' && elementInfo?.attributes?.type === 'date';
if (isDateInput) { if (isDateInput) {
// Notify client to show datepicker overlay
this.socket.emit('showDatePicker', { this.socket.emit('showDatePicker', {
coordinates, coordinates,
selector selector
@@ -640,8 +618,6 @@ export class WorkflowGenerator {
} }
} }
//const element = await getElementMouseIsOver(page, coordinates);
//logger.log('debug', `Element: ${JSON.stringify(element, null, 2)}`);
if (selector) { if (selector) {
where.selectors = [selector]; where.selectors = [selector];
} }
@@ -679,37 +655,6 @@ export class WorkflowGenerator {
await this.addPairToWorkflowAndNotifyClient(pair, page); await this.addPairToWorkflowAndNotifyClient(pair, page);
}; };
/**
* Generates a pair for the keypress event.
* @param key The key to be pressed.
* @param coordinates The coordinates of the keypress event.
* @param page The page to use for obtaining the needed data.
* @returns {Promise<void>}
*/
public onKeyboardInput = async (key: string, coordinates: Coordinates, page: Page) => {
let where: WhereWhatPair["where"] = { url: this.getBestUrl(page.url()) };
const selector = await this.generateSelector(page, coordinates, ActionType.Keydown);
const elementInfo = await getElementInformation(page, coordinates, '', false);
const inputType = elementInfo?.attributes?.type || "text";
if (selector) {
where.selectors = [selector];
}
const pair: WhereWhatPair = {
where,
what: [{
action: 'press',
args: [selector, encrypt(key), inputType],
}],
}
if (selector) {
this.generatedData.lastUsedSelector = selector;
this.generatedData.lastAction = 'press';
}
await this.addPairToWorkflowAndNotifyClient(pair, page);
};
/** /**
* Returns tag name and text content for the specified selector * Returns tag name and text content for the specified selector
* used in customAction for decision modal * used in customAction for decision modal
@@ -940,7 +885,7 @@ export class WorkflowGenerator {
return pair; return pair;
}) })
.filter((pair) => pair !== null) as WhereWhatPair[]; // Remove null entries .filter((pair) => pair !== null) as WhereWhatPair[];
if (actionWasRemoved) { if (actionWasRemoved) {
logger.log("info", `Action with actionId ${actionId} removed from workflow`); logger.log("info", `Action with actionId ${actionId} removed from workflow`);
@@ -996,24 +941,6 @@ export class WorkflowGenerator {
} }
} }
/**
* Returns the currently generated workflow without all the generated flag actions.
* @param workflow The workflow for removing the generated flag actions from.
* @private
* @returns {WorkflowFile}
*/
private removeAllGeneratedFlags = (workflow: WorkflowFile): WorkflowFile => {
for (let i = 0; i < workflow.workflow.length; i++) {
if (
workflow.workflow[i].what[0] &&
workflow.workflow[i].what[0].action === 'flag' &&
workflow.workflow[i].what[0].args?.includes('generated')) {
workflow.workflow[i].what.splice(0, 1);
}
}
return workflow;
};
/** /**
* Adds generated flag actions to the workflow's pairs' what conditions. * Adds generated flag actions to the workflow's pairs' what conditions.
* @param workflow The workflow for adding the generated flag actions from. * @param workflow The workflow for adding the generated flag actions from.
@@ -1127,7 +1054,6 @@ export class WorkflowGenerator {
: await getSelectors(page, coordinates); : await getSelectors(page, coordinates);
if (this.paginationMode && selectorBasedOnCustomAction) { if (this.paginationMode && selectorBasedOnCustomAction) {
// Chain selectors in specific priority order
const selectors = selectorBasedOnCustomAction; const selectors = selectorBasedOnCustomAction;
const selectorChain = [ const selectorChain = [
selectors?.iframeSelector?.full, selectors?.iframeSelector?.full,
@@ -1324,17 +1250,15 @@ export class WorkflowGenerator {
if (this.workflowRecord.workflow[index].where.selectors?.includes(selector)) { if (this.workflowRecord.workflow[index].where.selectors?.includes(selector)) {
break; break;
} else { } else {
// add new selector to the where part of the overshadowing pair
this.workflowRecord.workflow[index].where.selectors?.push(selector); this.workflowRecord.workflow[index].where.selectors?.push(selector);
} }
} }
} }
// push the action automatically to the first/the closest rule which would be overShadowed
this.workflowRecord.workflow[index].what = this.workflowRecord.workflow[index].what =
this.workflowRecord.workflow[index].what.concat(pair.what); this.workflowRecord.workflow[index].what.concat(pair.what);
return true; return true;
} else { } else {
// notify client about overshadowing a further rule
return false; return false;
} }
} }
@@ -1352,8 +1276,7 @@ export class WorkflowGenerator {
const parsedUrl = new URL(url); const parsedUrl = new URL(url);
const protocol = parsedUrl.protocol === 'https:' || parsedUrl.protocol === 'http:' ? `${parsedUrl.protocol}//` : parsedUrl.protocol; const protocol = parsedUrl.protocol === 'https:' || parsedUrl.protocol === 'http:' ? `${parsedUrl.protocol}//` : parsedUrl.protocol;
const regex = new RegExp(/(?=.*[A-Z])/g) const regex = new RegExp(/(?=.*[A-Z])/g)
// remove all params with uppercase letters, they are most likely dynamically generated
// also escapes all regex characters from the params
const search = parsedUrl.search const search = parsedUrl.search
.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') .replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
.split('&').map((param, index) => { .split('&').map((param, index) => {
@@ -1380,8 +1303,6 @@ export class WorkflowGenerator {
* @param workflow The workflow to be checked. * @param workflow The workflow to be checked.
*/ */
private checkWorkflowForParams = (workflow: WorkflowFile): string[] | null => { private checkWorkflowForParams = (workflow: WorkflowFile): string[] | null => {
// for now the where condition cannot have any params, so we're checking only what part of the pair
// where only the args part of what condition can have a parameter
for (const pair of workflow.workflow) { for (const pair of workflow.workflow) {
for (const condition of pair.what) { for (const condition of pair.what) {
if (condition.args) { if (condition.args) {

View File

@@ -7,7 +7,6 @@ import { useBrowserSteps, TextStep, ListStep } from '../../context/browserSteps'
import { useGlobalInfoStore } from '../../context/globalInfo'; import { useGlobalInfoStore } from '../../context/globalInfo';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { AuthContext } from '../../context/auth'; import { AuthContext } from '../../context/auth';
import { coordinateMapper } from '../../helpers/coordinateMapper';
import { useBrowserDimensionsStore } from '../../context/browserDimensions'; import { useBrowserDimensionsStore } from '../../context/browserDimensions';
import { clientSelectorGenerator, ElementFingerprint } from "../../helpers/clientSelectorGenerator"; import { clientSelectorGenerator, ElementFingerprint } from "../../helpers/clientSelectorGenerator";
import { capturedElementHighlighter } from "../../helpers/capturedElementHighlighter"; import { capturedElementHighlighter } from "../../helpers/capturedElementHighlighter";
@@ -36,79 +35,6 @@ interface AttributeOption {
value: string; value: string;
} }
interface ViewportInfo {
width: number;
height: number;
}
interface RRWebSnapshot {
type: number;
childNodes?: RRWebSnapshot[];
tagName?: string;
attributes?: Record<string, string>;
textContent: string;
id: number;
[key: string]: any;
}
interface ProcessedSnapshot {
snapshot: RRWebSnapshot;
resources: {
stylesheets: Array<{
href: string;
content: string;
media?: string;
}>;
images: Array<{
src: string;
dataUrl: string;
alt?: string;
}>;
fonts: Array<{
url: string;
dataUrl: string;
format?: string;
}>;
scripts: Array<{
src: string;
content: string;
type?: string;
}>;
media: Array<{
src: string;
dataUrl: string;
type: string;
}>;
};
baseUrl: string;
viewport: { width: number; height: number };
timestamp: number;
processingStats: {
totalReplacements: number;
discoveredResources: {
images: number;
stylesheets: number;
scripts: number;
fonts: number;
media: number;
};
cachedResources: {
stylesheets: number;
images: number;
fonts: number;
scripts: number;
media: number;
};
totalCacheSize: number;
};
}
interface RRWebDOMCastData {
snapshotData: ProcessedSnapshot;
userId: string;
timestamp: number;
}
const getAttributeOptions = (tagName: string, elementInfo: ElementInfo | null): AttributeOption[] => { const getAttributeOptions = (tagName: string, elementInfo: ElementInfo | null): AttributeOption[] => {
if (!elementInfo) return []; if (!elementInfo) return [];
switch (tagName.toLowerCase()) { switch (tagName.toLowerCase()) {
@@ -154,8 +80,6 @@ export const BrowserWindow = () => {
const [attributeOptions, setAttributeOptions] = useState<AttributeOption[]>([]); const [attributeOptions, setAttributeOptions] = useState<AttributeOption[]>([]);
const [selectedElement, setSelectedElement] = useState<{ selector: string, info: ElementInfo | null } | null>(null); const [selectedElement, setSelectedElement] = useState<{ selector: string, info: ElementInfo | null } | null>(null);
const [currentListId, setCurrentListId] = useState<number | null>(null); const [currentListId, setCurrentListId] = useState<number | null>(null);
const [viewportInfo, setViewportInfo] = useState<ViewportInfo>({ width: browserWidth, height: browserHeight });
const [isLoading, setIsLoading] = useState(false);
const [cachedChildSelectors, setCachedChildSelectors] = useState<string[]>([]); const [cachedChildSelectors, setCachedChildSelectors] = useState<string[]>([]);
const [processingGroupCoordinates, setProcessingGroupCoordinates] = useState<Array<{ element: HTMLElement; rect: DOMRect }>>([]); const [processingGroupCoordinates, setProcessingGroupCoordinates] = useState<Array<{ element: HTMLElement; rect: DOMRect }>>([]);
const [listSelector, setListSelector] = useState<string | null>(null); const [listSelector, setListSelector] = useState<string | null>(null);
@@ -177,7 +101,7 @@ export const BrowserWindow = () => {
const [manuallyAddedFieldIds, setManuallyAddedFieldIds] = useState<Set<number>>(new Set()); const [manuallyAddedFieldIds, setManuallyAddedFieldIds] = useState<Set<number>>(new Set());
const { socket } = useSocketStore(); const { socket } = useSocketStore();
const { notify, currentTextActionId, currentListActionId, updateDOMMode, isDOMMode, currentSnapshot } = useGlobalInfoStore(); const { notify, currentTextActionId, currentListActionId, updateDOMMode, isDOMMode } = useGlobalInfoStore();
const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext(); const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext();
const { addTextStep, addListStep, browserSteps } = useBrowserSteps(); const { addTextStep, addListStep, browserSteps } = useBrowserSteps();
@@ -258,27 +182,11 @@ export const BrowserWindow = () => {
[] []
); );
const rrwebSnapshotHandler = useCallback(
(data: RRWebDOMCastData) => {
if (!data.userId || data.userId === user?.id) {
if (data.snapshotData && data.snapshotData.snapshot) {
updateDOMMode(true, data.snapshotData);
socket?.emit("dom-mode-enabled");
setIsLoading(false);
} else {
setIsLoading(false);
}
}
},
[user?.id, socket, updateDOMMode]
);
const domModeHandler = useCallback( const domModeHandler = useCallback(
(data: any) => { (data: any) => {
if (!data.userId || data.userId === user?.id) { if (!data.userId || data.userId === user?.id) {
updateDOMMode(true); updateDOMMode(true);
socket?.emit("dom-mode-enabled"); socket?.emit("dom-mode-enabled");
setIsLoading(false);
} }
}, },
[user?.id, socket, updateDOMMode] [user?.id, socket, updateDOMMode]
@@ -288,18 +196,21 @@ export const BrowserWindow = () => {
(data: any) => { (data: any) => {
if (!data.userId || data.userId === user?.id) { if (!data.userId || data.userId === user?.id) {
updateDOMMode(false); updateDOMMode(false);
setIsLoading(false);
if (data.error) {
notify("error", data.error);
}
} }
}, },
[user?.id, updateDOMMode] [user?.id, updateDOMMode, notify]
); );
useEffect(() => { useEffect(() => {
if (isDOMMode) { if (isDOMMode) {
clientSelectorGenerator.setGetList(getList); clientSelectorGenerator.setGetList(getList);
clientSelectorGenerator.setListSelector(listSelector || ""); clientSelectorGenerator.setListSelector(listSelector || "");
clientSelectorGenerator.setPaginationMode(paginationMode); clientSelectorGenerator.setPaginationMode(paginationMode);
} }
}, [isDOMMode, getList, listSelector, paginationMode]); }, [isDOMMode, getList, listSelector, paginationMode]);
const createFieldsFromChildSelectors = useCallback( const createFieldsFromChildSelectors = useCallback(
@@ -698,7 +609,7 @@ export const BrowserWindow = () => {
return finalFields; return finalFields;
}, },
[currentSnapshot] []
); );
const removeParentChildDuplicates = ( const removeParentChildDuplicates = (
@@ -798,7 +709,7 @@ export const BrowserWindow = () => {
clientSelectorGenerator.setListSelector(listSelector); clientSelectorGenerator.setListSelector(listSelector);
if (currentSnapshot && cachedListSelector !== listSelector) { if (cachedListSelector !== listSelector) {
setCachedChildSelectors([]); setCachedChildSelectors([]);
setIsCachingChildSelectors(true); setIsCachingChildSelectors(true);
setCachedListSelector(listSelector); setCachedListSelector(listSelector);
@@ -878,7 +789,6 @@ export const BrowserWindow = () => {
listSelector, listSelector,
socket, socket,
getList, getList,
currentSnapshot,
cachedListSelector, cachedListSelector,
pendingNotification, pendingNotification,
notify, notify,
@@ -989,10 +899,6 @@ export const BrowserWindow = () => {
} }
}, [browserSteps, getText, getList, listSelector, currentTextActionId, currentListActionId, isDOMMode, manuallyAddedFieldIds]); }, [browserSteps, getText, getList, listSelector, currentTextActionId, currentListActionId, isDOMMode, manuallyAddedFieldIds]);
useEffect(() => {
coordinateMapper.updateDimensions(dimensions.width, dimensions.height, viewportInfo.width, viewportInfo.height);
}, [viewportInfo, dimensions.width, dimensions.height]);
useEffect(() => { useEffect(() => {
if (listSelector) { if (listSelector) {
sessionStorage.setItem('recordingListSelector', listSelector); sessionStorage.setItem('recordingListSelector', listSelector);
@@ -1027,21 +933,18 @@ export const BrowserWindow = () => {
useEffect(() => { useEffect(() => {
if (socket) { if (socket) {
socket.on("domcast", rrwebSnapshotHandler);
socket.on("dom-mode-enabled", domModeHandler); socket.on("dom-mode-enabled", domModeHandler);
socket.on("dom-mode-error", domModeErrorHandler); socket.on("dom-mode-error", domModeErrorHandler);
} }
return () => { return () => {
if (socket) { if (socket) {
socket.off("domcast", rrwebSnapshotHandler);
socket.off("dom-mode-enabled", domModeHandler); socket.off("dom-mode-enabled", domModeHandler);
socket.off("dom-mode-error", domModeErrorHandler); socket.off("dom-mode-error", domModeErrorHandler);
} }
}; };
}, [ }, [
socket, socket,
rrwebSnapshotHandler,
domModeHandler, domModeHandler,
domModeErrorHandler, domModeErrorHandler,
]); ]);
@@ -1066,14 +969,15 @@ export const BrowserWindow = () => {
isDOMMode?: boolean; isDOMMode?: boolean;
}) => { }) => {
if (paginationMode && paginationSelector) { if (paginationMode && paginationSelector) {
return; return;
} }
if (!getText && !getList) { if (!getText && !getList) {
setHighlighterData(null); setHighlighterData(null);
return; return;
} }
if (!isDOMMode || !currentSnapshot) { if (!isDOMMode) {
return; return;
} }
@@ -1187,7 +1091,6 @@ export const BrowserWindow = () => {
}, },
[ [
isDOMMode, isDOMMode,
currentSnapshot,
getText, getText,
getList, getList,
socket, socket,
@@ -1200,105 +1103,6 @@ export const BrowserWindow = () => {
] ]
); );
const highlighterHandler = useCallback((data: { rect: DOMRect, selector: string, elementInfo: ElementInfo | null, childSelectors?: string[], isDOMMode?: boolean; }) => {
if (paginationMode && paginationSelector) {
return;
}
if (isDOMMode || data.isDOMMode) {
domHighlighterHandler(data);
return;
}
const now = performance.now();
if (now - highlighterUpdateRef.current < 16) {
return;
}
highlighterUpdateRef.current = now;
const mappedRect = new DOMRect(
data.rect.x,
data.rect.y,
data.rect.width,
data.rect.height
);
const mappedData = {
...data,
rect: mappedRect
};
if (getList === true) {
if (listSelector) {
socket?.emit('listSelector', { selector: listSelector });
const hasValidChildSelectors = Array.isArray(mappedData.childSelectors) && mappedData.childSelectors.length > 0;
if (limitMode) {
setHighlighterData(null);
} else if (paginationMode) {
if (paginationType !== '' && !['none', 'scrollDown', 'scrollUp'].includes(paginationType)) {
setHighlighterData(mappedData);
} else {
setHighlighterData(null);
}
} else if (mappedData.childSelectors && mappedData.childSelectors.includes(mappedData.selector)) {
setHighlighterData(mappedData);
} else if (mappedData.elementInfo?.isIframeContent && mappedData.childSelectors) {
const isIframeChild = mappedData.childSelectors.some(childSelector =>
mappedData.selector.includes(':>>') &&
childSelector.split(':>>').some(part =>
mappedData.selector.includes(part.trim())
)
);
setHighlighterData(isIframeChild ? mappedData : null);
} else if (mappedData.selector.includes(':>>') && hasValidChildSelectors) {
const selectorParts = mappedData.selector.split(':>>').map(part => part.trim());
const isValidMixedSelector = selectorParts.some(part =>
mappedData.childSelectors!.some(childSelector =>
childSelector.includes(part)
)
);
setHighlighterData(isValidMixedSelector ? mappedData : null);
} else if (mappedData.elementInfo?.isShadowRoot && mappedData.childSelectors) {
const isShadowChild = mappedData.childSelectors.some(childSelector =>
mappedData.selector.includes('>>') &&
childSelector.split('>>').some(part =>
mappedData.selector.includes(part.trim())
)
);
setHighlighterData(isShadowChild ? mappedData : null);
} else if (mappedData.selector.includes('>>') && hasValidChildSelectors) {
const selectorParts = mappedData.selector.split('>>').map(part => part.trim());
const isValidMixedSelector = selectorParts.some(part =>
mappedData.childSelectors!.some(childSelector =>
childSelector.includes(part)
)
);
setHighlighterData(isValidMixedSelector ? mappedData : null);
} else {
setHighlighterData(null);
}
} else {
setHighlighterData(mappedData);
}
} else {
setHighlighterData(mappedData);
}
}, [getList, socket, listSelector, paginationMode, paginationType, limitMode]);
useEffect(() => {
document.addEventListener("mousemove", onMouseMove, false);
if (socket) {
socket.off("highlighter", highlighterHandler);
socket.on("highlighter", highlighterHandler);
}
return () => {
document.removeEventListener("mousemove", onMouseMove);
if (socket) {
socket.off("highlighter", highlighterHandler);
}
};
}, [socket, highlighterHandler, getList, listSelector]);
useEffect(() => { useEffect(() => {
if (socket && listSelector) { if (socket && listSelector) {
socket.emit('setGetList', { getList: true }); socket.emit('setGetList', { getList: true });
@@ -1533,7 +1337,6 @@ export const BrowserWindow = () => {
] ]
); );
const handleClick = (e: React.MouseEvent<HTMLDivElement>) => { const handleClick = (e: React.MouseEvent<HTMLDivElement>) => {
if (highlighterData) { if (highlighterData) {
const shouldProcessClick = true; const shouldProcessClick = true;
@@ -1720,49 +1523,50 @@ export const BrowserWindow = () => {
default: default:
data = selectedElement.info?.innerText || ''; data = selectedElement.info?.innerText || '';
} }
{
if (getText === true) { if (getText === true) {
addTextStep('', data, { addTextStep('', data, {
selector: selectedElement.selector,
tag: selectedElement.info?.tagName,
isShadow: highlighterData?.isShadow || selectedElement.info?.isShadowRoot,
attribute: attribute
}, currentTextActionId || `text-${crypto.randomUUID()}`);
}
if (getList === true && listSelector && currentListId) {
const newField: TextStep = {
id: Date.now(),
type: 'text',
label: `Label ${Object.keys(fields).length + 1}`,
data: data,
selectorObj: {
selector: selectedElement.selector, selector: selectedElement.selector,
tag: selectedElement.info?.tagName, tag: selectedElement.info?.tagName,
isShadow: highlighterData?.isShadow || selectedElement.info?.isShadowRoot, isShadow: highlighterData?.isShadow || highlighterData?.elementInfo?.isShadowRoot,
attribute: attribute attribute: attribute
}, currentTextActionId || `text-${crypto.randomUUID()}`);
}
if (getList === true && listSelector && currentListId) {
const newField: TextStep = {
id: Date.now(),
type: 'text',
label: `Label ${Object.keys(fields).length + 1}`,
data: data,
selectorObj: {
selector: selectedElement.selector,
tag: selectedElement.info?.tagName,
isShadow: highlighterData?.isShadow || highlighterData?.elementInfo?.isShadowRoot,
attribute: attribute
}
};
const updatedFields = {
...fields,
[newField.id]: newField
};
setFields(updatedFields);
if (listSelector) {
addListStep(
listSelector,
updatedFields,
currentListId,
currentListActionId || `list-${crypto.randomUUID()}`,
{ type: "", selector: paginationSelector, isShadow: highlighterData?.isShadow },
undefined,
highlighterData?.isShadow
);
} }
};
const updatedFields = {
...fields,
[newField.id]: newField
};
setFields(updatedFields);
if (listSelector) {
addListStep(
listSelector,
updatedFields,
currentListId,
currentListActionId || `list-${crypto.randomUUID()}`,
{ type: "", selector: paginationSelector, isShadow: highlighterData?.isShadow },
undefined,
highlighterData?.isShadow
);
} }
} }
} }
setShowAttributeModal(false); setShowAttributeModal(false);
@@ -1817,7 +1621,6 @@ export const BrowserWindow = () => {
style={{ width: browserWidth }} style={{ width: browserWidth }}
id="browser-window" id="browser-window"
> >
{/* Attribute selection modal */}
{(getText === true || getList === true) && ( {(getText === true || getList === true) && (
<GenericModal <GenericModal
isOpen={showAttributeModal} isOpen={showAttributeModal}
@@ -1906,17 +1709,15 @@ export const BrowserWindow = () => {
/> />
)} )}
{/* Main content area */}
<div <div
style={{ style={{
position: "relative", position: "relative",
width: "100%", width: "100%",
height: dimensions.height, height: dimensions.height,
overflow: "hidden", overflow: "hidden",
borderRadius: "0px 0px 5px 5px", borderRadius: "0px 0px 5px 5px",
}} }}
> >
{/* Add CSS for the spinner animation */}
<style>{` <style>{`
@keyframes spin { @keyframes spin {
0% { transform: rotate(0deg); } 0% { transform: rotate(0deg); }
@@ -1933,13 +1734,12 @@ export const BrowserWindow = () => {
id="dom-highlight-overlay" id="dom-highlight-overlay"
style={{ style={{
position: "absolute", position: "absolute",
inset: 0, // top:0; right:0; bottom:0; left:0 inset: 0,
overflow: "hidden", // clip everything within iframe area overflow: "hidden",
pointerEvents: "none", pointerEvents: "none",
zIndex: 1000, zIndex: 1000,
}} }}
> >
{/* Individual element highlight (for non-group or hovered element) */}
{((getText && !listSelector) || {((getText && !listSelector) ||
(getList && paginationMode && !paginationSelector && paginationType !== "" && (getList && paginationMode && !paginationSelector && paginationType !== "" &&
!["none", "scrollDown", "scrollUp"].includes(paginationType))) && ( !["none", "scrollDown", "scrollUp"].includes(paginationType))) && (
@@ -1960,7 +1760,6 @@ export const BrowserWindow = () => {
/> />
)} )}
{/* Grouped list element highlights */}
{getList && {getList &&
!listSelector && !listSelector &&
currentGroupInfo?.isGroupElement && currentGroupInfo?.isGroupElement &&
@@ -2029,7 +1828,6 @@ export const BrowserWindow = () => {
}} }}
/> />
{/* Label for similar element */}
<div <div
style={{ style={{
position: "absolute", position: "absolute",
@@ -2054,23 +1852,22 @@ export const BrowserWindow = () => {
)} )}
</> </>
)} )}
{/* --- Main DOM Renderer Section --- */}
<div <div
id="iframe-wrapper" id="iframe-wrapper"
style={{ style={{
position: "relative", position: "relative",
width: "100%", width: "100%",
height: "100%", height: "100%",
overflow: "hidden", // key: confine everything below overflow: "hidden",
borderRadius: "0px 0px 5px 5px", borderRadius: "0px 0px 5px 5px",
}} }}
> >
{currentSnapshot ? ( {isDOMMode ? (
<> <>
<DOMBrowserRenderer <DOMBrowserRenderer
width={dimensions.width} width={dimensions.width}
height={dimensions.height} height={dimensions.height}
snapshot={currentSnapshot}
getList={getList} getList={getList}
getText={getText} getText={getText}
listSelector={listSelector} listSelector={listSelector}
@@ -2088,7 +1885,6 @@ export const BrowserWindow = () => {
onShowDateTimePicker={handleShowDateTimePicker} onShowDateTimePicker={handleShowDateTimePicker}
/> />
{/* --- Loading overlay --- */}
{isCachingChildSelectors && ( {isCachingChildSelectors && (
<> <>
<div <div
@@ -2209,7 +2005,6 @@ export const BrowserWindow = () => {
const DOMLoadingIndicator: React.FC = () => { const DOMLoadingIndicator: React.FC = () => {
const [progress, setProgress] = useState(0); const [progress, setProgress] = useState(0);
const [pendingRequests, setPendingRequests] = useState(0);
const [hasStartedLoading, setHasStartedLoading] = useState(false); const [hasStartedLoading, setHasStartedLoading] = useState(false);
const { socket } = useSocketStore(); const { socket } = useSocketStore();
const { state } = useContext(AuthContext); const { state } = useContext(AuthContext);
@@ -2225,15 +2020,12 @@ const DOMLoadingIndicator: React.FC = () => {
userId: string; userId: string;
}) => { }) => {
if (!data.userId || data.userId === user?.id) { if (!data.userId || data.userId === user?.id) {
// Once loading has started, never reset progress to 0
if (!hasStartedLoading && data.progress > 0) { if (!hasStartedLoading && data.progress > 0) {
setHasStartedLoading(true); setHasStartedLoading(true);
} }
// Only update progress if we haven't started or if new progress is higher
if (!hasStartedLoading || data.progress >= progress) { if (!hasStartedLoading || data.progress >= progress) {
setProgress(data.progress); setProgress(data.progress);
setPendingRequests(data.pendingRequests);
} }
} }
}; };
@@ -2259,7 +2051,6 @@ const DOMLoadingIndicator: React.FC = () => {
gap: "15px", gap: "15px",
}} }}
> >
{/* Loading text with percentage */}
<div <div
style={{ style={{
fontSize: "18px", fontSize: "18px",
@@ -2270,7 +2061,6 @@ const DOMLoadingIndicator: React.FC = () => {
Loading {progress}% Loading {progress}%
</div> </div>
{/* Progress bar */}
<div <div
style={{ style={{
width: "240px", width: "240px",

View File

@@ -1,6 +1,10 @@
import React, { useState } from 'react'; import React, { useState } from 'react';
import { useSocketStore } from '../../context/socket'; import { useSocketStore } from '../../context/socket';
import { Coordinates } from '../recorder/Canvas';
interface Coordinates {
x: number;
y: number;
};
interface DatePickerProps { interface DatePickerProps {
coordinates: Coordinates; coordinates: Coordinates;

View File

@@ -1,6 +1,10 @@
import React, { useState } from 'react'; import React, { useState } from 'react';
import { useSocketStore } from '../../context/socket'; import { useSocketStore } from '../../context/socket';
import { Coordinates } from '../recorder/Canvas';
interface Coordinates {
x: number;
y: number;
};
interface DateTimeLocalPickerProps { interface DateTimeLocalPickerProps {
coordinates: Coordinates; coordinates: Coordinates;

View File

@@ -1,6 +1,10 @@
import React, { useState } from 'react'; import React, { useState } from 'react';
import { useSocketStore } from '../../context/socket'; import { useSocketStore } from '../../context/socket';
import { Coordinates } from '../recorder/Canvas';
interface Coordinates {
x: number;
y: number;
};
interface DropdownProps { interface DropdownProps {
coordinates: Coordinates; coordinates: Coordinates;

View File

@@ -1,6 +1,10 @@
import React, { useState } from 'react'; import React, { useState } from 'react';
import { useSocketStore } from '../../context/socket'; import { useSocketStore } from '../../context/socket';
import { Coordinates } from '../recorder/Canvas';
interface Coordinates {
x: number;
y: number;
};
interface TimePickerProps { interface TimePickerProps {
coordinates: Coordinates; coordinates: Coordinates;

View File

@@ -7,9 +7,8 @@ import React, {
} from "react"; } from "react";
import { useSocketStore } from "../../context/socket"; import { useSocketStore } from "../../context/socket";
import { useGlobalInfoStore } from "../../context/globalInfo"; import { useGlobalInfoStore } from "../../context/globalInfo";
import { useTranslation } from "react-i18next";
import { AuthContext } from "../../context/auth"; import { AuthContext } from "../../context/auth";
import { rebuild, createMirror } from "rrweb-snapshot"; import { Replayer } from "rrweb"
import { import {
ActionType, ActionType,
clientSelectorGenerator, clientSelectorGenerator,
@@ -29,72 +28,9 @@ interface ElementInfo {
isDOMMode?: boolean; isDOMMode?: boolean;
} }
interface ProcessedSnapshot {
snapshot: RRWebSnapshot;
resources: {
stylesheets: Array<{
href: string;
content: string;
media?: string;
}>;
images: Array<{
src: string;
dataUrl: string;
alt?: string;
}>;
fonts: Array<{
url: string;
dataUrl: string;
format?: string;
}>;
scripts: Array<{
src: string;
content: string;
type?: string;
}>;
media: Array<{
src: string;
dataUrl: string;
type: string;
}>;
};
baseUrl: string;
viewport: { width: number; height: number };
timestamp: number;
processingStats: {
totalReplacements: number;
discoveredResources: {
images: number;
stylesheets: number;
scripts: number;
fonts: number;
media: number;
};
cachedResources: {
stylesheets: number;
images: number;
fonts: number;
scripts: number;
media: number;
};
totalCacheSize: number;
};
}
interface RRWebSnapshot {
type: number;
childNodes?: RRWebSnapshot[];
tagName?: string;
attributes?: Record<string, string>;
textContent: string;
id: number;
[key: string]: any;
}
interface RRWebDOMBrowserRendererProps { interface RRWebDOMBrowserRendererProps {
width: number; width: number;
height: number; height: number;
snapshot: ProcessedSnapshot;
getList?: boolean; getList?: boolean;
getText?: boolean; getText?: boolean;
listSelector?: string | null; listSelector?: string | null;
@@ -148,7 +84,6 @@ interface RRWebDOMBrowserRendererProps {
export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
width, width,
height, height,
snapshot,
getList = false, getList = false,
getText = false, getText = false,
listSelector = null, listSelector = null,
@@ -165,8 +100,9 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
onShowTimePicker, onShowTimePicker,
onShowDateTimePicker, onShowDateTimePicker,
}) => { }) => {
const { t } = useTranslation();
const containerRef = useRef<HTMLDivElement>(null); const containerRef = useRef<HTMLDivElement>(null);
const replayerIframeRef = useRef<HTMLIFrameElement | null>(null);
const replayerRef = useRef<any>(null);
const iframeRef = useRef<HTMLIFrameElement>(null); const iframeRef = useRef<HTMLIFrameElement>(null);
const [isRendered, setIsRendered] = useState(false); const [isRendered, setIsRendered] = useState(false);
const [lastMousePosition, setLastMousePosition] = useState({ x: 0, y: 0 }); const [lastMousePosition, setLastMousePosition] = useState({ x: 0, y: 0 });
@@ -184,7 +120,7 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
const { state } = useContext(AuthContext); const { state } = useContext(AuthContext);
const { user } = state; const { user } = state;
const MOUSE_MOVE_THROTTLE = 16; // ~60fps const MOUSE_MOVE_THROTTLE = 16;
const lastMouseMoveTime = useRef(0); const lastMouseMoveTime = useRef(0);
const notifyLastAction = (action: string) => { const notifyLastAction = (action: string) => {
@@ -313,7 +249,7 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
isShadow, isShadow,
childSelectors, childSelectors,
groupInfo, groupInfo,
similarElements, // Pass similar elements data similarElements,
}); });
} }
} }
@@ -337,12 +273,14 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
getList, getList,
listSelector, listSelector,
paginationMode, paginationMode,
paginationSelector,
cachedChildSelectors, cachedChildSelectors,
paginationType, paginationType,
limitMode, limitMode,
onHighlight, onHighlight,
] ]
); );
/** /**
* Set up enhanced interaction handlers for DOM mode * Set up enhanced interaction handlers for DOM mode
*/ */
@@ -379,7 +317,7 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
const iframeX = mouseEvent.clientX; const iframeX = mouseEvent.clientX;
const iframeY = mouseEvent.clientY; const iframeY = mouseEvent.clientY;
const iframe = iframeRef.current; const iframe = replayerIframeRef.current;
if (iframe) { if (iframe) {
const iframeRect = iframe.getBoundingClientRect(); const iframeRect = iframe.getBoundingClientRect();
setLastMousePosition({ setLastMousePosition({
@@ -407,7 +345,6 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
e.stopPropagation(); e.stopPropagation();
if (currentHighlight && onElementSelect) { if (currentHighlight && onElementSelect) {
// Get the group info for the current highlight
const highlighterData = const highlighterData =
clientSelectorGenerator.generateDataForHighlighter( clientSelectorGenerator.generateDataForHighlighter(
{ x: iframeX, y: iframeY }, { x: iframeX, y: iframeY },
@@ -438,6 +375,7 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
e.stopPropagation(); e.stopPropagation();
const href = linkElement.href; const href = linkElement.href;
const originalTarget = linkElement.target;
if (linkElement.target) { if (linkElement.target) {
linkElement.target = ""; linkElement.target = "";
@@ -447,12 +385,17 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
linkElement.removeAttribute("href"); linkElement.removeAttribute("href");
setTimeout(() => { setTimeout(() => {
linkElement.setAttribute("href", originalHref); try {
linkElement.setAttribute("href", originalHref);
if (originalTarget) {
linkElement.setAttribute("target", originalTarget);
}
} catch (error) {
console.warn("Could not restore link attributes:", error);
}
}, 100); }, 100);
const isSPALink = const isSPALink = href.startsWith('#');
href.endsWith("#") ||
(href.includes("#") && new URL(href).hash !== "");
const selector = clientSelectorGenerator.generateSelector( const selector = clientSelectorGenerator.generateSelector(
iframeDoc, iframeDoc,
@@ -470,7 +413,6 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
if (selector && socket) { if (selector && socket) {
socket.emit("dom:click", { socket.emit("dom:click", {
selector, selector,
url: snapshot.baseUrl,
userId: user?.id || "unknown", userId: user?.id || "unknown",
elementInfo, elementInfo,
coordinates: undefined, coordinates: undefined,
@@ -592,17 +534,14 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
socket.emit("dom:click", { socket.emit("dom:click", {
selector, selector,
url: snapshot.baseUrl,
userId: user?.id || "unknown", userId: user?.id || "unknown",
elementInfo, elementInfo,
coordinates: { x: relativeX, y: relativeY }, coordinates: { x: relativeX, y: relativeY },
isSPA: false, isSPA: false,
}); });
} else if (elementInfo?.tagName !== "SELECT") { } else if (elementInfo?.tagName !== "SELECT") {
// Handle other elements normally
socket.emit("dom:click", { socket.emit("dom:click", {
selector, selector,
url: snapshot.baseUrl,
userId: user?.id || "unknown", userId: user?.id || "unknown",
elementInfo, elementInfo,
coordinates: { x: iframeX, y: iframeY }, coordinates: { x: iframeX, y: iframeY },
@@ -632,21 +571,19 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
const keyboardEvent = e as KeyboardEvent; const keyboardEvent = e as KeyboardEvent;
const target = keyboardEvent.target as HTMLElement; const target = keyboardEvent.target as HTMLElement;
if (!isInCaptureMode && socket && snapshot?.baseUrl) { if (!isInCaptureMode && socket) {
const iframe = iframeRef.current; const iframe = replayerIframeRef.current;
if (iframe) { if (iframe) {
const focusedElement = iframeDoc.activeElement as HTMLElement; const focusedElement = iframeDoc.activeElement as HTMLElement;
let coordinates = { x: 0, y: 0 }; let coordinates = { x: 0, y: 0 };
if (focusedElement && focusedElement !== iframeDoc.body) { if (focusedElement && focusedElement !== iframeDoc.body) {
// Get coordinates from the focused element
const rect = focusedElement.getBoundingClientRect(); const rect = focusedElement.getBoundingClientRect();
coordinates = { coordinates = {
x: rect.left + rect.width / 2, x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2 y: rect.top + rect.height / 2
}; };
} else { } else {
// Fallback to last mouse position if no focused element
const iframeRect = iframe.getBoundingClientRect(); const iframeRect = iframe.getBoundingClientRect();
coordinates = { coordinates = {
x: lastMousePosition.x - iframeRect.left, x: lastMousePosition.x - iframeRect.left,
@@ -671,7 +608,6 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
socket.emit("dom:keypress", { socket.emit("dom:keypress", {
selector, selector,
key: keyboardEvent.key, key: keyboardEvent.key,
url: snapshot.baseUrl,
userId: user?.id || "unknown", userId: user?.id || "unknown",
inputType: elementInfo?.attributes?.type || "text", inputType: elementInfo?.attributes?.type || "text",
}); });
@@ -768,12 +704,11 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
iframeDoc.addEventListener(event, handler, options); iframeDoc.addEventListener(event, handler, options);
}); });
// Store handlers for cleanup
(iframeDoc as any)._domRendererHandlers = handlers; (iframeDoc as any)._domRendererHandlers = handlers;
// Make iframe focusable for keyboard events const iframe = replayerIframeRef.current;
if (iframeRef.current) { if (iframe) {
iframeRef.current.tabIndex = 0; iframe.tabIndex = 0;
} }
}, },
[ [
@@ -784,287 +719,219 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
currentHighlight, currentHighlight,
onElementSelect, onElementSelect,
isInCaptureMode, isInCaptureMode,
snapshot,
user?.id, user?.id,
onShowDatePicker, onShowDatePicker,
onShowDropdown, onShowDropdown,
onShowTimePicker, onShowTimePicker,
onShowDateTimePicker, onShowDateTimePicker,
cachedChildSelectors
] ]
); );
/** /**
* Render DOM snapshot using rrweb * Cleanup replayer on unmount
*/ */
const renderRRWebSnapshot = useCallback(
(snapshotData: ProcessedSnapshot) => {
if (!iframeRef.current) {
console.warn("No iframe reference available");
return;
}
if (isInCaptureMode || isCachingChildSelectors) {
return; // Skip rendering in capture mode
}
try {
setIsRendered(false);
const iframe = iframeRef.current!;
let iframeDoc: Document;
try {
iframeDoc = iframe.contentDocument!;
if (!iframeDoc) {
throw new Error("Cannot access iframe document");
}
} catch (crossOriginError) {
console.warn("Cross-origin iframe access blocked, recreating iframe");
const newIframe = document.createElement('iframe');
newIframe.style.cssText = iframe.style.cssText;
newIframe.sandbox = iframe.sandbox.value;
newIframe.title = iframe.title;
newIframe.tabIndex = iframe.tabIndex;
newIframe.id = iframe.id;
iframe.parentNode?.replaceChild(newIframe, iframe);
Object.defineProperty(iframeRef, 'current', {
value: newIframe,
writable: false,
enumerable: true,
configurable: true
});
iframeDoc = newIframe.contentDocument!;
if (!iframeDoc) {
throw new Error("Cannot access new iframe document");
}
}
const styleTags = Array.from(
document.querySelectorAll('link[rel="stylesheet"], style')
)
.map((tag) => tag.outerHTML)
.join("\n");
const enhancedCSS = `
/* rrweb rebuilt content styles */
html, body {
margin: 0 !important;
padding: 8px !important;
overflow-x: hidden !important;
}
html::-webkit-scrollbar,
body::-webkit-scrollbar {
display: none !important;
width: 0 !important;
height: 0 !important;
background: transparent !important;
}
/* Hide scrollbars for all elements */
*::-webkit-scrollbar {
display: none !important;
width: 0 !important;
height: 0 !important;
background: transparent !important;
}
* {
scrollbar-width: none !important; /* Firefox */
-ms-overflow-style: none !important; /* Internet Explorer 10+ */
}
/* Make everything interactive */
* {
cursor: "pointer" !important;
}
`;
const skeleton = `
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<base href="${snapshotData.baseUrl}">
${styleTags}
<style>${enhancedCSS}</style>
</head>
<body></body>
</html>
`;
if (!iframeDoc) {
throw new Error("Cannot access iframe document");
}
// Write the skeleton into the iframe
iframeDoc.open();
iframeDoc.write(skeleton);
iframeDoc.close();
const mirror = createMirror();
try {
rebuild(snapshotData.snapshot, {
doc: iframeDoc,
mirror: mirror,
hackCss: false,
cache: { stylesWithHoverClass: new Map() },
afterAppend: (node) => {
if (node.nodeType === Node.TEXT_NODE && node.textContent) {
const text = node.textContent.trim();
if (
text.startsWith("<") &&
text.includes(">") &&
text.length > 50
) {
if (node.parentNode) {
node.parentNode.removeChild(node);
}
}
}
},
});
} catch (rebuildError) {
console.error("rrweb rebuild failed:", rebuildError);
throw new Error(`rrweb rebuild failed: ${rebuildError}`);
}
setIsRendered(true);
setupIframeInteractions(iframeDoc);
} catch (error) {
console.error("Error rendering rrweb snapshot:", error);
}
},
[setupIframeInteractions, isInCaptureMode, isCachingChildSelectors]
);
useEffect(() => {
if (snapshot && iframeRef.current) {
renderRRWebSnapshot(snapshot);
}
}, [snapshot]);
useEffect(() => {
if (isRendered && iframeRef.current) {
const iframeDoc = iframeRef.current.contentDocument;
if (iframeDoc) {
setupIframeInteractions(iframeDoc);
}
}
}, [getText, getList, listSelector, isRendered, setupIframeInteractions]);
useEffect(() => { useEffect(() => {
return () => { return () => {
if (iframeRef.current) { if (replayerRef.current) {
const iframeDoc = iframeRef.current.contentDocument; replayerRef.current.pause();
if (iframeDoc) { replayerRef.current = null;
const handlers = (iframeDoc as any)._domRendererHandlers;
if (handlers) {
Object.entries(handlers).forEach(([event, handler]) => {
const options: boolean | AddEventListenerOptions = ['wheel', 'touchstart', 'touchmove'].includes(event)
? { passive: false }
: false;
iframeDoc.removeEventListener(
event,
handler as EventListener,
options
);
});
}
}
} }
}; };
}, []); }, []);
/**
* Listen for rrweb events from backend and add to replayer
*/
useEffect(() => {
if (!socket) {
console.warn('No socket available, skipping event listener setup');
return;
}
const handleRRWebEvent = (event: any) => {
if (!replayerRef.current && event.type === 2) {
const container = document.getElementById('mirror-container');
if (!container) {
console.warn('Container #mirror-container not found');
return;
}
const replayer = new Replayer([], {
root: container,
liveMode: true,
mouseTail: false
});
replayer.startLive();
replayer.addEvent(event);
replayerRef.current = replayer;
setTimeout(() => {
const replayerWrapper = container.querySelector('.replayer-wrapper');
const replayerIframe = replayerWrapper?.querySelector('iframe') as HTMLIFrameElement;
if (replayerIframe) {
replayerIframe.style.width = '100%';
replayerIframe.style.height = '100%';
replayerIframe.style.border = 'none';
replayerIframe.style.position = 'absolute';
replayerIframe.style.top = '0';
replayerIframe.style.left = '0';
replayerIframe.style.backgroundColor = '#ffffff';
replayerIframe.style.display = 'block';
replayerIframe.style.pointerEvents = 'auto';
replayerIframe.id = 'dom-browser-iframe';
replayerIframeRef.current = replayerIframe;
try {
const iframeDoc = replayerIframe.contentDocument;
if (iframeDoc) {
setupIframeInteractions(iframeDoc);
}
} catch (err) {
console.warn('Error accessing iframe:', err);
}
replayer.on('fullsnapshot-rebuilded', () => {
const iframe = replayerIframeRef.current;
if (iframe && iframe.contentDocument) {
setupIframeInteractions(iframe.contentDocument);
iframe.style.pointerEvents = 'auto';
const wrapper = container.querySelector('.replayer-wrapper') as HTMLElement;
if(wrapper) wrapper.style.pointerEvents = 'auto';
setIsRendered(true);
}
});
} else {
console.warn('Could not find iframe in replayer-wrapper');
}
}, 150);
} else if (replayerRef.current) {
replayerRef.current.addEvent(event);
}
};
socket.on('rrweb-event', handleRRWebEvent);
socket.emit('request-refresh');
return () => {
socket.off('rrweb-event', handleRRWebEvent);
};
}, [socket, setupIframeInteractions]);
useEffect(() => {
const iframe = replayerIframeRef.current;
if (iframe && iframe.contentDocument) {
setupIframeInteractions(iframe.contentDocument);
}
}, [setupIframeInteractions]);
return ( return (
<div <div
id="mirror-container"
ref={containerRef} ref={containerRef}
style={{ style={{
width: width, width: width,
height: height, height: height,
overflow: "hidden !important",
position: "relative", position: "relative",
borderRadius: "0px 0px 5px 5px", backgroundColor: "#ffffff",
backgroundColor: "white", overflow: "hidden",
isolation: "isolate",
}} }}
> >
<iframe
ref={iframeRef}
id="dom-browser-iframe"
style={{
width: "100%",
height: "100%",
border: "none",
display: "block",
overflow: isCachingChildSelectors ? "hidden !important" : "hidden !important",
pointerEvents: isCachingChildSelectors ? "none" : "auto",
}}
sandbox="allow-same-origin allow-forms allow-scripts"
title="DOM Browser Content"
tabIndex={0}
/>
{/* Loading indicator */}
{!isRendered && ( {!isRendered && (
<div <DOMLoadingIndicator />
style={{
position: "absolute",
top: 0,
left: 0,
right: 0,
bottom: 0,
background: "rgba(255, 255, 255, 0.9)",
display: "flex",
alignItems: "center",
justifyContent: "center",
fontSize: "18px",
color: "#666",
flexDirection: "column",
gap: "10px",
}}
>
<div
style={{
width: "40px",
height: "40px",
border: "3px solid #ff00c3",
borderTop: "3px solid transparent",
borderRadius: "50%",
animation: "spin 1s linear infinite",
}}
/>
<div>Loading website...</div>
<style>{`
@keyframes spin {
0% { transform: rotate(0deg); }
100% { transform: rotate(360deg); }
}
`}</style>
</div>
)}
{/* Capture mode overlay */}
{isInCaptureMode && (
<div
style={{
position: "absolute",
top: 0,
left: 0,
right: 0,
bottom: 0,
cursor: "pointer",
pointerEvents: "none",
zIndex: 999,
borderRadius: "0px 0px 5px 5px",
}}
/>
)} )}
</div> </div>
); );
}; };
const DOMLoadingIndicator: React.FC = () => {
const [progress, setProgress] = useState(0);
const [hasStartedLoading, setHasStartedLoading] = useState(false);
const { socket } = useSocketStore();
const { state } = useContext(AuthContext);
const { user } = state;
useEffect(() => {
if (!socket) return;
const handleLoadingProgress = (data: {
progress: number;
pendingRequests: number;
userId: string;
}) => {
if (!data.userId || data.userId === user?.id) {
if (!hasStartedLoading && data.progress > 0) {
setHasStartedLoading(true);
}
if (!hasStartedLoading || data.progress >= progress) {
setProgress(data.progress);
}
}
};
socket.on("domLoadingProgress", handleLoadingProgress);
return () => {
socket.off("domLoadingProgress", handleLoadingProgress);
};
}, [socket, user?.id, hasStartedLoading, progress]);
return (
<div
style={{
position: "absolute",
top: 0,
left: 0,
right: 0,
bottom: 0,
background: "#f5f5f5",
display: "flex",
alignItems: "center",
justifyContent: "center",
flexDirection: "column",
gap: "15px",
zIndex: 9999,
}}
>
<div
style={{
fontSize: "18px",
fontWeight: "500",
color: "#333",
}}
>
Loading {progress}%
</div>
<div
style={{
width: "240px",
height: "6px",
background: "#e0e0e0",
borderRadius: "3px",
overflow: "hidden",
}}
>
<div
style={{
width: `${progress}%`,
height: "100%",
background: "linear-gradient(90deg, #ff00c3, #ff66d9)",
borderRadius: "3px",
transition: "width 0.3s ease-out",
}}
/>
</div>
</div>
);
};

View File

@@ -36,10 +36,6 @@ interface RightSidePanelProps {
} }
export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture }) => { export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture }) => {
const [textLabels, setTextLabels] = useState<{ [id: string]: string }>({});
const [errors, setErrors] = useState<{ [id: string]: string }>({});
const [confirmedTextSteps, setConfirmedTextSteps] = useState<{ [id: string]: boolean }>({});
const [confirmedListTextFields, setConfirmedListTextFields] = useState<{ [listId: string]: { [fieldKey: string]: boolean } }>({});
const [showCaptureList, setShowCaptureList] = useState(true); const [showCaptureList, setShowCaptureList] = useState(true);
const [showCaptureScreenshot, setShowCaptureScreenshot] = useState(true); const [showCaptureScreenshot, setShowCaptureScreenshot] = useState(true);
const [showCaptureText, setShowCaptureText] = useState(true); const [showCaptureText, setShowCaptureText] = useState(true);
@@ -52,7 +48,7 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
} | null>(null); } | null>(null);
const autoDetectionRunRef = useRef<string | null>(null); const autoDetectionRunRef = useRef<string | null>(null);
const { lastAction, notify, currentWorkflowActionsState, setCurrentWorkflowActionsState, resetInterpretationLog, currentListActionId, setCurrentListActionId, currentTextActionId, setCurrentTextActionId, currentScreenshotActionId, setCurrentScreenshotActionId, isDOMMode, setIsDOMMode, currentSnapshot, setCurrentSnapshot, updateDOMMode, initialUrl, setRecordingUrl, currentTextGroupName } = useGlobalInfoStore(); const { notify, currentWorkflowActionsState, setCurrentWorkflowActionsState, resetInterpretationLog, currentListActionId, setCurrentListActionId, currentTextActionId, setCurrentTextActionId, currentScreenshotActionId, setCurrentScreenshotActionId, isDOMMode, updateDOMMode, currentTextGroupName } = useGlobalInfoStore();
const { const {
getText, startGetText, stopGetText, getText, startGetText, stopGetText,
getList, startGetList, stopGetList, getList, startGetList, stopGetList,
@@ -65,8 +61,7 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
showPaginationOptions, setShowPaginationOptions, showPaginationOptions, setShowPaginationOptions,
showLimitOptions, setShowLimitOptions, showLimitOptions, setShowLimitOptions,
workflow, setWorkflow, workflow, setWorkflow,
activeAction, setActiveAction, activeAction, setActiveAction, finishAction
startAction, finishAction
} = useActionContext(); } = useActionContext();
const { browserSteps, addScreenshotStep, updateListStepLimit, updateListStepPagination, deleteStepsByActionId, updateListStepData, updateScreenshotStepData, emitActionForStep } = useBrowserSteps(); const { browserSteps, addScreenshotStep, updateListStepLimit, updateListStepPagination, deleteStepsByActionId, updateListStepData, updateScreenshotStepData, emitActionForStep } = useBrowserSteps();
@@ -154,20 +149,10 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
} }
}; };
const domcastHandler = (data: any) => {
if (!data.userId || data.userId === id) {
if (data.snapshotData && data.snapshotData.snapshot) {
updateDOMMode(true, data.snapshotData);
}
}
};
socket.on("dom-mode-enabled", domModeHandler); socket.on("dom-mode-enabled", domModeHandler);
socket.on("domcast", domcastHandler);
return () => { return () => {
socket.off("dom-mode-enabled", domModeHandler); socket.off("dom-mode-enabled", domModeHandler);
socket.off("domcast", domcastHandler);
}; };
} }
}, [socket, id, updateDOMMode]); }, [socket, id, updateDOMMode]);
@@ -176,11 +161,9 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
if (socket) { if (socket) {
socket.on("workflow", workflowHandler); socket.on("workflow", workflowHandler);
} }
// fetch the workflow every time the id changes
if (id) { if (id) {
fetchWorkflow(id, workflowHandler); fetchWorkflow(id, workflowHandler);
} }
// fetch workflow in 15min intervals
let interval = setInterval(() => { let interval = setInterval(() => {
if (id) { if (id) {
fetchWorkflow(id, workflowHandler); fetchWorkflow(id, workflowHandler);
@@ -267,7 +250,7 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
fields: Record<string, any>, fields: Record<string, any>,
currentListId: number currentListId: number
) => { ) => {
if (isDOMMode && currentSnapshot) { if (isDOMMode) {
try { try {
let iframeElement = document.querySelector( let iframeElement = document.querySelector(
"#dom-browser-iframe" "#dom-browser-iframe"
@@ -318,25 +301,9 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
console.error("Error in client-side data extraction:", error); console.error("Error in client-side data extraction:", error);
notify("error", "Failed to extract data client-side"); notify("error", "Failed to extract data client-side");
} }
} else {
if (!socket) {
console.error("Socket not available for backend extraction");
return;
}
try {
socket.emit("extractListData", {
listSelector,
fields,
currentListId,
pagination: { type: "", selector: "" },
});
} catch (error) {
console.error("Error in backend data extraction:", error);
}
} }
}, },
[isDOMMode, currentSnapshot, updateListStepData, socket, notify, currentWorkflowActionsState] [isDOMMode, updateListStepData, socket, notify, currentWorkflowActionsState]
); );
useEffect(() => { useEffect(() => {

View File

@@ -1,4 +1,4 @@
import React, { createContext, useContext, useState } from "react"; import { createContext, useContext, useState } from "react";
import { AlertSnackbarProps } from "../components/ui/AlertSnackbar"; import { AlertSnackbarProps } from "../components/ui/AlertSnackbar";
import { WhereWhatPair } from "maxun-core"; import { WhereWhatPair } from "maxun-core";
import { QueryClient, QueryClientProvider, useQuery, useQueryClient } from '@tanstack/react-query'; import { QueryClient, QueryClientProvider, useQuery, useQueryClient } from '@tanstack/react-query';
@@ -49,41 +49,6 @@ interface ScheduleConfig {
cronExpression?: string; cronExpression?: string;
} }
interface ProcessedSnapshot {
snapshot: any;
resources: {
stylesheets: Array<{
href: string;
content: string;
media?: string;
}>;
images: Array<{
src: string;
dataUrl: string;
alt?: string;
}>;
fonts: Array<{
url: string;
dataUrl: string;
format?: string;
}>;
scripts: Array<{
src: string;
content: string;
type?: string;
}>;
media: Array<{
src: string;
dataUrl: string;
type: string;
}>;
};
baseUrl: string;
viewport: { width: number; height: number };
timestamp: number;
processingStats: any;
}
export interface RobotSettings { export interface RobotSettings {
id: string; id: string;
userId?: number; userId?: number;
@@ -147,9 +112,7 @@ interface GlobalInfo {
setCurrentTextGroupName: (name: string) => void; setCurrentTextGroupName: (name: string) => void;
isDOMMode: boolean; isDOMMode: boolean;
setIsDOMMode: (isDOMMode: boolean) => void; setIsDOMMode: (isDOMMode: boolean) => void;
currentSnapshot: ProcessedSnapshot | null; updateDOMMode: (isDOMMode: boolean) => void;
setCurrentSnapshot: (snapshot: ProcessedSnapshot | null) => void;
updateDOMMode: (isDOMMode: boolean, snapshot?: ProcessedSnapshot | null) => void;
}; };
class GlobalInfoStore implements Partial<GlobalInfo> { class GlobalInfoStore implements Partial<GlobalInfo> {
@@ -181,7 +144,6 @@ class GlobalInfoStore implements Partial<GlobalInfo> {
currentScreenshotActionId = ''; currentScreenshotActionId = '';
currentTextGroupName = 'Text Data'; currentTextGroupName = 'Text Data';
isDOMMode = false; isDOMMode = false;
currentSnapshot = null;
}; };
const globalInfoStore = new GlobalInfoStore(); const globalInfoStore = new GlobalInfoStore();
@@ -272,8 +234,7 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
const [rerenderRuns, setRerenderRuns] = useState<boolean>(globalInfoStore.rerenderRuns); const [rerenderRuns, setRerenderRuns] = useState<boolean>(globalInfoStore.rerenderRuns);
const [rerenderRobots, setRerenderRobots] = useState<boolean>(globalInfoStore.rerenderRobots); const [rerenderRobots, setRerenderRobots] = useState<boolean>(globalInfoStore.rerenderRobots);
const [recordingLength, setRecordingLength] = useState<number>(globalInfoStore.recordingLength); const [recordingLength, setRecordingLength] = useState<number>(globalInfoStore.recordingLength);
// const [recordingId, setRecordingId] = useState<string | null>(globalInfoStore.recordingId); const [recordingId, setRecordingId] = useState<string | null>(() => {
const [recordingId, setRecordingId] = useState<string | null>(() => {
try { try {
const stored = sessionStorage.getItem('recordingId'); const stored = sessionStorage.getItem('recordingId');
return stored ? JSON.parse(stored) : globalInfoStore.recordingId; return stored ? JSON.parse(stored) : globalInfoStore.recordingId;
@@ -282,7 +243,6 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
} }
}); });
// Create a wrapped setter that persists to sessionStorage
const setPersistedRecordingId = (newRecordingId: string | null) => { const setPersistedRecordingId = (newRecordingId: string | null) => {
setRecordingId(newRecordingId); setRecordingId(newRecordingId);
try { try {
@@ -307,7 +267,6 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
const [currentScreenshotActionId, setCurrentScreenshotActionId] = useState<string>(''); const [currentScreenshotActionId, setCurrentScreenshotActionId] = useState<string>('');
const [currentTextGroupName, setCurrentTextGroupName] = useState<string>('Text Data'); const [currentTextGroupName, setCurrentTextGroupName] = useState<string>('Text Data');
const [isDOMMode, setIsDOMMode] = useState<boolean>(globalInfoStore.isDOMMode); const [isDOMMode, setIsDOMMode] = useState<boolean>(globalInfoStore.isDOMMode);
const [currentSnapshot, setCurrentSnapshot] = useState<ProcessedSnapshot | null>(globalInfoStore.currentSnapshot);
const notify = (severity: 'error' | 'warning' | 'info' | 'success', message: string) => { const notify = (severity: 'error' | 'warning' | 'info' | 'success', message: string) => {
setNotification({ severity, message, isOpen: true }); setNotification({ severity, message, isOpen: true });
@@ -326,22 +285,13 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
const resetInterpretationLog = () => { const resetInterpretationLog = () => {
setShouldResetInterpretationLog(true); setShouldResetInterpretationLog(true);
// Reset the flag after a short delay to allow components to respond
setTimeout(() => { setTimeout(() => {
setShouldResetInterpretationLog(false); setShouldResetInterpretationLog(false);
}, 100); }, 100);
} }
const updateDOMMode = (mode: boolean, snapshot?: ProcessedSnapshot | null) => { const updateDOMMode = (mode: boolean) => {
setIsDOMMode(mode); setIsDOMMode(mode);
if (snapshot !== undefined) {
setCurrentSnapshot(snapshot);
}
if (!mode) {
setCurrentSnapshot(null);
}
} }
const [dataCacheClient] = useState(() => createDataCacheClient()); const [dataCacheClient] = useState(() => createDataCacheClient());
@@ -391,8 +341,6 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
setCurrentTextGroupName, setCurrentTextGroupName,
isDOMMode, isDOMMode,
setIsDOMMode, setIsDOMMode,
currentSnapshot,
setCurrentSnapshot,
updateDOMMode, updateDOMMode,
}} }}
> >

View File

@@ -59,7 +59,12 @@ class CapturedElementHighlighter {
* Get the iframe document * Get the iframe document
*/ */
private getIframeDocument(): Document | null { private getIframeDocument(): Document | null {
const iframeElement = document.querySelector('#dom-browser-iframe') as HTMLIFrameElement; let iframeElement = document.querySelector('#dom-browser-iframe') as HTMLIFrameElement;
if (!iframeElement) {
iframeElement = document.querySelector('.replayer-wrapper iframe') as HTMLIFrameElement;
}
return iframeElement?.contentDocument || null; return iframeElement?.contentDocument || null;
} }

View File

@@ -56,8 +56,6 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
}); });
}; };
useEffect(() => changeBrowserDimensions(), [isLoaded])
useEffect(() => { useEffect(() => {
if (darkMode) { if (darkMode) {
@@ -118,20 +116,6 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
} }
}, [setId, recordingUrl, setRecordingUrl, setRecordingName, setRetrainRobotId]); }, [setId, recordingUrl, setRecordingUrl, setRecordingName, setRetrainRobotId]);
const changeBrowserDimensions = useCallback(() => {
if (browserContentRef.current) {
const currentWidth = Math.floor(browserContentRef.current.getBoundingClientRect().width);
const innerHeightWithoutNavBar = window.innerHeight - 54.5;
if (innerHeightWithoutNavBar <= (currentWidth / 1.6)) {
setWidth(currentWidth - 10);
setHasScrollbar(true);
} else {
setWidth(currentWidth);
}
socket?.emit("rerender");
}
}, [socket]);
const handleLoaded = useCallback(() => { const handleLoaded = useCallback(() => {
if (recordingName && browserId && recordingId) { if (recordingName && browserId && recordingId) {
editRecordingFromStorage(browserId, recordingId).then(() => setIsLoaded(true)); editRecordingFromStorage(browserId, recordingId).then(() => setIsLoaded(true));