diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index d04dcefe..fe782210 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -17,6 +17,73 @@ import { WorkflowInterpreter } from "../../workflow-management/classes/Interpret import { getDecryptedProxyConfig } from '../../routes/proxy'; import { getInjectableScript } from 'idcac-playwright'; +declare global { + interface Window { + rrwebSnapshot?: any; + } +} + +interface RRWebSnapshot { + type: number; + childNodes?: RRWebSnapshot[]; + tagName?: string; + attributes?: Record; + textContent?: string; + id: number; + [key: string]: any; +} + +interface ProcessedSnapshot { + snapshot: RRWebSnapshot; + resources: { + stylesheets: Array<{ + href: string; + content: string; + media?: string; + }>; + images: Array<{ + src: string; + dataUrl: string; + alt?: string; + }>; + fonts: Array<{ + url: string; + dataUrl: string; + format?: string; + }>; + scripts: Array<{ + src: string; + content: string; + type?: string; + }>; + media: Array<{ + src: string; + dataUrl: string; + type: string; + }>; + }; + baseUrl: string; + viewport: { width: number; height: number }; + timestamp: number; + processingStats: { + discoveredResources: { + images: number; + stylesheets: number; + scripts: number; + fonts: number; + media: number; + }; + cachedResources: { + stylesheets: number; + images: number; + fonts: number; + scripts: number; + media: number; + }; + totalCacheSize: number; + }; +} + chromium.use(stealthPlugin()); const MEMORY_CONFIG = { @@ -123,6 +190,45 @@ export class RemoteBrowser { private screencastInterval: NodeJS.Timeout | null = null private isScreencastActive: boolean = false; + private isDOMStreamingActive: boolean = false; + private domUpdateInterval: NodeJS.Timeout | null = null; + private renderingMode: "screenshot" | "dom" = "screenshot"; + + private lastScrollPosition = { x: 0, y: 0 }; + private scrollThreshold = 200; // pixels + private snapshotDebounceTimeout: NodeJS.Timeout | null = null; + private isScrollTriggeredSnapshot = false; + + /** + * Cache for network resources captured via CDP + * @private + */ + private networkResourceCache: Map< + string, + { + url: string; + content: string; + mimeType: string; + base64Encoded: boolean; + timestamp: number; + resourceType?: string; + statusCode?: number; + headers?: Record; + } + > = new Map(); + + /** + * Set to track active network requests + * @private + */ + private activeRequests: Set = new Set(); + + /** + * Flag to indicate if network monitoring is active + * @private + */ + private isNetworkMonitoringActive: boolean = false; + /** * Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and * assigns the socket instance everywhere. @@ -148,6 +254,682 @@ export class RemoteBrowser { }, 30000); // Every 30 seconds } + private processCSS( + cssContent: string, + cssUrl: string, + baseUrl: string, + resources?: any + ): string { + try { + let processedContent = cssContent; + + logger.debug(`Processing CSS from: ${cssUrl}`); + + // Process @font-face declarations and collect font resources + processedContent = processedContent.replace( + /@font-face\s*\{([^}]*)\}/gi, + (fontFaceMatch, fontFaceContent) => { + let newFontFaceContent = fontFaceContent; + + logger.debug( + `Processing @font-face block: ${fontFaceContent.substring( + 0, + 100 + )}...` + ); + + newFontFaceContent = newFontFaceContent.replace( + /src\s*:\s*([^;}]+)[;}]/gi, + (srcMatch: any, srcValue: any) => { + let newSrcValue = srcValue; + + newSrcValue = newSrcValue.replace( + /url\s*\(\s*['"]?([^'")]+)['"]?\s*\)(\s*format\s*\(\s*['"]?[^'")]*['"]?\s*\))?/gi, + (urlMatch: any, url: string, formatPart: any) => { + const originalUrl = url.trim(); + + logger.debug(`Found font URL in @font-face: ${originalUrl}`); + + if ( + originalUrl.startsWith("data:") || + originalUrl.startsWith("blob:") + ) { + return urlMatch; + } + + try { + let absoluteUrl: string; + try { + absoluteUrl = new URL(originalUrl).href; + } catch (e) { + absoluteUrl = new URL(originalUrl, cssUrl || baseUrl) + .href; + } + + const cachedResource = + this.networkResourceCache.get(absoluteUrl); + if (cachedResource && resources) { + const dataUrl = cachedResource.base64Encoded + ? `data:${cachedResource.mimeType};base64,${cachedResource.content}` + : `data:${cachedResource.mimeType};base64,${Buffer.from( + cachedResource.content, + "utf-8" + ).toString("base64")}`; + + resources.fonts.push({ + url: absoluteUrl, + dataUrl, + format: originalUrl.split(".").pop()?.split("?")[0], + }); + } + + // Keep original URL in CSS + return urlMatch; + } catch (e) { + logger.warn( + "Failed to process font URL in @font-face:", + originalUrl, + e + ); + return urlMatch; + } + } + ); + + return `src: ${newSrcValue};`; + } + ); + + return `@font-face {${newFontFaceContent}}`; + } + ); + + // Process other url() references and collect resources + processedContent = processedContent.replace( + /url\s*\(\s*['"]?([^'")]+)['"]?\s*\)/gi, + (match, url) => { + const originalUrl = url.trim(); + + if ( + originalUrl.startsWith("data:") || + originalUrl.startsWith("blob:") + ) { + return match; + } + + try { + let absoluteUrl: string; + try { + absoluteUrl = new URL(originalUrl).href; + } catch (e) { + absoluteUrl = new URL(originalUrl, cssUrl || baseUrl).href; + } + + const cachedResource = this.networkResourceCache.get(absoluteUrl); + if (cachedResource && resources) { + const lowerMimeType = cachedResource.mimeType.toLowerCase(); + + if (lowerMimeType.includes("image/")) { + const dataUrl = cachedResource.base64Encoded + ? `data:${cachedResource.mimeType};base64,${cachedResource.content}` + : `data:${cachedResource.mimeType};base64,${Buffer.from( + cachedResource.content, + "utf-8" + ).toString("base64")}`; + + resources.images.push({ + src: absoluteUrl, + dataUrl, + alt: "", + }); + } else if ( + lowerMimeType.includes("font/") || + lowerMimeType.includes("application/font") + ) { + const dataUrl = cachedResource.base64Encoded + ? `data:${cachedResource.mimeType};base64,${cachedResource.content}` + : `data:${cachedResource.mimeType};base64,${Buffer.from( + cachedResource.content, + "utf-8" + ).toString("base64")}`; + + resources.fonts.push({ + url: absoluteUrl, + dataUrl, + format: originalUrl.split(".").pop()?.split("?")[0], + }); + } + } + + // Keep original URL in CSS + return match; + } catch (e) { + logger.warn(`Failed to process CSS URL: ${originalUrl}`, e); + return match; + } + } + ); + + // Process @import statements and collect stylesheets + processedContent = processedContent.replace( + /@import\s+(?:url\s*\(\s*)?['"]?([^'")]+)['"]?\s*\)?([^;]*);?/gi, + (match, url, mediaQuery) => { + const originalUrl = url.trim(); + + if ( + originalUrl.startsWith("data:") || + originalUrl.startsWith("blob:") + ) { + return match; + } + + try { + let absoluteUrl: string; + try { + absoluteUrl = new URL(originalUrl).href; + } catch (e) { + absoluteUrl = new URL(originalUrl, cssUrl || baseUrl).href; + } + + const cachedResource = this.networkResourceCache.get(absoluteUrl); + if ( + cachedResource && + resources && + cachedResource.mimeType.includes("css") + ) { + const content = cachedResource.base64Encoded + ? Buffer.from(cachedResource.content, "base64").toString( + "utf-8" + ) + : cachedResource.content; + + resources.stylesheets.push({ + href: absoluteUrl, + content: this.processCSS( + content, + absoluteUrl, + baseUrl, + resources + ), + media: mediaQuery ? mediaQuery.trim() : "all", + }); + } + + // Keep original @import + return match; + } catch (e) { + logger.warn(`Failed to process CSS @import: ${originalUrl}`, e); + return match; + } + } + ); + + logger.debug(`CSS processing completed for: ${cssUrl}`); + return processedContent; + } catch (error) { + logger.error("Failed to process CSS content:", error); + return cssContent; // Return original content if processing fails + } + } + + private async processRRWebSnapshot( + snapshot: RRWebSnapshot + ): Promise { + const baseUrl = this.currentPage?.url() || ""; + + const resources = { + stylesheets: [] as Array<{ + href: string; + content: string; + media?: string; + }>, + images: [] as Array<{ src: string; dataUrl: string; alt?: string }>, + fonts: [] as Array<{ url: string; dataUrl: string; format?: string }>, + scripts: [] as Array<{ src: string; content: string; type?: string }>, + media: [] as Array<{ src: string; dataUrl: string; type: string }>, + }; + + const processNode = (node: RRWebSnapshot): RRWebSnapshot => { + const processedNode = { ...node }; + + // Process attributes if they exist + if (node.attributes) { + const newAttributes = { ...node.attributes }; + + // Process common attributes that contain URLs + const urlAttributes = ["src", "href", "data", "poster", "background"]; + + for (const attr of urlAttributes) { + if (newAttributes[attr]) { + const originalUrl = newAttributes[attr]; + + // Categorize and collect the resource instead of proxying + const lowerAttr = attr.toLowerCase(); + const lowerUrl = originalUrl.toLowerCase(); + + if (lowerAttr === "src" && node.tagName?.toLowerCase() === "img") { + const cachedResource = this.networkResourceCache.get(originalUrl); + if ( + cachedResource && + cachedResource.mimeType.includes("image/") + ) { + const dataUrl = cachedResource.base64Encoded + ? `data:${cachedResource.mimeType};base64,${cachedResource.content}` + : `data:${cachedResource.mimeType};base64,${Buffer.from( + cachedResource.content, + "utf-8" + ).toString("base64")}`; + + resources.images.push({ + src: originalUrl, + dataUrl, + alt: newAttributes.alt, + }); + } + } else if ( + lowerAttr === "href" && + node.tagName?.toLowerCase() === "link" + ) { + const rel = newAttributes.rel?.toLowerCase() || ""; + + if (rel.includes("stylesheet")) { + const cachedResource = + this.networkResourceCache.get(originalUrl); + if (cachedResource && cachedResource.mimeType.includes("css")) { + let content = cachedResource.base64Encoded + ? Buffer.from(cachedResource.content, "base64").toString( + "utf-8" + ) + : cachedResource.content; + + // Process CSS to collect embedded resources + content = this.processCSS( + content, + originalUrl, + baseUrl, + resources + ); + + resources.stylesheets.push({ + href: originalUrl, + content, + media: newAttributes.media || "all", + }); + } + } else if ( + rel.includes("font") || + lowerUrl.match(/\.(woff2?|ttf|otf|eot)(\?.*)?$/i) + ) { + const cachedResource = + this.networkResourceCache.get(originalUrl); + if (cachedResource) { + const dataUrl = cachedResource.base64Encoded + ? `data:${cachedResource.mimeType};base64,${cachedResource.content}` + : `data:${cachedResource.mimeType};base64,${Buffer.from( + cachedResource.content, + "utf-8" + ).toString("base64")}`; + + resources.fonts.push({ + url: originalUrl, + dataUrl, + format: lowerUrl.split(".").pop()?.split("?")[0], + }); + } + } + } + } + } + + // Process srcset attribute - collect resources but keep original URLs + if (newAttributes.srcset) { + const originalSrcset = newAttributes.srcset; + originalSrcset.split(",").forEach((srcsetItem) => { + const parts = srcsetItem.trim().split(/\s+/); + const url = parts[0]; + + if (url && !url.startsWith("data:") && !url.startsWith("blob:")) { + const cachedResource = this.networkResourceCache.get(url); + if ( + cachedResource && + cachedResource.mimeType.includes("image/") + ) { + const dataUrl = cachedResource.base64Encoded + ? `data:${cachedResource.mimeType};base64,${cachedResource.content}` + : `data:${cachedResource.mimeType};base64,${Buffer.from( + cachedResource.content, + "utf-8" + ).toString("base64")}`; + + resources.images.push({ + src: url, + dataUrl, + alt: newAttributes.alt, + }); + } + } + }); + } + + processedNode.attributes = newAttributes; + } + + // Process text content for style elements + if (node.tagName?.toLowerCase() === "style" && node.textContent) { + let content = node.textContent; + + // Process CSS content to collect embedded resources + content = this.processCSS(content, "", baseUrl, resources); + processedNode.textContent = content; + } + + // Recursively process child nodes + if (node.childNodes) { + processedNode.childNodes = node.childNodes.map(processNode); + } + + return processedNode; + }; + + const processedSnapshot = processNode(snapshot); + + // Add cached scripts and media + for (const [url, resource] of this.networkResourceCache.entries()) { + if (resource.mimeType.toLowerCase().includes("javascript")) { + const content = resource.base64Encoded + ? Buffer.from(resource.content, "base64").toString("utf-8") + : resource.content; + + resources.scripts.push({ + src: url, + content, + type: "text/javascript", + }); + } else if ( + resource.mimeType.toLowerCase().includes("video/") || + resource.mimeType.toLowerCase().includes("audio/") + ) { + const dataUrl = resource.base64Encoded + ? `data:${resource.mimeType};base64,${resource.content}` + : `data:${resource.mimeType};base64,${Buffer.from( + resource.content, + "utf-8" + ).toString("base64")}`; + + resources.media.push({ + src: url, + dataUrl, + type: resource.mimeType, + }); + } + } + + const viewport = (await this.currentPage?.viewportSize()) || { + width: 1280, + height: 720, + }; + + return { + snapshot: processedSnapshot, + resources, + baseUrl, + viewport, + timestamp: Date.now(), + processingStats: { + discoveredResources: { + images: resources.images.length, + stylesheets: resources.stylesheets.length, + scripts: resources.scripts.length, + fonts: resources.fonts.length, + media: resources.media.length, + }, + cachedResources: { + stylesheets: resources.stylesheets.length, + images: resources.images.length, + fonts: resources.fonts.length, + scripts: resources.scripts.length, + media: resources.media.length, + }, + totalCacheSize: this.networkResourceCache.size, + }, + }; + } + + /** + * Check if a resource should be cached based on its MIME type and URL + * @private + */ + private shouldCacheResource(mimeType: string, url: string): boolean { + const lowerMimeType = mimeType.toLowerCase(); + const lowerUrl = url.toLowerCase(); + + // CSS Resources + if ( + lowerMimeType.includes("text/css") || + lowerMimeType.includes("application/css") || + lowerUrl.endsWith(".css") + ) { + return true; + } + + // Font Resources + if ( + lowerMimeType.includes("font/") || + lowerMimeType.includes("application/font") || + lowerMimeType.includes("application/x-font") || + lowerUrl.match(/\.(woff2?|ttf|otf|eot)(\?.*)?$/) + ) { + return true; + } + + // Image Resources + if ( + lowerMimeType.includes("image/") || + lowerUrl.match(/\.(jpg|jpeg|png|gif|webp|svg|ico|bmp|tiff|avif)(\?.*)?$/) + ) { + return true; + } + + // JavaScript Resources + if ( + lowerMimeType.includes("javascript") || + lowerMimeType.includes("text/js") || + lowerMimeType.includes("application/js") || + lowerUrl.match(/\.js(\?.*)?$/) + ) { + return true; + } + + // Media Resources + if ( + lowerMimeType.includes("video/") || + lowerMimeType.includes("audio/") || + lowerUrl.match( + /\.(mp4|webm|ogg|avi|mov|wmv|flv|mp3|wav|m4a|aac|flac)(\?.*)?$/ + ) + ) { + return true; + } + + // Document Resources + if ( + lowerMimeType.includes("application/pdf") || + lowerMimeType.includes("application/msword") || + lowerMimeType.includes("application/vnd.ms-") || + lowerMimeType.includes("application/vnd.openxmlformats-") || + lowerUrl.match(/\.(pdf|doc|docx|xls|xlsx|ppt|pptx)(\?.*)?$/) + ) { + return true; + } + + // Manifest and Icon Resources + if ( + lowerMimeType.includes("application/manifest+json") || + lowerUrl.includes("manifest.json") || + lowerUrl.includes("browserconfig.xml") + ) { + return true; + } + + // SVG Resources (can be images or fonts) + if (lowerMimeType.includes("image/svg+xml") || lowerUrl.endsWith(".svg")) { + return true; + } + + // Other common web resources + if ( + lowerMimeType.includes("application/octet-stream") && + lowerUrl.match(/\.(woff2?|ttf|otf|eot|css|js)(\?.*)?$/) + ) { + return true; + } + + return false; + } + + /** + * Clean up old cached resources to prevent memory leaks + * @private + */ + private cleanupResourceCache(): void { + const now = Date.now(); + const maxAge = 5 * 60 * 1000; // 5 minutes + + for (const [url, resource] of this.networkResourceCache.entries()) { + if (now - resource.timestamp > maxAge) { + this.networkResourceCache.delete(url); + } + } + + if (this.networkResourceCache.size > 200) { + const entries = Array.from(this.networkResourceCache.entries()); + entries.sort((a, b) => a[1].timestamp - b[1].timestamp); + + for (let i = 0; i < 50; i++) { + this.networkResourceCache.delete(entries[i][0]); + } + } + + logger.debug( + `Resource cache cleaned up. Current size: ${this.networkResourceCache.size}` + ); + } + + /** + * Initialize network monitoring via CDP to capture all resources + * @private + */ + private async initializeNetworkMonitoring(): Promise { + if (!this.client || this.isNetworkMonitoringActive) { + return; + } + + try { + await this.client.send("Network.enable"); + await this.client.send("Runtime.enable"); + await this.client.send("Page.enable"); + + await this.client.send("Network.setRequestInterception", { + patterns: [ + { urlPattern: "*", resourceType: "Stylesheet" }, + { urlPattern: "*", resourceType: "Image" }, + { urlPattern: "*", resourceType: "Font" }, + { urlPattern: "*", resourceType: "Script" }, + { urlPattern: "*", resourceType: "Media" }, + { urlPattern: "*", resourceType: "Document" }, + { urlPattern: "*", resourceType: "Manifest" }, + { urlPattern: "*", resourceType: "Other" }, + ], + }); + + this.isNetworkMonitoringActive = true; + logger.info("Enhanced network monitoring enabled via CDP"); + + this.client.on( + "Network.responseReceived", + async ({ requestId, response, type }) => { + const mimeType = response.mimeType?.toLowerCase() || ""; + const url = response.url; + const resourceType = type; + + logger.debug( + `Resource received: ${resourceType} - ${mimeType} - ${url}` + ); + + if (this.shouldCacheResource(mimeType, url)) { + this.activeRequests.add(requestId); + + try { + const { body, base64Encoded } = await this.client!.send( + "Network.getResponseBody", + { requestId } + ); + + this.networkResourceCache.set(url, { + url, + content: body, + mimeType: response.mimeType || "application/octet-stream", + base64Encoded, + timestamp: Date.now(), + resourceType, + statusCode: response.status, + headers: response.headers, + }); + + logger.debug( + `Cached ${resourceType} resource: ${url} (${mimeType})` + ); + } catch (error) { + logger.warn( + `Failed to capture ${resourceType} resource body for ${url}:`, + error + ); + } finally { + this.activeRequests.delete(requestId); + } + } + } + ); + + this.client.on( + "Network.requestIntercepted", + async ({ interceptionId, request }) => { + try { + await this.client!.send("Network.continueInterceptedRequest", { + interceptionId, + }); + logger.debug(`Request intercepted and continued: ${request.url}`); + } catch (error) { + logger.warn( + `Failed to continue intercepted request for ${request.url}:`, + error + ); + } + } + ); + + this.client.on( + "Network.loadingFailed", + ({ requestId, errorText, type }) => { + this.activeRequests.delete(requestId); + logger.debug(`Network request failed (${type}): ${errorText}`); + } + ); + + this.client.on("Network.loadingFinished", ({ requestId }) => { + this.activeRequests.delete(requestId); + }); + + // Clean up cache periodically + setInterval(() => { + this.cleanupResourceCache(); + }, 60000); + } catch (error) { + logger.error("Failed to initialize enhanced network monitoring:", error); + this.isNetworkMonitoringActive = false; + } + } + private initializeMemoryManagement(): void { setInterval(() => { const memoryUsage = process.memoryUsage(); @@ -239,6 +1021,90 @@ export class RemoteBrowser { return normalizedNew !== normalizedLast; } + /** + * Setup scroll event listener to track user scrolling + */ + private setupScrollEventListener(): void { + this.socket.on( + "dom:scroll", + async (data: { deltaX: number; deltaY: number }) => { + if (!this.isDOMStreamingActive || !this.currentPage) return; + + try { + logger.debug( + `Received scroll event: deltaX=${data.deltaX}, deltaY=${data.deltaY}` + ); + + await this.currentPage.mouse.wheel(data.deltaX, data.deltaY); + + const scrollInfo = await this.currentPage.evaluate(() => ({ + x: window.scrollX, + y: window.scrollY, + maxX: Math.max( + 0, + document.documentElement.scrollWidth - window.innerWidth + ), + maxY: Math.max( + 0, + document.documentElement.scrollHeight - window.innerHeight + ), + documentHeight: document.documentElement.scrollHeight, + viewportHeight: window.innerHeight, + })); + + const scrollDelta = + Math.abs(scrollInfo.y - this.lastScrollPosition.y) + + Math.abs(scrollInfo.x - this.lastScrollPosition.x); + + logger.debug( + `Scroll delta: ${scrollDelta}, threshold: ${this.scrollThreshold}` + ); + + if (scrollDelta > this.scrollThreshold) { + this.lastScrollPosition = { x: scrollInfo.x, y: scrollInfo.y }; + this.isScrollTriggeredSnapshot = true; + + if (this.snapshotDebounceTimeout) { + clearTimeout(this.snapshotDebounceTimeout); + } + + this.snapshotDebounceTimeout = setTimeout(async () => { + logger.info( + `Triggering snapshot due to scroll. Position: ${scrollInfo.y}/${scrollInfo.maxY}` + ); + + await this.makeAndEmitDOMSnapshot(); + }, 300); + } + } catch (error) { + logger.error("Error handling scroll event:", error); + } + } + ); + } + + private setupPageChangeListeners(): void { + if (!this.currentPage) return; + + this.currentPage.on("domcontentloaded", async () => { + logger.info("DOM content loaded - triggering snapshot"); + await this.makeAndEmitDOMSnapshot(); + }); + + this.currentPage.on("response", async (response) => { + const url = response.url(); + if ( + response.request().resourceType() === "document" || + url.includes("api/") || + url.includes("ajax") + ) { + setTimeout(async () => { + await this.makeAndEmitDOMSnapshot(); + }, 800); + } + }); + } + private async setupPageEventListeners(page: Page) { page.on('framenavigated', async (frame) => { if (frame === page.mainFrame()) { @@ -389,6 +1255,9 @@ export class RemoteBrowser { ); this.currentPage = await this.context.newPage(); + + await this.currentPage.addInitScript({ path: './server/src/browser-management/classes/rrweb-bundle.js' }); + await this.setupPageEventListeners(this.currentPage); const viewportSize = await this.currentPage.viewportSize(); @@ -406,10 +1275,20 @@ export class RemoteBrowser { this.client = await this.currentPage.context().newCDPSession(this.currentPage); await blocker.disableBlockingInPage(this.currentPage); console.log('Adblocker initialized'); + + if (this.client) { + await this.initializeNetworkMonitoring(); + logger.info("Network monitoring initialized successfully"); + } } catch (error: any) { console.warn('Failed to initialize adblocker, continuing without it:', error.message); // Still need to set up the CDP session even if blocker fails this.client = await this.currentPage.context().newCDPSession(this.currentPage); + + if (this.client) { + await this.initializeNetworkMonitoring(); + logger.info("Network monitoring initialized successfully"); + } } success = true; @@ -1146,6 +2025,204 @@ export class RemoteBrowser { }); }; + /** + * Subscribe to DOM streaming - simplified version following screenshot pattern + */ + public async subscribeToDOM(): Promise { + if (!this.client) { + logger.warn("DOM streaming requires scraping browser with CDP client"); + return; + } + + try { + // Enable required CDP domains + await this.client.send("DOM.enable"); + await this.client.send("CSS.enable"); + + this.isDOMStreamingActive = true; + logger.info("DOM streaming started successfully"); + + // Initial DOM snapshot + await this.makeAndEmitDOMSnapshot(); + + this.setupScrollEventListener(); + this.setupPageChangeListeners(); + } catch (error) { + logger.error("Failed to start DOM streaming:", error); + this.isDOMStreamingActive = false; + } + } + + /** + * Wait for network requests to become idle + * @private + */ + private async waitForNetworkIdle(timeout: number = 2000): Promise { + const startTime = Date.now(); + + while (this.activeRequests.size > 0 && Date.now() - startTime < timeout) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + if (this.activeRequests.size > 0) { + logger.debug( + `Network idle timeout reached with ${this.activeRequests.size} pending requests` + ); + } + } + + /** + * Stop network monitoring + * @private + */ + private async stopNetworkMonitoring(): Promise { + if (!this.client || !this.isNetworkMonitoringActive) { + return; + } + + try { + await this.client.send("Network.disable"); + this.isNetworkMonitoringActive = false; + this.networkResourceCache.clear(); + this.activeRequests.clear(); + logger.info("Network monitoring stopped"); + } catch (error) { + logger.error("Error stopping network monitoring:", error); + } + } + + /** + * CDP-based DOM snapshot creation using captured network resources + */ + public async makeAndEmitDOMSnapshot(): Promise { + if ( + !this.currentPage || + !this.isDOMStreamingActive + ) { + return; + } + + try { + // Check if page is still valid and not closed + if (this.currentPage.isClosed()) { + logger.debug("Skipping DOM snapshot - page is closed"); + return; + } + + // Wait for network to become idle + await this.waitForNetworkIdle(); + + // Double-check page state after network wait + if (this.currentPage.isClosed()) { + logger.debug("Skipping DOM snapshot - page closed during network wait"); + return; + } + + // Get current scroll position + const currentScrollInfo = await this.currentPage.evaluate(() => ({ + x: window.scrollX, + y: window.scrollY, + maxX: Math.max( + 0, + document.documentElement.scrollWidth - window.innerWidth + ), + maxY: Math.max( + 0, + document.documentElement.scrollHeight - window.innerHeight + ), + documentHeight: document.documentElement.scrollHeight, + })); + + logger.info( + `Creating rrweb snapshot at scroll position: ${currentScrollInfo.y}/${currentScrollInfo.maxY}` + ); + + // Update our tracked scroll position + this.lastScrollPosition = { + x: currentScrollInfo.x, + y: currentScrollInfo.y, + }; + + // Final check before snapshot + if (this.currentPage.isClosed()) { + logger.debug("Skipping DOM snapshot - page closed before snapshot"); + return; + } + + // Capture snapshot using rrweb + const rawSnapshot = await this.currentPage.evaluate(() => { + if (typeof window.rrwebSnapshot === "undefined") { + throw new Error("rrweb-snapshot library not available"); + } + return window.rrwebSnapshot.snapshot(document); + }); + + // Process the snapshot to proxy resources + const processedSnapshot = await this.processRRWebSnapshot(rawSnapshot); + + // Add scroll position information + const enhancedSnapshot = { + ...processedSnapshot, + scrollPosition: currentScrollInfo, + captureTime: Date.now(), + }; + + // Emit the processed snapshot + this.emitRRWebSnapshot(enhancedSnapshot); + } catch (error) { + // Handle navigation context destruction gracefully + if (error instanceof Error && + (error.message.includes("Execution context was destroyed") || + error.message.includes("most likely because of a navigation") || + error.message.includes("Target closed"))) { + logger.debug("DOM snapshot skipped due to page navigation or closure"); + return; // Don't emit error for navigation - this is expected + } + + logger.error("Failed to create rrweb snapshot:", error); + this.socket.emit("dom-mode-error", { + userId: this.userId, + message: "Failed to create rrweb snapshot", + error: error instanceof Error ? error.message : String(error), + timestamp: Date.now(), + }); + } + } + + /** + * Emit DOM snapshot to client - following screenshot pattern + */ + private emitRRWebSnapshot(processedSnapshot: ProcessedSnapshot): void { + this.socket.emit("domcast", { + snapshotData: processedSnapshot, + userId: this.userId, + timestamp: Date.now(), + }); + } + + /** + * Stop DOM streaming - following screencast pattern + */ + private async stopDOM(): Promise { + this.isDOMStreamingActive = false; + + if (this.domUpdateInterval) { + clearInterval(this.domUpdateInterval); + this.domUpdateInterval = null; + } + + if (this.client) { + try { + await this.client.send("DOM.disable"); + await this.client.send("CSS.disable"); + } catch (error) { + logger.warn("Error stopping DOM stream:", error); + } + } + + logger.info("DOM streaming stopped successfully"); + } + /** * Terminates the screencast session and closes the remote browser. * If an interpretation was running it will be stopped. @@ -1154,6 +2231,7 @@ export class RemoteBrowser { public async switchOff(): Promise { try { this.isScreencastActive = false; + this.isDOMStreamingActive = false; await this.interpreter.stopInterpretation(); @@ -1163,6 +2241,8 @@ export class RemoteBrowser { if (this.client) { await this.stopScreencast(); + await this.stopDOM(); + await this.stopNetworkMonitoring(); } if (this.browser) { @@ -1298,6 +2378,8 @@ export class RemoteBrowser { await this.stopScreencast(); this.currentPage = page; + await this.currentPage.addInitScript({ path: './server/src/browser-management/classes/rrweb-bundle.js' }); + await this.setupPageEventListeners(this.currentPage); //await this.currentPage.setViewportSize({ height: 400, width: 900 }) @@ -1330,10 +2412,16 @@ export class RemoteBrowser { await this.currentPage?.close(); this.currentPage = newPage; if (this.currentPage) { + await this.currentPage.addInitScript({ path: './server/src/browser-management/classes/rrweb-bundle.js' }); + await this.setupPageEventListeners(this.currentPage); this.client = await this.currentPage.context().newCDPSession(this.currentPage); - await this.subscribeToScreencast(); + if (this.renderingMode === "dom") { + await this.subscribeToDOM(); + } else { + await this.subscribeToScreencast(); + } } else { logger.log('error', 'Could not get a new page, returned undefined'); } diff --git a/server/src/browser-management/classes/bundle-rrweb.js b/server/src/browser-management/classes/bundle-rrweb.js new file mode 100644 index 00000000..c2fe8b8a --- /dev/null +++ b/server/src/browser-management/classes/bundle-rrweb.js @@ -0,0 +1,10 @@ +const esbuild = require('esbuild'); + +esbuild.build({ + entryPoints: ['rrweb-entry.js'], + bundle: true, + minify: true, + outfile: 'rrweb-bundle.js', + format: 'iife', // so that rrwebSnapshot is available on window + globalName: 'rrwebSnapshotBundle' +}).catch(() => process.exit(1)); diff --git a/server/src/browser-management/classes/rrweb-bundle.js b/server/src/browser-management/classes/rrweb-bundle.js new file mode 100644 index 00000000..58532e0a --- /dev/null +++ b/server/src/browser-management/classes/rrweb-bundle.js @@ -0,0 +1 @@ +"use strict";var rrwebSnapshotBundle=(()=>{var g;(function(e){e[e.Document=0]="Document",e[e.DocumentType=1]="DocumentType",e[e.Element=2]="Element",e[e.Text=3]="Text",e[e.CDATA=4]="CDATA",e[e.Comment=5]="Comment"})(g||(g={}));function ue(e){return e.nodeType===e.ELEMENT_NODE}function fe(e){var t=e?.host;return t?.shadowRoot===e}function Q(e){return Object.prototype.toString.call(e)==="[object ShadowRoot]"}function de(e){return e.includes(" background-clip: text;")&&!e.includes(" -webkit-background-clip: text;")&&(e=e.replace(" background-clip: text;"," -webkit-background-clip: text; background-clip: text;")),e}function Z(e){try{var t=e.rules||e.cssRules;return t?de(Array.from(t).map(me).join("")):null}catch{return null}}function me(e){var t=e.cssText;if(pe(e))try{t=Z(e.styleSheet)||t}catch{}return t}function pe(e){return"styleSheet"in e}var he=function(){function e(){this.idNodeMap=new Map,this.nodeMetaMap=new WeakMap}return e.prototype.getId=function(t){var r;if(!t)return-1;var a=(r=this.getMeta(t))===null||r===void 0?void 0:r.id;return a??-1},e.prototype.getNode=function(t){return this.idNodeMap.get(t)||null},e.prototype.getIds=function(){return Array.from(this.idNodeMap.keys())},e.prototype.getMeta=function(t){return this.nodeMetaMap.get(t)||null},e.prototype.removeNodeFromMap=function(t){var r=this,a=this.getId(t);this.idNodeMap.delete(a),t.childNodes&&t.childNodes.forEach(function(i){return r.removeNodeFromMap(i)})},e.prototype.has=function(t){return this.idNodeMap.has(t)},e.prototype.hasNode=function(t){return this.nodeMetaMap.has(t)},e.prototype.add=function(t,r){var a=r.id;this.idNodeMap.set(a,t),this.nodeMetaMap.set(t,r)},e.prototype.replace=function(t,r){var a=this.getNode(t);if(a){var i=this.nodeMetaMap.get(a);i&&this.nodeMetaMap.set(r,i)}this.idNodeMap.set(t,r)},e.prototype.reset=function(){this.idNodeMap=new Map,this.nodeMetaMap=new WeakMap},e}();function ve(e){var t=e.maskInputOptions,r=e.tagName,a=e.type,i=e.value,n=e.maskInputFn,l=i||"";return(t[r.toLowerCase()]||t[a])&&(n?l=n(l):l="*".repeat(l.length)),l}var oe="__rrweb_original__";function ge(e){var t=e.getContext("2d");if(!t)return!0;for(var r=50,a=0;a-1?t=e.split("/").slice(0,3).join("/"):t=e.split("/")[0],t=t.split("?")[0],t}var H,le,Te=/url\((?:(')([^']*)'|(")(.*?)"|([^)]*))\)/gm,Ie=/^(?!www\.|(?:http|ftp)s?:\/\/|[A-Za-z]:\\|\/\/|#).*/,Ee=/^(data:)([^,]*),(.*)/i;function $(e,t){return(e||"").replace(Te,function(r,a,i,n,l,u){var o=i||l||u,f=a||n||"";if(!o)return r;if(!Ie.test(o)||Ee.test(o))return"url(".concat(f).concat(o).concat(f,")");if(o[0]==="/")return"url(".concat(f).concat(we(t)+o).concat(f,")");var c=t.split("/"),p=o.split("/");c.pop();for(var C=0,k=p;C=t.length);){var n=a(Le);if(n.slice(-1)===",")n=j(e,n.substring(0,n.length-1)),i.push(n);else{var l="";n=j(e,n);for(var u=!1;;){var o=t.charAt(r);if(o===""){i.push((n+l).trim());break}else if(u)o===")"&&(u=!1);else if(o===","){r+=1,i.push((n+l).trim());break}else o==="("&&(u=!0);l+=o,r+=1}}}return i.join(", ")}function j(e,t){if(!t||t.trim()==="")return t;var r=e.createElement("a");return r.href=t,r.href}function Me(e){return!!(e.tagName==="svg"||e.ownerSVGElement)}function ee(){var e=document.createElement("a");return e.href="",e.href}function Oe(e,t,r,a){return r==="src"||r==="href"&&a&&!(t==="use"&&a[0]==="#")||r==="xlink:href"&&a&&a[0]!=="#"||r==="background"&&a&&(t==="table"||t==="td"||t==="th")?j(e,a):r==="srcset"&&a?Ne(e,a):r==="style"&&a?$(a,ee()):t==="object"&&r==="data"&&a?j(e,a):a}function Re(e,t,r){if(typeof t=="string"){if(e.classList.contains(t))return!0}else for(var a=e.classList.length;a--;){var i=e.classList[a];if(t.test(i))return!0}return r?e.matches(r):!1}function J(e,t,r){if(!e)return!1;if(e.nodeType!==e.ELEMENT_NODE)return r?J(e.parentNode,t,r):!1;for(var a=e.classList.length;a--;){var i=e.classList[a];if(t.test(i))return!0}return r?J(e.parentNode,t,r):!1}function Ae(e,t,r){var a=e.nodeType===e.ELEMENT_NODE?e:e.parentElement;if(a===null)return!1;if(typeof t=="string"){if(a.classList.contains(t)||a.closest(".".concat(t)))return!0}else if(J(a,t,!0))return!0;return!!(r&&(a.matches(r)||a.closest(r)))}function De(e,t,r){var a=e.contentWindow;if(a){var i=!1,n;try{n=a.document.readyState}catch{return}if(n!=="complete"){var l=setTimeout(function(){i||(t(),i=!0)},r);e.addEventListener("load",function(){clearTimeout(l),i=!0,t()});return}var u="about:blank";if(a.location.href!==u||e.src===u||e.src==="")return setTimeout(t,0),e.addEventListener("load",t);e.addEventListener("load",t)}}function Fe(e,t,r){var a=!1,i;try{i=e.sheet}catch{return}if(!i){var n=setTimeout(function(){a||(t(),a=!0)},r);e.addEventListener("load",function(){clearTimeout(n),a=!0,t()})}}function Ue(e,t){var r=t.doc,a=t.mirror,i=t.blockClass,n=t.blockSelector,l=t.maskTextClass,u=t.maskTextSelector,o=t.inlineStylesheet,f=t.maskInputOptions,c=f===void 0?{}:f,p=t.maskTextFn,C=t.maskInputFn,k=t.dataURLOptions,S=k===void 0?{}:k,T=t.inlineImages,I=t.recordCanvas,E=t.keepIframeSrcFn,m=t.newlyAddedElement,s=m===void 0?!1:m,y=We(r,a);switch(e.nodeType){case e.DOCUMENT_NODE:return e.compatMode!=="CSS1Compat"?{type:g.Document,childNodes:[],compatMode:e.compatMode}:{type:g.Document,childNodes:[]};case e.DOCUMENT_TYPE_NODE:return{type:g.DocumentType,name:e.name,publicId:e.publicId,systemId:e.systemId,rootId:y};case e.ELEMENT_NODE:return _e(e,{doc:r,blockClass:i,blockSelector:n,inlineStylesheet:o,maskInputOptions:c,maskInputFn:C,dataURLOptions:S,inlineImages:T,recordCanvas:I,keepIframeSrcFn:E,newlyAddedElement:s,rootId:y});case e.TEXT_NODE:return Pe(e,{maskTextClass:l,maskTextSelector:u,maskTextFn:p,rootId:y});case e.CDATA_SECTION_NODE:return{type:g.CDATA,textContent:"",rootId:y};case e.COMMENT_NODE:return{type:g.Comment,textContent:e.textContent||"",rootId:y};default:return!1}}function We(e,t){if(t.hasNode(e)){var r=t.getId(e);return r===1?void 0:r}}function Pe(e,t){var r,a=t.maskTextClass,i=t.maskTextSelector,n=t.maskTextFn,l=t.rootId,u=e.parentNode&&e.parentNode.tagName,o=e.textContent,f=u==="STYLE"?!0:void 0,c=u==="SCRIPT"?!0:void 0;if(f&&o){try{e.nextSibling||e.previousSibling||!((r=e.parentNode.sheet)===null||r===void 0)&&r.cssRules&&(o=Ce(e.parentNode.sheet))}catch(p){console.warn("Cannot get CSS styles from text's parentNode. Error: ".concat(p),e)}o=$(o,ee())}return c&&(o=""),!f&&!c&&o&&Ae(e,a,i)&&(o=n?n(o):o.replace(/[\S]/g,"*")),{type:g.Text,textContent:o||"",isStyle:f,rootId:l}}function _e(e,t){for(var r=t.doc,a=t.blockClass,i=t.blockSelector,n=t.inlineStylesheet,l=t.maskInputOptions,u=l===void 0?{}:l,o=t.maskInputFn,f=t.dataURLOptions,c=f===void 0?{}:f,p=t.inlineImages,C=t.recordCanvas,k=t.keepIframeSrcFn,S=t.newlyAddedElement,T=S===void 0?!1:S,I=t.rootId,E=Re(e,a,i),m=be(e),s={},y=e.attributes.length,A=0;A { +export const initializeRemoteBrowserForRecording = (userId: string, mode: string = "dom"): string => { const id = getActiveBrowserIdByState(userId, "recording") || uuid(); createSocketConnection( io.of(id), @@ -37,7 +37,15 @@ export const initializeRemoteBrowserForRecording = (userId: string): string => { browserSession.interpreter.subscribeToPausing(); await browserSession.initialize(userId); await browserSession.registerEditorEvents(); - await browserSession.subscribeToScreencast(); + + if (mode === "dom") { + await browserSession.subscribeToDOM(); + logger.info('DOM streaming started for scraping browser in recording mode'); + } else { + await browserSession.subscribeToScreencast(); + logger.info('Screenshot streaming started for local browser in recording mode'); + } + browserPool.addRemoteBrowser(id, browserSession, userId, false, "recording"); } socket.emit('loaded'); diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 6b5016f3..b0be33e6 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -11,6 +11,8 @@ import { WorkflowGenerator } from "../workflow-management/classes/Generator"; import { Page } from "playwright"; import { throttle } from "../../../src/helpers/inputHelpers"; import { CustomActions } from "../../../src/shared/types"; +import { WhereWhatPair } from "maxun-core"; +import { RemoteBrowser } from './classes/RemoteBrowser'; /** * A wrapper function for handling user input. @@ -27,7 +29,7 @@ import { CustomActions } from "../../../src/shared/types"; */ const handleWrapper = async ( handleCallback: ( - generator: WorkflowGenerator, + activeBrowser: RemoteBrowser, page: Page, args?: any ) => Promise, @@ -44,9 +46,9 @@ const handleWrapper = async ( const currentPage = activeBrowser?.getCurrentPage(); if (currentPage && activeBrowser) { if (args) { - await handleCallback(activeBrowser.generator, currentPage, args); + await handleCallback(activeBrowser, currentPage, args); } else { - await handleCallback(activeBrowser.generator, currentPage); + await handleCallback(activeBrowser, currentPage); } } else { logger.log('warn', `No active page for browser ${id}`); @@ -85,8 +87,19 @@ const onGenerateAction = async (customActionEventData: CustomActionEventData, us * @category BrowserManagement */ const handleGenerateAction = - async (generator: WorkflowGenerator, page: Page, { action, settings }: CustomActionEventData) => { - await generator.customAction(action, settings, page); + async (activeBrowser: RemoteBrowser, page: Page, { action, settings }: CustomActionEventData) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring generate action event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await generator.customAction(action, settings, page); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling generate action event: ${message}`); + } } /** @@ -104,40 +117,51 @@ const onMousedown = async (coordinates: Coordinates, userId: string) => { * A mousedown event handler. * Reproduces the click on the remote browser instance * and generates pair data for the recorded workflow. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param x - the x coordinate of the mousedown event * @param y - the y coordinate of the mousedown event * @category BrowserManagement */ -const handleMousedown = async (generator: WorkflowGenerator, page: Page, { x, y }: Coordinates) => { +const handleMousedown = async (activeBrowser: RemoteBrowser, page: Page, { x, y }: Coordinates) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring mousedown event: page is closed`); + return; + } + + const generator = activeBrowser.generator; await generator.onClick({ x, y }, page); const previousUrl = page.url(); const tabsBeforeClick = page.context().pages().length; await page.mouse.click(x, y); // try if the click caused a navigation to a new url try { - await page.waitForNavigation({ timeout: 2000 }); - const currentUrl = page.url(); - if (currentUrl !== previousUrl) { - generator.notifyUrlChange(currentUrl); - } + await page.waitForNavigation({ timeout: 2000 }); + const currentUrl = page.url(); + if (currentUrl !== previousUrl) { + generator.notifyUrlChange(currentUrl); + } } catch (e) { - const { message } = e as Error; + const { message } = e as Error; } //ignore possible timeouts // check if any new page was opened by the click const tabsAfterClick = page.context().pages().length; const numOfNewPages = tabsAfterClick - tabsBeforeClick; if (numOfNewPages > 0) { - for (let i = 1; i <= numOfNewPages; i++) { - const newPage = page.context().pages()[tabsAfterClick - i]; - if (newPage) { - generator.notifyOnNewTab(newPage, tabsAfterClick - i); - } + for (let i = 1; i <= numOfNewPages; i++) { + const newPage = page.context().pages()[tabsAfterClick - i]; + if (newPage) { + generator.notifyOnNewTab(newPage, tabsAfterClick - i); } + } } - logger.log('debug', `Clicked on position x:${x}, y:${y}`); + logger.log("debug", `Clicked on position x:${x}, y:${y}`); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling mousedown event: ${message}`); + } }; /** @@ -156,15 +180,16 @@ const onWheel = async (scrollDeltas: ScrollDeltas, userId: string) => { * Reproduces the wheel event on the remote browser instance. * Scroll is not generated for the workflow pair. This is because * Playwright scrolls elements into focus on any action. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param deltaX - the delta x of the wheel event * @param deltaY - the delta y of the wheel event * @category BrowserManagement */ -const handleWheel = async (generator: WorkflowGenerator, page: Page, { deltaX, deltaY }: ScrollDeltas) => { +const handleWheel = async (activeBrowser: RemoteBrowser, page: Page, { deltaX, deltaY }: ScrollDeltas) => { try { if (page.isClosed()) { + logger.log("debug", `Ignoring wheel event: page is closed`); return; } @@ -194,28 +219,30 @@ const onMousemove = async (coordinates: Coordinates, userId: string) => { * Reproduces the mousemove event on the remote browser instance * and generates data for the client's highlighter. * Mousemove is also not reflected in the workflow. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param x - the x coordinate of the mousemove event * @param y - the y coordinate of the mousemove event * @category BrowserManagement */ -const handleMousemove = async (generator: WorkflowGenerator, page: Page, { x, y }: Coordinates) => { +const handleMousemove = async (activeBrowser: RemoteBrowser, page: Page, { x, y }: Coordinates) => { try { if (page.isClosed()) { - logger.log('debug', `Ignoring mousemove event: page is closed`); + logger.log("debug", `Ignoring mousemove event: page is closed`); return; } + + const generator = activeBrowser.generator; await page.mouse.move(x, y); throttle(async () => { if (!page.isClosed()) { await generator.generateDataForHighlighter(page, { x, y }); } }, 100)(); - logger.log('debug', `Moved over position x:${x}, y:${y}`); + logger.log("debug", `Moved over position x:${x}, y:${y}`); } catch (e) { const { message } = e as Error; - logger.log('error', message); + logger.log("error", message); } } @@ -234,28 +261,50 @@ const onKeydown = async (keyboardInput: KeyboardInput, userId: string) => { * A keydown event handler. * Reproduces the keydown event on the remote browser instance * and generates the workflow pair data. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param key - the pressed key * @param coordinates - the coordinates, where the keydown event happened * @category BrowserManagement */ -const handleKeydown = async (generator: WorkflowGenerator, page: Page, { key, coordinates }: KeyboardInput) => { - await page.keyboard.down(key); - await generator.onKeyboardInput(key, coordinates, page); - logger.log('debug', `Key ${key} pressed`); +const handleKeydown = async (activeBrowser: RemoteBrowser, page: Page, { key, coordinates }: KeyboardInput) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring keydown event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await page.keyboard.down(key); + await generator.onKeyboardInput(key, coordinates, page); + logger.log("debug", `Key ${key} pressed`); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling keydown event: ${message}`); + } }; /** * Handles the date selection event. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param data - the data of the date selection event {@link DatePickerEventData} * @category BrowserManagement */ -const handleDateSelection = async (generator: WorkflowGenerator, page: Page, data: DatePickerEventData) => { - await generator.onDateSelection(page, data); - logger.log('debug', `Date ${data.value} selected`); +const handleDateSelection = async (activeBrowser: RemoteBrowser, page: Page, data: DatePickerEventData) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring date selection event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await generator.onDateSelection(page, data); + logger.log("debug", `Date ${data.value} selected`); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling date selection event: ${message}`); + } } /** @@ -271,14 +320,25 @@ const onDateSelection = async (data: DatePickerEventData, userId: string) => { /** * Handles the dropdown selection event. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param data - the data of the dropdown selection event * @category BrowserManagement */ -const handleDropdownSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { - await generator.onDropdownSelection(page, data); - logger.log('debug', `Dropdown value ${data.value} selected`); +const handleDropdownSelection = async (activeBrowser: RemoteBrowser, page: Page, data: { selector: string, value: string }) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring dropdown selection event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await generator.onDropdownSelection(page, data); + logger.log("debug", `Dropdown value ${data.value} selected`); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling dropdown selection event: ${message}`); + } } /** @@ -294,14 +354,25 @@ const onDropdownSelection = async (data: { selector: string, value: string }, us /** * Handles the time selection event. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param data - the data of the time selection event * @category BrowserManagement */ -const handleTimeSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { - await generator.onTimeSelection(page, data); - logger.log('debug', `Time value ${data.value} selected`); +const handleTimeSelection = async (activeBrowser: RemoteBrowser, page: Page, data: { selector: string, value: string }) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring time selection event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await generator.onTimeSelection(page, data); + logger.log("debug", `Time value ${data.value} selected`); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling time selection event: ${message}`); + } } /** @@ -317,14 +388,31 @@ const onTimeSelection = async (data: { selector: string, value: string }, userId /** * Handles the datetime-local selection event. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param data - the data of the datetime-local selection event * @category BrowserManagement */ -const handleDateTimeLocalSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { - await generator.onDateTimeLocalSelection(page, data); - logger.log('debug', `DateTime Local value ${data.value} selected`); +const handleDateTimeLocalSelection = async (activeBrowser: RemoteBrowser, page: Page, data: { selector: string, value: string }) => { + try { + if (page.isClosed()) { + logger.log( + "debug", + `Ignoring datetime-local selection event: page is closed` + ); + return; + } + + const generator = activeBrowser.generator; + await generator.onDateTimeLocalSelection(page, data); + logger.log("debug", `DateTime Local value ${data.value} selected`); + } catch (e) { + const { message } = e as Error; + logger.log( + "warn", + `Error handling datetime-local selection event: ${message}` + ); + } } /** @@ -353,14 +441,24 @@ const onKeyup = async (keyboardInput: KeyboardInput, userId: string) => { * A keyup event handler. * Reproduces the keyup event on the remote browser instance. * Does not generate any data - keyup is not reflected in the workflow. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param key - the released key * @category BrowserManagement */ -const handleKeyup = async (generator: WorkflowGenerator, page: Page, key: string) => { - await page.keyboard.up(key); - logger.log('debug', `Key ${key} unpressed`); +const handleKeyup = async (activeBrowser: RemoteBrowser, page: Page, key: string) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring keyup event: page is closed`); + return; + } + + await page.keyboard.up(key); + logger.log("debug", `Key ${key} unpressed`); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling keyup event: ${message}`); + } }; /** @@ -377,23 +475,35 @@ const onChangeUrl = async (url: string, userId: string) => { /** * An url change event handler. * Navigates the page to the given url and generates data for the workflow. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @param url - the new url of the page * @category BrowserManagement */ -const handleChangeUrl = async (generator: WorkflowGenerator, page: Page, url: string) => { - if (url) { - await generator.onChangeUrl(url, page); - try { - await page.goto(url, { waitUntil: 'networkidle', timeout: 100000 }); - logger.log('debug', `Went to ${url}`); - } catch (e) { - const { message } = e as Error; - logger.log('error', message); +const handleChangeUrl = async (activeBrowser: RemoteBrowser, page: Page, url: string) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring change url event: page is closed`); + return; } - } else { - logger.log('warn', `No url provided`); + + if (url) { + const generator = activeBrowser.generator; + await generator.onChangeUrl(url, page); + + try { + await page.goto(url, { waitUntil: "networkidle", timeout: 100000 }); + logger.log("debug", `Went to ${url}`); + } catch (e) { + const { message } = e as Error; + logger.log("error", message); + } + } else { + logger.log("warn", `No url provided`); + } + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling change url event: ${message}`); } }; @@ -410,13 +520,23 @@ const onRefresh = async (userId: string) => { /** * A refresh event handler. * Refreshes the page. This is not reflected in the workflow. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @category BrowserManagement */ -const handleRefresh = async (generator: WorkflowGenerator, page: Page) => { - await page.reload(); - logger.log('debug', `Page refreshed.`); +const handleRefresh = async (activeBrowser: RemoteBrowser, page: Page) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring refresh event: page is closed`); + return; + } + + await page.reload(); + logger.log("debug", `Page refreshed.`); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling refresh event: ${message}`); + } }; /** @@ -432,14 +552,25 @@ const onGoBack = async (userId: string) => { /** * A go back event handler. * Navigates the page back and generates data for the workflow. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @category BrowserManagement */ -const handleGoBack = async (generator: WorkflowGenerator, page: Page) => { - await page.goBack({ waitUntil: 'commit' }); - generator.onGoBack(page.url()); - logger.log('debug', 'Page went back') +const handleGoBack = async (activeBrowser: RemoteBrowser, page: Page) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring go back event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await page.goBack({ waitUntil: "commit" }); + generator.onGoBack(page.url()); + logger.log("debug", "Page went back"); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling go back event: ${message}`); + } }; /** @@ -455,14 +586,207 @@ const onGoForward = async (userId: string) => { /** * A go forward event handler. * Navigates the page forward and generates data for the workflow. - * @param generator - the workflow generator {@link Generator} + * @param activeBrowser - the active remote browser {@link RemoteBrowser} * @param page - the active page of the remote browser * @category BrowserManagement */ -const handleGoForward = async (generator: WorkflowGenerator, page: Page) => { - await page.goForward({ waitUntil: 'commit' }); - generator.onGoForward(page.url()); - logger.log('debug', 'Page went forward'); +const handleGoForward = async (activeBrowser: RemoteBrowser, page: Page) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring go forward event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await page.goForward({ waitUntil: "commit" }); + generator.onGoForward(page.url()); + logger.log("debug", "Page went forward"); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling go forward event: ${message}`); + } +}; + +/** + * Handles the click action event. + * @param activeBrowser - the active remote browser {@link RemoteBrowser} + * @param page - the active page of the remote browser + * @param data - the data of the click action event + * @category BrowserManagement + */ +const handleClickAction = async ( + activeBrowser: RemoteBrowser, + page: Page, + data: { + selector: string; + url: string; + userId: string; + elementInfo?: any; + coordinates?: { x: number; y: number }; + isSPA?: boolean; + } +) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring click action event: page is closed`); + return; + } + + const { selector, url, elementInfo, coordinates, isSPA = false } = data; + const currentUrl = page.url(); + + await page.click(selector); + + const generator = activeBrowser.generator; + await generator.onDOMClickAction(page, data); + + logger.log("debug", `Click action processed: ${selector}`); + + if (isSPA) { + logger.log("debug", `SPA interaction detected for selector: ${selector}`); + + await new Promise((resolve) => setTimeout(resolve, 1500)); + } else { + const newUrl = page.url(); + const hasNavigated = newUrl !== currentUrl && !newUrl.endsWith("/#"); + + if (hasNavigated) { + logger.log("debug", `Navigation detected: ${currentUrl} -> ${newUrl}`); + + await generator.onDOMNavigation(page, { + url: newUrl, + currentUrl: currentUrl, + userId: data.userId, + }); + } + } + + await new Promise((resolve) => setTimeout(resolve, 2000)); + await activeBrowser.makeAndEmitDOMSnapshot(); + } catch (e) { + const { message } = e as Error; + logger.log( + "warn", + `Error handling enhanced click action event: ${message}` + ); + } +}; + +/** + * A wrapper function for handling the click action event. + * @param socket The socket connection + * @param data - the data of the click action event + * @category HelperFunctions + */ +const onDOMClickAction = async ( + data: { + selector: string; + url: string; + userId: string; + elementInfo?: any; + coordinates?: { x: number; y: number }; + }, + userId: string +) => { + logger.log("debug", "Handling click action event emitted from client"); + await handleWrapper(handleClickAction, userId, data); +}; + +/** + * Handles the keyboard action event. + * @param activeBrowser - the active remote browser {@link RemoteBrowser} + * @param page - the active page of the remote browser + * @param data - the data of the keyboard action event + * @category BrowserManagement + */ +const handleKeyboardAction = async ( + activeBrowser: RemoteBrowser, + page: Page, + data: { + selector: string; + key: string; + url: string; + userId: string; + inputType?: string; + } +) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring keyboard action event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await generator.onDOMKeyboardAction(page, data); + logger.log( + "debug", + `Keyboard action processed: ${data.key} on ${data.selector}` + ); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling keyboard action event: ${message}`); + } +}; + +/** + * A wrapper function for handling the keyboard action event. + * @param socket The socket connection + * @param data - the data of the keyboard action event + * @category HelperFunctions + */ +const onDOMKeyboardAction = async ( + data: { + selector: string; + key: string; + url: string; + userId: string; + inputType?: string; + }, + userId: string +) => { + logger.log("debug", "Handling keyboard action event emitted from client"); + await handleWrapper(handleKeyboardAction, userId, data); +}; + +/** + * Handles the workflow pair event. + * @param activeBrowser - the active remote browser {@link RemoteBrowser} + * @param page - the active page of the remote browser + * @param data - the data of the workflow pair event + * @category BrowserManagement + */ +const handleWorkflowPair = async ( + activeBrowser: RemoteBrowser, + page: Page, + data: { pair: WhereWhatPair; userId: string } +) => { + try { + if (page.isClosed()) { + logger.log("debug", `Ignoring workflow pair event: page is closed`); + return; + } + + const generator = activeBrowser.generator; + await generator.onDOMWorkflowPair(page, data); + logger.log("debug", `Workflow pair processed from frontend`); + } catch (e) { + const { message } = e as Error; + logger.log("warn", `Error handling workflow pair event: ${message}`); + } +}; + +/** + * A wrapper function for handling the workflow pair event. + * @param socket The socket connection + * @param data - the data of the workflow pair event + * @category HelperFunctions + */ +const onDOMWorkflowPair = async ( + data: { pair: WhereWhatPair; userId: string }, + userId: string +) => { + logger.log("debug", "Handling workflow pair event emitted from client"); + await handleWrapper(handleWorkflowPair, userId, data); }; /** @@ -493,6 +817,10 @@ const registerInputHandlers = (socket: Socket, userId: string) => { socket.on("input:time", (data) => onTimeSelection(data, userId)); socket.on("input:datetime-local", (data) => onDateTimeLocalSelection(data, userId)); socket.on("action", (data) => onGenerateAction(data, userId)); + + socket.on("dom:click", (data) => onDOMClickAction(data, userId)); + socket.on("dom:keypress", (data) => onDOMKeyboardAction(data, userId)); + socket.on("dom:addpair", (data) => onDOMWorkflowPair(data, userId)); }; export default registerInputHandlers; diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 27123e22..c229b183 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -82,6 +82,7 @@ export class WorkflowGenerator { this.poolId = poolId; this.registerEventHandlers(socket); this.initializeSocketListeners(); + this.initializeDOMListeners(); } /** @@ -92,6 +93,8 @@ export class WorkflowGenerator { workflow: [], }; + private isDOMMode: boolean = false; + /** * Metadata of the currently recorded workflow. * @private @@ -134,6 +137,18 @@ export class WorkflowGenerator { }) } + private initializeDOMListeners() { + this.socket.on('dom-mode-enabled', () => { + this.isDOMMode = true; + logger.log('debug', 'Generator: DOM mode enabled'); + }); + + this.socket.on('screenshot-mode-enabled', () => { + this.isDOMMode = false; + logger.log('debug', 'Generator: Screenshot mode enabled'); + }); + } + /** * Registers the event handlers for all generator-related events on the socket. * @param socket The socket used to communicate with the client. @@ -348,6 +363,96 @@ export class WorkflowGenerator { await this.addPairToWorkflowAndNotifyClient(pair, page); }; + // Handles click events on the DOM, generating a pair for the click action + public onDOMClickAction = async (page: Page, data: { + selector: string, + url: string, + userId: string, + elementInfo?: any, + coordinates?: { x: number, y: number } + }) => { + const { selector, url, elementInfo, coordinates } = data; + + const pair: WhereWhatPair = { + where: { + url: this.getBestUrl(url), + selectors: [selector] + }, + what: [{ + action: 'click', + args: [selector], + }], + }; + + // Handle special input elements with cursor positioning + if (elementInfo && coordinates && + (elementInfo.tagName === 'INPUT' || elementInfo.tagName === 'TEXTAREA')) { + pair.what[0] = { + action: 'click', + args: [selector, { position: coordinates }, { cursorIndex: 0 }], + }; + } + + this.generatedData.lastUsedSelector = selector; + this.generatedData.lastAction = 'click'; + + await this.addPairToWorkflowAndNotifyClient(pair, page); + }; + + // Handles keyboard actions on the DOM, generating a pair for the key press action + public onDOMKeyboardAction = async (page: Page, data: { + selector: string, + key: string, + url: string, + userId: string, + inputType?: string + }) => { + const { selector, key, url, inputType } = data; + + const pair: WhereWhatPair = { + where: { + url: this.getBestUrl(url), + selectors: [selector] + }, + what: [{ + action: 'press', + args: [selector, encrypt(key), inputType || 'text'], + }], + }; + + this.generatedData.lastUsedSelector = selector; + this.generatedData.lastAction = 'press'; + + await this.addPairToWorkflowAndNotifyClient(pair, page); + }; + + // Handles navigation events on the DOM, generating a pair for the navigation action + public onDOMNavigation = async (page: Page, data: { + url: string, + currentUrl: string, + userId: string + }) => { + const { url, currentUrl } = data; + + const pair: WhereWhatPair = { + where: { url: this.getBestUrl(currentUrl) }, + what: [{ + action: 'goto', + args: [url], + }], + }; + + this.generatedData.lastUsedSelector = ''; + await this.addPairToWorkflowAndNotifyClient(pair, page); + }; + + // Handles workflow pair events on the DOM + public onDOMWorkflowPair = async (page: Page, data: { pair: WhereWhatPair, userId: string }) => { + const { pair } = data; + + await this.addPairToWorkflowAndNotifyClient(pair, page); + }; + /** * Generates a pair for the click event. * @param coordinates The coordinates of the click event. @@ -357,6 +462,7 @@ export class WorkflowGenerator { public onClick = async (coordinates: Coordinates, page: Page) => { let where: WhereWhatPair["where"] = { url: this.getBestUrl(page.url()) }; const selector = await this.generateSelector(page, coordinates, ActionType.Click); + console.log("COOORDINATES: ", coordinates); logger.log('debug', `Element's selector: ${selector}`); const elementInfo = await getElementInformation(page, coordinates, '', false); @@ -708,6 +814,7 @@ export class WorkflowGenerator { this.socket = socket; this.registerEventHandlers(socket); this.initializeSocketListeners(); + this.initializeDOMListeners(); }; /** diff --git a/src/components/browser/BrowserWindow.tsx b/src/components/browser/BrowserWindow.tsx index a0065e7b..5aec1e61 100644 --- a/src/components/browser/BrowserWindow.tsx +++ b/src/components/browser/BrowserWindow.tsx @@ -11,6 +11,12 @@ import { useTranslation } from 'react-i18next'; import { AuthContext } from '../../context/auth'; import { coordinateMapper } from '../../helpers/coordinateMapper'; import { useBrowserDimensionsStore } from '../../context/browserDimensions'; +import { clientSelectorGenerator } from "../../helpers/clientSelectorGenerator"; +import DatePicker from "../pickers/DatePicker"; +import Dropdown from "../pickers/Dropdown"; +import TimePicker from "../pickers/TimePicker"; +import DateTimeLocalPicker from "../pickers/DateTimeLocalPicker"; +import { DOMBrowserRenderer } from '../recorder/DOMBrowserRenderer'; interface ElementInfo { tagName: string; @@ -23,6 +29,7 @@ interface ElementInfo { attributes?: Record; innerHTML?: string; outerHTML?: string; + isDOMMode?: boolean; } interface AttributeOption { @@ -41,6 +48,73 @@ interface ViewportInfo { height: number; } +interface RRWebSnapshot { + type: number; + childNodes?: RRWebSnapshot[]; + tagName?: string; + attributes?: Record; + textContent: string; + id: number; + [key: string]: any; +} + +interface ProcessedSnapshot { + snapshot: RRWebSnapshot; + resources: { + stylesheets: Array<{ + href: string; + content: string; + media?: string; + }>; + images: Array<{ + src: string; + dataUrl: string; + alt?: string; + }>; + fonts: Array<{ + url: string; + dataUrl: string; + format?: string; + }>; + scripts: Array<{ + src: string; + content: string; + type?: string; + }>; + media: Array<{ + src: string; + dataUrl: string; + type: string; + }>; + }; + baseUrl: string; + viewport: { width: number; height: number }; + timestamp: number; + processingStats: { + totalReplacements: number; + discoveredResources: { + images: number; + stylesheets: number; + scripts: number; + fonts: number; + media: number; + }; + cachedResources: { + stylesheets: number; + images: number; + fonts: number; + scripts: number; + media: number; + }; + totalCacheSize: number; + }; +} + +interface RRWebDOMCastData { + snapshotData: ProcessedSnapshot; + userId: string; + timestamp: number; +} const getAttributeOptions = (tagName: string, elementInfo: ElementInfo | null): AttributeOption[] => { if (!elementInfo) return []; @@ -79,6 +153,9 @@ export const BrowserWindow = () => { const [selectedElement, setSelectedElement] = useState<{ selector: string, info: ElementInfo | null } | null>(null); const [currentListId, setCurrentListId] = useState(null); const [viewportInfo, setViewportInfo] = useState({ width: browserWidth, height: browserHeight }); + const [isDOMMode, setIsDOMMode] = useState(false); + const [currentSnapshot, setCurrentSnapshot] = useState(null); + const [isLoading, setIsLoading] = useState(false); const [listSelector, setListSelector] = useState(null); const [fields, setFields] = useState>({}); @@ -94,11 +171,142 @@ export const BrowserWindow = () => { const { state } = useContext(AuthContext); const { user } = state; + const [datePickerInfo, setDatePickerInfo] = useState<{ + coordinates: { x: number; y: number }; + selector: string; + } | null>(null); + + const [dropdownInfo, setDropdownInfo] = useState<{ + coordinates: { x: number; y: number }; + selector: string; + options: Array<{ + value: string; + text: string; + disabled: boolean; + selected: boolean; + }>; + } | null>(null); + + const [timePickerInfo, setTimePickerInfo] = useState<{ + coordinates: { x: number; y: number }; + selector: string; + } | null>(null); + + const [dateTimeLocalInfo, setDateTimeLocalInfo] = useState<{ + coordinates: { x: number; y: number }; + selector: string; + } | null>(null); + const dimensions = { width: browserWidth, height: browserHeight }; + const handleShowDatePicker = useCallback( + (info: { coordinates: { x: number; y: number }; selector: string }) => { + setDatePickerInfo(info); + }, + [] + ); + + const handleShowDropdown = useCallback( + (info: { + coordinates: { x: number; y: number }; + selector: string; + options: Array<{ + value: string; + text: string; + disabled: boolean; + selected: boolean; + }>; + }) => { + setDropdownInfo(info); + }, + [] + ); + + const handleShowTimePicker = useCallback( + (info: { coordinates: { x: number; y: number }; selector: string }) => { + setTimePickerInfo(info); + }, + [] + ); + + const handleShowDateTimePicker = useCallback( + (info: { coordinates: { x: number; y: number }; selector: string }) => { + setDateTimeLocalInfo(info); + }, + [] + ); + + const rrwebSnapshotHandler = useCallback( + (data: RRWebDOMCastData) => { + if (!data.userId || data.userId === user?.id) { + if (data.snapshotData && data.snapshotData.snapshot) { + setCurrentSnapshot(data.snapshotData); + setIsDOMMode(true); + socket?.emit("dom-mode-enabled"); + + setIsLoading(false); + } else { + setIsLoading(false); + } + } + }, + [user?.id, socket] + ); + + const domModeHandler = useCallback( + (data: any) => { + if (!data.userId || data.userId === user?.id) { + setIsDOMMode(true); + socket?.emit("dom-mode-enabled"); + setIsLoading(false); + } + }, + [user?.id, socket] + ); + + const screenshotModeHandler = useCallback( + (data: any) => { + if (!data.userId || data.userId === user?.id) { + setIsDOMMode(false); + socket?.emit("screenshot-mode-enabled"); + setCurrentSnapshot(null); + setIsLoading(false); + } + }, + [user?.id] + ); + + const domModeErrorHandler = useCallback( + (data: any) => { + if (!data.userId || data.userId === user?.id) { + setIsDOMMode(false); + setCurrentSnapshot(null); + setIsLoading(false); + } + }, + [user?.id] + ); + + useEffect(() => { + if (isDOMMode) { + clientSelectorGenerator.setGetList(getList); + clientSelectorGenerator.setListSelector(listSelector || ""); + clientSelectorGenerator.setPaginationMode(paginationMode); + } + }, [isDOMMode, getList, listSelector, paginationMode]); + + useEffect(() => { + if (isDOMMode && listSelector) { + socket?.emit("setGetList", { getList: true }); + socket?.emit("listSelector", { selector: listSelector }); + + clientSelectorGenerator.setListSelector(listSelector); + } + }, [isDOMMode, listSelector, socket, getList]); + useEffect(() => { coordinateMapper.updateDimensions(dimensions.width, dimensions.height, viewportInfo.width, viewportInfo.height); }, [viewportInfo, dimensions.width, dimensions.height]); @@ -162,16 +370,185 @@ export const BrowserWindow = () => { useEffect(() => { if (socket) { socket.on("screencast", screencastHandler); + socket.on("domcast", rrwebSnapshotHandler); + socket.on("dom-mode-enabled", domModeHandler); + socket.on("screenshot-mode-enabled", screenshotModeHandler); + socket.on("dom-mode-error", domModeErrorHandler); } - if (canvasRef?.current) { + + if (canvasRef?.current && !isDOMMode && screenShot) { drawImage(screenShot, canvasRef.current); - } else { - console.log('Canvas is not initialized'); } + return () => { - socket?.off("screencast", screencastHandler); - } - }, [screenShot, canvasRef, socket, screencastHandler]); + if (socket) { + console.log("Cleaning up DOM streaming event listeners"); + socket.off("screencast", screencastHandler); + socket.off("domcast", rrwebSnapshotHandler); + socket.off("dom-mode-enabled", domModeHandler); + socket.off("screenshot-mode-enabled", screenshotModeHandler); + socket.off("dom-mode-error", domModeErrorHandler); + } + }; + }, [ + socket, + screenShot, + canvasRef, + isDOMMode, + screencastHandler, + rrwebSnapshotHandler, + domModeHandler, + screenshotModeHandler, + domModeErrorHandler, + ]); + + const domHighlighterHandler = useCallback( + (data: { + rect: DOMRect; + selector: string; + elementInfo: ElementInfo | null; + childSelectors?: string[]; + isDOMMode?: boolean; + }) => { + if (!isDOMMode || !currentSnapshot) { + return; + } + + let iframeElement = document.querySelector( + "#dom-browser-iframe" + ) as HTMLIFrameElement; + + if (!iframeElement) { + iframeElement = document.querySelector( + "#browser-window iframe" + ) as HTMLIFrameElement; + } + + if (!iframeElement) { + const browserWindow = document.querySelector("#browser-window"); + if (browserWindow) { + iframeElement = browserWindow.querySelector( + "iframe" + ) as HTMLIFrameElement; + } + } + + if (!iframeElement) { + console.error("Could not find iframe element for DOM highlighting"); + return; + } + + const iframeRect = iframeElement.getBoundingClientRect(); + const IFRAME_BODY_PADDING = 16; + + const absoluteRect = new DOMRect( + data.rect.x + iframeRect.left - IFRAME_BODY_PADDING, + data.rect.y + iframeRect.top - IFRAME_BODY_PADDING, + data.rect.width, + data.rect.height + ); + + const mappedData = { + ...data, + rect: absoluteRect, + }; + + if (getList === true) { + if (listSelector) { + socket?.emit("listSelector", { selector: listSelector }); + const hasValidChildSelectors = + Array.isArray(mappedData.childSelectors) && + mappedData.childSelectors.length > 0; + + if (limitMode) { + setHighlighterData(null); + } else if (paginationMode) { + if ( + paginationType !== "" && + !["none", "scrollDown", "scrollUp"].includes(paginationType) + ) { + setHighlighterData(mappedData); + } else { + setHighlighterData(null); + } + } else if ( + mappedData.childSelectors && + mappedData.childSelectors.includes(mappedData.selector) + ) { + setHighlighterData(mappedData); + } else if ( + mappedData.elementInfo?.isIframeContent && + mappedData.childSelectors + ) { + const isIframeChild = mappedData.childSelectors.some( + (childSelector) => + mappedData.selector.includes(":>>") && + childSelector + .split(":>>") + .some((part) => mappedData.selector.includes(part.trim())) + ); + setHighlighterData(isIframeChild ? mappedData : null); + } else if ( + mappedData.selector.includes(":>>") && + hasValidChildSelectors + ) { + const selectorParts = mappedData.selector + .split(":>>") + .map((part) => part.trim()); + const isValidMixedSelector = selectorParts.some((part) => + mappedData.childSelectors!.some((childSelector) => + childSelector.includes(part) + ) + ); + setHighlighterData(isValidMixedSelector ? mappedData : null); + } else if ( + mappedData.elementInfo?.isShadowRoot && + mappedData.childSelectors + ) { + const isShadowChild = mappedData.childSelectors.some( + (childSelector) => + mappedData.selector.includes(">>") && + childSelector + .split(">>") + .some((part) => mappedData.selector.includes(part.trim())) + ); + setHighlighterData(isShadowChild ? mappedData : null); + } else if ( + mappedData.selector.includes(">>") && + hasValidChildSelectors + ) { + const selectorParts = mappedData.selector + .split(">>") + .map((part) => part.trim()); + const isValidMixedSelector = selectorParts.some((part) => + mappedData.childSelectors!.some((childSelector) => + childSelector.includes(part) + ) + ); + + setHighlighterData(isValidMixedSelector ? mappedData : null); + } else { + setHighlighterData(null); + } + } else { + setHighlighterData(mappedData); + } + } else { + // getText mode + setHighlighterData(mappedData); + } + }, + [ + isDOMMode, + currentSnapshot, + getList, + socket, + listSelector, + paginationMode, + paginationType, + limitMode, + ] + ); const highlighterHandler = useCallback((data: { rect: DOMRect, selector: string, elementInfo: ElementInfo | null, childSelectors?: string[] }) => { const now = performance.now(); @@ -260,20 +637,6 @@ export const BrowserWindow = () => { } }, [getList, socket, listSelector, paginationMode, paginationType, limitMode]); - useEffect(() => { - if (socket) { - socket.on('listDataExtracted', (response) => { - const { currentListId, data } = response; - - updateListStepData(currentListId, data); - }); - } - - return () => { - socket?.off('listDataExtracted'); - }; - }, [socket]); - useEffect(() => { document.addEventListener('mousemove', onMouseMove, false); if (socket) { @@ -304,6 +667,188 @@ export const BrowserWindow = () => { } }, [captureStage, listSelector, socket]); + const handleDOMElementSelection = useCallback( + (highlighterData: { + rect: DOMRect; + selector: string; + elementInfo: ElementInfo | null; + childSelectors?: string[]; + }) => { + setShowAttributeModal(false); + setSelectedElement(null); + setAttributeOptions([]); + + const options = getAttributeOptions( + highlighterData.elementInfo?.tagName || "", + highlighterData.elementInfo + ); + + if (getText === true) { + if (options.length === 1) { + const attribute = options[0].value; + const data = + attribute === "href" + ? highlighterData.elementInfo?.url || "" + : attribute === "src" + ? highlighterData.elementInfo?.imageUrl || "" + : highlighterData.elementInfo?.innerText || ""; + + addTextStep( + "", + data, + { + selector: highlighterData.selector, + tag: highlighterData.elementInfo?.tagName, + shadow: highlighterData.elementInfo?.isShadowRoot, + attribute, + }, + currentTextActionId || `text-${crypto.randomUUID()}` + ); + } else { + setAttributeOptions(options); + setSelectedElement({ + selector: highlighterData.selector, + info: highlighterData.elementInfo, + }); + setShowAttributeModal(true); + } + } + + if (paginationMode && getList) { + if ( + paginationType !== "" && + paginationType !== "scrollDown" && + paginationType !== "scrollUp" && + paginationType !== "none" + ) { + setPaginationSelector(highlighterData.selector); + notify( + `info`, + t( + "browser_window.attribute_modal.notifications.pagination_select_success" + ) + ); + addListStep( + listSelector!, + fields, + currentListId || 0, + currentListActionId || `list-${crypto.randomUUID()}`, + { type: paginationType, selector: highlighterData.selector } + ); + socket?.emit("setPaginationMode", { pagination: false }); + } + return; + } + + if (getList === true && !listSelector) { + let cleanedSelector = highlighterData.selector; + if (cleanedSelector.includes("nth-child")) { + cleanedSelector = cleanedSelector.replace(/:nth-child\(\d+\)/g, ""); + } + + setListSelector(cleanedSelector); + notify( + `info`, + t("browser_window.attribute_modal.notifications.list_select_success") + ); + setCurrentListId(Date.now()); + setFields({}); + + socket?.emit("setGetList", { getList: true }); + socket?.emit("listSelector", { selector: cleanedSelector }); + } else if (getList === true && listSelector && currentListId) { + if (options.length === 1) { + const attribute = options[0].value; + let currentSelector = highlighterData.selector; + + if (currentSelector.includes(">")) { + const [firstPart, ...restParts] = currentSelector + .split(">") + .map((p) => p.trim()); + const listSelectorRightPart = listSelector + .split(">") + .pop() + ?.trim() + .replace(/:nth-child\(\d+\)/g, ""); + + if ( + firstPart.includes("nth-child") && + firstPart.replace(/:nth-child\(\d+\)/g, "") === + listSelectorRightPart + ) { + currentSelector = `${firstPart.replace( + /:nth-child\(\d+\)/g, + "" + )} > ${restParts.join(" > ")}`; + } + } + + const data = + attribute === "href" + ? highlighterData.elementInfo?.url || "" + : attribute === "src" + ? highlighterData.elementInfo?.imageUrl || "" + : highlighterData.elementInfo?.innerText || ""; + + const newField: TextStep = { + id: Date.now(), + type: "text", + label: `Label ${Object.keys(fields).length + 1}`, + data: data, + selectorObj: { + selector: currentSelector, + tag: highlighterData.elementInfo?.tagName, + shadow: highlighterData.elementInfo?.isShadowRoot, + attribute, + }, + }; + + const updatedFields = { + ...fields, + [newField.id]: newField, + }; + + setFields(updatedFields); + + if (listSelector) { + addListStep( + listSelector, + updatedFields, + currentListId, + currentListActionId || `list-${crypto.randomUUID()}`, + { type: "", selector: paginationSelector } + ); + } + } else { + setAttributeOptions(options); + setSelectedElement({ + selector: highlighterData.selector, + info: highlighterData.elementInfo, + }); + setShowAttributeModal(true); + } + } + }, + [ + getText, + getList, + listSelector, + paginationMode, + paginationType, + fields, + currentListId, + currentTextActionId, + currentListActionId, + addTextStep, + addListStep, + notify, + socket, + t, + paginationSelector, + ] + ); + + const handleClick = (e: React.MouseEvent) => { if (highlighterData && canvasRef?.current) { const canvasRect = canvasRef.current.getBoundingClientRect(); @@ -409,13 +954,6 @@ export const BrowserWindow = () => { setFields(updatedFields); if (listSelector) { - socket?.emit('extractListData', { - listSelector, - fields: updatedFields, - currentListId, - pagination: { type: '', selector: paginationSelector } - }); - addListStep( listSelector, updatedFields, @@ -482,13 +1020,6 @@ export const BrowserWindow = () => { setFields(updatedFields); if (listSelector) { - socket?.emit('extractListData', { - listSelector, - fields: updatedFields, - currentListId, - pagination: { type: '', selector: paginationSelector } - }); - addListStep( listSelector, updatedFields, @@ -500,7 +1031,14 @@ export const BrowserWindow = () => { } } } + setShowAttributeModal(false); + setSelectedElement(null); + setAttributeOptions([]); + + setTimeout(() => { + setShowAttributeModal(false); + }, 0); }; const resetPaginationSelector = useCallback(() => { @@ -519,8 +1057,12 @@ export const BrowserWindow = () => { getText === true || getList === true ? ( { }} - canBeClosed={false} + onClose={() => { + setShowAttributeModal(false); + setSelectedElement(null); + setAttributeOptions([]); + }} + canBeClosed={true} modalStyle={modalStyle} >
@@ -560,21 +1102,150 @@ export const BrowserWindow = () => { ) : null } -
- {((getText === true || getList === true) && !showAttributeModal && highlighterData?.rect != null && highlighterData?.rect.top != null) && canvasRef?.current ? - setDatePickerInfo(null)} + /> + )} + {dropdownInfo && ( + setDropdownInfo(null)} + /> + )} + {timePickerInfo && ( + setTimePickerInfo(null)} + /> + )} + {dateTimeLocalInfo && ( + setDateTimeLocalInfo(null)} + /> + )} + +
+ {(getText === true || getList === true) && + !showAttributeModal && + highlighterData?.rect != null && ( + <> + {!isDOMMode && canvasRef?.current && ( + + )} + + {isDOMMode && highlighterData && ( + <> +
+ + )} + + )} + + {isDOMMode ? ( + currentSnapshot ? ( + { + domHighlighterHandler(data); + }} + onElementSelect={handleDOMElementSelection} + onShowDatePicker={handleShowDatePicker} + onShowDropdown={handleShowDropdown} + onShowTimePicker={handleShowTimePicker} + onShowDateTimePicker={handleShowDateTimePicker} + /> + ) : ( +
+
+
+ Loading website... +
+ +
+ ) + ) : ( + /* Screenshot mode canvas */ + - : null} - + )}
); diff --git a/src/components/pickers/DatePicker.tsx b/src/components/pickers/DatePicker.tsx index 00687115..ec4334e0 100644 --- a/src/components/pickers/DatePicker.tsx +++ b/src/components/pickers/DatePicker.tsx @@ -16,12 +16,58 @@ const DatePicker: React.FC = ({ coordinates, selector, onClose setSelectedDate(e.target.value); }; + const updateDOMElement = (selector: string, value: string) => { + try { + let iframeElement = document.querySelector('#dom-browser-iframe') as HTMLIFrameElement; + + if (!iframeElement) { + iframeElement = document.querySelector('#browser-window iframe') as HTMLIFrameElement; + } + + if (!iframeElement) { + const browserWindow = document.querySelector('#browser-window'); + if (browserWindow) { + iframeElement = browserWindow.querySelector('iframe') as HTMLIFrameElement; + } + } + + if (!iframeElement) { + console.error('Could not find iframe element for DOM update'); + return; + } + + const iframeDoc = iframeElement.contentDocument; + if (!iframeDoc) { + console.error('Could not access iframe document'); + return; + } + + const element = iframeDoc.querySelector(selector) as HTMLInputElement; + if (element) { + element.value = value; + + const changeEvent = new Event('change', { bubbles: true }); + element.dispatchEvent(changeEvent); + + const inputEvent = new Event('input', { bubbles: true }); + element.dispatchEvent(inputEvent); + } else { + console.warn(`Could not find element with selector: ${selector}`); + } + } catch (error) { + console.error('Error updating DOM element:', error); + } + }; + const handleConfirm = () => { if (socket && selectedDate) { socket.emit('input:date', { selector, value: selectedDate }); + + updateDOMElement(selector, selectedDate); + onClose(); } }; diff --git a/src/components/pickers/DateTimeLocalPicker.tsx b/src/components/pickers/DateTimeLocalPicker.tsx index c51e3540..0cc952bc 100644 --- a/src/components/pickers/DateTimeLocalPicker.tsx +++ b/src/components/pickers/DateTimeLocalPicker.tsx @@ -16,12 +16,58 @@ const DateTimeLocalPicker: React.FC = ({ coordinates, setSelectedDateTime(e.target.value); }; + const updateDOMElement = (selector: string, value: string) => { + try { + let iframeElement = document.querySelector('#dom-browser-iframe') as HTMLIFrameElement; + + if (!iframeElement) { + iframeElement = document.querySelector('#browser-window iframe') as HTMLIFrameElement; + } + + if (!iframeElement) { + const browserWindow = document.querySelector('#browser-window'); + if (browserWindow) { + iframeElement = browserWindow.querySelector('iframe') as HTMLIFrameElement; + } + } + + if (!iframeElement) { + console.error('Could not find iframe element for DOM update'); + return; + } + + const iframeDoc = iframeElement.contentDocument; + if (!iframeDoc) { + console.error('Could not access iframe document'); + return; + } + + const element = iframeDoc.querySelector(selector) as HTMLInputElement; + if (element) { + element.value = value; + + const changeEvent = new Event('change', { bubbles: true }); + element.dispatchEvent(changeEvent); + + const inputEvent = new Event('input', { bubbles: true }); + element.dispatchEvent(inputEvent); + } else { + console.warn(`Could not find element with selector: ${selector}`); + } + } catch (error) { + console.error('Error updating DOM element:', error); + } + }; + const handleConfirm = () => { if (socket && selectedDateTime) { socket.emit('input:datetime-local', { selector, value: selectedDateTime }); + + updateDOMElement(selector, selectedDateTime); + onClose(); } }; @@ -58,8 +104,8 @@ const DateTimeLocalPicker: React.FC = ({ coordinates, onClick={handleConfirm} disabled={!selectedDateTime} className={`px-3 py-1 text-sm rounded ${selectedDateTime - ? 'bg-blue-500 text-white hover:bg-blue-600' - : 'bg-gray-300 text-gray-500 cursor-not-allowed' + ? 'bg-blue-500 text-white hover:bg-blue-600' + : 'bg-gray-300 text-gray-500 cursor-not-allowed' }`} > Confirm diff --git a/src/components/pickers/Dropdown.tsx b/src/components/pickers/Dropdown.tsx index df695efa..743d721e 100644 --- a/src/components/pickers/Dropdown.tsx +++ b/src/components/pickers/Dropdown.tsx @@ -18,9 +18,65 @@ const Dropdown = ({ coordinates, selector, options, onClose }: DropdownProps) => const { socket } = useSocketStore(); const [hoveredIndex, setHoveredIndex] = useState(null); + const updateDOMElement = (selector: string, value: string) => { + try { + let iframeElement = document.querySelector('#dom-browser-iframe') as HTMLIFrameElement; + + if (!iframeElement) { + iframeElement = document.querySelector('#browser-window iframe') as HTMLIFrameElement; + } + + if (!iframeElement) { + const browserWindow = document.querySelector('#browser-window'); + if (browserWindow) { + iframeElement = browserWindow.querySelector('iframe') as HTMLIFrameElement; + } + } + + if (!iframeElement) { + console.error('Could not find iframe element for DOM update'); + return; + } + + const iframeDoc = iframeElement.contentDocument; + if (!iframeDoc) { + console.error('Could not access iframe document'); + return; + } + + const selectElement = iframeDoc.querySelector(selector) as HTMLSelectElement; + if (selectElement) { + selectElement.value = value; + + const optionElements = selectElement.querySelectorAll('option'); + optionElements.forEach(option => { + if (option.value === value) { + option.selected = true; + option.setAttribute('selected', 'selected'); + } else { + option.selected = false; + option.removeAttribute('selected'); + } + }); + + const changeEvent = new Event('change', { bubbles: true }); + selectElement.dispatchEvent(changeEvent); + + const inputEvent = new Event('input', { bubbles: true }); + selectElement.dispatchEvent(inputEvent); + } else { + console.warn(`Could not find select element with selector: ${selector}`); + } + } catch (error) { + console.error('Error updating DOM select element:', error); + } + }; + const handleSelect = (value: string) => { if (socket) { socket.emit('input:dropdown', { selector, value }); + + updateDOMElement(selector, value); } onClose(); }; diff --git a/src/components/recorder/DOMBrowserRenderer.tsx b/src/components/recorder/DOMBrowserRenderer.tsx new file mode 100644 index 00000000..c18c9885 --- /dev/null +++ b/src/components/recorder/DOMBrowserRenderer.tsx @@ -0,0 +1,1139 @@ +import React, { + useCallback, + useContext, + useEffect, + useState, + useRef, +} from "react"; +import { useSocketStore } from "../../context/socket"; +import { useGlobalInfoStore } from "../../context/globalInfo"; +import { useTranslation } from "react-i18next"; +import { AuthContext } from "../../context/auth"; +import { rebuild, createMirror } from "rrweb-snapshot"; +import { + ActionType, + clientSelectorGenerator, +} from "../../helpers/clientSelectorGenerator"; + +interface ElementInfo { + tagName: string; + hasOnlyText?: boolean; + isIframeContent?: boolean; + isShadowRoot?: boolean; + innerText?: string; + url?: string; + imageUrl?: string; + attributes?: Record; + innerHTML?: string; + outerHTML?: string; + isDOMMode?: boolean; +} + +interface ProcessedSnapshot { + snapshot: RRWebSnapshot; + resources: { + stylesheets: Array<{ + href: string; + content: string; + media?: string; + }>; + images: Array<{ + src: string; + dataUrl: string; + alt?: string; + }>; + fonts: Array<{ + url: string; + dataUrl: string; + format?: string; + }>; + scripts: Array<{ + src: string; + content: string; + type?: string; + }>; + media: Array<{ + src: string; + dataUrl: string; + type: string; + }>; + }; + baseUrl: string; + viewport: { width: number; height: number }; + timestamp: number; + processingStats: { + totalReplacements: number; + discoveredResources: { + images: number; + stylesheets: number; + scripts: number; + fonts: number; + media: number; + }; + cachedResources: { + stylesheets: number; + images: number; + fonts: number; + scripts: number; + media: number; + }; + totalCacheSize: number; + }; +} + +interface RRWebSnapshot { + type: number; + childNodes?: RRWebSnapshot[]; + tagName?: string; + attributes?: Record; + textContent: string; + id: number; + [key: string]: any; +} + +interface RRWebDOMBrowserRendererProps { + width: number; + height: number; + snapshot: ProcessedSnapshot; + getList?: boolean; + getText?: boolean; + listSelector?: string | null; + paginationMode?: boolean; + paginationType?: string; + limitMode?: boolean; + onHighlight?: (data: { + rect: DOMRect; + selector: string; + elementInfo: ElementInfo | null; + childSelectors?: string[]; + }) => void; + onElementSelect?: (data: { + rect: DOMRect; + selector: string; + elementInfo: ElementInfo | null; + childSelectors?: string[]; + }) => void; + onShowDatePicker?: (info: { + coordinates: { x: number; y: number }; + selector: string; + }) => void; + onShowDropdown?: (info: { + coordinates: { x: number; y: number }; + selector: string; + options: Array<{ + value: string; + text: string; + disabled: boolean; + selected: boolean; + }>; + }) => void; + onShowTimePicker?: (info: { + coordinates: { x: number; y: number }; + selector: string; + }) => void; + onShowDateTimePicker?: (info: { + coordinates: { x: number; y: number }; + selector: string; + }) => void; +} + +export const DOMBrowserRenderer: React.FC = ({ + width, + height, + snapshot, + getList = false, + getText = false, + listSelector = null, + paginationMode = false, + paginationType = "", + limitMode = false, + onHighlight, + onElementSelect, + onShowDatePicker, + onShowDropdown, + onShowTimePicker, + onShowDateTimePicker, +}) => { + const { t } = useTranslation(); + const containerRef = useRef(null); + const iframeRef = useRef(null); + const [isRendered, setIsRendered] = useState(false); + const [renderError, setRenderError] = useState(null); + const [lastMousePosition, setLastMousePosition] = useState({ x: 0, y: 0 }); + const [currentHighlight, setCurrentHighlight] = useState<{ + element: Element; + rect: DOMRect; + selector: string; + elementInfo: ElementInfo; + childSelectors?: string[]; + } | null>(null); + + const { socket } = useSocketStore(); + const { setLastAction, lastAction } = useGlobalInfoStore(); + + const { state } = useContext(AuthContext); + const { user } = state; + + const MOUSE_MOVE_THROTTLE = 16; // ~60fps + const lastMouseMoveTime = useRef(0); + + const notifyLastAction = (action: string) => { + if (lastAction !== action) { + setLastAction(action); + } + }; + + const isInCaptureMode = getText || getList; + + useEffect(() => { + clientSelectorGenerator.setGetList(getList); + clientSelectorGenerator.setListSelector(listSelector || ""); + clientSelectorGenerator.setPaginationMode(paginationMode); + }, [getList, listSelector, paginationMode]); + + useEffect(() => { + if (listSelector) { + clientSelectorGenerator.setListSelector(listSelector); + clientSelectorGenerator.setGetList(getList); + clientSelectorGenerator.setPaginationMode(paginationMode); + } + }, [listSelector, getList, paginationMode]); + + /** + * Handle client-side highlighting for DOM mode using complete backend logic + */ + const handleDOMHighlighting = useCallback( + (x: number, y: number, iframeDoc: Document) => { + try { + const highlighterData = + clientSelectorGenerator.generateDataForHighlighter( + { x, y }, + iframeDoc, + true + ); + + if (!highlighterData) { + setCurrentHighlight(null); + if (onHighlight) { + onHighlight({ + rect: new DOMRect(0, 0, 0, 0), + selector: "", + elementInfo: null, + }); + } + return; + } + + const { rect, selector, elementInfo, childSelectors } = highlighterData; + + let shouldHighlight = false; + + if (getList) { + if (listSelector) { + const hasValidChildSelectors = + Array.isArray(childSelectors) && childSelectors.length > 0; + + if (limitMode) { + shouldHighlight = false; + } else if (paginationMode) { + if ( + paginationType !== "" && + !["none", "scrollDown", "scrollUp"].includes(paginationType) + ) { + shouldHighlight = true; + } else { + shouldHighlight = false; + } + } else if (childSelectors && childSelectors.includes(selector)) { + shouldHighlight = true; + } else if (elementInfo?.isIframeContent && childSelectors) { + const isIframeChild = childSelectors.some( + (childSelector: string) => + selector.includes(":>>") && + childSelector + .split(":>>") + .some((part) => selector.includes(part.trim())) + ); + shouldHighlight = isIframeChild; + } else if (selector.includes(":>>") && hasValidChildSelectors) { + const selectorParts = selector + .split(":>>") + .map((part: string) => part.trim()); + const isValidMixedSelector = selectorParts.some((part: any) => + childSelectors!.some((childSelector) => + childSelector.includes(part) + ) + ); + } else if (elementInfo?.isShadowRoot && childSelectors) { + const isShadowChild = childSelectors.some( + (childSelector: string) => + selector.includes(">>") && + childSelector + .split(">>") + .some((part) => selector.includes(part.trim())) + ); + } else if (selector.includes(">>") && hasValidChildSelectors) { + const selectorParts = selector + .split(">>") + .map((part: string) => part.trim()); + const isValidMixedSelector = selectorParts.some((part: any) => + childSelectors!.some((childSelector) => + childSelector.includes(part) + ) + ); + } else { + shouldHighlight = false; + } + } else { + shouldHighlight = true; + } + } else { + shouldHighlight = true; + } + + if (shouldHighlight) { + const element = iframeDoc.elementFromPoint(x, y); + if (element) { + setCurrentHighlight({ + element, + rect: rect, + selector, + elementInfo: { + ...elementInfo, + tagName: elementInfo?.tagName ?? "", + isDOMMode: true, + }, + childSelectors, + }); + + if (onHighlight) { + onHighlight({ + rect: rect, + elementInfo: { + ...elementInfo, + tagName: elementInfo?.tagName ?? "", + isDOMMode: true, + }, + selector, + childSelectors, + }); + } + } + } else { + setCurrentHighlight(null); + if (onHighlight) { + onHighlight({ + rect: new DOMRect(0, 0, 0, 0), + selector: "", + elementInfo: null, + }); + } + } + } catch (error) { + console.error("Error in DOM highlighting:", error); + setCurrentHighlight(null); + } + }, + [ + getList, + listSelector, + paginationMode, + paginationType, + limitMode, + onHighlight, + ] + ); + + /** + * Set up enhanced interaction handlers for DOM mode + */ + const setupIframeInteractions = useCallback( + (iframeDoc: Document) => { + const existingHandlers = (iframeDoc as any)._domRendererHandlers; + if (existingHandlers) { + Object.entries(existingHandlers).forEach(([event, handler]) => { + iframeDoc.removeEventListener(event, handler as EventListener, false); // Changed to false + }); + } + + const handlers: { [key: string]: EventListener } = {}; + + const mouseMoveHandler: EventListener = (e: Event) => { + if (e.target && !iframeDoc.contains(e.target as Node)) { + return; + } + + const now = performance.now(); + if (now - lastMouseMoveTime.current < MOUSE_MOVE_THROTTLE) { + return; + } + lastMouseMoveTime.current = now; + + const mouseEvent = e as MouseEvent; + const iframeX = mouseEvent.clientX; + const iframeY = mouseEvent.clientY; + + const iframe = iframeRef.current; + if (iframe) { + const iframeRect = iframe.getBoundingClientRect(); + setLastMousePosition({ + x: iframeX + iframeRect.left, + y: iframeY + iframeRect.top, + }); + } + + handleDOMHighlighting(iframeX, iframeY, iframeDoc); + notifyLastAction("move"); + }; + + const mouseDownHandler: EventListener = (e: Event) => { + if (e.target && !iframeDoc.contains(e.target as Node)) { + return; + } + + const mouseEvent = e as MouseEvent; + const target = mouseEvent.target as Element; + const iframeX = mouseEvent.clientX; + const iframeY = mouseEvent.clientY; + + if (isInCaptureMode) { + e.preventDefault(); + e.stopPropagation(); + + if (currentHighlight && onElementSelect) { + onElementSelect({ + rect: currentHighlight.rect, + selector: currentHighlight.selector, + elementInfo: currentHighlight.elementInfo, + childSelectors: currentHighlight.childSelectors || [], + }); + } + notifyLastAction("select element"); + return; + } + + const linkElement = target.closest("a[href]") as HTMLAnchorElement; + if (linkElement && linkElement.href && socket) { + e.preventDefault(); + e.stopPropagation(); + + const href = linkElement.href; + + if (linkElement.target) { + linkElement.target = ""; + } + + const originalHref = linkElement.href; + linkElement.removeAttribute("href"); + + setTimeout(() => { + linkElement.setAttribute("href", originalHref); + }, 100); + + const isSPALink = + href.endsWith("#") || + (href.includes("#") && new URL(href).hash !== ""); + + const selector = clientSelectorGenerator.generateSelector( + iframeDoc, + { x: iframeX, y: iframeY }, + ActionType.Click + ); + + const elementInfo = clientSelectorGenerator.getElementInformation( + iframeDoc, + { x: iframeX, y: iframeY }, + clientSelectorGenerator.getCurrentState().listSelector, + clientSelectorGenerator.getCurrentState().getList + ); + + if (selector && socket) { + socket.emit("dom:click", { + selector, + url: snapshot.baseUrl, + userId: user?.id || "unknown", + elementInfo, + coordinates: undefined, + isSPA: isSPALink, + }); + + notifyLastAction( + isSPALink ? `SPA navigation to ${href}` : `navigate to ${href}` + ); + } + return; + } + + const selector = clientSelectorGenerator.generateSelector( + iframeDoc, + { x: iframeX, y: iframeY }, + ActionType.Click + ); + + const elementInfo = clientSelectorGenerator.getElementInformation( + iframeDoc, + { x: iframeX, y: iframeY }, + clientSelectorGenerator.getCurrentState().listSelector, + clientSelectorGenerator.getCurrentState().getList + ); + + if (selector && elementInfo && socket) { + if (elementInfo?.tagName === "SELECT" && elementInfo.innerHTML) { + const inputElement = target as HTMLInputElement; + inputElement.blur(); + + const wasDisabled = inputElement.disabled; + inputElement.disabled = true; + + setTimeout(() => { + inputElement.disabled = wasDisabled; + }, 100); + + const options = elementInfo.innerHTML + .split(" { + const valueMatch = optionHtml.match(/value="([^"]*)"/); + const textMatch = optionHtml.match(/>([^<]*) { + inputElement.disabled = wasDisabled; + }, 100); + + const pickerInfo = { + coordinates: { x: iframeX, y: iframeY }, + selector, + }; + + switch (inputType) { + case "date": + case "month": + case "week": + if (onShowDatePicker) { + onShowDatePicker(pickerInfo); + } + break; + case "time": + if (onShowTimePicker) { + onShowTimePicker(pickerInfo); + } + break; + case "datetime-local": + if (onShowDateTimePicker) { + onShowDateTimePicker(pickerInfo); + } + break; + } + + notifyLastAction(`${inputType} picker opened`); + return; + } + } + + if ( + elementInfo?.tagName !== "INPUT" && + elementInfo?.tagName !== "SELECT" + ) { + socket.emit("dom:click", { + selector, + url: snapshot.baseUrl, + userId: user?.id || "unknown", + elementInfo, + coordinates: { x: iframeX, y: iframeY }, + isSPA: false, + }); + } + } + + notifyLastAction("click"); + }; + + const mouseUpHandler: EventListener = (e: Event) => { + if (e.target && !iframeDoc.contains(e.target as Node)) { + return; + } + + if (!isInCaptureMode) { + notifyLastAction("release"); + } + }; + + const keyDownHandler: EventListener = (e: Event) => { + if (e.target && !iframeDoc.contains(e.target as Node)) { + return; + } + + const keyboardEvent = e as KeyboardEvent; + const target = keyboardEvent.target as HTMLElement; + + if (!isInCaptureMode && socket && snapshot?.baseUrl) { + const iframe = iframeRef.current; + if (iframe) { + const iframeRect = iframe.getBoundingClientRect(); + const iframeX = lastMousePosition.x - iframeRect.left; + const iframeY = lastMousePosition.y - iframeRect.top; + + const selector = clientSelectorGenerator.generateSelector( + iframeDoc, + { x: iframeX, y: iframeY }, + ActionType.Keydown + ); + + const elementInfo = clientSelectorGenerator.getElementInformation( + iframeDoc, + { x: iframeX, y: iframeY }, + clientSelectorGenerator.getCurrentState().listSelector, + clientSelectorGenerator.getCurrentState().getList + ); + + if (selector) { + socket.emit("dom:keypress", { + selector, + key: keyboardEvent.key, + url: snapshot.baseUrl, + userId: user?.id || "unknown", + inputType: elementInfo?.attributes?.type || "text", + }); + } + } + + notifyLastAction(`${keyboardEvent.key} typed`); + } + + if ( + ["INPUT", "TEXTAREA"].includes(target.tagName) && + !isInCaptureMode + ) { + return; + } + }; + + const keyUpHandler: EventListener = (e: Event) => { + if (e.target && !iframeDoc.contains(e.target as Node)) { + return; + } + + const keyboardEvent = e as KeyboardEvent; + + if (!isInCaptureMode && socket) { + socket.emit("input:keyup", { key: keyboardEvent.key }); + } + }; + + const wheelHandler: EventListener = (e: Event) => { + if (e.target && !iframeDoc.contains(e.target as Node)) { + return; + } + + e.preventDefault(); + + if (!isInCaptureMode) { + const wheelEvent = e as WheelEvent; + const deltaX = Math.round(wheelEvent.deltaX / 10) * 10; + const deltaY = Math.round(wheelEvent.deltaY / 10) * 10; + + if (Math.abs(deltaX) > 5 || Math.abs(deltaY) > 5) { + if (socket) { + socket.emit("dom:scroll", { + deltaX, + deltaY + }) + } + notifyLastAction("scroll"); + } + } + }; + + const clickHandler: EventListener = (e: Event) => { + if (e.target && !iframeDoc.contains(e.target as Node)) { + return; + } + + if (isInCaptureMode) { + e.preventDefault(); + e.stopPropagation(); + return; + } + }; + + const preventDefaults = (e: Event) => { + if (e.target && !iframeDoc.contains(e.target as Node)) { + return; + } + + e.preventDefault(); + e.stopPropagation(); + return false; + }; + + handlers.mousedown = mouseDownHandler; + handlers.mouseup = mouseUpHandler; + handlers.mousemove = mouseMoveHandler; + handlers.wheel = wheelHandler; + handlers.keydown = keyDownHandler; + handlers.keyup = keyUpHandler; + handlers.click = clickHandler; + handlers.submit = preventDefaults; + handlers.beforeunload = preventDefaults; + + Object.entries(handlers).forEach(([event, handler]) => { + iframeDoc.addEventListener(event, handler, false); + }); + + // Store handlers for cleanup + (iframeDoc as any)._domRendererHandlers = handlers; + + // Make iframe focusable for keyboard events + if (iframeRef.current) { + iframeRef.current.tabIndex = 0; + } + }, + [ + socket, + lastMousePosition, + notifyLastAction, + handleDOMHighlighting, + currentHighlight, + onElementSelect, + isInCaptureMode, + snapshot, + user?.id, + onShowDatePicker, + onShowDropdown, + onShowTimePicker, + onShowDateTimePicker, + ] + ); + + /** + * Render DOM snapshot using rrweb + */ + const renderRRWebSnapshot = useCallback( + (snapshotData: ProcessedSnapshot) => { + if (!iframeRef.current) { + console.warn("No iframe reference available"); + return; + } + + const iframe = iframeRef.current; + + try { + setRenderError(null); + setIsRendered(false); + + const tempDoc = + document.implementation.createHTMLDocument("RRWeb Snapshot"); + + const mirror = createMirror(); + + try { + rebuild(snapshotData.snapshot, { + doc: tempDoc, + mirror: mirror, + cache: { stylesWithHoverClass: new Map() }, + afterAppend: (node) => { + if (node.nodeType === Node.TEXT_NODE && node.textContent) { + const text = node.textContent.trim(); + + if ( + text.startsWith("<") && + text.includes(">") && + text.length > 50 + ) { + if (node.parentNode) { + node.parentNode.removeChild(node); + } + } + } + }, + }); + } catch (rebuildError) { + console.error("rrweb rebuild failed:", rebuildError); + throw new Error(`rrweb rebuild failed: ${rebuildError}`); + } + + let rebuiltHTML = tempDoc.documentElement.outerHTML; + + rebuiltHTML = "\n" + rebuiltHTML; + + const additionalCSS = []; + + if (snapshotData.resources.fonts?.length > 0) { + const fontCSS = snapshotData.resources.fonts + .map((font) => { + const format = font.format || "woff2"; + return ` + @font-face { + font-family: 'ProxiedFont-${ + font.url.split("/").pop()?.split(".")[0] || + "unknown" + }'; + src: url("${font.dataUrl}") format("${format}"); + font-display: swap; + } + `; + }) + .join("\n"); + additionalCSS.push(fontCSS); + } + + if (snapshotData.resources.stylesheets?.length > 0) { + const externalCSS = snapshotData.resources.stylesheets + .map((stylesheet) => stylesheet.content) + .join("\n\n"); + additionalCSS.push(externalCSS); + } + + const enhancedCSS = ` + /* rrweb rebuilt content styles */ + html, body { + margin: 0 !important; + padding: 8px !important; + font-family: system-ui, -apple-system, BlinkMacSystemFont, sans-serif !important; + background: white !important; + overflow-x: hidden !important; + } + + html::-webkit-scrollbar, + body::-webkit-scrollbar { + display: none !important; + width: 0 !important; + height: 0 !important; + background: transparent !important; + } + + /* Hide scrollbars for all elements */ + *::-webkit-scrollbar { + display: none !important; + width: 0 !important; + height: 0 !important; + background: transparent !important; + } + + * { + scrollbar-width: none !important; /* Firefox */ + -ms-overflow-style: none !important; /* Internet Explorer 10+ */ + } + + img { + max-width: 100% !important; + height: auto !important; + } + + /* Make everything interactive */ + * { + cursor: ${isInCaptureMode ? "crosshair" : "pointer"} !important; + } + + /* Additional CSS from resources */ + ${additionalCSS.join("\n\n")} + `; + + const headTagRegex = /]*>/i; + const cssInjection = ` + + + + + `; + + if (headTagRegex.test(rebuiltHTML)) { + rebuiltHTML = rebuiltHTML.replace( + headTagRegex, + `${cssInjection}` + ); + } else { + rebuiltHTML = rebuiltHTML.replace( + /]*>/i, + `${cssInjection}` + ); + } + + rebuiltHTML = rebuiltHTML + .replace(/)<[^<]*)*<\/script>/gi, "") + .replace(/\s*on\w+\s*=\s*"[^"]*"/gi, "") + .replace(/\s*on\w+\s*=\s*'[^']*'/gi, "") + .replace(/javascript:/gi, "void:") + .replace(/ { + setIsRendered(true); + setupIframeInteractions(iframeDoc); + }; + } catch (error) { + console.error("Error rendering rrweb snapshot:", error); + setRenderError(error instanceof Error ? error.message : String(error)); + showErrorInIframe(error); + } + }, + [setupIframeInteractions, isInCaptureMode] + ); + + useEffect(() => { + if (snapshot && iframeRef.current) { + renderRRWebSnapshot(snapshot); + } + }, [snapshot]); + + useEffect(() => { + if (isRendered && iframeRef.current) { + const iframeDoc = iframeRef.current.contentDocument; + if (iframeDoc) { + setupIframeInteractions(iframeDoc); + } + } + }, [getText, getList, listSelector, isRendered, setupIframeInteractions]); + + /** + * Show error message in iframe + */ + const showErrorInIframe = (error: any) => { + if (!iframeRef.current) return; + + const iframe = iframeRef.current; + const iframeDoc = iframe.contentDocument || iframe.contentWindow?.document; + + if (iframeDoc) { + try { + iframeDoc.open(); + iframeDoc.write(` + + + + + +
+

Error Loading DOM Content

+

Failed to render the page in DOM mode.

+

Common causes:

+
    +
  • Page is still loading or navigating
    • +
  • Resource proxy timeouts or failures
    • +
  • Network connectivity issues
    • +
  • Invalid HTML structure
    • +
+

Solutions:

+
    +
  • Try switching back to Screenshot mode
    • +
  • Wait for the page to fully load and try again
    • +
  • Check your network connection
    • +
  • Refresh the browser page
    • +
+ +
+ Technical details + 
${error.toString()}
+
+
+ + + `); + iframeDoc.close(); + + window.addEventListener("message", (event) => { + if (event.data === "retry-dom-mode") { + if (socket) { + socket.emit("enable-dom-streaming"); + } + } + }); + } catch (e) { + console.error("Failed to write error message to iframe:", e); + } + } + }; + + useEffect(() => { + return () => { + if (iframeRef.current) { + const iframeDoc = iframeRef.current.contentDocument; + if (iframeDoc) { + const handlers = (iframeDoc as any)._domRendererHandlers; + if (handlers) { + Object.entries(handlers).forEach(([event, handler]) => { + iframeDoc.removeEventListener( + event, + handler as EventListener, + true + ); + }); + } + } + } + }; + }, []); + + return ( +
+