From f17c7299e35ce0b718a2983cc4c891fa016aae7c Mon Sep 17 00:00:00 2001 From: Rohit Date: Mon, 30 Jun 2025 16:13:54 +0530 Subject: [PATCH] feat: rm network caching logic --- .../classes/RemoteBrowser.ts | 344 +----------------- 1 file changed, 5 insertions(+), 339 deletions(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 2ae24d04..52281fd3 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -198,6 +198,10 @@ export class RemoteBrowser { private snapshotDebounceTimeout: NodeJS.Timeout | null = null; private isScrollTriggeredSnapshot = false; + private networkRequestTimeout: NodeJS.Timeout | null = null; + private pendingNetworkRequests: string[] = []; + private readonly NETWORK_QUIET_PERIOD = 8000; + /** * Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and * assigns the socket instance everywhere. @@ -223,224 +227,6 @@ export class RemoteBrowser { }, 30000); // Every 30 seconds } - private processCSS( - cssContent: string, - cssUrl: string, - baseUrl: string, - resources?: any - ): string { - try { - let processedContent = cssContent; - - logger.debug(`Processing CSS from: ${cssUrl}`); - - // Process @font-face declarations and collect font resources - processedContent = processedContent.replace( - /@font-face\s*\{([^}]*)\}/gi, - (fontFaceMatch, fontFaceContent) => { - let newFontFaceContent = fontFaceContent; - - logger.debug( - `Processing @font-face block: ${fontFaceContent.substring( - 0, - 100 - )}...` - ); - - newFontFaceContent = newFontFaceContent.replace( - /src\s*:\s*([^;}]+)[;}]/gi, - (srcMatch: any, srcValue: any) => { - let newSrcValue = srcValue; - - newSrcValue = newSrcValue.replace( - /url\s*\(\s*['"]?([^'")]+)['"]?\s*\)(\s*format\s*\(\s*['"]?[^'")]*['"]?\s*\))?/gi, - (urlMatch: any, url: string, formatPart: any) => { - const originalUrl = url.trim(); - - logger.debug(`Found font URL in @font-face: ${originalUrl}`); - - if ( - originalUrl.startsWith("data:") || - originalUrl.startsWith("blob:") - ) { - return urlMatch; - } - - try { - let absoluteUrl: string; - try { - absoluteUrl = new URL(originalUrl).href; - } catch (e) { - absoluteUrl = new URL(originalUrl, cssUrl || baseUrl) - .href; - } - - const cachedResource = - this.networkResourceCache.get(absoluteUrl); - if (cachedResource && resources) { - const dataUrl = cachedResource.base64Encoded - ? `data:${cachedResource.mimeType};base64,${cachedResource.content}` - : `data:${cachedResource.mimeType};base64,${Buffer.from( - cachedResource.content, - "utf-8" - ).toString("base64")}`; - - resources.fonts.push({ - url: absoluteUrl, - dataUrl, - format: originalUrl.split(".").pop()?.split("?")[0], - }); - } - - // Keep original URL in CSS - return urlMatch; - } catch (e) { - logger.warn( - "Failed to process font URL in @font-face:", - originalUrl, - e - ); - return urlMatch; - } - } - ); - - return `src: ${newSrcValue};`; - } - ); - - return `@font-face {${newFontFaceContent}}`; - } - ); - - // Process other url() references and collect resources - processedContent = processedContent.replace( - /url\s*\(\s*['"]?([^'")]+)['"]?\s*\)/gi, - (match, url) => { - const originalUrl = url.trim(); - - if ( - originalUrl.startsWith("data:") || - originalUrl.startsWith("blob:") - ) { - return match; - } - - try { - let absoluteUrl: string; - try { - absoluteUrl = new URL(originalUrl).href; - } catch (e) { - absoluteUrl = new URL(originalUrl, cssUrl || baseUrl).href; - } - - const cachedResource = this.networkResourceCache.get(absoluteUrl); - if (cachedResource && resources) { - const lowerMimeType = cachedResource.mimeType.toLowerCase(); - - if (lowerMimeType.includes("image/")) { - const dataUrl = cachedResource.base64Encoded - ? `data:${cachedResource.mimeType};base64,${cachedResource.content}` - : `data:${cachedResource.mimeType};base64,${Buffer.from( - cachedResource.content, - "utf-8" - ).toString("base64")}`; - - resources.images.push({ - src: absoluteUrl, - dataUrl, - alt: "", - }); - } else if ( - lowerMimeType.includes("font/") || - lowerMimeType.includes("application/font") - ) { - const dataUrl = cachedResource.base64Encoded - ? `data:${cachedResource.mimeType};base64,${cachedResource.content}` - : `data:${cachedResource.mimeType};base64,${Buffer.from( - cachedResource.content, - "utf-8" - ).toString("base64")}`; - - resources.fonts.push({ - url: absoluteUrl, - dataUrl, - format: originalUrl.split(".").pop()?.split("?")[0], - }); - } - } - - // Keep original URL in CSS - return match; - } catch (e) { - logger.warn(`Failed to process CSS URL: ${originalUrl}`, e); - return match; - } - } - ); - - // Process @import statements and collect stylesheets - processedContent = processedContent.replace( - /@import\s+(?:url\s*\(\s*)?['"]?([^'")]+)['"]?\s*\)?([^;]*);?/gi, - (match, url, mediaQuery) => { - const originalUrl = url.trim(); - - if ( - originalUrl.startsWith("data:") || - originalUrl.startsWith("blob:") - ) { - return match; - } - - try { - let absoluteUrl: string; - try { - absoluteUrl = new URL(originalUrl).href; - } catch (e) { - absoluteUrl = new URL(originalUrl, cssUrl || baseUrl).href; - } - - const cachedResource = this.networkResourceCache.get(absoluteUrl); - if ( - cachedResource && - resources && - cachedResource.mimeType.includes("css") - ) { - const content = cachedResource.base64Encoded - ? Buffer.from(cachedResource.content, "base64").toString( - "utf-8" - ) - : cachedResource.content; - - resources.stylesheets.push({ - href: absoluteUrl, - content: this.processCSS( - content, - absoluteUrl, - baseUrl, - resources - ), - media: mediaQuery ? mediaQuery.trim() : "all", - }); - } - - // Keep original @import - return match; - } catch (e) { - logger.warn(`Failed to process CSS @import: ${originalUrl}`, e); - return match; - } - } - ); - - logger.debug(`CSS processing completed for: ${cssUrl}`); - return processedContent; - } catch (error) { - logger.error("Failed to process CSS content:", error); - return cssContent; // Return original content if processing fails - } - } - private async processRRWebSnapshot( snapshot: RRWebSnapshot ): Promise { @@ -464,7 +250,7 @@ export class RemoteBrowser { }; return { - snapshot: snapshot, + snapshot, resources, baseUrl, viewport, @@ -488,126 +274,6 @@ export class RemoteBrowser { }; } - /** - * Check if a resource should be cached based on its MIME type and URL - * @private - */ - private shouldCacheResource(mimeType: string, url: string): boolean { - const lowerMimeType = mimeType.toLowerCase(); - const lowerUrl = url.toLowerCase(); - - // CSS Resources - if ( - lowerMimeType.includes("text/css") || - lowerMimeType.includes("application/css") || - lowerUrl.endsWith(".css") - ) { - return true; - } - - // Font Resources - if ( - lowerMimeType.includes("font/") || - lowerMimeType.includes("application/font") || - lowerMimeType.includes("application/x-font") || - lowerUrl.match(/\.(woff2?|ttf|otf|eot)(\?.*)?$/) - ) { - return true; - } - - // Image Resources - if ( - lowerMimeType.includes("image/") || - lowerUrl.match(/\.(jpg|jpeg|png|gif|webp|svg|ico|bmp|tiff|avif)(\?.*)?$/) - ) { - return true; - } - - // JavaScript Resources - if ( - lowerMimeType.includes("javascript") || - lowerMimeType.includes("text/js") || - lowerMimeType.includes("application/js") || - lowerUrl.match(/\.js(\?.*)?$/) - ) { - return true; - } - - // Media Resources - if ( - lowerMimeType.includes("video/") || - lowerMimeType.includes("audio/") || - lowerUrl.match( - /\.(mp4|webm|ogg|avi|mov|wmv|flv|mp3|wav|m4a|aac|flac)(\?.*)?$/ - ) - ) { - return true; - } - - // Document Resources - if ( - lowerMimeType.includes("application/pdf") || - lowerMimeType.includes("application/msword") || - lowerMimeType.includes("application/vnd.ms-") || - lowerMimeType.includes("application/vnd.openxmlformats-") || - lowerUrl.match(/\.(pdf|doc|docx|xls|xlsx|ppt|pptx)(\?.*)?$/) - ) { - return true; - } - - // Manifest and Icon Resources - if ( - lowerMimeType.includes("application/manifest+json") || - lowerUrl.includes("manifest.json") || - lowerUrl.includes("browserconfig.xml") - ) { - return true; - } - - // SVG Resources (can be images or fonts) - if (lowerMimeType.includes("image/svg+xml") || lowerUrl.endsWith(".svg")) { - return true; - } - - // Other common web resources - if ( - lowerMimeType.includes("application/octet-stream") && - lowerUrl.match(/\.(woff2?|ttf|otf|eot|css|js)(\?.*)?$/) - ) { - return true; - } - - return false; - } - - /** - * Clean up old cached resources to prevent memory leaks - * @private - */ - private cleanupResourceCache(): void { - const now = Date.now(); - const maxAge = 5 * 60 * 1000; // 5 minutes - - for (const [url, resource] of this.networkResourceCache.entries()) { - if (now - resource.timestamp > maxAge) { - this.networkResourceCache.delete(url); - } - } - - if (this.networkResourceCache.size > 200) { - const entries = Array.from(this.networkResourceCache.entries()); - entries.sort((a, b) => a[1].timestamp - b[1].timestamp); - - for (let i = 0; i < 50; i++) { - this.networkResourceCache.delete(entries[i][0]); - } - } - - logger.debug( - `Resource cache cleaned up. Current size: ${this.networkResourceCache.size}` - ); - } - private initializeMemoryManagement(): void { setInterval(() => { const memoryUsage = process.memoryUsage();