feat: rm network caching logic
This commit is contained in:
Rohit
@@ -198,6 +198,10 @@ export class RemoteBrowser {
|
||||
private snapshotDebounceTimeout: NodeJS.Timeout | null = null;
|
||||
private isScrollTriggeredSnapshot = false;
|
||||
|
||||
private networkRequestTimeout: NodeJS.Timeout | null = null;
|
||||
private pendingNetworkRequests: string[] = [];
|
||||
private readonly NETWORK_QUIET_PERIOD = 8000;
|
||||
|
||||
/**
|
||||
* Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and
|
||||
* assigns the socket instance everywhere.
|
||||
@@ -223,224 +227,6 @@ export class RemoteBrowser {
|
||||
}, 30000); // Every 30 seconds
|
||||
}
|
||||
|
||||
private processCSS(
|
||||
cssContent: string,
|
||||
cssUrl: string,
|
||||
baseUrl: string,
|
||||
resources?: any
|
||||
): string {
|
||||
try {
|
||||
let processedContent = cssContent;
|
||||
|
||||
logger.debug(`Processing CSS from: ${cssUrl}`);
|
||||
|
||||
// Process @font-face declarations and collect font resources
|
||||
processedContent = processedContent.replace(
|
||||
/@font-face\s*\{([^}]*)\}/gi,
|
||||
(fontFaceMatch, fontFaceContent) => {
|
||||
let newFontFaceContent = fontFaceContent;
|
||||
|
||||
logger.debug(
|
||||
`Processing @font-face block: ${fontFaceContent.substring(
|
||||
0,
|
||||
100
|
||||
)}...`
|
||||
);
|
||||
|
||||
newFontFaceContent = newFontFaceContent.replace(
|
||||
/src\s*:\s*([^;}]+)[;}]/gi,
|
||||
(srcMatch: any, srcValue: any) => {
|
||||
let newSrcValue = srcValue;
|
||||
|
||||
newSrcValue = newSrcValue.replace(
|
||||
/url\s*\(\s*['"]?([^'")]+)['"]?\s*\)(\s*format\s*\(\s*['"]?[^'")]*['"]?\s*\))?/gi,
|
||||
(urlMatch: any, url: string, formatPart: any) => {
|
||||
const originalUrl = url.trim();
|
||||
|
||||
logger.debug(`Found font URL in @font-face: ${originalUrl}`);
|
||||
|
||||
if (
|
||||
originalUrl.startsWith("data:") ||
|
||||
originalUrl.startsWith("blob:")
|
||||
) {
|
||||
return urlMatch;
|
||||
}
|
||||
|
||||
try {
|
||||
let absoluteUrl: string;
|
||||
try {
|
||||
absoluteUrl = new URL(originalUrl).href;
|
||||
} catch (e) {
|
||||
absoluteUrl = new URL(originalUrl, cssUrl || baseUrl)
|
||||
.href;
|
||||
}
|
||||
|
||||
const cachedResource =
|
||||
this.networkResourceCache.get(absoluteUrl);
|
||||
if (cachedResource && resources) {
|
||||
const dataUrl = cachedResource.base64Encoded
|
||||
? `data:${cachedResource.mimeType};base64,${cachedResource.content}`
|
||||
: `data:${cachedResource.mimeType};base64,${Buffer.from(
|
||||
cachedResource.content,
|
||||
"utf-8"
|
||||
).toString("base64")}`;
|
||||
|
||||
resources.fonts.push({
|
||||
url: absoluteUrl,
|
||||
dataUrl,
|
||||
format: originalUrl.split(".").pop()?.split("?")[0],
|
||||
});
|
||||
}
|
||||
|
||||
// Keep original URL in CSS
|
||||
return urlMatch;
|
||||
} catch (e) {
|
||||
logger.warn(
|
||||
"Failed to process font URL in @font-face:",
|
||||
originalUrl,
|
||||
e
|
||||
);
|
||||
return urlMatch;
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
return `src: ${newSrcValue};`;
|
||||
}
|
||||
);
|
||||
|
||||
return `@font-face {${newFontFaceContent}}`;
|
||||
}
|
||||
);
|
||||
|
||||
// Process other url() references and collect resources
|
||||
processedContent = processedContent.replace(
|
||||
/url\s*\(\s*['"]?([^'")]+)['"]?\s*\)/gi,
|
||||
(match, url) => {
|
||||
const originalUrl = url.trim();
|
||||
|
||||
if (
|
||||
originalUrl.startsWith("data:") ||
|
||||
originalUrl.startsWith("blob:")
|
||||
) {
|
||||
return match;
|
||||
}
|
||||
|
||||
try {
|
||||
let absoluteUrl: string;
|
||||
try {
|
||||
absoluteUrl = new URL(originalUrl).href;
|
||||
} catch (e) {
|
||||
absoluteUrl = new URL(originalUrl, cssUrl || baseUrl).href;
|
||||
}
|
||||
|
||||
const cachedResource = this.networkResourceCache.get(absoluteUrl);
|
||||
if (cachedResource && resources) {
|
||||
const lowerMimeType = cachedResource.mimeType.toLowerCase();
|
||||
|
||||
if (lowerMimeType.includes("image/")) {
|
||||
const dataUrl = cachedResource.base64Encoded
|
||||
? `data:${cachedResource.mimeType};base64,${cachedResource.content}`
|
||||
: `data:${cachedResource.mimeType};base64,${Buffer.from(
|
||||
cachedResource.content,
|
||||
"utf-8"
|
||||
).toString("base64")}`;
|
||||
|
||||
resources.images.push({
|
||||
src: absoluteUrl,
|
||||
dataUrl,
|
||||
alt: "",
|
||||
});
|
||||
} else if (
|
||||
lowerMimeType.includes("font/") ||
|
||||
lowerMimeType.includes("application/font")
|
||||
) {
|
||||
const dataUrl = cachedResource.base64Encoded
|
||||
? `data:${cachedResource.mimeType};base64,${cachedResource.content}`
|
||||
: `data:${cachedResource.mimeType};base64,${Buffer.from(
|
||||
cachedResource.content,
|
||||
"utf-8"
|
||||
).toString("base64")}`;
|
||||
|
||||
resources.fonts.push({
|
||||
url: absoluteUrl,
|
||||
dataUrl,
|
||||
format: originalUrl.split(".").pop()?.split("?")[0],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Keep original URL in CSS
|
||||
return match;
|
||||
} catch (e) {
|
||||
logger.warn(`Failed to process CSS URL: ${originalUrl}`, e);
|
||||
return match;
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// Process @import statements and collect stylesheets
|
||||
processedContent = processedContent.replace(
|
||||
/@import\s+(?:url\s*\(\s*)?['"]?([^'")]+)['"]?\s*\)?([^;]*);?/gi,
|
||||
(match, url, mediaQuery) => {
|
||||
const originalUrl = url.trim();
|
||||
|
||||
if (
|
||||
originalUrl.startsWith("data:") ||
|
||||
originalUrl.startsWith("blob:")
|
||||
) {
|
||||
return match;
|
||||
}
|
||||
|
||||
try {
|
||||
let absoluteUrl: string;
|
||||
try {
|
||||
absoluteUrl = new URL(originalUrl).href;
|
||||
} catch (e) {
|
||||
absoluteUrl = new URL(originalUrl, cssUrl || baseUrl).href;
|
||||
}
|
||||
|
||||
const cachedResource = this.networkResourceCache.get(absoluteUrl);
|
||||
if (
|
||||
cachedResource &&
|
||||
resources &&
|
||||
cachedResource.mimeType.includes("css")
|
||||
) {
|
||||
const content = cachedResource.base64Encoded
|
||||
? Buffer.from(cachedResource.content, "base64").toString(
|
||||
"utf-8"
|
||||
)
|
||||
: cachedResource.content;
|
||||
|
||||
resources.stylesheets.push({
|
||||
href: absoluteUrl,
|
||||
content: this.processCSS(
|
||||
content,
|
||||
absoluteUrl,
|
||||
baseUrl,
|
||||
resources
|
||||
),
|
||||
media: mediaQuery ? mediaQuery.trim() : "all",
|
||||
});
|
||||
}
|
||||
|
||||
// Keep original @import
|
||||
return match;
|
||||
} catch (e) {
|
||||
logger.warn(`Failed to process CSS @import: ${originalUrl}`, e);
|
||||
return match;
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
logger.debug(`CSS processing completed for: ${cssUrl}`);
|
||||
return processedContent;
|
||||
} catch (error) {
|
||||
logger.error("Failed to process CSS content:", error);
|
||||
return cssContent; // Return original content if processing fails
|
||||
}
|
||||
}
|
||||
|
||||
private async processRRWebSnapshot(
|
||||
snapshot: RRWebSnapshot
|
||||
): Promise<ProcessedSnapshot> {
|
||||
@@ -464,7 +250,7 @@ export class RemoteBrowser {
|
||||
};
|
||||
|
||||
return {
|
||||
snapshot: snapshot,
|
||||
snapshot,
|
||||
resources,
|
||||
baseUrl,
|
||||
viewport,
|
||||
@@ -488,126 +274,6 @@ export class RemoteBrowser {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a resource should be cached based on its MIME type and URL
|
||||
* @private
|
||||
*/
|
||||
private shouldCacheResource(mimeType: string, url: string): boolean {
|
||||
const lowerMimeType = mimeType.toLowerCase();
|
||||
const lowerUrl = url.toLowerCase();
|
||||
|
||||
// CSS Resources
|
||||
if (
|
||||
lowerMimeType.includes("text/css") ||
|
||||
lowerMimeType.includes("application/css") ||
|
||||
lowerUrl.endsWith(".css")
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Font Resources
|
||||
if (
|
||||
lowerMimeType.includes("font/") ||
|
||||
lowerMimeType.includes("application/font") ||
|
||||
lowerMimeType.includes("application/x-font") ||
|
||||
lowerUrl.match(/\.(woff2?|ttf|otf|eot)(\?.*)?$/)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Image Resources
|
||||
if (
|
||||
lowerMimeType.includes("image/") ||
|
||||
lowerUrl.match(/\.(jpg|jpeg|png|gif|webp|svg|ico|bmp|tiff|avif)(\?.*)?$/)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// JavaScript Resources
|
||||
if (
|
||||
lowerMimeType.includes("javascript") ||
|
||||
lowerMimeType.includes("text/js") ||
|
||||
lowerMimeType.includes("application/js") ||
|
||||
lowerUrl.match(/\.js(\?.*)?$/)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Media Resources
|
||||
if (
|
||||
lowerMimeType.includes("video/") ||
|
||||
lowerMimeType.includes("audio/") ||
|
||||
lowerUrl.match(
|
||||
/\.(mp4|webm|ogg|avi|mov|wmv|flv|mp3|wav|m4a|aac|flac)(\?.*)?$/
|
||||
)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Document Resources
|
||||
if (
|
||||
lowerMimeType.includes("application/pdf") ||
|
||||
lowerMimeType.includes("application/msword") ||
|
||||
lowerMimeType.includes("application/vnd.ms-") ||
|
||||
lowerMimeType.includes("application/vnd.openxmlformats-") ||
|
||||
lowerUrl.match(/\.(pdf|doc|docx|xls|xlsx|ppt|pptx)(\?.*)?$/)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Manifest and Icon Resources
|
||||
if (
|
||||
lowerMimeType.includes("application/manifest+json") ||
|
||||
lowerUrl.includes("manifest.json") ||
|
||||
lowerUrl.includes("browserconfig.xml")
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// SVG Resources (can be images or fonts)
|
||||
if (lowerMimeType.includes("image/svg+xml") || lowerUrl.endsWith(".svg")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Other common web resources
|
||||
if (
|
||||
lowerMimeType.includes("application/octet-stream") &&
|
||||
lowerUrl.match(/\.(woff2?|ttf|otf|eot|css|js)(\?.*)?$/)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old cached resources to prevent memory leaks
|
||||
* @private
|
||||
*/
|
||||
private cleanupResourceCache(): void {
|
||||
const now = Date.now();
|
||||
const maxAge = 5 * 60 * 1000; // 5 minutes
|
||||
|
||||
for (const [url, resource] of this.networkResourceCache.entries()) {
|
||||
if (now - resource.timestamp > maxAge) {
|
||||
this.networkResourceCache.delete(url);
|
||||
}
|
||||
}
|
||||
|
||||
if (this.networkResourceCache.size > 200) {
|
||||
const entries = Array.from(this.networkResourceCache.entries());
|
||||
entries.sort((a, b) => a[1].timestamp - b[1].timestamp);
|
||||
|
||||
for (let i = 0; i < 50; i++) {
|
||||
this.networkResourceCache.delete(entries[i][0]);
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
`Resource cache cleaned up. Current size: ${this.networkResourceCache.size}`
|
||||
);
|
||||
}
|
||||
|
||||
private initializeMemoryManagement(): void {
|
||||
setInterval(() => {
|
||||
const memoryUsage = process.memoryUsage();
|
||||
|
||||
Reference in New Issue
Block a user