vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: simplify process snapshot logic

Browse Source
This commit is contained in:
Karishma Shukla
2025-06-30 15:10:50 +05:30
committed by GitHub
parent 3ccf58cdce
commit a770d69094
1 changed files with 1 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

177
server/src/browser-management/classes/RemoteBrowser.ts
View File

@@ -488,182 +488,7 @@ export class RemoteBrowser {
scripts: [] as Array<{ src: string; content: string; type?: string }>, scripts: [] as Array<{ src: string; content: string; type?: string }>,
media: [] as Array<{ src: string; dataUrl: string; type: string }>, media: [] as Array<{ src: string; dataUrl: string; type: string }>,
}; };
const processNode = (node: RRWebSnapshot): RRWebSnapshot => {
const processedNode = { ...node };
// Process attributes if they exist
if (node.attributes) {
const newAttributes = { ...node.attributes };
// Process common attributes that contain URLs
const urlAttributes = ["src", "href", "data", "poster", "background"];
for (const attr of urlAttributes) {
if (newAttributes[attr]) {
const originalUrl = newAttributes[attr];
// Categorize and collect the resource instead of proxying
const lowerAttr = attr.toLowerCase();
const lowerUrl = originalUrl.toLowerCase();
if (lowerAttr === "src" && node.tagName?.toLowerCase() === "img") {
const cachedResource = this.networkResourceCache.get(originalUrl);
if (
cachedResource &&
cachedResource.mimeType.includes("image/")
) {
const dataUrl = cachedResource.base64Encoded
? `data:${cachedResource.mimeType};base64,${cachedResource.content}`
: `data:${cachedResource.mimeType};base64,${Buffer.from(
cachedResource.content,
"utf-8"
).toString("base64")}`;
resources.images.push({
src: originalUrl,
dataUrl,
alt: newAttributes.alt,
});
}
} else if (
lowerAttr === "href" &&
node.tagName?.toLowerCase() === "link"
) {
const rel = newAttributes.rel?.toLowerCase() || "";
if (rel.includes("stylesheet")) {
const cachedResource =
this.networkResourceCache.get(originalUrl);
if (cachedResource && cachedResource.mimeType.includes("css")) {
let content = cachedResource.base64Encoded
? Buffer.from(cachedResource.content, "base64").toString(
"utf-8"
)
: cachedResource.content;
// Process CSS to collect embedded resources
content = this.processCSS(
content,
originalUrl,
baseUrl,
resources
);
resources.stylesheets.push({
href: originalUrl,
content,
media: newAttributes.media || "all",
});
}
} else if (
rel.includes("font") ||
lowerUrl.match(/\.(woff2?|ttf|otf|eot)(\?.*)?$/i)
) {
const cachedResource =
this.networkResourceCache.get(originalUrl);
if (cachedResource) {
const dataUrl = cachedResource.base64Encoded
? `data:${cachedResource.mimeType};base64,${cachedResource.content}`
: `data:${cachedResource.mimeType};base64,${Buffer.from(
cachedResource.content,
"utf-8"
).toString("base64")}`;
resources.fonts.push({
url: originalUrl,
dataUrl,
format: lowerUrl.split(".").pop()?.split("?")[0],
});
}
}
}
}
}
// Process srcset attribute - collect resources but keep original URLs
if (newAttributes.srcset) {
const originalSrcset = newAttributes.srcset;
originalSrcset.split(",").forEach((srcsetItem) => {
const parts = srcsetItem.trim().split(/\s+/);
const url = parts[0];
if (url && !url.startsWith("data:") && !url.startsWith("blob:")) {
const cachedResource = this.networkResourceCache.get(url);
if (
cachedResource &&
cachedResource.mimeType.includes("image/")
) {
const dataUrl = cachedResource.base64Encoded
? `data:${cachedResource.mimeType};base64,${cachedResource.content}`
: `data:${cachedResource.mimeType};base64,${Buffer.from(
cachedResource.content,
"utf-8"
).toString("base64")}`;
resources.images.push({
src: url,
dataUrl,
alt: newAttributes.alt,
});
}
}
});
}
processedNode.attributes = newAttributes;
}
// Process text content for style elements
if (node.tagName?.toLowerCase() === "style" && node.textContent) {
let content = node.textContent;
// Process CSS content to collect embedded resources
content = this.processCSS(content, "", baseUrl, resources);
processedNode.textContent = content;
}
// Recursively process child nodes
if (node.childNodes) {
processedNode.childNodes = node.childNodes.map(processNode);
}
return processedNode;
};
const processedSnapshot = processNode(snapshot);
// Add cached scripts and media
for (const [url, resource] of this.networkResourceCache.entries()) {
if (resource.mimeType.toLowerCase().includes("javascript")) {
const content = resource.base64Encoded
? Buffer.from(resource.content, "base64").toString("utf-8")
: resource.content;
resources.scripts.push({
src: url,
content,
type: "text/javascript",
});
} else if (
resource.mimeType.toLowerCase().includes("video/") ||
resource.mimeType.toLowerCase().includes("audio/")
) {
const dataUrl = resource.base64Encoded
? `data:${resource.mimeType};base64,${resource.content}`
: `data:${resource.mimeType};base64,${Buffer.from(
resource.content,
"utf-8"
).toString("base64")}`;
resources.media.push({
src: url,
dataUrl,
type: resource.mimeType,
});
}
}
const viewport = (await this.currentPage?.viewportSize()) || { const viewport = (await this.currentPage?.viewportSize()) || {
width: 1280, width: 1280,
height: 720, height: 720,