abort scraping when too many elements (#3678)

This commit is contained in:
LawyZheng
2025-10-10 15:46:55 +08:00
committed by GitHub
parent 7cd92f6972
commit e08778993e

View File

@@ -1533,9 +1533,13 @@ async function buildTreeFromBody(
) { ) {
window.GlobalSkyvernFrameIndex = frame_index; window.GlobalSkyvernFrameIndex = frame_index;
} }
const maxElementNumber = 15000;
const elementsAndResultArray = await buildElementTree( const elementsAndResultArray = await buildElementTree(
document.documentElement, document.documentElement,
frame, frame,
false,
undefined,
maxElementNumber,
); );
DomUtils.elementListCache = elementsAndResultArray[0]; DomUtils.elementListCache = elementsAndResultArray[0];
return elementsAndResultArray; return elementsAndResultArray;
@@ -1546,6 +1550,7 @@ async function buildElementTree(
frame, frame,
full_tree = false, full_tree = false,
hoverStylesMap = undefined, hoverStylesMap = undefined,
maxElementNumber = 0,
) { ) {
// Generate hover styles map at the start // Generate hover styles map at the start
if (hoverStylesMap === undefined) { if (hoverStylesMap === undefined) {
@@ -1570,6 +1575,13 @@ async function buildElementTree(
return; return;
} }
if (maxElementNumber > 0 && elements.length >= maxElementNumber) {
_jsConsoleWarn(
"Max element number reached, aborting the element tree building",
);
return;
}
const tagName = element.tagName?.toLowerCase(); const tagName = element.tagName?.toLowerCase();
if (!tagName) { if (!tagName) {
_jsConsoleLog("get a null tagName"); _jsConsoleLog("get a null tagName");