abort scraping when too many elements (#3678)
This commit is contained in:
@@ -1533,9 +1533,13 @@ async function buildTreeFromBody(
|
|||||||
) {
|
) {
|
||||||
window.GlobalSkyvernFrameIndex = frame_index;
|
window.GlobalSkyvernFrameIndex = frame_index;
|
||||||
}
|
}
|
||||||
|
const maxElementNumber = 15000;
|
||||||
const elementsAndResultArray = await buildElementTree(
|
const elementsAndResultArray = await buildElementTree(
|
||||||
document.documentElement,
|
document.documentElement,
|
||||||
frame,
|
frame,
|
||||||
|
false,
|
||||||
|
undefined,
|
||||||
|
maxElementNumber,
|
||||||
);
|
);
|
||||||
DomUtils.elementListCache = elementsAndResultArray[0];
|
DomUtils.elementListCache = elementsAndResultArray[0];
|
||||||
return elementsAndResultArray;
|
return elementsAndResultArray;
|
||||||
@@ -1546,6 +1550,7 @@ async function buildElementTree(
|
|||||||
frame,
|
frame,
|
||||||
full_tree = false,
|
full_tree = false,
|
||||||
hoverStylesMap = undefined,
|
hoverStylesMap = undefined,
|
||||||
|
maxElementNumber = 0,
|
||||||
) {
|
) {
|
||||||
// Generate hover styles map at the start
|
// Generate hover styles map at the start
|
||||||
if (hoverStylesMap === undefined) {
|
if (hoverStylesMap === undefined) {
|
||||||
@@ -1570,6 +1575,13 @@ async function buildElementTree(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (maxElementNumber > 0 && elements.length >= maxElementNumber) {
|
||||||
|
_jsConsoleWarn(
|
||||||
|
"Max element number reached, aborting the element tree building",
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const tagName = element.tagName?.toLowerCase();
|
const tagName = element.tagName?.toLowerCase();
|
||||||
if (!tagName) {
|
if (!tagName) {
|
||||||
_jsConsoleLog("get a null tagName");
|
_jsConsoleLog("get a null tagName");
|
||||||
|
|||||||
Reference in New Issue
Block a user