vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: remove browser side scroll down

Browse Source
This commit is contained in:
karishmas6
2024-09-20 20:19:55 +05:30
parent 7ab668633c
commit e9f1ff0294
1 changed files with 0 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
maxun-core/src/browserSide/scraper.js
View File

@@ -126,25 +126,6 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
return out; return out;
} }
async function scrollDownToLoadMore(selector, limit) {
let previousHeight = 0;
let itemsLoaded = 0;
while (itemsLoaded < limit) {
window.scrollBy(0, window.innerHeight);
await new Promise(resolve => setTimeout(resolve, 1000));
const currentHeight = document.body.scrollHeight;
if (currentHeight === previousHeight) {
break; // No more items to load
}
previousHeight = currentHeight;
itemsLoaded += document.querySelectorAll(selector).length;
}
}
/** /**
* Returns a "scrape" result from the current page. * Returns a "scrape" result from the current page.
* @returns {Array<Object>} *Curated* array of scraped information (with sparse rows removed) * @returns {Array<Object>} *Curated* array of scraped information (with sparse rows removed)