vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: scrape after scroll completion

Browse Source
This commit is contained in:
karishmas6
2024-08-18 22:32:39 +05:30
parent ccc68ceb09
commit 1c93e5430a
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
maxun-core/src/interpret.ts
View File

@@ -384,7 +384,10 @@ export default class Interpreter extends EventEmitter {
// Check if new content was loaded // Check if new content was loaded
const currentHeight = await page.evaluate(() => document.body.scrollHeight); const currentHeight = await page.evaluate(() => document.body.scrollHeight);
if (currentHeight === previousHeight) { if (currentHeight === previousHeight) {
// No new content loaded, exit loop // No new content loaded, scrape final results and exit loop
const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
allResults = allResults.concat(finalResults);
return allResults;
return allResults; return allResults;
} }
previousHeight = currentHeight; previousHeight = currentHeight;