vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: lint

Browse Source
This commit is contained in:
karishmas6
2024-08-18 22:46:10 +05:30
parent 5d1747c874
commit 9a18f470f9
1 changed files with 8 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
maxun-core/src/interpret.ts
View File

@@ -384,7 +384,7 @@ export default class Interpreter extends EventEmitter {
// Check if new content was loaded // Check if new content was loaded
const currentHeight = await page.evaluate(() => document.body.scrollHeight); const currentHeight = await page.evaluate(() => document.body.scrollHeight);
if (currentHeight === previousHeight) { if (currentHeight === previousHeight) {
// No new content loaded, scrape final results and exit loop // No new content loaded, scrape final results and exit loop
const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config); const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
allResults = allResults.concat(finalResults); allResults = allResults.concat(finalResults);
return allResults; return allResults;
@@ -422,20 +422,16 @@ export default class Interpreter extends EventEmitter {
} }
// Wait a bit before next iteration to ensure content is loaded // Wait a bit before next iteration to ensure content is loaded
await page.waitForTimeout(1000); await page.waitForTimeout(1000);
// Check if new items were loaded // Scrape the current page after scrolling/clicking
const newItemsLoaded = await page.evaluate((prevCount, listSelector) => { const pageResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
const currentCount = document.querySelectorAll(listSelector).length; allResults = allResults.concat(pageResults);
return currentCount > prevCount;
}, allResults.length, config.listSelector);
if (!newItemsLoaded) { if (config.limit && allResults.length >= config.limit) {
return allResults; // No new items, end pagination allResults = allResults.slice(0, config.limit);
break;
} }
currentPage++;
await page.waitForTimeout(1000); // Wait for page to load
} }
return allResults; return allResults;