vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add programmatic click event for clickNext

Browse Source
This commit is contained in:
RohitR311
2024-12-09 20:13:46 +05:30
parent 308c527530
commit 386e7c9a98
1 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
maxun-core/src/interpret.ts
View File

@@ -475,6 +475,8 @@ export default class Interpreter extends EventEmitter {
case 'clickNext': case 'clickNext':
const pageResults = await page.evaluate((cfg) => window.scrapeList(cfg), config); const pageResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
// console.log("Page results:", pageResults);
// Filter out already scraped items // Filter out already scraped items
const newResults = pageResults.filter(item => { const newResults = pageResults.filter(item => {
const uniqueKey = JSON.stringify(item); const uniqueKey = JSON.stringify(item);
@@ -482,9 +484,9 @@ export default class Interpreter extends EventEmitter {
scrapedItems.add(uniqueKey); // Mark as scraped scrapedItems.add(uniqueKey); // Mark as scraped
return true; return true;
}); });
allResults = allResults.concat(newResults); allResults = allResults.concat(newResults);
if (config.limit && allResults.length >= config.limit) { if (config.limit && allResults.length >= config.limit) {
return allResults.slice(0, config.limit); return allResults.slice(0, config.limit);
} }
@@ -494,7 +496,7 @@ export default class Interpreter extends EventEmitter {
return allResults; // No more pages to scrape return allResults; // No more pages to scrape
} }
await Promise.all([ await Promise.all([
nextButton.click(), nextButton.dispatchEvent('click'),
page.waitForNavigation({ waitUntil: 'networkidle' }) page.waitForNavigation({ waitUntil: 'networkidle' })
]); ]);