feat: add programmatic click event for clickNext

This commit is contained in:
RohitR311
2024-12-09 20:13:46 +05:30
parent 308c527530
commit 386e7c9a98

View File

@@ -475,6 +475,8 @@ export default class Interpreter extends EventEmitter {
case 'clickNext':
const pageResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
// console.log("Page results:", pageResults);
// Filter out already scraped items
const newResults = pageResults.filter(item => {
const uniqueKey = JSON.stringify(item);
@@ -482,9 +484,9 @@ export default class Interpreter extends EventEmitter {
scrapedItems.add(uniqueKey); // Mark as scraped
return true;
});
allResults = allResults.concat(newResults);
if (config.limit && allResults.length >= config.limit) {
return allResults.slice(0, config.limit);
}
@@ -494,7 +496,7 @@ export default class Interpreter extends EventEmitter {
return allResults; // No more pages to scrape
}
await Promise.all([
nextButton.click(),
nextButton.dispatchEvent('click'),
page.waitForNavigation({ waitUntil: 'networkidle' })
]);