fix: remove while loop

This commit is contained in:
karishmas6
2024-08-20 21:48:11 +05:30
parent e4b7ca5235
commit 8857de0b8f

View File

@@ -374,7 +374,7 @@ export default class Interpreter extends EventEmitter {
let previousHeight = 0; let previousHeight = 0;
let currentPage = 1; let currentPage = 1;
// track unique items per page to avoid re-scraping // track unique items per page to avoid re-scraping
let scrapedItems: Set<string> = new Set<string>(); // Track unique items across all pages let scrapedItems: Set<string> = new Set<string>();
while (true) { while (true) {
switch (config.pagination.type) { switch (config.pagination.type) {
@@ -394,7 +394,6 @@ export default class Interpreter extends EventEmitter {
case 'scrollUp': case 'scrollUp':
break; break;
case 'clickNext': case 'clickNext':
while (true) {
// Scrape the current page // Scrape the current page
const pageResults = await page.evaluate((cfg) => window.scrapeList(cfg), config); const pageResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
@@ -425,9 +424,10 @@ export default class Interpreter extends EventEmitter {
page.waitForNavigation({ waitUntil: 'networkidle' }) page.waitForNavigation({ waitUntil: 'networkidle' })
]); ]);
currentPage++; // Increment page count and proceed // Wait a bit for the content to load
await page.waitForTimeout(1000);
break; break;
}
case 'clickLoadMore': case 'clickLoadMore':
const loadMoreButton = await page.$(config.pagination.selector); const loadMoreButton = await page.$(config.pagination.selector);
if (!loadMoreButton) { if (!loadMoreButton) {