fix: remove while loop
This commit is contained in:
@@ -374,7 +374,7 @@ export default class Interpreter extends EventEmitter {
|
|||||||
let previousHeight = 0;
|
let previousHeight = 0;
|
||||||
let currentPage = 1;
|
let currentPage = 1;
|
||||||
// track unique items per page to avoid re-scraping
|
// track unique items per page to avoid re-scraping
|
||||||
let scrapedItems: Set<string> = new Set<string>(); // Track unique items across all pages
|
let scrapedItems: Set<string> = new Set<string>();
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
switch (config.pagination.type) {
|
switch (config.pagination.type) {
|
||||||
@@ -394,7 +394,6 @@ export default class Interpreter extends EventEmitter {
|
|||||||
case 'scrollUp':
|
case 'scrollUp':
|
||||||
break;
|
break;
|
||||||
case 'clickNext':
|
case 'clickNext':
|
||||||
while (true) {
|
|
||||||
// Scrape the current page
|
// Scrape the current page
|
||||||
const pageResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
|
const pageResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
|
||||||
|
|
||||||
@@ -425,9 +424,10 @@ export default class Interpreter extends EventEmitter {
|
|||||||
page.waitForNavigation({ waitUntil: 'networkidle' })
|
page.waitForNavigation({ waitUntil: 'networkidle' })
|
||||||
]);
|
]);
|
||||||
|
|
||||||
currentPage++; // Increment page count and proceed
|
// Wait a bit for the content to load
|
||||||
|
await page.waitForTimeout(1000);
|
||||||
|
|
||||||
break;
|
break;
|
||||||
}
|
|
||||||
case 'clickLoadMore':
|
case 'clickLoadMore':
|
||||||
const loadMoreButton = await page.$(config.pagination.selector);
|
const loadMoreButton = await page.$(config.pagination.selector);
|
||||||
if (!loadMoreButton) {
|
if (!loadMoreButton) {
|
||||||
|
|||||||
Reference in New Issue
Block a user