vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: handle chain selector logic for click load more pagination

Browse Source
This commit is contained in:
RohitR311
2025-01-11 22:53:39 +05:30
parent 681aa8045e
commit 03e3098b58
1 changed files with 15 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
maxun-core/src/interpret.ts
View File

@@ -611,10 +611,13 @@ export default class Interpreter extends EventEmitter {
} }
} catch (error) { } catch (error) {
console.log(`Selector failed: ${selector}`); console.log(`Selector failed: ${selector}`);
continue;
} }
} }
if (!workingSelector) {
return allResults;
}
// const nextButton = await page.$(config.pagination.selector); // const nextButton = await page.$(config.pagination.selector);
const nextButton = await page.$(workingSelector); const nextButton = await page.$(workingSelector);
if (!nextButton) { if (!nextButton) {
@@ -673,10 +676,16 @@ export default class Interpreter extends EventEmitter {
} }
} catch (error) { } catch (error) {
console.log(`Selector failed: ${selector}`); console.log(`Selector failed: ${selector}`);
continue;
} }
} }
if (!workingSelector) {
// No more working selectors available, so scrape the remaining items
const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
allResults = allResults.concat(finalResults);
return allResults;
}
const loadMoreButton = await page.$(workingSelector); const loadMoreButton = await page.$(workingSelector);
if (!loadMoreButton) { if (!loadMoreButton) {
// No more "Load More" button, so scrape the remaining items // No more "Load More" button, so scrape the remaining items
@@ -687,8 +696,9 @@ export default class Interpreter extends EventEmitter {
const selectorIndex = availableSelectors.indexOf(workingSelector!); const selectorIndex = availableSelectors.indexOf(workingSelector!);
availableSelectors = availableSelectors.slice(selectorIndex); availableSelectors = availableSelectors.slice(selectorIndex);
// Click the 'Load More' button to load additional items // Click the 'Load More' button to load additional items
// await loadMoreButton.dispatchEvent('click');
try { try {
await Promise.race([ await Promise.race([
loadMoreButton.click(), loadMoreButton.click(),
@@ -696,13 +706,12 @@ export default class Interpreter extends EventEmitter {
]); ]);
} catch (error) { } catch (error) {
console.log('Both click attempts failed'); console.log('Both click attempts failed');
continue;
} }
await page.waitForTimeout(2000); // Wait for new items to load await page.waitForTimeout(2000); // Wait for new items to load
// After clicking 'Load More', scroll down to load more items // After clicking 'Load More', scroll down to load more items
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight)); await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
await page.waitForTimeout(2000); await page.waitForTimeout(2000);
// Check if more items are available // Check if more items are available
const currentHeight = await page.evaluate(() => document.body.scrollHeight); const currentHeight = await page.evaluate(() => document.body.scrollHeight);
if (currentHeight === previousHeight) { if (currentHeight === previousHeight) {
@@ -712,8 +721,7 @@ export default class Interpreter extends EventEmitter {
return allResults; return allResults;
} }
previousHeight = currentHeight; previousHeight = currentHeight;
console.log("Results so far:", allResults.length);
if (config.limit && allResults.length >= config.limit) { if (config.limit && allResults.length >= config.limit) {
// If limit is set and reached, return the limited results // If limit is set and reached, return the limited results
allResults = allResults.slice(0, config.limit); allResults = allResults.slice(0, config.limit);