vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: handle chained selectors for click load more pagination

Browse Source
This commit is contained in:
RohitR311
2025-01-10 12:49:14 +05:30
parent c7d6dea845
commit 73bbdeb6bb
1 changed files with 26 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
maxun-core/src/interpret.ts
View File

@@ -651,7 +651,25 @@ export default class Interpreter extends EventEmitter {
break;
case 'clickLoadMore':
while (true) {
const loadMoreButton = await page.$(config.pagination.selector);
let checkButton = null;
let workingSelector = null;
for (let i = 0; i < availableSelectors.length; i++) {
const selector = availableSelectors[i];
try {
// Wait for selector with a short timeout
checkButton = await page.waitForSelector(selector, { state: 'attached', timeout: 10000 });
if (checkButton) {
workingSelector = selector;
break;
}
} catch (error) {
console.log(`Selector failed: ${selector}`);
continue;
}
}
const loadMoreButton = await page.$(workingSelector);
if (!loadMoreButton) {
// No more "Load More" button, so scrape the remaining items
const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
@@ -659,8 +677,14 @@ export default class Interpreter extends EventEmitter {
return allResults;
}
// Click the 'Load More' button to load additional items
await loadMoreButton.dispatchEvent('click');
try {
await loadMoreButton.click();
} catch {
console.log('Regular click failed, trying dispatchEvent');
await loadMoreButton.dispatchEvent('click')
}
await page.waitForTimeout(2000); // Wait for new items to load
// After clicking 'Load More', scroll down to load more items
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
await page.waitForTimeout(2000);