vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: track pages

Browse Source
This commit is contained in:
karishmas6
2024-08-19 02:47:41 +05:30
parent 56bd3c38ae
commit 1b3bfddcb1
1 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
maxun-core/src/interpret.ts
View File

@@ -374,6 +374,7 @@ export default class Interpreter extends EventEmitter {
let previousHeight = 0; let previousHeight = 0;
// track unique items to avoid re-scraping // track unique items to avoid re-scraping
let scrapedItems: Set<string> = new Set(); let scrapedItems: Set<string> = new Set();
let currentPage = 1
while (true) { while (true) {
switch (config.pagination.type) { switch (config.pagination.type) {
@@ -414,6 +415,8 @@ export default class Interpreter extends EventEmitter {
nextButton.click(), nextButton.click(),
page.waitForNavigation({ waitUntil: 'networkidle' }) page.waitForNavigation({ waitUntil: 'networkidle' })
]); ]);
currentPage += 1;
break; break;
case 'clickLoadMore': case 'clickLoadMore':
const loadMoreButton = await page.$(config.pagination.selector); const loadMoreButton = await page.$(config.pagination.selector);