vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: initialize new set for unique items per page

Browse Source
This commit is contained in:
karishmas6
2024-08-19 18:18:30 +05:30
parent 1b3bfddcb1
commit a715c56afe
1 changed files with 33 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
maxun-core/src/interpret.ts
View File

@@ -372,9 +372,9 @@ export default class Interpreter extends EventEmitter {
private async handlePagination(page: Page, config: { listSelector: string, fields: any, limit?: number, pagination: any }) { private async handlePagination(page: Page, config: { listSelector: string, fields: any, limit?: number, pagination: any }) {
let allResults: Record<string, any>[] = []; let allResults: Record<string, any>[] = [];
let previousHeight = 0; let previousHeight = 0;
// track unique items to avoid re-scraping let currentPage = 1;
let scrapedItems: Set<string> = new Set(); // track unique items per page to avoid re-scraping
let currentPage = 1 let scrapedItemsPerPage: Set<string>[] = [];
while (true) { while (true) {
switch (config.pagination.type) { switch (config.pagination.type) {
@@ -395,29 +395,37 @@ export default class Interpreter extends EventEmitter {
break; break;
case 'clickNext': case 'clickNext':
const pageResults = await page.evaluate((cfg) => window.scrapeList(cfg), config); const pageResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
// filter out items that have already been scraped
const newResults = pageResults.filter(item => { // Initialize a new Set for the current page if it doesn't exist
const uniqueKey = JSON.stringify(item); if (!scrapedItemsPerPage[currentPage - 1]) {
if (scrapedItems.has(uniqueKey)) return false; scrapedItemsPerPage[currentPage - 1] = new Set<string>();
scrapedItems.add(uniqueKey); }
return true;
});
allResults = allResults.concat(newResults);
// if the limit is reached, return the required number of items
if (config.limit && allResults.length >= config.limit) {
return allResults.slice(0, config.limit);
}
const nextButton = await page.$(config.pagination.selector);
if (!nextButton) {
return allResults;
}
await Promise.all([
nextButton.click(),
page.waitForNavigation({ waitUntil: 'networkidle' })
]);
currentPage += 1; const newResults = pageResults.filter(item => {
break; const uniqueKey = JSON.stringify(item);
if (scrapedItemsPerPage[currentPage - 1].has(uniqueKey)) return false;
scrapedItemsPerPage[currentPage - 1].add(uniqueKey);
return true;
});
allResults = allResults.concat(newResults);
if (config.limit && allResults.length >= config.limit) {
return allResults.slice(0, config.limit);
}
const nextButton = await page.$(config.pagination.selector);
if (!nextButton) {
return allResults;
}
await Promise.all([
nextButton.click(),
page.waitForNavigation({ waitUntil: 'networkidle' })
]);
currentPage++;
break;
case 'clickLoadMore': case 'clickLoadMore':
const loadMoreButton = await page.$(config.pagination.selector); const loadMoreButton = await page.$(config.pagination.selector);
if (!loadMoreButton) { if (!loadMoreButton) {