vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #684 from getmaxun/allow-duplicates

Browse Source 
feat(maxun-core): remove max duplicate item logic
This commit is contained in:
Karishma Shukla
2025-07-16 01:06:28 +05:30
committed by GitHub
parent 6c2e2d7171 cfd3c174fd
commit fd7d0df386
1 changed files with 16 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
maxun-core/src/interpret.ts
View File

@@ -950,9 +950,9 @@ export default class Interpreter extends EventEmitter {
if (checkLimit()) return allResults; if (checkLimit()) return allResults;
let loadMoreCounter = 0; let loadMoreCounter = 0;
let previousResultCount = allResults.length; // let previousResultCount = allResults.length;
let noNewItemsCounter = 0; // let noNewItemsCounter = 0;
const MAX_NO_NEW_ITEMS = 2; // const MAX_NO_NEW_ITEMS = 2;
while (true) { while (true) {
// Find working button with retry mechanism // Find working button with retry mechanism
@@ -1019,21 +1019,21 @@ export default class Interpreter extends EventEmitter {
await scrapeCurrentPage(); await scrapeCurrentPage();
const currentResultCount = allResults.length; // const currentResultCount = allResults.length;
const newItemsAdded = currentResultCount > previousResultCount; // const newItemsAdded = currentResultCount > previousResultCount;
if (!newItemsAdded) { // if (!newItemsAdded) {
noNewItemsCounter++; // noNewItemsCounter++;
debugLog(`No new items added after click (${noNewItemsCounter}/${MAX_NO_NEW_ITEMS})`); // debugLog(`No new items added after click (${noNewItemsCounter}/${MAX_NO_NEW_ITEMS})`);
if (noNewItemsCounter >= MAX_NO_NEW_ITEMS) { // if (noNewItemsCounter >= MAX_NO_NEW_ITEMS) {
debugLog(`Stopping after ${MAX_NO_NEW_ITEMS} clicks with no new items`); // debugLog(`Stopping after ${MAX_NO_NEW_ITEMS} clicks with no new items`);
return allResults; // return allResults;
} // }
} else { // } else {
noNewItemsCounter = 0; // noNewItemsCounter = 0;
previousResultCount = currentResultCount; // previousResultCount = currentResultCount;
} // }
if (checkLimit()) return allResults; if (checkLimit()) return allResults;