vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: lint

Browse Source
This commit is contained in:
karishmas6
2024-08-16 23:43:51 +05:30
parent 87476c3e35
commit b2678759db
1 changed files with 84 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
maxun-core/src/browserSide/scraper.js
View File

@@ -200,7 +200,7 @@ async function clickNextPagination(selector, scrapedData, limit) {
throw new Error("No more items to load or pagination has ended.");
}
//return clicked; // Indicate whether pagination occurred
return clicked; // Indicate whether pagination occurred
}
}
@@ -355,7 +355,6 @@ async function clickNextPagination(selector, scrapedData, limit) {
for (const [label, { selector, attribute }] of Object.entries(fields)) {
const fieldElement = parent.querySelector(selector);
// Depending on the attribute specified, extract the data
if (fieldElement) {
if (attribute === 'innerText') {
record[label] = fieldElement.innerText.trim();
@@ -366,7 +365,6 @@ async function clickNextPagination(selector, scrapedData, limit) {
} else if (attribute === 'href') {
record[label] = fieldElement.href;
} else {
// Default to attribute retrieval
record[label] = fieldElement.getAttribute(attribute);
}
}
@@ -376,19 +374,25 @@ async function clickNextPagination(selector, scrapedData, limit) {
scrapedData.push(record);
}
// Check if we need to paginate
if (pagination && scrapedData.length < limit) {
let paginated = false;
switch (pagination.type) {
case 'scrollDown':
await scrollDownToLoadMore(listSelector, limit);
paginated = true;
break;
case 'scrollUp':
await scrollUpToLoadMore(listSelector, limit);
paginated = true;
break;
case 'clickNext':
await clickNextPagination(pagination.selector, scrapedData, limit);
paginated = await clickNextPagination(pagination.selector, scrapedData, limit);
break;
case 'clickLoadMore':
//await clickLoadMorePagination(pagination.selector);
//paginated = true;
break;
case 'none':
// No more items to load
@@ -397,7 +401,12 @@ async function clickNextPagination(selector, scrapedData, limit) {
console.warn("Unknown pagination type");
break;
}
if (paginated) {
await new Promise(resolve => setTimeout(resolve, 2000)); // Wait for content to load
} else {
break; // No further pagination needed
}
} else {
break; // No more items to load or no pagination
}