chore: lint

This commit is contained in:
karishmas6
2024-08-18 22:19:24 +05:30
parent 3771de92ca
commit 1260730e56

View File

@@ -329,7 +329,7 @@ export default class Interpreter extends EventEmitter {
const x = new AsyncFunction('page', 'log', code); const x = new AsyncFunction('page', 'log', code);
await x(page, this.log); await x(page, this.log);
}, },
flag: async () => new Promise((res) => { flag: async () => new Promise((res) => {
this.emit('flag', page, res); this.emit('flag', page, res);
}), }),
@@ -368,48 +368,48 @@ export default class Interpreter extends EventEmitter {
let allResults: Record<string, any>[] = []; let allResults: Record<string, any>[] = [];
let currentPage = 1; let currentPage = 1;
let previousHeight = 0 let previousHeight = 0
while (true) { while (true) {
// Scrape current page // Scrape current page
const pageResults = await page.evaluate((cfg) => window.scrapeList(cfg), config); const pageResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
allResults = allResults.concat(pageResults); allResults = allResults.concat(pageResults);
if (config.limit && allResults.length >= config.limit) { if (config.limit && allResults.length >= config.limit) {
allResults = allResults.slice(0, config.limit); allResults = allResults.slice(0, config.limit);
break; break;
} }
switch (config.pagination.type) { switch (config.pagination.type) {
case 'scrollDown': case 'scrollDown':
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight)); await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
// Wait for potential lazy-loaded content // Wait for potential lazy-loaded content
await page.waitForTimeout(2000); await page.waitForTimeout(2000);
// Check if new content was loaded // Check if new content was loaded
const currentHeight = await page.evaluate(() => document.body.scrollHeight); const currentHeight = await page.evaluate(() => document.body.scrollHeight);
if (currentHeight === previousHeight) { if (currentHeight === previousHeight) {
// No new content loaded, exit loop // No new content loaded, exit loop
return allResults; return allResults;
} }
previousHeight = currentHeight; previousHeight = currentHeight;
break; break;
case 'scrollUp': case 'scrollUp':
await page.evaluate(() => window.scrollUp(config.listSelector, config.limit)); await page.evaluate(() => window.scrollUp(config.listSelector, config.limit));
break; break;
case 'clickNext': case 'clickNext':
const nextButton = await page.$(config.pagination.selector); const nextButton = await page.$(config.pagination.selector);
if (!nextButton) { if (!nextButton) {
return allResults; // No more pages return allResults; // No more pages
} }
// Capture the current URL to check if it changes after clicking next // Capture the current URL to check if it changes after clicking next
const currentURL = page.url(); const currentURL = page.url();
await Promise.all([ await Promise.all([
nextButton.click(), nextButton.click(),
page.waitForNavigation({ waitUntil: 'load' }) // Wait for page navigation page.waitForNavigation({ waitUntil: 'load' }) // Wait for page navigation
]); ]);
break; break;
case 'clickLoadMore': case 'clickLoadMore':
const loadMoreButton = await page.$(config.pagination.selector); const loadMoreButton = await page.$(config.pagination.selector);
if (!loadMoreButton) { if (!loadMoreButton) {
@@ -420,21 +420,21 @@ export default class Interpreter extends EventEmitter {
default: default:
return allResults; // No pagination or unknown type return allResults; // No pagination or unknown type
} }
// Check if new items were loaded // Check if new items were loaded
const newItemsLoaded = await page.evaluate((prevCount, listSelector) => { const newItemsLoaded = await page.evaluate((prevCount, listSelector) => {
const currentCount = document.querySelectorAll(listSelector).length; const currentCount = document.querySelectorAll(listSelector).length;
return currentCount > prevCount; return currentCount > prevCount;
}, allResults.length, config.listSelector); }, allResults.length, config.listSelector);
if (!newItemsLoaded) { if (!newItemsLoaded) {
return allResults; // No new items, end pagination return allResults; // No new items, end pagination
} }
currentPage++; currentPage++;
await page.waitForTimeout(1000); // Wait for page to load await page.waitForTimeout(1000); // Wait for page to load
} }
return allResults; return allResults;
} }