chore: lint
This commit is contained in:
@@ -329,7 +329,7 @@ export default class Interpreter extends EventEmitter {
|
|||||||
const x = new AsyncFunction('page', 'log', code);
|
const x = new AsyncFunction('page', 'log', code);
|
||||||
await x(page, this.log);
|
await x(page, this.log);
|
||||||
},
|
},
|
||||||
|
|
||||||
flag: async () => new Promise((res) => {
|
flag: async () => new Promise((res) => {
|
||||||
this.emit('flag', page, res);
|
this.emit('flag', page, res);
|
||||||
}),
|
}),
|
||||||
@@ -368,48 +368,48 @@ export default class Interpreter extends EventEmitter {
|
|||||||
let allResults: Record<string, any>[] = [];
|
let allResults: Record<string, any>[] = [];
|
||||||
let currentPage = 1;
|
let currentPage = 1;
|
||||||
let previousHeight = 0
|
let previousHeight = 0
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
// Scrape current page
|
// Scrape current page
|
||||||
const pageResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
|
const pageResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
|
||||||
allResults = allResults.concat(pageResults);
|
allResults = allResults.concat(pageResults);
|
||||||
|
|
||||||
if (config.limit && allResults.length >= config.limit) {
|
if (config.limit && allResults.length >= config.limit) {
|
||||||
allResults = allResults.slice(0, config.limit);
|
allResults = allResults.slice(0, config.limit);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
switch (config.pagination.type) {
|
switch (config.pagination.type) {
|
||||||
case 'scrollDown':
|
case 'scrollDown':
|
||||||
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
||||||
// Wait for potential lazy-loaded content
|
// Wait for potential lazy-loaded content
|
||||||
await page.waitForTimeout(2000);
|
await page.waitForTimeout(2000);
|
||||||
|
|
||||||
// Check if new content was loaded
|
// Check if new content was loaded
|
||||||
const currentHeight = await page.evaluate(() => document.body.scrollHeight);
|
const currentHeight = await page.evaluate(() => document.body.scrollHeight);
|
||||||
if (currentHeight === previousHeight) {
|
if (currentHeight === previousHeight) {
|
||||||
// No new content loaded, exit loop
|
// No new content loaded, exit loop
|
||||||
return allResults;
|
return allResults;
|
||||||
}
|
}
|
||||||
previousHeight = currentHeight;
|
previousHeight = currentHeight;
|
||||||
break;
|
break;
|
||||||
case 'scrollUp':
|
case 'scrollUp':
|
||||||
await page.evaluate(() => window.scrollUp(config.listSelector, config.limit));
|
await page.evaluate(() => window.scrollUp(config.listSelector, config.limit));
|
||||||
break;
|
break;
|
||||||
case 'clickNext':
|
case 'clickNext':
|
||||||
const nextButton = await page.$(config.pagination.selector);
|
const nextButton = await page.$(config.pagination.selector);
|
||||||
if (!nextButton) {
|
if (!nextButton) {
|
||||||
return allResults; // No more pages
|
return allResults; // No more pages
|
||||||
}
|
}
|
||||||
|
|
||||||
// Capture the current URL to check if it changes after clicking next
|
// Capture the current URL to check if it changes after clicking next
|
||||||
const currentURL = page.url();
|
const currentURL = page.url();
|
||||||
|
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
nextButton.click(),
|
nextButton.click(),
|
||||||
page.waitForNavigation({ waitUntil: 'load' }) // Wait for page navigation
|
page.waitForNavigation({ waitUntil: 'load' }) // Wait for page navigation
|
||||||
]);
|
]);
|
||||||
break;
|
break;
|
||||||
case 'clickLoadMore':
|
case 'clickLoadMore':
|
||||||
const loadMoreButton = await page.$(config.pagination.selector);
|
const loadMoreButton = await page.$(config.pagination.selector);
|
||||||
if (!loadMoreButton) {
|
if (!loadMoreButton) {
|
||||||
@@ -420,21 +420,21 @@ export default class Interpreter extends EventEmitter {
|
|||||||
default:
|
default:
|
||||||
return allResults; // No pagination or unknown type
|
return allResults; // No pagination or unknown type
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if new items were loaded
|
// Check if new items were loaded
|
||||||
const newItemsLoaded = await page.evaluate((prevCount, listSelector) => {
|
const newItemsLoaded = await page.evaluate((prevCount, listSelector) => {
|
||||||
const currentCount = document.querySelectorAll(listSelector).length;
|
const currentCount = document.querySelectorAll(listSelector).length;
|
||||||
return currentCount > prevCount;
|
return currentCount > prevCount;
|
||||||
}, allResults.length, config.listSelector);
|
}, allResults.length, config.listSelector);
|
||||||
|
|
||||||
if (!newItemsLoaded) {
|
if (!newItemsLoaded) {
|
||||||
return allResults; // No new items, end pagination
|
return allResults; // No new items, end pagination
|
||||||
}
|
}
|
||||||
|
|
||||||
currentPage++;
|
currentPage++;
|
||||||
await page.waitForTimeout(1000); // Wait for page to load
|
await page.waitForTimeout(1000); // Wait for page to load
|
||||||
}
|
}
|
||||||
|
|
||||||
return allResults;
|
return allResults;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user