fix: return scrapedData after crawlee.run

This commit is contained in:
karishmas6
2024-04-19 03:21:35 +05:30
parent 0bc497b933
commit 20b40562b5

View File

@@ -1,28 +1,25 @@
import { PlaywrightCrawler } from 'crawlee';
async function scrapeData(url, selectors, waitForSeconds = 2) {
const scrapedData = [];
const crawler = new PlaywrightCrawler({
requestHandler: async ({ page }) => {
await page.goto(url);
// Wait for specific time (optional)
await page.waitForTimeout(waitForSeconds * 1000);
console.log('Received selectors:', selectors);
const scrapedData = [];
for (const selector of selectors) {
const elementData = await page.$$eval(selector, elements => elements.map(el => el.textContent.trim()));
scrapedData.push(...elementData);
}
console.log('Scraped data:', scrapedData);
return { data: scrapedData };
},
});
await crawler.run([{ url }]);
return scrapedData;
}
export default scrapeData
export default scrapeData;