vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: set persistStorage to false

Browse Source
This commit is contained in:
karishmas6
2024-04-20 00:47:50 +05:30
parent e385142428
commit b4cb0568fb
1 changed files with 10 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
scraper/src/scraper.js
View File

@@ -1,9 +1,10 @@
import { PlaywrightCrawler } from 'crawlee'; import { PlaywrightCrawler, Configuration } from 'crawlee';
async function scrapeData(url, selectors, waitForSeconds = 2) { async function scrapeData(url, selectors, waitForSeconds = 2) {
const scrapedData = []; const scrapedData = [];
const crawler = new PlaywrightCrawler({ const crawler = new PlaywrightCrawler({
requestHandler: async ({ page }) => { requestHandler: async ({ page, request }) => {
console.log('Request object:', request)
await page.goto(url); await page.goto(url);
await page.waitForTimeout(waitForSeconds * 1000); await page.waitForTimeout(waitForSeconds * 1000);
@@ -17,9 +18,14 @@ async function scrapeData(url, selectors, waitForSeconds = 2) {
console.log('Scraped data:', scrapedData); console.log('Scraped data:', scrapedData);
}, },
}); },
await crawler.run([{ url }]); new Configuration({
persistStorage: false,
}));
await crawler.run([ url ]);
return scrapedData; return scrapedData;
} }
export default scrapeData; export default scrapeData;