chore: setup scraper engine
This commit is contained in:
15
scraper/src/main.ts
Normal file
15
scraper/src/main.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
// For more information, see https://crawlee.dev/
|
||||
import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';
|
||||
|
||||
import { router } from './routes.js';
|
||||
|
||||
const startUrls = ['https://crawlee.dev'];
|
||||
|
||||
const crawler = new PlaywrightCrawler({
|
||||
// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
|
||||
requestHandler: router,
|
||||
// Comment this option to scrape the full website.
|
||||
maxRequestsPerCrawl: 20,
|
||||
});
|
||||
|
||||
await crawler.run(startUrls);
|
||||
Reference in New Issue
Block a user