16 lines
461 B
TypeScript
16 lines
461 B
TypeScript
// For more information, see https://crawlee.dev/
|
|
import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';
|
|
|
|
import { router } from './routes.js';
|
|
|
|
const startUrls = ['https://crawlee.dev'];
|
|
|
|
const crawler = new PlaywrightCrawler({
|
|
// proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
|
|
requestHandler: router,
|
|
// Comment this option to scrape the full website.
|
|
maxRequestsPerCrawl: 20,
|
|
});
|
|
|
|
await crawler.run(startUrls);
|