chore: setup scraper engine

scraper/src/main.ts

@@ -0,0 +1,15 @@
+// For more information, see https://crawlee.dev/
+import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';
+
+import { router } from './routes.js';
+
+const startUrls = ['https://crawlee.dev'];
+
+const crawler = new PlaywrightCrawler({
+    // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
+    requestHandler: router,
+    // Comment this option to scrape the full website.
+    maxRequestsPerCrawl: 20,
+});
+
+await crawler.run(startUrls);

scraper/src/routes.ts

@@ -0,0 +1,21 @@
+import { createPlaywrightRouter } from 'crawlee';
+
+export const router = createPlaywrightRouter();
+
+router.addDefaultHandler(async ({ enqueueLinks, log }) => {
+    log.info(`enqueueing new URLs`);
+    await enqueueLinks({
+        globs: ['https://crawlee.dev/**'],
+        label: 'detail',
+    });
+});
+
+router.addHandler('detail', async ({ request, page, log, pushData }) => {
+    const title = await page.title();
+    log.info(`${title}`, { url: request.loadedUrl });
+
+    await pushData({
+        url: request.loadedUrl,
+        title,
+    });
+});