feat(engine): stale

scraper/src/scraper.js

@@ -1,31 +1,35 @@
-import playwright from 'playwright';
+import playwright from 'playwright'
 
-async function scraper(url, selectors) {
+fastify.post('/scrape', async (request, reply) => {
-const browser = await playwright.chromium.launch({
+    const { url, selectors } = request.body;
-    headless: false // setting this to true will not run the UI
+  
-});
+    try {
-  const page = await browser.newPage();
+      const browser = await playwright.chromium.launch({ headless: true }); // Launch headless browser
-
+      const page = await browser.newPage();
-  try {
+  
-    await page.goto(url);
+      try {
-    // Handle any required interactions (logins, captchas, etc.)
+        await page.goto(url);
-
+        // Handle any required interactions (logins, captchas, etc.)
-    const scrapedData = await page.evaluate((selectors) => {
+  
-      const data = [];
+        const scrapedData = await page.evaluate((selectors) => {
-      for (const selector of selectors) {
+          const data = [];
-        const elements = Array.from(document.querySelectorAll(selector));
+          for (const selector of selectors) {
-        const elementData = elements.map((el) => el.textContent);
+            const elements = Array.from(document.querySelectorAll(selector));
-        data.push(elementData);
+            const elementData = elements.map((el) => el.textContent);
+            data.push(elementData);
+          }
+          return data;
+        }, selectors);
+  
+        await browser.close();
+        reply.send(scrapedData);
+      } catch (error) {
+        console.error('Error scraping:', error);
+        await browser.close();
+        reply.status(500).send({ error: 'Failed to scrape data' });
       }
-      return data;
+    } catch (error) {
-    }, selectors);
+      console.error('Error launching browser:', error);
-
+      reply.status(500).send({ error: 'Failed to initiate scraping' });
-    await browser.close();
+    }
-    return scrapedData;
+  });
-  } catch (error) {
-    await browser.close();
-    throw error;
-  }
-}
-
-export default scraper;