From b4cb0568fbe2695be8a7abd004d451885a733970 Mon Sep 17 00:00:00 2001
From: karishmas6 <carishmashukla28@gmail.com>
Date: Sat, 20 Apr 2024 00:47:50 +0530
Subject: [PATCH] feat: set persistStorage to false

---
 scraper/src/scraper.js | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/scraper/src/scraper.js b/scraper/src/scraper.js
index f9f93d76..bff62cfe 100644
--- a/scraper/src/scraper.js
+++ b/scraper/src/scraper.js
@@ -1,9 +1,10 @@
-import { PlaywrightCrawler } from 'crawlee';
+import { PlaywrightCrawler, Configuration } from 'crawlee';
 
 async function scrapeData(url, selectors, waitForSeconds = 2) {
     const scrapedData = [];
     const crawler = new PlaywrightCrawler({
-        requestHandler: async ({ page }) => {
+        requestHandler: async ({ page, request }) => {
+            console.log('Request object:', request)
             await page.goto(url);
   
             await page.waitForTimeout(waitForSeconds * 1000);
@@ -17,9 +18,14 @@ async function scrapeData(url, selectors, waitForSeconds = 2) {
   
             console.log('Scraped data:', scrapedData);
         },
-    });
-    await crawler.run([{ url }]);
+    },
+    new Configuration({
+        persistStorage: false,
+    }));
+
+    await crawler.run([ url ]);
     return scrapedData;
+   
 }
 
 export default scrapeData;
\ No newline at end of file