From fb6ce1069312c95fa42c0327b90de669f73cf81b Mon Sep 17 00:00:00 2001
From: karishmas6 <carishmashukla28@gmail.com>
Date: Fri, 20 Sep 2024 23:35:49 +0530
Subject: [PATCH] feat: click load more

---
 maxun-core/src/interpret.ts | 38 +++++++++++++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 4 deletions(-)

diff --git a/maxun-core/src/interpret.ts b/maxun-core/src/interpret.ts
index 8a5b3788..38cf9cb0 100644
--- a/maxun-core/src/interpret.ts
+++ b/maxun-core/src/interpret.ts
@@ -447,11 +447,41 @@ export default class Interpreter extends EventEmitter {
           await page.waitForTimeout(1000);
           break;
         case 'clickLoadMore':
-          const loadMoreButton = await page.$(config.pagination.selector);
-          if (!loadMoreButton) {
-            return allResults;
+          while (true) {
+              // Find and click the 'Load More' button
+              const loadMoreButton = await page.$(config.pagination.selector);
+              if (!loadMoreButton) {
+                  // No more "Load More" button, so scrape the remaining items
+                  const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
+                  allResults = allResults.concat(finalResults);
+                  return allResults;
+              }
+      
+              // Click the 'Load More' button to load additional items
+              await loadMoreButton.click();
+              await page.waitForTimeout(2000); // Wait for new items to load
+      
+              // After clicking 'Load More', scroll down to load more items
+              await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+              await page.waitForTimeout(2000);
+      
+              // Check if more items are available
+              const currentHeight = await page.evaluate(() => document.body.scrollHeight);
+              if (currentHeight === previousHeight) {
+                  // No more items loaded, return the scraped results
+                  const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
+                  allResults = allResults.concat(finalResults);
+                  return allResults;
+              }
+      
+              previousHeight = currentHeight;
+      
+              if (config.limit && allResults.length >= config.limit) {
+                  // If limit is set and reached, return the limited results
+                  allResults = allResults.slice(0, config.limit);
+                  break;
+              }
           }
-          await loadMoreButton.click();
           break;
         default:
           const results = await page.evaluate((cfg) => window.scrapeList(cfg), config);