From bf292f05167cd6875fb68722f1adca1be8137a5c Mon Sep 17 00:00:00 2001
From: karishmas6 <carishmashukla28@gmail.com>
Date: Sun, 18 Aug 2024 22:18:03 +0530
Subject: [PATCH] feat: scroll donw on server side

---
 maxun-core/src/interpret.ts | 34 ++++++++++++++++++++++++++--------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/maxun-core/src/interpret.ts b/maxun-core/src/interpret.ts
index 4edc2131..ca273ac1 100644
--- a/maxun-core/src/interpret.ts
+++ b/maxun-core/src/interpret.ts
@@ -380,18 +380,36 @@ export default class Interpreter extends EventEmitter {
   
       switch (config.pagination.type) {
         case 'scrollDown':
-          await page.evaluate(() => window.scrollDown(config.listSelector, config.limit));
+          let previousHeight = 0
+          await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+        // Wait for potential lazy-loaded content
+        await page.waitForTimeout(2000);
+
+        // Check if new content was loaded
+        const currentHeight = await page.evaluate(() => document.body.scrollHeight);
+        if (currentHeight === previousHeight) {
+          // No new content loaded, exit loop
+          return allResults;
+        }
+        previousHeight = currentHeight;
           break;
         case 'scrollUp':
           await page.evaluate(() => window.scrollUp(config.listSelector, config.limit));
           break;
-        case 'clickNext':
-          const nextButton = await page.$(config.pagination.selector);
-          if (!nextButton) {
-            return allResults; // No more pages
-          }
-          await nextButton.click();
-          break;
+          case 'clickNext':
+            const nextButton = await page.$(config.pagination.selector);
+            if (!nextButton) {
+                return allResults; // No more pages
+            }
+
+            // Capture the current URL to check if it changes after clicking next
+            const currentURL = page.url();
+
+            await Promise.all([
+                nextButton.click(),
+                page.waitForNavigation({ waitUntil: 'load' }) // Wait for page navigation
+            ]);
+            break;
         case 'clickLoadMore':
           const loadMoreButton = await page.$(config.pagination.selector);
           if (!loadMoreButton) {