From fa961c5f034aef3b4e46db2bc362aca857d0ca07 Mon Sep 17 00:00:00 2001 From: Rohit Rajan Date: Fri, 21 Nov 2025 13:21:18 +0530 Subject: [PATCH 01/22] feat: reuse existing page instance --- server/src/api/record.ts | 24 +++--- server/src/markdownify/scrape.ts | 84 ++++++++++++++----- server/src/pgboss-worker.ts | 54 ++++++------ .../workflow-management/scheduler/index.ts | 24 +++--- 4 files changed, 116 insertions(+), 70 deletions(-) diff --git a/server/src/api/record.ts b/server/src/api/record.ts index 7c665001..3d95e1c7 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -662,6 +662,16 @@ async function executeRun(id: string, userId: string, requestedFormats?: string[ }; } + browser = browserPool.getRemoteBrowser(plainRun.browserId); + if (!browser) { + throw new Error('Could not access browser'); + } + + let currentPage = await browser.getCurrentPage(); + if (!currentPage) { + throw new Error('Could not create a new page'); + } + if (recording.recording_meta.type === 'scrape') { logger.log('info', `Executing scrape robot for API run ${id}`); @@ -690,13 +700,13 @@ async function executeRun(id: string, userId: string, requestedFormats?: string[ // Markdown conversion if (formats.includes('markdown')) { - markdown = await convertPageToMarkdown(url); + markdown = await convertPageToMarkdown(url, currentPage); serializableOutput.markdown = [{ content: markdown }]; } // HTML conversion if (formats.includes('html')) { - html = await convertPageToHTML(url); + html = await convertPageToHTML(url, currentPage); serializableOutput.html = [{ content: html }]; } @@ -824,16 +834,6 @@ async function executeRun(id: string, userId: string, requestedFormats?: string[ plainRun.status = 'running'; - browser = browserPool.getRemoteBrowser(plainRun.browserId); - if (!browser) { - throw new Error('Could not access browser'); - } - - let currentPage = await browser.getCurrentPage(); - if (!currentPage) { - throw new Error('Could not create a new page'); - } - const workflow = AddGeneratedFlags(recording.recording); browser.interpreter.setRunId(plainRun.runId); diff --git a/server/src/markdownify/scrape.ts b/server/src/markdownify/scrape.ts index 6821bfdb..f78bba80 100644 --- a/server/src/markdownify/scrape.ts +++ b/server/src/markdownify/scrape.ts @@ -1,17 +1,46 @@ -import { chromium } from "playwright"; +import { chromium, Page } from "playwright"; import { parseMarkdown } from "./markdown"; +import logger from "../logger"; + +async function gotoWithFallback(page: any, url: string) { + try { + return await page.goto(url, { + waitUntil: "networkidle", + timeout: 100000, + }); + } catch (err) { + // fallback: JS-heavy or unstable sites + return await page.goto(url, { + waitUntil: "domcontentloaded", + timeout: 100000, + }); + } +} /** * Fetches a webpage, strips scripts/styles/images/etc, * returns clean Markdown using parser. + * @param url - The URL to convert + * @param existingPage - Optional existing Playwright page instance to reuse */ -export async function convertPageToMarkdown(url: string): Promise { - const browser = await chromium.launch(); - const page = await browser.newPage(); +export async function convertPageToMarkdown(url: string, existingPage?: Page): Promise { + let browser: any = null; + let page: Page; + let shouldCloseBrowser = false; - await page.goto(url, { waitUntil: "networkidle", timeout: 100000 }); + if (existingPage) { + logger.log('info', `[Scrape] Reusing existing Playwright page instance for markdown conversion of ${url}`); + page = existingPage; + } else { + logger.log('info', `[Scrape] Creating new Chromium browser instance for markdown conversion of ${url}`); + browser = await chromium.launch(); + page = await browser.newPage(); + shouldCloseBrowser = true; + } - await page.addInitScript(() => { + await gotoWithFallback(page, url); + + const cleanedHtml = await page.evaluate(() => { const selectors = [ "script", "style", @@ -42,14 +71,16 @@ export async function convertPageToMarkdown(url: string): Promise { } }); }); - }); - // Re-extract HTML after cleanup - const cleanedHtml = await page.evaluate(() => { return document.documentElement.outerHTML; }); - await browser.close(); + if (shouldCloseBrowser && browser) { + logger.log('info', `[Scrape] Closing browser instance created for markdown conversion`); + await browser.close(); + } else { + logger.log('info', `[Scrape] Keeping existing browser instance open after markdown conversion`); + } // Convert cleaned HTML → Markdown const markdown = await parseMarkdown(cleanedHtml, url); @@ -59,14 +90,27 @@ export async function convertPageToMarkdown(url: string): Promise { /** * Fetches a webpage, strips scripts/styles/images/etc, * returns clean HTML. + * @param url - The URL to convert + * @param existingPage - Optional existing Playwright page instance to reuse */ -export async function convertPageToHTML(url: string): Promise { - const browser = await chromium.launch(); - const page = await browser.newPage(); +export async function convertPageToHTML(url: string, existingPage?: Page): Promise { + let browser: any = null; + let page: Page; + let shouldCloseBrowser = false; - await page.goto(url, { waitUntil: "networkidle", timeout: 100000 }); + if (existingPage) { + logger.log('info', `[Scrape] Reusing existing Playwright page instance for HTML conversion of ${url}`); + page = existingPage; + } else { + logger.log('info', `[Scrape] Creating new Chromium browser instance for HTML conversion of ${url}`); + browser = await chromium.launch(); + page = await browser.newPage(); + shouldCloseBrowser = true; + } - await page.addInitScript(() => { + await gotoWithFallback(page, url); + + const cleanedHtml = await page.evaluate(() => { const selectors = [ "script", "style", @@ -97,14 +141,16 @@ export async function convertPageToHTML(url: string): Promise { } }); }); - }); - // Re-extract HTML after cleanup - const cleanedHtml = await page.evaluate(() => { return document.documentElement.outerHTML; }); - await browser.close(); + if (shouldCloseBrowser && browser) { + logger.log('info', `[Scrape] Closing browser instance created for HTML conversion`); + await browser.close(); + } else { + logger.log('info', `[Scrape] Keeping existing browser instance open after HTML conversion`); + } // Return cleaned HTML directly return cleanedHtml; diff --git a/server/src/pgboss-worker.ts b/server/src/pgboss-worker.ts index f5d719b4..415eea91 100644 --- a/server/src/pgboss-worker.ts +++ b/server/src/pgboss-worker.ts @@ -181,7 +181,7 @@ async function processRunExecution(job: Job) { logger.log('info', `Browser ${browserId} found and ready for execution`); - try { + try { // Find the recording const recording = await Robot.findOne({ where: { 'recording_meta.id': plainRun.robotMetaId }, raw: true }); @@ -189,6 +189,30 @@ async function processRunExecution(job: Job) { throw new Error(`Recording for run ${data.runId} not found`); } + let currentPage = browser.getCurrentPage(); + + const pageWaitStart = Date.now(); + let lastPageLogTime = 0; + let pageAttempts = 0; + const MAX_PAGE_ATTEMPTS = 15; + + while (!currentPage && (Date.now() - pageWaitStart) < BROWSER_PAGE_TIMEOUT && pageAttempts < MAX_PAGE_ATTEMPTS) { + const currentTime = Date.now(); + pageAttempts++; + + if (currentTime - lastPageLogTime > 5000) { + logger.log('info', `Page not ready for browser ${browserId}, waiting... (${Math.round((currentTime - pageWaitStart) / 1000)}s elapsed)`); + lastPageLogTime = currentTime; + } + + await new Promise(resolve => setTimeout(resolve, 1000)); + currentPage = browser.getCurrentPage(); + } + + if (!currentPage) { + throw new Error(`No current page available for browser ${browserId} after ${BROWSER_PAGE_TIMEOUT/1000}s timeout`); + } + if (recording.recording_meta.type === 'scrape') { logger.log('info', `Executing scrape robot for run ${data.runId}`); @@ -212,13 +236,13 @@ async function processRunExecution(job: Job) { // Markdown conversion if (formats.includes('markdown')) { - markdown = await convertPageToMarkdown(url); + markdown = await convertPageToMarkdown(url, currentPage); serializableOutput.markdown = [{ content: markdown }]; } // HTML conversion if (formats.includes('html')) { - html = await convertPageToHTML(url); + html = await convertPageToHTML(url, currentPage); serializableOutput.html = [{ content: html }]; } @@ -328,30 +352,6 @@ async function processRunExecution(job: Job) { } }; - let currentPage = browser.getCurrentPage(); - - const pageWaitStart = Date.now(); - let lastPageLogTime = 0; - let pageAttempts = 0; - const MAX_PAGE_ATTEMPTS = 15; - - while (!currentPage && (Date.now() - pageWaitStart) < BROWSER_PAGE_TIMEOUT && pageAttempts < MAX_PAGE_ATTEMPTS) { - const currentTime = Date.now(); - pageAttempts++; - - if (currentTime - lastPageLogTime > 5000) { - logger.log('info', `Page not ready for browser ${browserId}, waiting... (${Math.round((currentTime - pageWaitStart) / 1000)}s elapsed)`); - lastPageLogTime = currentTime; - } - - await new Promise(resolve => setTimeout(resolve, 1000)); - currentPage = browser.getCurrentPage(); - } - - if (!currentPage) { - throw new Error(`No current page available for browser ${browserId} after ${BROWSER_PAGE_TIMEOUT/1000}s timeout`); - } - logger.log('info', `Starting workflow execution for run ${data.runId}`); await run.update({ diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 470cdacb..95d366ae 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -208,6 +208,16 @@ async function executeRun(id: string, userId: string) { } } + browser = browserPool.getRemoteBrowser(plainRun.browserId); + if (!browser) { + throw new Error('Could not access browser'); + } + + let currentPage = await browser.getCurrentPage(); + if (!currentPage) { + throw new Error('Could not create a new page'); + } + if (recording.recording_meta.type === 'scrape') { logger.log('info', `Executing scrape robot for scheduled run ${id}`); @@ -252,13 +262,13 @@ async function executeRun(id: string, userId: string) { // Markdown conversion if (formats.includes('markdown')) { - markdown = await convertPageToMarkdown(url); + markdown = await convertPageToMarkdown(url, currentPage); serializableOutput.markdown = [{ content: markdown }]; } // HTML conversion if (formats.includes('html')) { - html = await convertPageToHTML(url); + html = await convertPageToHTML(url, currentPage); serializableOutput.html = [{ content: html }]; } @@ -391,16 +401,6 @@ async function executeRun(id: string, userId: string) { logger.log('warn', `Failed to send run-started notification for run ${plainRun.runId}: ${socketError.message}`); } - browser = browserPool.getRemoteBrowser(plainRun.browserId); - if (!browser) { - throw new Error('Could not access browser'); - } - - let currentPage = await browser.getCurrentPage(); - if (!currentPage) { - throw new Error('Could not create a new page'); - } - const workflow = AddGeneratedFlags(recording.recording); // Set run ID for real-time data persistence From 1ec86242af41d883bb6c82bc701106de44e25a96 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 27 Nov 2025 00:03:01 +0530 Subject: [PATCH 02/22] feat: store render outputs --- src/components/robot/pages/RobotEditPage.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/components/robot/pages/RobotEditPage.tsx b/src/components/robot/pages/RobotEditPage.tsx index 53424bb2..280ce9dd 100644 --- a/src/components/robot/pages/RobotEditPage.tsx +++ b/src/components/robot/pages/RobotEditPage.tsx @@ -791,6 +791,9 @@ export const RobotEditPage = ({ handleStart }: RobotSettingsProps) => { navigate(basePath); }; + const scrapeListLimitFields = renderScrapeListLimitFields(); + const actionNameFields = renderActionNameFields(); + return ( Date: Thu, 27 Nov 2025 00:04:42 +0530 Subject: [PATCH 03/22] feat: conditionally render last two --- src/components/robot/pages/RobotEditPage.tsx | 21 +++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/components/robot/pages/RobotEditPage.tsx b/src/components/robot/pages/RobotEditPage.tsx index 280ce9dd..3a63f8f1 100644 --- a/src/components/robot/pages/RobotEditPage.tsx +++ b/src/components/robot/pages/RobotEditPage.tsx @@ -791,9 +791,6 @@ export const RobotEditPage = ({ handleStart }: RobotSettingsProps) => { navigate(basePath); }; - const scrapeListLimitFields = renderScrapeListLimitFields(); - const actionNameFields = renderActionNameFields(); - return ( { onChange={(e) => handleTargetUrlChange(e.target.value)} style={{ marginBottom: "20px" }} /> - - {renderScrapeListLimitFields()} - - {renderActionNameFields()} + {renderScrapeListLimitFields() && ( + <> + + {renderScrapeListLimitFields()} + +)} + +{renderActionNameFields() && ( + <> + + {renderActionNameFields()} + +)} + )} From d133858f584c38cebe69c78acb9b7d027361ca48 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 27 Nov 2025 00:05:01 +0530 Subject: [PATCH 04/22] fix: format --- src/components/robot/pages/RobotEditPage.tsx | 27 ++++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/components/robot/pages/RobotEditPage.tsx b/src/components/robot/pages/RobotEditPage.tsx index 3a63f8f1..178c5ad3 100644 --- a/src/components/robot/pages/RobotEditPage.tsx +++ b/src/components/robot/pages/RobotEditPage.tsx @@ -500,7 +500,7 @@ export const RobotEditPage = ({ handleStart }: RobotSettingsProps) => { {scrapeListLimits.map((limitInfo, index) => { // Get the corresponding scrapeList action to extract its name const scrapeListAction = robot?.recording?.workflow?.[limitInfo.pairIndex]?.what?.[limitInfo.actionIndex]; - const actionName = + const actionName = scrapeListAction?.name || `List Limit ${index + 1}`; @@ -821,20 +821,19 @@ export const RobotEditPage = ({ handleStart }: RobotSettingsProps) => { onChange={(e) => handleTargetUrlChange(e.target.value)} style={{ marginBottom: "20px" }} /> - {renderScrapeListLimitFields() && ( - <> - - {renderScrapeListLimitFields()} - -)} - -{renderActionNameFields() && ( - <> - - {renderActionNameFields()} - -)} + {renderScrapeListLimitFields() && ( + <> + + {renderScrapeListLimitFields()} + + )} + {renderActionNameFields() && ( + <> + + {renderActionNameFields()} + + )} )} From bf1a6f1aa7a2522136fbcfedbb8f94e95f54db0d Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 28 Nov 2025 15:03:43 +0530 Subject: [PATCH 05/22] fix(core): clean socket handlers causing memory leaks --- maxun-core/src/interpret.ts | 72 +++++++++++++++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/maxun-core/src/interpret.ts b/maxun-core/src/interpret.ts index a34777d8..c1df57ba 100644 --- a/maxun-core/src/interpret.ts +++ b/maxun-core/src/interpret.ts @@ -460,8 +460,9 @@ export default class Interpreter extends EventEmitter { for (const link of links) { // eslint-disable-next-line this.concurrency.addJob(async () => { + let newPage = null; try { - const newPage = await context.newPage(); + newPage = await context.newPage(); await newPage.goto(link); await newPage.waitForLoadState('networkidle'); await this.runLoop(newPage, this.initializedWorkflow!); @@ -470,6 +471,14 @@ export default class Interpreter extends EventEmitter { // but newPage(), goto() and waitForLoadState() don't (and will kill // the interpreter by throwing). this.log(e, Level.ERROR); + } finally { + if (newPage && !newPage.isClosed()) { + try { + await newPage.close(); + } catch (closeError) { + this.log('Failed to close enqueued page', Level.WARN); + } + } } }); } @@ -1463,41 +1472,57 @@ export default class Interpreter extends EventEmitter { * User-requested concurrency should be entirely managed by the concurrency manager, * e.g. via `enqueueLinks`. */ - p.on('popup', (popup) => { + const popupHandler = (popup) => { this.concurrency.addJob(() => this.runLoop(popup, workflowCopy)); - }); + }; + p.on('popup', popupHandler); /* eslint no-constant-condition: ["warn", { "checkLoops": false }] */ let loopIterations = 0; const MAX_LOOP_ITERATIONS = 1000; // Circuit breaker + + // Cleanup function to remove popup listener + const cleanup = () => { + try { + if (!p.isClosed()) { + p.removeListener('popup', popupHandler); + } + } catch (cleanupError) { + } + }; while (true) { if (this.isAborted) { this.log('Workflow aborted during step execution', Level.WARN); + cleanup(); return; } // Circuit breaker to prevent infinite loops if (++loopIterations > MAX_LOOP_ITERATIONS) { this.log('Maximum loop iterations reached, terminating to prevent infinite loop', Level.ERROR); + cleanup(); return; } // Checks whether the page was closed from outside, // or the workflow execution has been stopped via `interpreter.stop()` if (p.isClosed() || !this.stopper) { + cleanup(); return; } try { await p.waitForLoadState(); } catch (e) { + cleanup(); await p.close(); return; } if (workflowCopy.length === 0) { this.log('All actions completed. Workflow finished.', Level.LOG); + cleanup(); return; } @@ -1589,6 +1614,7 @@ export default class Interpreter extends EventEmitter { } } else { //await this.disableAdBlocker(p); + cleanup(); return; } } @@ -1681,4 +1707,44 @@ export default class Interpreter extends EventEmitter { throw new Error('Cannot stop, there is no running workflow!'); } } + /** + * Cleanup method to release resources and prevent memory leaks + * Call this when the interpreter is no longer needed + */ + public async cleanup(): Promise { + try { + // Stop any running workflows first + if (this.stopper) { + try { + await this.stop(); + } catch (error: any) { + this.log(`Error stopping workflow during cleanup: ${error.message}`, Level.WARN); + } + } + + // Clear ad-blocker resources + if (this.blocker) { + try { + this.blocker = null; + this.log('Ad-blocker resources cleared', Level.DEBUG); + } catch (error: any) { + this.log(`Error cleaning up ad-blocker: ${error.message}`, Level.WARN); + } + } + + // Clear accumulated data to free memory + this.cumulativeResults = []; + this.namedResults = {}; + this.serializableDataByType = { scrapeList: {}, scrapeSchema: {} }; + + // Reset state + this.isAborted = false; + this.initializedWorkflow = null; + + this.log('Interpreter cleanup completed', Level.DEBUG); + } catch (error: any) { + this.log(`Error during interpreter cleanup: ${error.message}`, Level.ERROR); + throw error; + } + } } \ No newline at end of file From 77e7f375d13b070a21d434d76f68ae80750f22b0 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 29 Nov 2025 16:22:49 +0530 Subject: [PATCH 06/22] chore: remove fortawesome --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 65a2f87a..16a9d058 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,6 @@ "dotenv": "^16.0.0", "express": "^4.17.2", "express-session": "^1.18.1", - "fortawesome": "^0.0.1-security", "google-auth-library": "^9.14.1", "googleapis": "^144.0.0", "i18next": "^24.0.2", From 0c44e4d0433075cfa8b16a1c9069a37d3071b85d Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 29 Nov 2025 16:23:28 +0530 Subject: [PATCH 07/22] chore: remove ioredis --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 16a9d058..eb963aa8 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,6 @@ "i18next-browser-languagedetector": "^8.0.0", "i18next-http-backend": "^3.0.1", "idcac-playwright": "^0.1.3", - "ioredis": "^5.4.1", "joi": "^17.6.0", "joplin-turndown-plugin-gfm": "^1.0.12", "jsonwebtoken": "^9.0.2", From 088e7c85d4289a7fd3b5feab2cb29f005ee70c66 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 29 Nov 2025 16:24:07 +0530 Subject: [PATCH 08/22] chore: remove --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index eb963aa8..d37af508 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,6 @@ "node-cron": "^3.0.3", "pg": "^8.13.0", "pg-boss": "^10.1.6", - "pkce-challenge": "^4.1.0", "playwright": "^1.48.2", "playwright-extra": "^4.3.6", "posthog-node": "^4.2.1", From 0ec7ab8a1b01799149aa1932a00f6baba4b5629d Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 29 Nov 2025 16:24:32 +0530 Subject: [PATCH 09/22] chore: remove --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index d37af508..59fe414a 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,6 @@ "playwright": "^1.48.2", "playwright-extra": "^4.3.6", "posthog-node": "^4.2.1", - "prismjs": "^1.28.0", "puppeteer-extra-plugin-recaptcha": "^3.6.8", "puppeteer-extra-plugin-stealth": "^2.11.2", "react": "^18.0.0", From a49f0c994e8f38bd3ca0cb13d6066a95bcf44afc Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 29 Nov 2025 16:24:55 +0530 Subject: [PATCH 10/22] chore: remove puppeteer-extra-plugin-recaptcha --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 59fe414a..57d5a242 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,6 @@ "playwright": "^1.48.2", "playwright-extra": "^4.3.6", "posthog-node": "^4.2.1", - "puppeteer-extra-plugin-recaptcha": "^3.6.8", "puppeteer-extra-plugin-stealth": "^2.11.2", "react": "^18.0.0", "react-dom": "^18.0.0", From 2610ea8b042770b55943e8ceb9b34b8762eb8dcc Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 29 Nov 2025 16:25:23 +0530 Subject: [PATCH 11/22] chore: remove react-highlight --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 57d5a242..f6eb4f6b 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,6 @@ "puppeteer-extra-plugin-stealth": "^2.11.2", "react": "^18.0.0", "react-dom": "^18.0.0", - "react-highlight": "0.15.0", "react-i18next": "^15.1.3", "react-router-dom": "^6.26.1", "react-simple-code-editor": "^0.11.2", From 70ba91478267a859b3cfc894ebb03de77f936b3c Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 29 Nov 2025 16:25:40 +0530 Subject: [PATCH 12/22] chore: remove react-simple-code-editor --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index f6eb4f6b..8ddc1235 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,6 @@ "react-dom": "^18.0.0", "react-i18next": "^15.1.3", "react-router-dom": "^6.26.1", - "react-simple-code-editor": "^0.11.2", "react-transition-group": "^4.4.2", "rrweb-snapshot": "^2.0.0-alpha.4", "sequelize": "^6.37.3", From c957374bf03acfa4c5f08018d60744bc4d779e67 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 29 Nov 2025 16:26:06 +0530 Subject: [PATCH 13/22] chore: remove --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 8ddc1235..6b0679db 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,6 @@ "react-dom": "^18.0.0", "react-i18next": "^15.1.3", "react-router-dom": "^6.26.1", - "react-transition-group": "^4.4.2", "rrweb-snapshot": "^2.0.0-alpha.4", "sequelize": "^6.37.3", "sequelize-typescript": "^2.1.6", From 0b3450db6da3a861b06fb2c4f4c8fde4eccd9937 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 29 Nov 2025 16:26:39 +0530 Subject: [PATCH 14/22] chore: remove @types/prismjs --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 6b0679db..26bd5e97 100644 --- a/package.json +++ b/package.json @@ -115,7 +115,6 @@ "@types/node": "22.7.9", "@types/node-cron": "^3.0.11", "@types/node-fetch": "^2.6.12", - "@types/prismjs": "^1.26.0", "@types/react-highlight": "^0.12.5", "@types/react-transition-group": "^4.4.4", "@types/styled-components": "^5.1.23", From 02e2d20c5c35c14abab3c566d216f4951f26147c Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 29 Nov 2025 16:27:31 +0530 Subject: [PATCH 15/22] chore: remove @types/react-highlight --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 26bd5e97..4108eff7 100644 --- a/package.json +++ b/package.json @@ -115,7 +115,6 @@ "@types/node": "22.7.9", "@types/node-cron": "^3.0.11", "@types/node-fetch": "^2.6.12", - "@types/react-highlight": "^0.12.5", "@types/react-transition-group": "^4.4.4", "@types/styled-components": "^5.1.23", "@types/swagger-jsdoc": "^6.0.4", From 77690af2ff33ab96abaa7aa754d99582bce148f2 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 29 Nov 2025 16:27:46 +0530 Subject: [PATCH 16/22] chore: remove @types/react-transition-group --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 4108eff7..0a7be3d1 100644 --- a/package.json +++ b/package.json @@ -115,7 +115,6 @@ "@types/node": "22.7.9", "@types/node-cron": "^3.0.11", "@types/node-fetch": "^2.6.12", - "@types/react-transition-group": "^4.4.4", "@types/styled-components": "^5.1.23", "@types/swagger-jsdoc": "^6.0.4", "@types/swagger-ui-express": "^4.1.6", From 1a16101fedb4fb56d0843e41a03593a682ff9623 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 29 Nov 2025 16:28:28 +0530 Subject: [PATCH 17/22] chore: remove @testing-library/react --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 0a7be3d1..b8b50ab9 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,6 @@ "@mui/material": "^5.6.2", "@react-oauth/google": "^0.12.1", "@tanstack/react-query": "^5.90.2", - "@testing-library/react": "^13.1.1", "@testing-library/user-event": "^13.5.0", "@types/bcrypt": "^5.0.2", "@types/body-parser": "^1.19.5", From f63fd2f38f1bcc774ac9201b9b4b5c3f73ee382a Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 29 Nov 2025 16:28:58 +0530 Subject: [PATCH 18/22] chore: remove @testing-library/user-event --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index b8b50ab9..cddc87e3 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,6 @@ "@mui/material": "^5.6.2", "@react-oauth/google": "^0.12.1", "@tanstack/react-query": "^5.90.2", - "@testing-library/user-event": "^13.5.0", "@types/bcrypt": "^5.0.2", "@types/body-parser": "^1.19.5", "@types/csurf": "^1.11.5", From bde19c1255a2773bb6fcef5eac9011a15b764e9d Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 29 Nov 2025 16:37:30 +0530 Subject: [PATCH 19/22] fix: remove --- src/components/run/RunContent.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/components/run/RunContent.tsx b/src/components/run/RunContent.tsx index f01494c3..093fc5c1 100644 --- a/src/components/run/RunContent.tsx +++ b/src/components/run/RunContent.tsx @@ -22,7 +22,6 @@ import TableCell from '@mui/material/TableCell'; import TableContainer from '@mui/material/TableContainer'; import TableHead from '@mui/material/TableHead'; import TableRow from '@mui/material/TableRow'; -import 'highlight.js/styles/github.css'; import { useTranslation } from "react-i18next"; import { useThemeMode } from "../../context/theme-provider"; From 36d224b77463d4b4aa2ee324e43476933e0c17a4 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Sat, 29 Nov 2025 16:46:03 +0530 Subject: [PATCH 20/22] fix: remove highlight.js imports --- src/components/run/RunContent.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/components/run/RunContent.tsx b/src/components/run/RunContent.tsx index 093fc5c1..1d7d400d 100644 --- a/src/components/run/RunContent.tsx +++ b/src/components/run/RunContent.tsx @@ -10,7 +10,6 @@ import { AccordionSummary, AccordionDetails } from "@mui/material"; -import Highlight from "react-highlight"; import * as React from "react"; import { Data } from "./RunsTable"; import { TabPanel, TabContext } from "@mui/lab"; From 6cdeb0b0e2af70211e6df3dd69e930a80b0fd59c Mon Sep 17 00:00:00 2001 From: Rohit Rajan Date: Sun, 30 Nov 2025 14:48:03 +0530 Subject: [PATCH 21/22] feat: add auto pagination detection --- src/components/browser/BrowserWindow.tsx | 91 ++- .../recorder/DOMBrowserRenderer.tsx | 9 + src/components/recorder/RightSidePanel.tsx | 409 +++++++++++- src/context/browserSteps.tsx | 22 + src/helpers/clientPaginationDetector.ts | 586 ++++++++++++++++++ src/helpers/clientSelectorGenerator.ts | 41 ++ 6 files changed, 1149 insertions(+), 9 deletions(-) create mode 100644 src/helpers/clientPaginationDetector.ts diff --git a/src/components/browser/BrowserWindow.tsx b/src/components/browser/BrowserWindow.tsx index 8bdabaae..1d4f3ab0 100644 --- a/src/components/browser/BrowserWindow.tsx +++ b/src/components/browser/BrowserWindow.tsx @@ -1242,6 +1242,29 @@ export const BrowserWindow = () => { } }, [browserSteps, getList, listSelector, initialAutoFieldIds, currentListActionId, manuallyAddedFieldIds]); + useEffect(() => { + if (currentListActionId && browserSteps.length > 0) { + const activeStep = browserSteps.find( + s => s.type === 'list' && s.actionId === currentListActionId + ) as ListStep | undefined; + + if (activeStep) { + if (currentListId !== activeStep.id) { + setCurrentListId(activeStep.id); + } + if (listSelector !== activeStep.listSelector) { + setListSelector(activeStep.listSelector); + } + if (JSON.stringify(fields) !== JSON.stringify(activeStep.fields)) { + setFields(activeStep.fields); + } + if (activeStep.pagination?.selector && paginationSelector !== activeStep.pagination.selector) { + setPaginationSelector(activeStep.pagination.selector); + } + } + } + }, [currentListActionId, browserSteps, currentListId, listSelector, fields, paginationSelector]); + useEffect(() => { if (!isDOMMode) { capturedElementHighlighter.clearHighlights(); @@ -1637,6 +1660,22 @@ export const BrowserWindow = () => { paginationType !== "scrollUp" && paginationType !== "none" ) { + let targetListId = currentListId; + let targetFields = fields; + + if ((!targetListId || targetListId === 0) && currentListActionId) { + const activeStep = browserSteps.find( + s => s.type === 'list' && s.actionId === currentListActionId + ) as ListStep | undefined; + + if (activeStep) { + targetListId = activeStep.id; + if (Object.keys(targetFields).length === 0 && Object.keys(activeStep.fields).length > 0) { + targetFields = activeStep.fields; + } + } + } + setPaginationSelector(highlighterData.selector); notify( `info`, @@ -1646,8 +1685,8 @@ export const BrowserWindow = () => { ); addListStep( listSelector!, - fields, - currentListId || 0, + targetFields, + targetListId || 0, currentListActionId || `list-${crypto.randomUUID()}`, { type: paginationType, @@ -1812,6 +1851,8 @@ export const BrowserWindow = () => { socket, t, paginationSelector, + highlighterData, + browserSteps ] ); @@ -1864,6 +1905,22 @@ export const BrowserWindow = () => { paginationType !== "scrollUp" && paginationType !== "none" ) { + let targetListId = currentListId; + let targetFields = fields; + + if ((!targetListId || targetListId === 0) && currentListActionId) { + const activeStep = browserSteps.find( + s => s.type === 'list' && s.actionId === currentListActionId + ) as ListStep | undefined; + + if (activeStep) { + targetListId = activeStep.id; + if (Object.keys(targetFields).length === 0 && Object.keys(activeStep.fields).length > 0) { + targetFields = activeStep.fields; + } + } + } + setPaginationSelector(highlighterData.selector); notify( `info`, @@ -1873,8 +1930,8 @@ export const BrowserWindow = () => { ); addListStep( listSelector!, - fields, - currentListId || 0, + targetFields, + targetListId || 0, currentListActionId || `list-${crypto.randomUUID()}`, { type: paginationType, selector: highlighterData.selector, isShadow: highlighterData.isShadow }, undefined, @@ -2046,6 +2103,31 @@ export const BrowserWindow = () => { } }, [paginationMode, resetPaginationSelector]); + useEffect(() => { + if (paginationMode && currentListActionId) { + const currentListStep = browserSteps.find( + step => step.type === 'list' && step.actionId === currentListActionId + ) as (ListStep & { type: 'list' }) | undefined; + + const currentSelector = currentListStep?.pagination?.selector; + const currentType = currentListStep?.pagination?.type; + + if (['clickNext', 'clickLoadMore'].includes(paginationType)) { + if (!currentSelector || (currentType && currentType !== paginationType)) { + setPaginationSelector(''); + } + } + + const stepSelector = currentListStep?.pagination?.selector; + + if (stepSelector && !paginationSelector) { + setPaginationSelector(stepSelector); + } else if (!stepSelector && paginationSelector) { + setPaginationSelector(''); + } + } + }, [browserSteps, paginationMode, currentListActionId, paginationSelector]); + return (
{ listSelector={listSelector} cachedChildSelectors={cachedChildSelectors} paginationMode={paginationMode} + paginationSelector={paginationSelector} paginationType={paginationType} limitMode={limitMode} isCachingChildSelectors={isCachingChildSelectors} diff --git a/src/components/recorder/DOMBrowserRenderer.tsx b/src/components/recorder/DOMBrowserRenderer.tsx index 9e818e31..10fa4742 100644 --- a/src/components/recorder/DOMBrowserRenderer.tsx +++ b/src/components/recorder/DOMBrowserRenderer.tsx @@ -100,6 +100,7 @@ interface RRWebDOMBrowserRendererProps { listSelector?: string | null; cachedChildSelectors?: string[]; paginationMode?: boolean; + paginationSelector?: string; paginationType?: string; limitMode?: boolean; isCachingChildSelectors?: boolean; @@ -153,6 +154,7 @@ export const DOMBrowserRenderer: React.FC = ({ listSelector = null, cachedChildSelectors = [], paginationMode = false, + paginationSelector = "", paginationType = "", limitMode = false, isCachingChildSelectors = false, @@ -257,6 +259,13 @@ export const DOMBrowserRenderer: React.FC = ({ else if (listSelector) { if (limitMode) { shouldHighlight = false; + } else if ( + paginationMode && + paginationSelector && + paginationType !== "" && + !["none", "scrollDown", "scrollUp"].includes(paginationType) + ) { + shouldHighlight = false; } else if ( paginationMode && paginationType !== "" && diff --git a/src/components/recorder/RightSidePanel.tsx b/src/components/recorder/RightSidePanel.tsx index d5a7c29c..8159e149 100644 --- a/src/components/recorder/RightSidePanel.tsx +++ b/src/components/recorder/RightSidePanel.tsx @@ -1,4 +1,4 @@ -import React, { useState, useCallback, useEffect, useMemo } from 'react'; +import React, { useState, useCallback, useEffect, useRef, useMemo } from 'react'; import { Button, Paper, Box, TextField, IconButton, Tooltip } from "@mui/material"; import { WorkflowFile } from "maxun-core"; import Typography from "@mui/material/Typography"; @@ -15,9 +15,9 @@ import ActionDescriptionBox from '../action/ActionDescriptionBox'; import { useThemeMode } from '../../context/theme-provider'; import { useTranslation } from 'react-i18next'; import { useBrowserDimensionsStore } from '../../context/browserDimensions'; -import { emptyWorkflow } from '../../shared/constants'; import { clientListExtractor } from '../../helpers/clientListExtractor'; import { clientSelectorGenerator } from '../../helpers/clientSelectorGenerator'; +import { clientPaginationDetector } from '../../helpers/clientPaginationDetector'; const fetchWorkflow = (id: string, callback: (response: WorkflowFile) => void) => { getActiveWorkflow(id).then( @@ -45,6 +45,13 @@ export const RightSidePanel: React.FC = ({ onFinishCapture const [showCaptureText, setShowCaptureText] = useState(true); const { panelHeight } = useBrowserDimensionsStore(); + const [autoDetectedPagination, setAutoDetectedPagination] = useState<{ + type: PaginationType; + selector: string | null; + confidence: 'high' | 'medium' | 'low'; + } | null>(null); + const autoDetectionRunRef = useRef(null); + const { lastAction, notify, currentWorkflowActionsState, setCurrentWorkflowActionsState, resetInterpretationLog, currentListActionId, setCurrentListActionId, currentTextActionId, setCurrentTextActionId, currentScreenshotActionId, setCurrentScreenshotActionId, isDOMMode, setIsDOMMode, currentSnapshot, setCurrentSnapshot, updateDOMMode, initialUrl, setRecordingUrl, currentTextGroupName } = useGlobalInfoStore(); const { getText, startGetText, stopGetText, @@ -62,7 +69,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture startAction, finishAction } = useActionContext(); - const { browserSteps, updateBrowserTextStepLabel, deleteBrowserStep, addScreenshotStep, updateListTextFieldLabel, removeListTextField, updateListStepLimit, deleteStepsByActionId, updateListStepData, updateScreenshotStepData, emitActionForStep } = useBrowserSteps(); + const { browserSteps, addScreenshotStep, updateListStepLimit, updateListStepPagination, deleteStepsByActionId, updateListStepData, updateScreenshotStepData, emitActionForStep } = useBrowserSteps(); const { id, socket } = useSocketStore(); const { t } = useTranslation(); @@ -72,6 +79,73 @@ export const RightSidePanel: React.FC = ({ onFinishCapture setWorkflow(data); }, [setWorkflow]); + useEffect(() => { + if (!paginationType || !currentListActionId) return; + + const currentListStep = browserSteps.find( + step => step.type === 'list' && step.actionId === currentListActionId + ) as (BrowserStep & { type: 'list' }) | undefined; + + const currentSelector = currentListStep?.pagination?.selector; + const currentType = currentListStep?.pagination?.type; + + if (['clickNext', 'clickLoadMore'].includes(paginationType)) { + const needsSelector = !currentSelector && !currentType; + const typeChanged = currentType && currentType !== paginationType; + + if (typeChanged) { + const iframeElement = document.querySelector('#browser-window iframe') as HTMLIFrameElement; + if (iframeElement?.contentDocument && currentSelector) { + try { + function evaluateSelector(selector: string, doc: Document): Element[] { + if (selector.startsWith('//') || selector.startsWith('(//')) { + try { + const result = doc.evaluate(selector, doc, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null); + const elements: Element[] = []; + for (let i = 0; i < result.snapshotLength; i++) { + const node = result.snapshotItem(i); + if (node && node.nodeType === Node.ELEMENT_NODE) { + elements.push(node as Element); + } + } + return elements; + } catch (err) { + return []; + } + } else { + try { + return Array.from(doc.querySelectorAll(selector)); + } catch (err) { + return []; + } + } + } + + const elements = evaluateSelector(currentSelector, iframeElement.contentDocument); + elements.forEach((el: Element) => { + (el as HTMLElement).style.outline = ''; + (el as HTMLElement).style.outlineOffset = ''; + (el as HTMLElement).style.zIndex = ''; + }); + } catch (error) { + console.error('Error removing pagination highlight:', error); + } + } + + if (currentListStep) { + updateListStepPagination(currentListStep.id, { + type: paginationType, + selector: null, + }); + } + + startPaginationMode(); + } else if (needsSelector) { + startPaginationMode(); + } + } + }, [paginationType, currentListActionId, browserSteps, updateListStepPagination, startPaginationMode]); + useEffect(() => { if (socket) { const domModeHandler = (data: any) => { @@ -391,7 +465,182 @@ export const RightSidePanel: React.FC = ({ onFinishCapture return; } - startPaginationMode(); + const currentListStepForAutoDetect = browserSteps.find( + step => step.type === 'list' && step.actionId === currentListActionId + ) as (BrowserStep & { type: 'list'; listSelector?: string }) | undefined; + + if (currentListStepForAutoDetect?.listSelector) { + if (autoDetectionRunRef.current !== currentListActionId) { + autoDetectionRunRef.current = currentListActionId; + + notify('info', 'Detecting pagination...'); + + try { + socket?.emit('testPaginationScroll', { + listSelector: currentListStepForAutoDetect.listSelector + }); + + const handleScrollTestResult = (result: any) => { + if (result.success && result.contentLoaded) { + setAutoDetectedPagination({ + type: 'scrollDown', + selector: null, + confidence: 'high' + }); + updatePaginationType('scrollDown'); + + const latestListStep = browserSteps.find( + step => step.type === 'list' && step.actionId === currentListActionId + ); + if (latestListStep) { + updateListStepPagination(latestListStep.id, { + type: 'scrollDown', + selector: null, + isShadow: false + }); + } + } else if (result.success && !result.contentLoaded) { + const iframeElement = document.querySelector('#browser-window iframe') as HTMLIFrameElement; + const iframeDoc = iframeElement?.contentDocument; + + if (iframeDoc) { + const detectionResult = clientPaginationDetector.autoDetectPagination( + iframeDoc, + currentListStepForAutoDetect.listSelector!, + clientSelectorGenerator, + { disableScrollDetection: true } + ); + + if (detectionResult.type) { + setAutoDetectedPagination({ + type: detectionResult.type, + selector: detectionResult.selector, + confidence: detectionResult.confidence + }); + + const latestListStep = browserSteps.find( + step => step.type === 'list' && step.actionId === currentListActionId + ); + if (latestListStep) { + updateListStepPagination(latestListStep.id, { + type: detectionResult.type, + selector: detectionResult.selector, + isShadow: false + }); + } + + updatePaginationType(detectionResult.type); + + if (detectionResult.selector && (detectionResult.type === 'clickNext' || detectionResult.type === 'clickLoadMore')) { + try { + function evaluateSelector(selector: string, doc: Document): Element[] { + try { + const isXPath = selector.startsWith('//') || selector.startsWith('(//'); + if (isXPath) { + const result = doc.evaluate( + selector, + doc, + null, + XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, + null + ); + const elements: Element[] = []; + for (let i = 0; i < result.snapshotLength; i++) { + const node = result.snapshotItem(i); + if (node && node.nodeType === Node.ELEMENT_NODE) { + elements.push(node as Element); + } + } + return elements; + } else { + try { + const allElements = Array.from(doc.querySelectorAll(selector)); + if (allElements.length > 0) { + return allElements; + } + } catch (err) { + console.warn('[RightSidePanel] Full chained selector failed, trying individual selectors:', err); + } + + const selectorParts = selector.split(','); + for (const part of selectorParts) { + try { + const elements = Array.from(doc.querySelectorAll(part.trim())); + if (elements.length > 0) { + return elements; + } + } catch (err) { + console.warn('[RightSidePanel] Selector part failed:', part.trim(), err); + continue; + } + } + return []; + } + } catch (err) { + console.error('[RightSidePanel] Selector evaluation failed:', selector, err); + return []; + } + } + + const elements = evaluateSelector(detectionResult.selector, iframeDoc); + if (elements.length > 0) { + elements.forEach((el: Element) => { + (el as HTMLElement).style.outline = '3px dashed #ff00c3'; + (el as HTMLElement).style.outlineOffset = '2px'; + (el as HTMLElement).style.zIndex = '9999'; + }); + + const firstElement = elements[0] as HTMLElement; + const elementRect = firstElement.getBoundingClientRect(); + const iframeWindow = iframeElement.contentWindow; + if (iframeWindow) { + const targetY = elementRect.top + iframeWindow.scrollY - (iframeWindow.innerHeight / 2) + (elementRect.height / 2); + iframeWindow.scrollTo({ top: targetY, behavior: 'smooth' }); + } + + const paginationTypeLabel = detectionResult.type === 'clickNext' ? 'Next Button' : 'Load More Button'; + notify('info', `${paginationTypeLabel} has been auto-detected and highlighted on the page`); + } else { + console.warn(' No elements found for selector:', detectionResult.selector); + } + } catch (error) { + console.error('Error highlighting pagination button:', error); + } + } + } else { + setAutoDetectedPagination(null); + } + } + } else { + console.error('Scroll test failed:', result.error); + setAutoDetectedPagination(null); + } + + socket?.off('paginationScrollTestResult', handleScrollTestResult); + }; + + socket?.on('paginationScrollTestResult', handleScrollTestResult); + + setTimeout(() => { + socket?.off('paginationScrollTestResult', handleScrollTestResult); + }, 5000); + + } catch (error) { + console.error('Scroll test failed:', error); + setAutoDetectedPagination(null); + } + } + } + + const shouldSkipPaginationMode = autoDetectedPagination && ( + ['scrollDown', 'scrollUp'].includes(autoDetectedPagination.type) || + (['clickNext', 'clickLoadMore'].includes(autoDetectedPagination.type) && autoDetectedPagination.selector) + ); + + if (!shouldSkipPaginationMode) { + startPaginationMode(); + } + setShowPaginationOptions(true); setCaptureStage('pagination'); break; @@ -460,6 +709,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture case 'pagination': stopPaginationMode(); setShowPaginationOptions(false); + setAutoDetectedPagination(null); setCaptureStage('initial'); break; } @@ -495,17 +745,58 @@ export const RightSidePanel: React.FC = ({ onFinishCapture socket.emit('removeAction', { actionId: currentListActionId }); } } + + if (autoDetectedPagination?.selector) { + const iframeElement = document.querySelector('#browser-window iframe') as HTMLIFrameElement; + if (iframeElement?.contentDocument) { + try { + function evaluateSelector(selector: string, doc: Document): Element[] { + if (selector.startsWith('//') || selector.startsWith('(//')) { + try { + const result = doc.evaluate(selector, doc, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null); + const elements: Element[] = []; + for (let i = 0; i < result.snapshotLength; i++) { + const node = result.snapshotItem(i); + if (node && node.nodeType === Node.ELEMENT_NODE) { + elements.push(node as Element); + } + } + return elements; + } catch (err) { + return []; + } + } else { + try { + return Array.from(doc.querySelectorAll(selector)); + } catch (err) { + return []; + } + } + } + + const elements = evaluateSelector(autoDetectedPagination.selector, iframeElement.contentDocument); + elements.forEach((el: Element) => { + (el as HTMLElement).style.outline = ''; + (el as HTMLElement).style.outlineOffset = ''; + (el as HTMLElement).style.zIndex = ''; + }); + } catch (error) { + console.error('Error removing pagination highlight on discard:', error); + } + } + } resetListState(); stopPaginationMode(); stopLimitMode(); setShowPaginationOptions(false); setShowLimitOptions(false); + setAutoDetectedPagination(null); setCaptureStage('initial'); setCurrentListActionId(''); clientSelectorGenerator.cleanup(); notify('error', t('right_panel.errors.capture_list_discarded')); - }, [currentListActionId, browserSteps, stopGetList, deleteStepsByActionId, resetListState, setShowPaginationOptions, setShowLimitOptions, setCaptureStage, notify, t, stopPaginationMode, stopLimitMode, socket]); + }, [currentListActionId, browserSteps, stopGetList, deleteStepsByActionId, resetListState, setShowPaginationOptions, setShowLimitOptions, setCaptureStage, notify, t, stopPaginationMode, stopLimitMode, socket, autoDetectedPagination]); const captureScreenshot = (fullPage: boolean) => { const screenshotCount = browserSteps.filter(s => s.type === 'screenshot').length + 1; @@ -615,6 +906,114 @@ export const RightSidePanel: React.FC = ({ onFinishCapture {showPaginationOptions && ( {t('right_panel.pagination.title')} + + {autoDetectedPagination && autoDetectedPagination.type !== '' && ( + + + ✓ Auto-detected: { + autoDetectedPagination.type === 'clickNext' ? 'Click Next' : + autoDetectedPagination.type === 'clickLoadMore' ? 'Click Load More' : + autoDetectedPagination.type === 'scrollDown' ? 'Scroll Down' : + autoDetectedPagination.type === 'scrollUp' ? 'Scroll Up' : + autoDetectedPagination.type + } + + + You can continue with this or manually select a different pagination type below. + + {autoDetectedPagination.selector && ['clickNext', 'clickLoadMore'].includes(autoDetectedPagination.type) && ( + + )} + + )}