From a30eb6ea8b7ebc7806337d1e5dfdff918fb7f261 Mon Sep 17 00:00:00 2001 From: Rohit Date: Thu, 27 Feb 2025 16:47:12 +0530 Subject: [PATCH 01/54] feat: check for headers and append data --- .../integrations/gsheet.ts | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/server/src/workflow-management/integrations/gsheet.ts b/server/src/workflow-management/integrations/gsheet.ts index 47dda84e..858c2063 100644 --- a/server/src/workflow-management/integrations/gsheet.ts +++ b/server/src/workflow-management/integrations/gsheet.ts @@ -49,11 +49,7 @@ export async function updateGoogleSheet(robotId: string, runId: string) { if (plainRobot.google_sheet_email && spreadsheetId) { console.log(`Preparing to write data to Google Sheet for robot: ${robotId}, spreadsheetId: ${spreadsheetId}`); - const headers = Object.keys(data[0]); - const rows = data.map((row: { [key: string]: any }) => Object.values(row)); - const outputData = [headers, ...rows]; - - await writeDataToSheet(robotId, spreadsheetId, outputData); + await writeDataToSheet(robotId, spreadsheetId, data); console.log(`Data written to Google Sheet successfully for Robot: ${robotId} and Run: ${runId}`); } else { console.log('Google Sheets integration not configured.'); @@ -102,7 +98,38 @@ export async function writeDataToSheet(robotId: string, spreadsheetId: string, d const sheets = google.sheets({ version: 'v4', auth: oauth2Client }); - const resource = { values: data }; + const checkResponse = await sheets.spreadsheets.values.get({ + spreadsheetId, + range: 'Sheet1!1:1', + }); + + const expectedHeaders = Object.keys(data[0]); + + const rows = data.map(item => Object.values(item)); + + const existingHeaders = + checkResponse.data.values && + checkResponse.data.values[0] ? + checkResponse.data.values[0].map(String) : + []; + + const isSheetEmpty = existingHeaders.length === 0; + + const headersMatch = + !isSheetEmpty && + existingHeaders.length === expectedHeaders.length && + expectedHeaders.every((header, index) => existingHeaders[index] === header); + + let resource; + + if (isSheetEmpty || !headersMatch) { + resource = { values: [expectedHeaders, ...rows] }; + console.log('Including headers in the append operation.'); + } else { + resource = { values: rows }; + console.log('Headers already exist and match, only appending data rows.'); + } + console.log('Attempting to write to spreadsheet:', spreadsheetId); const response = await sheets.spreadsheets.values.append({ From 5f56d3d1e0d90e150e3cfa2a388bc338c1805605 Mon Sep 17 00:00:00 2001 From: RohitR311 Date: Fri, 28 Feb 2025 19:53:09 +0530 Subject: [PATCH 02/54] feat: add check if data empty and return --- server/src/workflow-management/integrations/gsheet.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/workflow-management/integrations/gsheet.ts b/server/src/workflow-management/integrations/gsheet.ts index 858c2063..b492a7df 100644 --- a/server/src/workflow-management/integrations/gsheet.ts +++ b/server/src/workflow-management/integrations/gsheet.ts @@ -103,6 +103,11 @@ export async function writeDataToSheet(robotId: string, spreadsheetId: string, d range: 'Sheet1!1:1', }); + if (!data || data.length === 0) { + console.log('No data to write. Exiting early.'); + return; + } + const expectedHeaders = Object.keys(data[0]); const rows = data.map(item => Object.values(item)); From 2217a9577e0d3617fcfad3b1391ff2146ba0c851 Mon Sep 17 00:00:00 2001 From: Rohit Date: Sun, 2 Mar 2025 22:45:08 +0530 Subject: [PATCH 03/54] feat: add rel selector for pagination elements --- server/src/workflow-management/selector.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts index 20a6a529..be497528 100644 --- a/server/src/workflow-management/selector.ts +++ b/server/src/workflow-management/selector.ts @@ -1427,6 +1427,7 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { const iframeSelector = genSelectorForIframe(element); const shadowSelector = genSelectorForShadowDOM(element); + const relSelector = genSelectorForAttributes(element, ['rel']); const hrefSelector = genSelectorForAttributes(element, ['href']); const formSelector = genSelectorForAttributes(element, [ 'name', @@ -1473,6 +1474,7 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { hrefSelector, accessibilitySelector, formSelector, + relSelector, iframeSelector: iframeSelector ? { full: iframeSelector.fullSelector, isIframe: iframeSelector.isFrameContent, @@ -1509,6 +1511,11 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { function genSelectorForAttributes(element: HTMLElement, attributes: string[]) { let selector = null; try { + if (attributes.includes('rel') && element.hasAttribute('rel')) { + const relValue = element.getAttribute('rel'); + return `[rel="${relValue}"]`; + } + selector = isAttributesDefined(element, attributes) ? finder(element, { idName: () => false, // Don't use the id to generate a selector From 2d6112932a060304ad842e129d4464c1847ed345 Mon Sep 17 00:00:00 2001 From: Rohit Date: Sun, 2 Mar 2025 22:46:19 +0530 Subject: [PATCH 04/54] feat: add rel selector for pagination element --- server/src/workflow-management/classes/Generator.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 2eb5f130..b4c72e96 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -825,6 +825,7 @@ export class WorkflowGenerator { selectors?.testIdSelector, selectors?.id, selectors?.hrefSelector, + selectors?.relSelector, selectors?.accessibilitySelector, selectors?.attrSelector ] From 1bde2c9fd89536513a1fc5cf028e2f5751136a07 Mon Sep 17 00:00:00 2001 From: Rohit Date: Mon, 3 Mar 2025 01:49:31 +0530 Subject: [PATCH 05/54] feat: revamp click load more scraping --- maxun-core/src/interpret.ts | 50 ++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/maxun-core/src/interpret.ts b/maxun-core/src/interpret.ts index 88047128..0adde74a 100644 --- a/maxun-core/src/interpret.ts +++ b/maxun-core/src/interpret.ts @@ -768,14 +768,20 @@ export default class Interpreter extends EventEmitter { } case 'clickLoadMore': { + await scrapeCurrentPage(); + if (checkLimit()) return allResults; + + let loadMoreCounter = 0; + let previousResultCount = allResults.length; + let noNewItemsCounter = 0; + const MAX_NO_NEW_ITEMS = 2; + while (true) { - // Find working button with retry mechanism, consistent with clickNext + // Find working button with retry mechanism const { button: loadMoreButton, workingSelector } = await findWorkingButton(availableSelectors); if (!workingSelector || !loadMoreButton) { debugLog('No working Load More selector found after retries'); - const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config); - allResults = allResults.concat(finalResults); return allResults; } @@ -808,6 +814,8 @@ export default class Interpreter extends EventEmitter { if (clickSuccess) { await page.waitForTimeout(1000); + loadMoreCounter++; + debugLog(`Successfully clicked Load More button (${loadMoreCounter} times)`); } } catch (error) { debugLog(`Click attempt ${retryCount + 1} failed completely.`); @@ -822,8 +830,6 @@ export default class Interpreter extends EventEmitter { if (!clickSuccess) { debugLog(`Load More clicking failed after ${MAX_RETRIES} attempts`); - const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config); - allResults = allResults.concat(finalResults); return allResults; } @@ -833,20 +839,34 @@ export default class Interpreter extends EventEmitter { await page.waitForTimeout(2000); const currentHeight = await page.evaluate(() => document.body.scrollHeight); - if (currentHeight === previousHeight) { - debugLog('No more items loaded after Load More'); - const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config); - allResults = allResults.concat(finalResults); - return allResults; - } + const heightChanged = currentHeight !== previousHeight; previousHeight = currentHeight; - if (config.limit && allResults.length >= config.limit) { - allResults = allResults.slice(0, config.limit); - break; + await scrapeCurrentPage(); + + const currentResultCount = allResults.length; + const newItemsAdded = currentResultCount > previousResultCount; + + if (!newItemsAdded) { + noNewItemsCounter++; + debugLog(`No new items added after click (${noNewItemsCounter}/${MAX_NO_NEW_ITEMS})`); + + if (noNewItemsCounter >= MAX_NO_NEW_ITEMS) { + debugLog(`Stopping after ${MAX_NO_NEW_ITEMS} clicks with no new items`); + return allResults; + } + } else { + noNewItemsCounter = 0; + previousResultCount = currentResultCount; + } + + if (checkLimit()) return allResults; + + if (!heightChanged) { + debugLog('No more items loaded after Load More'); + return allResults; } } - break; } default: { From f57a84c49f8593abd80c5053f33126edcc581b15 Mon Sep 17 00:00:00 2001 From: Rohit Date: Mon, 3 Mar 2025 18:01:17 +0530 Subject: [PATCH 06/54] feat: selector generation for frame elements --- server/src/workflow-management/selector.ts | 522 ++++++++++++++------- 1 file changed, 363 insertions(+), 159 deletions(-) diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts index be497528..8ace5779 100644 --- a/server/src/workflow-management/selector.ts +++ b/server/src/workflow-management/selector.ts @@ -24,54 +24,64 @@ export const getElementInformation = async ( const elementInfo = await page.evaluate( async ({ x, y }) => { const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // First, get the element at the clicked coordinates in the main document let element = document.elementFromPoint(x, y) as HTMLElement; if (!element) return null; - - // Track the deepest element found + let deepestElement = element; - - // Function to traverse shadow DOM + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; let shadowRoot = current.shadowRoot; let deepest = current; - - while (shadowRoot) { + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; shadowRoot = current.shadowRoot; + depth++; } - + return deepest; }; - - // Handle iframe traversal + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + if (element.tagName === 'IFRAME') { let currentIframe = element as HTMLIFrameElement; - - while (currentIframe) { + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; if (!iframeDocument) break; - + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Update deepest element and check for shadow DOM + deepestElement = traverseShadowDOM(iframeElement); - - // Continue traversing if we found another iframe + if (iframeElement.tagName === 'IFRAME') { currentIframe = iframeElement as HTMLIFrameElement; + depth++; } else { break; } @@ -80,28 +90,78 @@ export const getElementInformation = async ( break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); } else { - // If not an iframe, check for shadow DOM deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; - // Get the element and its iframe path const el = getDeepestElementFromPoint(x, y); if (el) { - // Handle potential anchor parent const { parentElement } = el; const targetElement = parentElement?.tagName === 'A' ? parentElement : el; - // Get containing context information const ownerDocument = targetElement.ownerDocument; const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; const isIframeContent = Boolean(frameElement); + const isFrameContent = frameElement?.tagName === 'FRAME'; - // Get the containing shadow root if any const containingShadowRoot = targetElement.getRootNode() as ShadowRoot; const isShadowRoot = containingShadowRoot instanceof ShadowRoot; @@ -115,8 +175,11 @@ export const getElementInformation = async ( innerHTML?: string; outerHTML?: string; isIframeContent?: boolean; + isFrameContent?: boolean; iframeURL?: string; + frameURL?: string; iframeIndex?: number; + frameIndex?: number; frameHierarchy?: string[]; isShadowRoot?: boolean; shadowRootMode?: string; @@ -124,12 +187,17 @@ export const getElementInformation = async ( } = { tagName: targetElement?.tagName ?? '', isIframeContent, + isFrameContent, isShadowRoot }; - if (isIframeContent) { + if (isIframeContent || isFrameContent) { // Include iframe specific information - info.iframeURL = frameElement.src; + if (isIframeContent) { + info.iframeURL = (frameElement as HTMLIFrameElement).src; + } else { + info.frameURL = (frameElement).src; + } // Calculate the frame's position in the hierarchy let currentFrame = frameElement; @@ -140,8 +208,9 @@ export const getElementInformation = async ( // Store the frame's identifier (src, id, or index) frameHierarchy.unshift( currentFrame.id || + currentFrame.getAttribute('name') || currentFrame.src || - `iframe[${frameIndex}]` + `${currentFrame.tagName.toLowerCase()}[${frameIndex}]` ); // Move up to parent frame if it exists @@ -151,7 +220,11 @@ export const getElementInformation = async ( } info.frameHierarchy = frameHierarchy; - info.iframeIndex = frameIndex - 1; // Adjust for 0-based index + if (isIframeContent) { + info.iframeIndex = frameIndex - 1; // Adjust for 0-based index + } else { + info.frameIndex = frameIndex - 1; + } } if (isShadowRoot) { @@ -429,54 +502,64 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector async ({ x, y }) => { // Enhanced helper function to get element from point including iframes const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // First, get the element at the clicked coordinates in the main document let element = document.elementFromPoint(x, y) as HTMLElement; if (!element) return null; - - // Track the deepest element found + let deepestElement = element; - - // Function to traverse shadow DOM + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; let shadowRoot = current.shadowRoot; let deepest = current; - - while (shadowRoot) { + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; shadowRoot = current.shadowRoot; + depth++; } - + return deepest; }; - - // Handle iframe traversal + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + if (element.tagName === 'IFRAME') { let currentIframe = element as HTMLIFrameElement; - - while (currentIframe) { + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; if (!iframeDocument) break; - + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Update deepest element and check for shadow DOM + deepestElement = traverseShadowDOM(iframeElement); - - // Continue traversing if we found another iframe + if (iframeElement.tagName === 'IFRAME') { currentIframe = iframeElement as HTMLIFrameElement; + depth++; } else { break; } @@ -485,11 +568,64 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); } else { - // If not an iframe, check for shadow DOM deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; @@ -1205,65 +1341,63 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { } const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // Helper function to traverse shadow DOM - const traverseShadowDOM = (element: HTMLElement, depth: number = 0): HTMLElement => { - const MAX_SHADOW_DEPTH = 4; - let current = element; - let deepest = current; - - while (current && depth < MAX_SHADOW_DEPTH) { - const shadowRoot = current.shadowRoot; - if (!shadowRoot) break; - - const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; - if (!shadowElement || shadowElement === current) break; - - deepest = shadowElement; - current = shadowElement; - depth++; - } - - return deepest; - }; - - // Start with the element at the specified coordinates let element = document.elementFromPoint(x, y) as HTMLElement; if (!element) return null; - - // Initialize tracking variables + let deepestElement = element; - let depth = 0; - const MAX_IFRAME_DEPTH = 4; - - // First check if the initial element has a shadow root - deepestElement = traverseShadowDOM(element); - - // If it's an iframe, traverse through iframe hierarchy - if (deepestElement.tagName === 'IFRAME') { - let currentIframe = deepestElement as HTMLIFrameElement; - + + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { + let current = element; + let shadowRoot = current.shadowRoot; + let deepest = current; + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { + const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; + if (!shadowElement || shadowElement === current) break; + + deepest = shadowElement; + current = shadowElement; + shadowRoot = current.shadowRoot; + depth++; + } + + return deepest; + }; + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + + if (element.tagName === 'IFRAME') { + let currentIframe = element as HTMLIFrameElement; + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - - // Access iframe's document - const iframeDoc = currentIframe.contentDocument || currentIframe.contentWindow?.document; - if (!iframeDoc) break; - - // Get element at transformed coordinates in iframe - const iframeElement = iframeDoc.elementFromPoint(iframeX, iframeY) as HTMLElement; + + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; + if (!iframeDocument) break; + + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Check for shadow DOM within iframe - const shadowResult = traverseShadowDOM(iframeElement); - deepestElement = shadowResult; - - // If we found another iframe, continue traversing - if (shadowResult.tagName === 'IFRAME') { - currentIframe = shadowResult as HTMLIFrameElement; + + deepestElement = traverseShadowDOM(iframeElement); + + if (iframeElement.tagName === 'IFRAME') { + currentIframe = iframeElement as HTMLIFrameElement; depth++; } else { break; @@ -1273,74 +1407,129 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); + } else { + deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; - const genSelectorForIframe = (element: HTMLElement) => { - // Helper function to get the complete iframe path up to document root - const getIframePath = (el: HTMLElement) => { - const path = []; - let current = el; - let depth = 0; - const MAX_DEPTH = 4; + + const genSelectorForFrame = (element: HTMLElement) => { + const getFramePath = (el: HTMLElement) => { + const path = []; + let current = el; + let depth = 0; + const MAX_DEPTH = 4; + + while (current && depth < MAX_DEPTH) { + const ownerDocument = current.ownerDocument; - while (current && depth < MAX_DEPTH) { - // Get the owner document of the current element - const ownerDocument = current.ownerDocument; - - // Check if this document belongs to an iframe - const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; - - if (frameElement) { - path.unshift({ - frame: frameElement, - document: ownerDocument, - element: current - }); - // Move up to the parent document's element (the iframe) - current = frameElement; - depth++; - } else { - break; - } + const frameElement = + ownerDocument?.defaultView?.frameElement as HTMLIFrameElement | HTMLFrameElement; + + if (frameElement) { + path.unshift({ + frame: frameElement, + document: ownerDocument, + element: current, + isFrame: frameElement.tagName === 'FRAME' + }); + + current = frameElement; + depth++; + } else { + break; } - return path; + } + return path; }; - - const iframePath = getIframePath(element); - if (iframePath.length === 0) return null; - + + const framePath = getFramePath(element); + if (framePath.length === 0) return null; + try { - const selectorParts: string[] = []; + const selectorParts: string[] = []; + + framePath.forEach((context, index) => { + const frameSelector = context.isFrame ? + `frame[name="${context.frame.getAttribute('name')}"]` : + finder(context.frame, { + root: index === 0 ? document.body : + (framePath[index - 1].document.body as Element) + }); - // Generate selector for each iframe boundary - iframePath.forEach((context, index) => { - // Get selector for the iframe element - const frameSelector = finder(context.frame, { - root: index === 0 ? document.body : - (iframePath[index - 1].document.body as Element) - }); - - // For the last context, get selector for target element - if (index === iframePath.length - 1) { - const elementSelector = finder(element, { - root: context.document.body as Element - }); - selectorParts.push(`${frameSelector} :>> ${elementSelector}`); - } else { - selectorParts.push(frameSelector); - } - }); - - return { - fullSelector: selectorParts.join(' :>> '), - isFrameContent: true - }; + if (index === framePath.length - 1) { + const elementSelector = finder(element, { + root: context.document.body as Element + }); + selectorParts.push(`${frameSelector} :>> ${elementSelector}`); + } else { + selectorParts.push(frameSelector); + } + }); + + return { + fullSelector: selectorParts.join(' :>> '), + isFrameContent: true + }; } catch (e) { - console.warn('Error generating iframe selector:', e); - return null; + console.warn('Error generating frame selector:', e); + return null; } }; @@ -1424,7 +1613,22 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { } - const iframeSelector = genSelectorForIframe(element); + let iframeSelector = null; + try { + // Check if element is within frame/iframe + const isInFrame = element.ownerDocument !== document; + const isInFrameset = () => { + let doc = element.ownerDocument; + return doc.querySelectorAll('frameset').length > 0; + }; + + if (isInFrame || isInFrameset()) { + iframeSelector = genSelectorForFrame(element); + } + } catch (e) { + console.warn('Error detecting frames:', e); + } + const shadowSelector = genSelectorForShadowDOM(element); const relSelector = genSelectorForAttributes(element, ['rel']); From 8f4e6a9f3eb1f6503c7e1c7e33e8cdb4600bc13a Mon Sep 17 00:00:00 2001 From: Rohit Date: Mon, 3 Mar 2025 18:09:27 +0530 Subject: [PATCH 07/54] feat: match state if url and selectors empty --- maxun-core/src/interpret.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/maxun-core/src/interpret.ts b/maxun-core/src/interpret.ts index 0adde74a..bc82d031 100644 --- a/maxun-core/src/interpret.ts +++ b/maxun-core/src/interpret.ts @@ -286,6 +286,12 @@ export default class Interpreter extends EventEmitter { ? arrayToObject(superset[key]) : superset[key]; + if ((key === 'url' || key === 'selectors') && + Array.isArray(value) && Array.isArray(superset[key]) && + value.length === 0 && (superset[key] as any[]).length === 0) { + return true; + } + if (key === 'selectors' && Array.isArray(value) && Array.isArray(superset[key])) { return value.some(selector => (superset[key] as any[]).includes(selector) From ccbf9d074bf98c964b85e0e3afd0f7ea1aca3a34 Mon Sep 17 00:00:00 2001 From: Rohit Date: Mon, 3 Mar 2025 18:11:19 +0530 Subject: [PATCH 08/54] feat: add scraping support for frame elements --- maxun-core/src/browserSide/scraper.js | 51 +++++++++++++++++++++------ 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/maxun-core/src/browserSide/scraper.js b/maxun-core/src/browserSide/scraper.js index f6b53da2..84649286 100644 --- a/maxun-core/src/browserSide/scraper.js +++ b/maxun-core/src/browserSide/scraper.js @@ -210,7 +210,6 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, return Array.from(document.querySelectorAll(config.selector)); } - // First handle iframe traversal if present if (config.selector.includes(':>>')) { const parts = config.selector.split(':>>').map(s => s.trim()); let currentElements = [document]; @@ -223,23 +222,44 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, for (const element of currentElements) { try { - // For document or iframe document const doc = element.contentDocument || element || element.contentWindow?.document; if (!doc) continue; - // Query elements in current context + if (part.startsWith('frame[name=') || part.startsWith('iframe[name=')) { + const nameMatch = part.match(/\[name=['"]([^'"]+)['"]\]/); + if (nameMatch && nameMatch[1]) { + const frameName = nameMatch[1]; + let foundFrames = []; + + if (doc.getElementsByName && typeof doc.getElementsByName === 'function') { + foundFrames = Array.from(doc.getElementsByName(frameName)) + .filter(el => el.tagName === 'FRAME' || el.tagName === 'IFRAME'); + } + + if (foundFrames.length === 0) { + const framesBySelector = Array.from(doc.querySelectorAll(`frame[name="${frameName}"], iframe[name="${frameName}"]`)); + foundFrames = framesBySelector; + } + + if (isLast) { + nextElements.push(...foundFrames); + } else { + nextElements.push(...foundFrames); + } + continue; + } + } + const found = Array.from(doc.querySelectorAll(part)); if (isLast) { - // If it's the last part, keep all matching elements nextElements.push(...found); } else { - // If not last, only keep iframes for next iteration - const iframes = found.filter(el => el.tagName === 'IFRAME'); - nextElements.push(...iframes); + const frames = found.filter(el => el.tagName === 'IFRAME' || el.tagName === 'FRAME'); + nextElements.push(...frames); } } catch (error) { - console.warn('Cannot access iframe content:', error, { + console.warn('Cannot access iframe/frame content:', error, { part, element, index: i @@ -285,12 +305,17 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, return []; } - // Modified to handle iframe context for URL resolution function getElementValue(element, attribute) { if (!element) return null; - // Get the base URL for resolving relative URLs - const baseURL = element.ownerDocument?.location?.href || window.location.origin; + let baseURL; + try { + baseURL = element.ownerDocument?.location?.href || + element.ownerDocument?.baseURI || + window.location.origin; + } catch (e) { + baseURL = window.location.origin; + } switch (attribute) { case 'href': { @@ -305,6 +330,10 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, return element.innerText?.trim(); case 'textContent': return element.textContent?.trim(); + case 'innerHTML': + return element.innerHTML; + case 'outerHTML': + return element.outerHTML; default: return element.getAttribute(attribute) || element.innerText?.trim(); } From 87b2613fb453936df3ed61fdb15f803169f6aef2 Mon Sep 17 00:00:00 2001 From: Karishma Shukla Date: Mon, 3 Mar 2025 23:48:47 +0530 Subject: [PATCH 09/54] feat: add AIRTABLE_CLIENT_ID --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 13c994a0..0a3cb3b1 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,7 @@ You can access the frontend at http://localhost:5173/ and backend at http://loca | `GOOGLE_CLIENT_ID` | No | Client ID for Google OAuth, used for Google Sheet integration authentication. | Google login will not work. | | `GOOGLE_CLIENT_SECRET`| No | Client Secret for Google OAuth. | Google login will not work. | | `GOOGLE_REDIRECT_URI` | No | Redirect URI for handling Google OAuth responses. | Google login will not work. | +| `AIRTABLE_CLIENT_ID` | No | Client ID for Airtable, used for Airtable integration authentication. | Airtable will not work. | | `REDIS_HOST` | Yes | Host address of the Redis server, used by BullMQ for scheduling robots. | Redis connection will fail. | | `REDIS_PORT` | Yes | Port number for the Redis server. | Redis connection will fail. | | `MAXUN_TELEMETRY` | No | Disables telemetry to stop sending anonymous usage data. Keeping it enabled helps us understand how the product is used and assess the impact of any new changes. Please keep it enabled. | Telemetry data will not be collected. | From 5af53017a1e37efa6f64d5ab8b4d9621d0156d2b Mon Sep 17 00:00:00 2001 From: Karishma Shukla Date: Mon, 3 Mar 2025 23:50:20 +0530 Subject: [PATCH 10/54] feat: add AIRTABLE_REDIRECT_URI --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a3cb3b1..3470a4ee 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,8 @@ You can access the frontend at http://localhost:5173/ and backend at http://loca | `GOOGLE_CLIENT_ID` | No | Client ID for Google OAuth, used for Google Sheet integration authentication. | Google login will not work. | | `GOOGLE_CLIENT_SECRET`| No | Client Secret for Google OAuth. | Google login will not work. | | `GOOGLE_REDIRECT_URI` | No | Redirect URI for handling Google OAuth responses. | Google login will not work. | -| `AIRTABLE_CLIENT_ID` | No | Client ID for Airtable, used for Airtable integration authentication. | Airtable will not work. | +| `AIRTABLE_CLIENT_ID` | No | Client ID for Airtable, used for Airtable integration authentication. | Airtable login will not work. | +| `AIRTABLE_REDIRECT_URI` | No | Redirect URI for handling Airtable OAuth responses. | Airtable login will not work. | | `REDIS_HOST` | Yes | Host address of the Redis server, used by BullMQ for scheduling robots. | Redis connection will fail. | | `REDIS_PORT` | Yes | Port number for the Redis server. | Redis connection will fail. | | `MAXUN_TELEMETRY` | No | Disables telemetry to stop sending anonymous usage data. Keeping it enabled helps us understand how the product is used and assess the impact of any new changes. Please keep it enabled. | Telemetry data will not be collected. | From 3543ed957c24e1a70217bf40efeebc8cb5586d5b Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 4 Mar 2025 02:23:07 +0530 Subject: [PATCH 11/54] fix: remove blue auto highlight in text fields --- src/index.css | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/index.css b/src/index.css index 252b737a..194ae0bb 100644 --- a/src/index.css +++ b/src/index.css @@ -14,6 +14,19 @@ body { } +input:-webkit-autofill, +input:-webkit-autofill:hover, +input:-webkit-autofill:focus, +textarea:-webkit-autofill, +textarea:-webkit-autofill:hover, +textarea:-webkit-autofill:focus, +select:-webkit-autofill, +select:-webkit-autofill:hover, +select:-webkit-autofill:focus { + -webkit-box-shadow: 0 0 0 1000px transparent inset !important; + transition: background-color 5000s ease-in-out 0s !important; +} + html { width: 100%; height: 100%; From 485e67f183fc1c8b776cb491e69db4717256c0dc Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 4 Mar 2025 02:23:26 +0530 Subject: [PATCH 12/54] fix: format --- src/index.css | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/index.css b/src/index.css index 194ae0bb..05052000 100644 --- a/src/index.css +++ b/src/index.css @@ -11,7 +11,7 @@ body { padding: 0; scrollbar-gutter: stable; overflow-y: auto; - + } input:-webkit-autofill, @@ -35,6 +35,7 @@ html { a { color: #ff00c3; + &:hover { color: #ff00c3; } @@ -42,7 +43,7 @@ a { code { font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', - monospace; + monospace; color: #ff00c3; } @@ -57,7 +58,7 @@ code { align-items: center; overflow: hidden; position: relative; - + } #browser-content { @@ -65,14 +66,14 @@ code { width: 100%; display: flex; flex-direction: column; - transform: scale(1); /* Ensure no scaling */ - transform-origin: top left; /* Keep the position fixed */ + transform: scale(1); + /* Ensure no scaling */ + transform-origin: top left; + /* Keep the position fixed */ } -#browser { - -} +#browser {} #browser-window { overflow-y: auto; @@ -176,4 +177,4 @@ code { height: calc(100vh - 2rem); margin: 1rem 55rem; } -} +} \ No newline at end of file From b81491130c27dba4bc01a956a65c366580952d19 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 4 Mar 2025 02:23:42 +0530 Subject: [PATCH 13/54] chore: cleanup usused class --- src/index.css | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/index.css b/src/index.css index 05052000..70cfbff1 100644 --- a/src/index.css +++ b/src/index.css @@ -72,9 +72,6 @@ code { /* Keep the position fixed */ } - -#browser {} - #browser-window { overflow-y: auto; height: 100%; From 0f59d59ef5a1d5f207ed747fe2de90ca6ace523d Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 4 Mar 2025 02:24:18 +0530 Subject: [PATCH 14/54] fix: whitespace --- src/index.css | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/index.css b/src/index.css index 70cfbff1..e63d18f7 100644 --- a/src/index.css +++ b/src/index.css @@ -11,7 +11,6 @@ body { padding: 0; scrollbar-gutter: stable; overflow-y: auto; - } input:-webkit-autofill, @@ -58,7 +57,6 @@ code { align-items: center; overflow: hidden; position: relative; - } #browser-content { From 9400ebe032948eb8bb574ebd7ac26abbb97b5a5f Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 4 Mar 2025 09:11:27 +0530 Subject: [PATCH 15/54] feat: frame selector support capture list --- server/src/workflow-management/selector.ts | 787 ++++++++++++++++----- 1 file changed, 609 insertions(+), 178 deletions(-) diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts index 8ace5779..4f325025 100644 --- a/server/src/workflow-management/selector.ts +++ b/server/src/workflow-management/selector.ts @@ -192,20 +192,17 @@ export const getElementInformation = async ( }; if (isIframeContent || isFrameContent) { - // Include iframe specific information if (isIframeContent) { info.iframeURL = (frameElement as HTMLIFrameElement).src; } else { info.frameURL = (frameElement).src; } - // Calculate the frame's position in the hierarchy let currentFrame = frameElement; const frameHierarchy: string[] = []; let frameIndex = 0; while (currentFrame) { - // Store the frame's identifier (src, id, or index) frameHierarchy.unshift( currentFrame.id || currentFrame.getAttribute('name') || @@ -213,7 +210,6 @@ export const getElementInformation = async ( `${currentFrame.tagName.toLowerCase()}[${frameIndex}]` ); - // Move up to parent frame if it exists const parentDoc = currentFrame.ownerDocument; currentFrame = parentDoc?.defaultView?.frameElement as HTMLIFrameElement; frameIndex++; @@ -221,19 +217,17 @@ export const getElementInformation = async ( info.frameHierarchy = frameHierarchy; if (isIframeContent) { - info.iframeIndex = frameIndex - 1; // Adjust for 0-based index + info.iframeIndex = frameIndex - 1; } else { info.frameIndex = frameIndex - 1; } } if (isShadowRoot) { - // Include shadow root specific information info.shadowRootMode = containingShadowRoot.mode; info.shadowRootContent = containingShadowRoot.innerHTML; } - // Collect element attributes and properties if (targetElement) { info.attributes = Array.from(targetElement.attributes).reduce( (acc, attr) => { @@ -279,56 +273,65 @@ export const getElementInformation = async ( } else { const elementInfo = await page.evaluate( async ({ x, y }) => { - // Enhanced helper function to get element from point including shadow DOM const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // First, get the element at the clicked coordinates in the main document let element = document.elementFromPoint(x, y) as HTMLElement; if (!element) return null; - - // Track the deepest element found + let deepestElement = element; - - // Function to traverse shadow DOM + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; let shadowRoot = current.shadowRoot; let deepest = current; - - while (shadowRoot) { + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; shadowRoot = current.shadowRoot; + depth++; } - + return deepest; }; - - // Handle iframe traversal + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + if (element.tagName === 'IFRAME') { let currentIframe = element as HTMLIFrameElement; - - while (currentIframe) { + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; if (!iframeDocument) break; - + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Update deepest element and check for shadow DOM + deepestElement = traverseShadowDOM(iframeElement); - - // Continue traversing if we found another iframe + if (iframeElement.tagName === 'IFRAME') { currentIframe = iframeElement as HTMLIFrameElement; + depth++; } else { break; } @@ -337,14 +340,67 @@ export const getElementInformation = async ( break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); } else { - // If not an iframe, check for shadow DOM deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; - + const originalEl = getDeepestElementFromPoint(x, y); if (originalEl) { let element = originalEl; @@ -374,7 +430,7 @@ export const getElementInformation = async ( const significantOverlap = (childRect.width * childRect.height) / - (parentRect.width * parentRect.height) > 0.5; + (parentRect.width * parentRect.height) > 0.1; if (fullyContained && significantOverlap) { const nextParent = element.parentElement; @@ -391,8 +447,9 @@ export const getElementInformation = async ( } const ownerDocument = element.ownerDocument; - const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; + const frameElement = ownerDocument?.defaultView?.frameElement; const isIframeContent = Boolean(frameElement); + const isFrameContent = frameElement?.tagName === 'FRAME'; const containingShadowRoot = element.getRootNode() as ShadowRoot; const isShadowRoot = containingShadowRoot instanceof ShadowRoot; @@ -407,8 +464,11 @@ export const getElementInformation = async ( innerHTML?: string; outerHTML?: string; isIframeContent?: boolean; + isFrameContent?: boolean; iframeURL?: string; + frameURL?: string; iframeIndex?: number; + frameIndex?: number; frameHierarchy?: string[]; isShadowRoot?: boolean; shadowRootMode?: string; @@ -416,44 +476,48 @@ export const getElementInformation = async ( } = { tagName: element?.tagName ?? '', isIframeContent, + isFrameContent, isShadowRoot }; - if (isIframeContent) { - // Include iframe specific information - info.iframeURL = frameElement.src; + if (isIframeContent || isFrameContent) { + if (isIframeContent && !isFrameContent) { + info.iframeURL = (frameElement as HTMLIFrameElement).src; + } else if (isFrameContent) { + info.frameURL = (frameElement as HTMLFrameElement).src; + } - // Calculate the frame's position in the hierarchy let currentFrame = frameElement; const frameHierarchy: string[] = []; let frameIndex = 0; while (currentFrame) { - // Store the frame's identifier (src, id, or index) frameHierarchy.unshift( currentFrame.id || - currentFrame.src || - `iframe[${frameIndex}]` + currentFrame.getAttribute('name') || + (currentFrame as HTMLFrameElement).src || + `${currentFrame.tagName.toLowerCase()}[${frameIndex}]` ); - // Move up to parent frame if it exists const parentDoc = currentFrame.ownerDocument; - currentFrame = parentDoc?.defaultView?.frameElement as HTMLIFrameElement; + currentFrame = parentDoc?.defaultView?.frameElement; frameIndex++; } info.frameHierarchy = frameHierarchy; - info.iframeIndex = frameIndex - 1; // Adjust for 0-based index - }; + if (isIframeContent && !isFrameContent) { + info.iframeIndex = frameIndex - 1; + } else if (isFrameContent) { + info.frameIndex = frameIndex - 1; + } + } if (isShadowRoot) { - // Include shadow root specific information info.shadowRootMode = containingShadowRoot.mode; info.shadowRootContent = containingShadowRoot.innerHTML; } if (element) { - // Get attributes including those from shadow DOM context info.attributes = Array.from(element.attributes).reduce( (acc, attr) => { acc[attr.name] = attr.value; @@ -462,14 +526,21 @@ export const getElementInformation = async ( {} as Record ); - // Handle specific element types if (element.tagName === 'A') { info.url = (element as HTMLAnchorElement).href; info.innerText = element.textContent ?? ''; } else if (element.tagName === 'IMG') { info.imageUrl = (element as HTMLImageElement).src; + } else if (element?.tagName === 'SELECT') { + const selectElement = element as HTMLSelectElement; + info.innerText = selectElement.options[selectElement.selectedIndex]?.text ?? ''; + info.attributes = { + ...info.attributes, + selectedValue: selectElement.value, + }; + } else if (element?.tagName === 'INPUT' && ((element as HTMLInputElement).type === 'time' || (element as HTMLInputElement).type === 'date')) { + info.innerText = (element as HTMLInputElement).value; } else { - // Handle text content with proper null checking info.hasOnlyText = element.children.length === 0 && (element.textContent !== null && element.textContent.trim().length > 0); @@ -500,7 +571,6 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector if (!getList || listSelector !== '') { const rect = await page.evaluate( async ({ x, y }) => { - // Enhanced helper function to get element from point including iframes const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { let element = document.elementFromPoint(x, y) as HTMLElement; if (!element) return null; @@ -693,54 +763,64 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector const rect = await page.evaluate( async ({ x, y }) => { const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // First, get the element at the clicked coordinates in the main document let element = document.elementFromPoint(x, y) as HTMLElement; if (!element) return null; - - // Track the deepest element found + let deepestElement = element; - - // Function to traverse shadow DOM + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; let shadowRoot = current.shadowRoot; let deepest = current; - - while (shadowRoot) { + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; shadowRoot = current.shadowRoot; + depth++; } - + return deepest; }; - - // Handle iframe traversal + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + if (element.tagName === 'IFRAME') { let currentIframe = element as HTMLIFrameElement; - - while (currentIframe) { + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; if (!iframeDocument) break; - + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Update deepest element and check for shadow DOM + deepestElement = traverseShadowDOM(iframeElement); - - // Continue traversing if we found another iframe + if (iframeElement.tagName === 'IFRAME') { currentIframe = iframeElement as HTMLIFrameElement; + depth++; } else { break; } @@ -749,11 +829,64 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); } else { - // If not an iframe, check for shadow DOM deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; @@ -786,7 +919,7 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector const significantOverlap = (childRect.width * childRect.height) / - (parentRect.width * parentRect.height) > 0.5; + (parentRect.width * parentRect.height) > 0.1; if (fullyContained && significantOverlap) { const nextParent = element.parentElement; @@ -827,12 +960,12 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector } }); - // Same coordinate adjustment for iframe elements as above + // For elements inside iframes or frames, adjust coordinates relative to the top window let adjustedRect = createRectObject(rectangle); let currentWindow = element.ownerDocument.defaultView; while (currentWindow !== window.top) { - const frameElement = currentWindow?.frameElement as HTMLIFrameElement; + const frameElement = currentWindow?.frameElement; if (!frameElement) break; const frameRect = frameElement.getBoundingClientRect(); @@ -1773,9 +1906,9 @@ interface SelectorResult { export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates, listSelector: string): Promise => { interface DOMContext { - type: 'iframe' | 'shadow'; + type: 'iframe' | 'frame' | 'shadow'; element: HTMLElement; - container: HTMLIFrameElement | ShadowRoot; + container: HTMLIFrameElement | HTMLFrameElement | ShadowRoot; host?: HTMLElement; document?: Document; } @@ -1784,53 +1917,63 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates if (!listSelector) { const selectors = await page.evaluate(({ x, y }: { x: number, y: number }) => { const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // Function to traverse shadow DOM + let element = document.elementFromPoint(x, y) as HTMLElement; + if (!element) return null; + + let deepestElement = element; + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; - let deepest = current; let shadowRoot = current.shadowRoot; - - while (shadowRoot) { + let deepest = current; + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; shadowRoot = current.shadowRoot; + depth++; } - + return deepest; }; - - // Start with the element at coordinates - let element = document.elementFromPoint(x, y) as HTMLElement; - if (!element) return null; - - let deepestElement = element; - let depth = 0; - const MAX_DEPTH = 4; - - // Handle iframe traversal + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + if (element.tagName === 'IFRAME') { let currentIframe = element as HTMLIFrameElement; - - while (currentIframe && depth < MAX_DEPTH) { + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - - const iframeDoc = currentIframe.contentDocument || currentIframe.contentWindow?.document; - if (!iframeDoc) break; - - const iframeElement = iframeDoc.elementFromPoint(iframeX, iframeY) as HTMLElement; + + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; + if (!iframeDocument) break; + + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Check for shadow DOM within iframe + deepestElement = traverseShadowDOM(iframeElement); - - if (deepestElement.tagName === 'IFRAME') { - currentIframe = deepestElement as HTMLIFrameElement; + + if (iframeElement.tagName === 'IFRAME') { + currentIframe = iframeElement as HTMLIFrameElement; depth++; } else { break; @@ -1840,25 +1983,134 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); } else { - // If not an iframe, check for shadow DOM deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; - // Basic selector generation function getNonUniqueSelector(element: HTMLElement): string { let selector = element.tagName.toLowerCase(); - + + if (selector === 'frame' || selector === 'iframe') { + let baseSelector = selector; + + if (element.className) { + const classes = element.className.split(/\s+/).filter(Boolean); + if (classes.length > 0) { + const validClasses = classes.filter(cls => !cls.startsWith('!') && !cls.includes(':')); + if (validClasses.length > 0) { + baseSelector += '.' + validClasses.map(cls => CSS.escape(cls)).join('.'); + } + } + } + + if (element.id) { + return `${selector}#${CSS.escape(element.id)}`; + } + + if (element.getAttribute('name')) { + return `${selector}[name="${CSS.escape(element.getAttribute('name')!)}"]`; + } + + if (element.parentElement && element.parentElement.tagName === 'FRAMESET') { + const frameIndex = Array.from(element.parentElement.children) + .filter(child => child.tagName.toLowerCase() === selector) + .indexOf(element) + 1; + + if (frameIndex > 0) { + return `${selector}:nth-of-type(${frameIndex})`; + } + } + + if (element.parentElement) { + const siblings = Array.from(element.parentElement.children); + const identicalSiblings = siblings.filter(sibling => { + if (sibling === element) return false; + + let siblingSelector = sibling.tagName.toLowerCase(); + const siblingClassName = typeof sibling.className === 'string' ? sibling.className : ''; + if (siblingClassName) { + const siblingClasses = siblingClassName.split(/\s+/).filter(Boolean); + const validSiblingClasses = siblingClasses.filter(cls => !cls.startsWith('!') && !cls.includes(':')); + if (validSiblingClasses.length > 0) { + siblingSelector += '.' + validSiblingClasses.map(cls => CSS.escape(cls)).join('.'); + } + } + + return siblingSelector === baseSelector; + }); + + if (identicalSiblings.length > 0) { + const position = siblings.indexOf(element) + 1; + return `${baseSelector}:nth-child(${position})`; + } + } + + return baseSelector; + } + if (selector === 'td' && element.parentElement) { - // Find position among td siblings const siblings = Array.from(element.parentElement.children); const position = siblings.indexOf(element) + 1; return `${selector}:nth-child(${position})`; } - + if (element.className) { const classes = element.className.split(/\s+/).filter((cls: string) => Boolean(cls)); if (classes.length > 0) { @@ -1868,9 +2120,8 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates } } } - + if (element.parentElement) { - // Look for identical siblings const siblings = Array.from(element.parentElement.children); const identicalSiblings = siblings.filter(sibling => { if (sibling === element) return false; @@ -1893,11 +2144,10 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates selector += `:nth-child(${position})`; } } - + return selector; } - function getContextPath(element: HTMLElement): DOMContext[] { const path: DOMContext[] = []; let current = element; @@ -1919,18 +2169,19 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates continue; } - // Check for iframe + // Check for iframe or frame const ownerDocument = current.ownerDocument; - const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; + const frameElement = ownerDocument?.defaultView?.frameElement; if (frameElement) { + const isFrame = frameElement.tagName === 'FRAME'; path.unshift({ - type: 'iframe', + type: isFrame ? 'frame' : 'iframe', element: current, - container: frameElement, + container: frameElement as (HTMLIFrameElement | HTMLFrameElement), document: ownerDocument }); - current = frameElement; + current = frameElement as HTMLElement; depth++; continue; } @@ -1997,7 +2248,6 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates } } - // if (listSelector === '') { if (element.tagName !== 'TABLE') { while (element.parentElement) { if (element.tagName.toLowerCase() === 'body' || @@ -2016,7 +2266,7 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates const significantOverlap = (childRect.width * childRect.height) / - (parentRect.width * parentRect.height) > 0.5; + (parentRect.width * parentRect.height) > 0.1; if (fullyContained && significantOverlap) { const nextParent = element.parentElement; @@ -2038,68 +2288,66 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates return selectors || { generalSelector: '' }; } else { - // When we have a list selector, we need special handling while maintaining shadow DOM support + // When we have a list selector, we need special handling while maintaining shadow DOM and frame support const selectors = await page.evaluate(({ x, y }: { x: number, y: number }) => { const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // Helper function to traverse shadow DOM - const traverseShadowDOM = (element: HTMLElement, depth: number = 0): HTMLElement => { - const MAX_SHADOW_DEPTH = 4; + let element = document.elementFromPoint(x, y) as HTMLElement; + if (!element) return null; + + let deepestElement = element; + + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; + let shadowRoot = current.shadowRoot; let deepest = current; - - while (current && depth < MAX_SHADOW_DEPTH) { - const shadowRoot = current.shadowRoot; - if (!shadowRoot) break; - + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; + shadowRoot = current.shadowRoot; depth++; } - + return deepest; }; - // Start with the element at the specified coordinates - let element = document.elementFromPoint(x, y) as HTMLElement; - if (!element) return null; + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; - // Initialize tracking variables - let deepestElement = element; - let depth = 0; - const MAX_IFRAME_DEPTH = 4; + if (element.tagName === 'IFRAME') { + let currentIframe = element as HTMLIFrameElement; + let depth = 0; + const MAX_IFRAME_DEPTH = 4; - // First check if the initial element has a shadow root - deepestElement = traverseShadowDOM(element); - - // If it's an iframe, traverse through iframe hierarchy - if (deepestElement.tagName === 'IFRAME') { - let currentIframe = deepestElement as HTMLIFrameElement; - while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - // Access iframe's document - const iframeDoc = currentIframe.contentDocument || currentIframe.contentWindow?.document; - if (!iframeDoc) break; + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; + if (!iframeDocument) break; - // Get element at transformed coordinates in iframe - const iframeElement = iframeDoc.elementFromPoint(iframeX, iframeY) as HTMLElement; + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - // Check for shadow DOM within iframe - const shadowResult = traverseShadowDOM(iframeElement); - deepestElement = shadowResult; + deepestElement = traverseShadowDOM(iframeElement); - // If we found another iframe, continue traversing - if (shadowResult.tagName === 'IFRAME') { - currentIframe = shadowResult as HTMLIFrameElement; + if (iframeElement.tagName === 'IFRAME') { + currentIframe = iframeElement as HTMLIFrameElement; depth++; } else { break; @@ -2109,33 +2357,145 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); + } else { + deepestElement = traverseShadowDOM(element); } return deepestElement; }; - // Generate basic selector from element's tag and classes function getNonUniqueSelector(element: HTMLElement): string { let selector = element.tagName.toLowerCase(); - + + if (selector === 'frame' || selector === 'iframe') { + let baseSelector = selector; + + if (element.className) { + const classes = element.className.split(/\s+/).filter(Boolean); + if (classes.length > 0) { + const validClasses = classes.filter(cls => !cls.startsWith('!') && !cls.includes(':')); + if (validClasses.length > 0) { + baseSelector += '.' + validClasses.map(cls => CSS.escape(cls)).join('.'); + } + } + } + + if (element.id) { + return `${selector}#${CSS.escape(element.id)}`; + } + + if (element.getAttribute('name')) { + return `${selector}[name="${CSS.escape(element.getAttribute('name')!)}"]`; + } + + if (element.parentElement && element.parentElement.tagName === 'FRAMESET') { + const frameIndex = Array.from(element.parentElement.children) + .filter(child => child.tagName.toLowerCase() === selector) + .indexOf(element) + 1; + + if (frameIndex > 0) { + return `${selector}:nth-of-type(${frameIndex})`; + } + } + + if (element.parentElement) { + const siblings = Array.from(element.parentElement.children); + const identicalSiblings = siblings.filter(sibling => { + if (sibling === element) return false; + + let siblingSelector = sibling.tagName.toLowerCase(); + const siblingClassName = typeof sibling.className === 'string' ? sibling.className : ''; + if (siblingClassName) { + const siblingClasses = siblingClassName.split(/\s+/).filter(Boolean); + const validSiblingClasses = siblingClasses.filter(cls => !cls.startsWith('!') && !cls.includes(':')); + if (validSiblingClasses.length > 0) { + siblingSelector += '.' + validSiblingClasses.map(cls => CSS.escape(cls)).join('.'); + } + } + + return siblingSelector === baseSelector; + }); + + if (identicalSiblings.length > 0) { + const position = siblings.indexOf(element) + 1; + return `${baseSelector}:nth-child(${position})`; + } + } + + return baseSelector; + } + if (selector === 'td' && element.parentElement) { const siblings = Array.from(element.parentElement.children); const position = siblings.indexOf(element) + 1; return `${selector}:nth-child(${position})`; } - + if (element.className) { - const classes = element.className.split(/\s+/).filter(Boolean); + const classes = element.className.split(/\s+/).filter((cls: string) => Boolean(cls)); if (classes.length > 0) { - const validClasses = classes.filter(cls => !cls.startsWith('!') && !cls.includes(':')); + const validClasses = classes.filter((cls: string) => !cls.startsWith('!') && !cls.includes(':')); if (validClasses.length > 0) { selector += '.' + validClasses.map(cls => CSS.escape(cls)).join('.'); } } } - + if (element.parentElement) { - // Look for identical siblings const siblings = Array.from(element.parentElement.children); const identicalSiblings = siblings.filter(sibling => { if (sibling === element) return false; @@ -2158,11 +2518,11 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates selector += `:nth-child(${position})`; } } - + return selector; } - // Get complete context path (both iframe and shadow DOM) + // Get complete context path (iframe, frame, and shadow DOM) function getContextPath(element: HTMLElement): DOMContext[] { const path: DOMContext[] = []; let current = element; @@ -2184,18 +2544,19 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates continue; } - // Check for iframe + // Check for iframe or frame const ownerDocument = current.ownerDocument; - const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; + const frameElement = ownerDocument?.defaultView?.frameElement; if (frameElement) { + const isFrame = frameElement.tagName === 'FRAME'; path.unshift({ - type: 'iframe', + type: isFrame ? 'frame' : 'iframe', element: current, - container: frameElement, + container: frameElement as (HTMLIFrameElement | HTMLFrameElement), document: ownerDocument }); - current = frameElement; + current = frameElement as HTMLElement; depth++; continue; } @@ -2256,9 +2617,9 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates const generalSelector = getSelectorPath(element); return { generalSelector }; - }, coordinates); + }, coordinates); - return selectors || { generalSelector: '' }; + return selectors || { generalSelector: '' }; } } catch (error) { console.error('Error in getNonUniqueSelectors:', error); @@ -2331,11 +2692,13 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro return `${hostSelector} >> ${elementSelector}`; } - // Check for iframe context + // Check for iframe/frame context const ownerDocument = element.ownerDocument; - const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; + const frameElement = ownerDocument?.defaultView?.frameElement; if (frameElement) { - const frameSelector = getNonUniqueSelector(frameElement); + const frameSelector = getNonUniqueSelector(frameElement as HTMLElement); + const isFrame = frameElement.tagName === 'FRAME'; + // Use the appropriate delimiter based on whether it's a frame or iframe return `${frameSelector} :>> ${elementSelector}`; } @@ -2344,8 +2707,7 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro return `${parentSelector} > ${elementSelector}`; } - - // Function to get all children from special contexts + // Function to get all children from special contexts including frames function getSpecialContextChildren(element: HTMLElement): HTMLElement[] { const children: HTMLElement[] = []; @@ -2371,10 +2733,43 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro } } + // Get frame children + const frames = Array.from(element.querySelectorAll('frame')) as HTMLFrameElement[]; + for (const frame of frames) { + try { + const frameDoc = frame.contentDocument || frame.contentWindow?.document; + if (frameDoc) { + const frameElements = Array.from(frameDoc.querySelectorAll('*')) as HTMLElement[]; + children.push(...frameElements); + } + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + + // Check for framesets + const framesets = Array.from(element.querySelectorAll('frameset')) as HTMLElement[]; + for (const frameset of framesets) { + const framesToCheck = Array.from(frameset.querySelectorAll('frame')) as HTMLFrameElement[]; + for (const frame of framesToCheck) { + try { + const frameDoc = frame.contentDocument || frame.contentWindow?.document; + if (frameDoc) { + const frameElements = Array.from(frameDoc.querySelectorAll('*')) as HTMLElement[]; + children.push(...frameElements); + } + } catch (error) { + console.warn('Cannot access frameset frame content:', error); + continue; + } + } + } + return children; } - // Function to recursively get all descendant selectors including shadow DOM and iframes + // Function to recursively get all descendant selectors function getAllDescendantSelectors(element: HTMLElement): string[] { let selectors: string[] = []; @@ -2388,7 +2783,7 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro // Process regular descendants selectors = selectors.concat(getAllDescendantSelectors(child)); - // Process special context children (shadow DOM and iframes) + // Process special context children (shadow DOM, iframes, and frames) const specialChildren = getSpecialContextChildren(child); for (const specialChild of specialChildren) { const specialPath = getSelectorPath(specialChild); @@ -2413,7 +2808,7 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro return selectors; } - // Handle both shadow DOM and iframe parent selectors + // Handle both shadow DOM, iframe, and frame parent selectors let parentElements: HTMLElement[] = []; // Check for special context traversal in parent selector @@ -2453,6 +2848,42 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro continue; } } + + // Check for frame + if (element.tagName === 'FRAME') { + try { + const frameDoc = (element as HTMLFrameElement).contentDocument || + (element as HTMLFrameElement).contentWindow?.document; + if (frameDoc) { + const frameChildren = Array.from( + frameDoc.querySelectorAll(selectorParts[i]) + ) as HTMLElement[]; + newParentElements.push(...frameChildren); + } + } catch (error) { + console.warn('Cannot access frame content during traversal:', error); + continue; + } + } + + // Check for frameset + if (element.tagName === 'FRAMESET') { + const frames = Array.from(element.querySelectorAll('frame')) as HTMLFrameElement[]; + for (const frame of frames) { + try { + const frameDoc = frame.contentDocument || frame.contentWindow?.document; + if (frameDoc) { + const frameChildren = Array.from( + frameDoc.querySelectorAll(selectorParts[i]) + ) as HTMLElement[]; + newParentElements.push(...frameChildren); + } + } catch (error) { + console.warn('Cannot access frameset frame during traversal:', error); + continue; + } + } + } } parentElements = newParentElements; From cb4dabfcf7bdd88d5d9c320c7350a87d81d8654e Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 4 Mar 2025 09:14:27 +0530 Subject: [PATCH 16/54] feat: add frame element support for scrapeList action --- maxun-core/src/browserSide/scraper.js | 62 ++++++++++++++------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/maxun-core/src/browserSide/scraper.js b/maxun-core/src/browserSide/scraper.js index 84649286..c04a9993 100644 --- a/maxun-core/src/browserSide/scraper.js +++ b/maxun-core/src/browserSide/scraper.js @@ -423,7 +423,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, * @returns {Array.>} Array of arrays of scraped items, one sub-array per list */ window.scrapeList = async function ({ listSelector, fields, limit = 10 }) { - // Enhanced query function to handle both iframe and shadow DOM + // Enhanced query function to handle iframe, frame and shadow DOM const queryElement = (rootElement, selector) => { if (!selector.includes('>>') && !selector.includes(':>>')) { return rootElement.querySelector(selector); @@ -435,14 +435,14 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, for (let i = 0; i < parts.length; i++) { if (!currentElement) return null; - // Handle iframe traversal - if (currentElement.tagName === 'IFRAME') { + // Handle iframe and frame traversal + if (currentElement.tagName === 'IFRAME' || currentElement.tagName === 'FRAME') { try { - const iframeDoc = currentElement.contentDocument || currentElement.contentWindow.document; - currentElement = iframeDoc.querySelector(parts[i]); + const frameDoc = currentElement.contentDocument || currentElement.contentWindow.document; + currentElement = frameDoc.querySelector(parts[i]); continue; } catch (e) { - console.warn('Cannot access iframe content:', e); + console.warn(`Cannot access ${currentElement.tagName.toLowerCase()} content:`, e); return null; } } @@ -485,13 +485,13 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, const nextElements = []; for (const element of currentElements) { - // Handle iframe traversal - if (element.tagName === 'IFRAME') { + // Handle iframe and frame traversal + if (element.tagName === 'IFRAME' || element.tagName === 'FRAME') { try { - const iframeDoc = element.contentDocument || element.contentWindow.document; - nextElements.push(...iframeDoc.querySelectorAll(part)); + const frameDoc = element.contentDocument || element.contentWindow.document; + nextElements.push(...frameDoc.querySelectorAll(part)); } catch (e) { - console.warn('Cannot access iframe content:', e); + console.warn(`Cannot access ${element.tagName.toLowerCase()} content:`, e); continue; } } else { @@ -566,8 +566,8 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, return { type: 'TR', element: currentElement }; } - // Handle iframe crossing - if (currentElement.tagName === 'IFRAME') { + // Handle iframe and frame crossing + if (currentElement.tagName === 'IFRAME' || currentElement.tagName === 'FRAME') { try { currentElement = currentElement.contentDocument.body; } catch (e) { @@ -611,7 +611,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, if (current.tagName === 'TH') return true; - if (current.tagName === 'IFRAME') { + if (current.tagName === 'IFRAME' || current.tagName === 'FRAME') { try { current = current.contentDocument.body; } catch (e) { @@ -667,14 +667,18 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, allElements.push(...shadowHost.getElementsByTagName(baseElement.tagName)); } - // Get elements from iframes - const iframes = document.getElementsByTagName('iframe'); - for (const iframe of iframes) { + // Get elements from iframes and frames + const frames = [ + ...Array.from(document.getElementsByTagName('iframe')), + ...Array.from(document.getElementsByTagName('frame')) + ]; + + for (const frame of frames) { try { - const iframeDoc = iframe.contentDocument || iframe.contentWindow.document; - allElements.push(...iframeDoc.getElementsByTagName(baseElement.tagName)); + const frameDoc = frame.contentDocument || frame.contentWindow.document; + allElements.push(...frameDoc.getElementsByTagName(baseElement.tagName)); } catch (e) { - console.warn('Cannot access iframe content:', e); + console.warn(`Cannot access ${frame.tagName.toLowerCase()} content:`, e); } } @@ -736,7 +740,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, const tableData = []; const nonTableData = []; - // Process table data with both iframe and shadow DOM support + // Process table data with support for iframes, frames, and shadow DOM for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) { const container = containers[containerIndex]; const { tableFields } = containerFields[containerIndex]; @@ -746,14 +750,14 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, const firstElement = queryElement(container, firstField.selector); let tableContext = firstElement; - // Find table context including both iframe and shadow DOM + // Find table context including iframe, frame and shadow DOM while (tableContext && tableContext.tagName !== 'TABLE' && tableContext !== container) { if (tableContext.getRootNode() instanceof ShadowRoot) { tableContext = tableContext.getRootNode().host; continue; } - if (tableContext.tagName === 'IFRAME') { + if (tableContext.tagName === 'IFRAME' || tableContext.tagName === 'FRAME') { try { tableContext = tableContext.contentDocument.body; } catch (e) { @@ -776,13 +780,13 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, rows.push(...tableContext.shadowRoot.getElementsByTagName('TR')); } - // Get rows from iframes - if (tableContext.tagName === 'IFRAME') { + // Get rows from iframes and frames + if (tableContext.tagName === 'IFRAME' || tableContext.tagName === 'FRAME') { try { - const iframeDoc = tableContext.contentDocument || tableContext.contentWindow.document; - rows.push(...iframeDoc.getElementsByTagName('TR')); + const frameDoc = tableContext.contentDocument || tableContext.contentWindow.document; + rows.push(...frameDoc.getElementsByTagName('TR')); } catch (e) { - console.warn('Cannot access iframe rows:', e); + console.warn(`Cannot access ${tableContext.tagName.toLowerCase()} rows:`, e); } } @@ -852,7 +856,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, } } - // Process non-table data with both contexts support + // Process non-table data with all contexts support for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) { if (nonTableData.length >= limit) break; From acb3da38afdf8998f86bcaec55afc6fc94c333fd Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 4 Mar 2025 10:53:57 +0530 Subject: [PATCH 17/54] feat: better find working button logic --- maxun-core/src/interpret.ts | 83 ++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 34 deletions(-) diff --git a/maxun-core/src/interpret.ts b/maxun-core/src/interpret.ts index bc82d031..2887b634 100644 --- a/maxun-core/src/interpret.ts +++ b/maxun-core/src/interpret.ts @@ -598,33 +598,52 @@ export default class Interpreter extends EventEmitter { }; // Enhanced button finder with retry mechanism - const findWorkingButton = async (selectors: string[], retryCount = 0): Promise<{ - button: ElementHandle | null, - workingSelector: string | null + const findWorkingButton = async (selectors: string[]): Promise<{ + button: ElementHandle | null, + workingSelector: string | null, + updatedSelectors: string[] }> => { - for (const selector of selectors) { - try { - const button = await page.waitForSelector(selector, { - state: 'attached', - timeout: 10000 // Reduced timeout for faster checks - }); - if (button) { - debugLog('Found working selector:', selector); - return { button, workingSelector: selector }; + let updatedSelectors = [...selectors]; + + for (let i = 0; i < selectors.length; i++) { + const selector = selectors[i]; + let retryCount = 0; + let selectorSuccess = false; + + while (retryCount < MAX_RETRIES && !selectorSuccess) { + try { + const button = await page.waitForSelector(selector, { + state: 'attached', + timeout: 10000 + }); + + if (button) { + debugLog('Found working selector:', selector); + return { + button, + workingSelector: selector, + updatedSelectors + }; + } + } catch (error) { + retryCount++; + debugLog(`Selector "${selector}" failed: attempt ${retryCount}/${MAX_RETRIES}`); + + if (retryCount < MAX_RETRIES) { + await page.waitForTimeout(RETRY_DELAY); + } else { + debugLog(`Removing failed selector "${selector}" after ${MAX_RETRIES} attempts`); + updatedSelectors = updatedSelectors.filter(s => s !== selector); + } } - } catch (error) { - debugLog(`Selector failed: ${selector}`); } } - - // Implement retry mechanism when no selectors work - if (selectors.length > 0 && retryCount < MAX_RETRIES) { - debugLog(`Retry attempt ${retryCount + 1} of ${MAX_RETRIES}`); - await page.waitForTimeout(RETRY_DELAY); - return findWorkingButton(selectors, retryCount + 1); - } - - return { button: null, workingSelector: null }; + + return { + button: null, + workingSelector: null, + updatedSelectors + }; }; const retryOperation = async (operation: () => Promise, retryCount = 0): Promise => { @@ -686,7 +705,10 @@ export default class Interpreter extends EventEmitter { await scrapeCurrentPage(); if (checkLimit()) return allResults; - const { button, workingSelector } = await findWorkingButton(availableSelectors); + const { button, workingSelector, updatedSelectors } = await findWorkingButton(availableSelectors); + + availableSelectors = updatedSelectors; + if (!button || !workingSelector) { // Final retry for navigation when no selectors work const success = await retryOperation(async () => { @@ -703,10 +725,6 @@ export default class Interpreter extends EventEmitter { break; } - availableSelectors = availableSelectors.slice( - availableSelectors.indexOf(workingSelector) - ); - let retryCount = 0; let navigationSuccess = false; @@ -784,18 +802,15 @@ export default class Interpreter extends EventEmitter { while (true) { // Find working button with retry mechanism - const { button: loadMoreButton, workingSelector } = await findWorkingButton(availableSelectors); + const { button: loadMoreButton, workingSelector, updatedSelectors } = await findWorkingButton(availableSelectors); + + availableSelectors = updatedSelectors; if (!workingSelector || !loadMoreButton) { debugLog('No working Load More selector found after retries'); return allResults; } - // Update available selectors to start from the working one - availableSelectors = availableSelectors.slice( - availableSelectors.indexOf(workingSelector) - ); - // Implement retry mechanism for clicking the button let retryCount = 0; let clickSuccess = false; From c4540168a3935b8233e78bcdb14933db3f9ab213 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 4 Mar 2025 13:28:15 +0530 Subject: [PATCH 18/54] feat: rm anchor tag priority logic --- server/src/workflow-management/selector.ts | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts index 4f325025..80ed55f3 100644 --- a/server/src/workflow-management/selector.ts +++ b/server/src/workflow-management/selector.ts @@ -154,8 +154,11 @@ export const getElementInformation = async ( const el = getDeepestElementFromPoint(x, y); if (el) { - const { parentElement } = el; - const targetElement = parentElement?.tagName === 'A' ? parentElement : el; + // Prioritize Link (DO NOT REMOVE) + // const { parentElement } = el; + // const targetElement = parentElement?.tagName === 'A' ? parentElement : el; + + const targetElement = el; const ownerDocument = targetElement.ownerDocument; const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; @@ -701,8 +704,11 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector const el = getDeepestElementFromPoint(x, y); if (el) { - const { parentElement } = el; - const element = parentElement?.tagName === 'A' ? parentElement : el; + // Prioritize Link (DO NOT REMOVE) + // const { parentElement } = el; + // const element = parentElement?.tagName === 'A' ? parentElement : el; + + const element = el; const rectangle = element?.getBoundingClientRect(); if (rectangle) { const createRectObject = (rect: DOMRect) => ({ @@ -1875,9 +1881,12 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { hoveredElement != null && !hoveredElement.closest('#overlay-controls') != null ) { - const { parentElement } = hoveredElement; + // Prioritize Link (DO NOT REMOVE) + // const { parentElement } = hoveredElement; // Match the logic in recorder.ts for link clicks - const element = parentElement?.tagName === 'A' ? parentElement : hoveredElement; + // const element = parentElement?.tagName === 'A' ? parentElement : hoveredElement; + + const element = hoveredElement; const generatedSelectors = genSelectors(element); return generatedSelectors; } From cce663c6fd853f424fd367bb291f6c68b0ad4068 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 5 Mar 2025 21:27:08 +0530 Subject: [PATCH 19/54] feat: pass userId --- server/src/api/record.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/src/api/record.ts b/server/src/api/record.ts index 05560487..b826806d 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -507,9 +507,9 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) { } } -async function readyForRunHandler(browserId: string, id: string) { +async function readyForRunHandler(browserId: string, id: string, userId: string){ try { - const result = await executeRun(id); + const result = await executeRun(id, userId); if (result && result.success) { logger.log('info', `Interpretation of ${id} succeeded`); @@ -546,7 +546,7 @@ function AddGeneratedFlags(workflow: WorkflowFile) { return copy; }; -async function executeRun(id: string) { +async function executeRun(id: string, userId: string) { try { const run = await Run.findOne({ where: { runId: id } }); if (!run) { @@ -568,7 +568,7 @@ async function executeRun(id: string) { plainRun.status = 'running'; - const browser = browserPool.getRemoteBrowser(plainRun.browserId); + const browser = browserPool.getRemoteBrowser(plainRun.browserId, userId); if (!browser) { throw new Error('Could not access browser'); } @@ -672,12 +672,12 @@ export async function handleRunRecording(id: string, userId: string) { rejectUnauthorized: false }); - socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId)); + socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId, userId)); logger.log('info', `Running Robot: ${id}`); socket.on('disconnect', () => { - cleanupSocketListeners(socket, browserId, newRunId); + cleanupSocketListeners(socket, browserId, newRunId, userId); }); // Return the runId immediately, so the client knows the run is started @@ -688,8 +688,8 @@ export async function handleRunRecording(id: string, userId: string) { } } -function cleanupSocketListeners(socket: Socket, browserId: string, id: string) { - socket.off('ready-for-run', () => readyForRunHandler(browserId, id)); +function cleanupSocketListeners(socket: Socket, browserId: string, id: string, userId: string) { + socket.off('ready-for-run', () => readyForRunHandler(browserId, id, userId)); logger.log('info', `Cleaned up listeners for browserId: ${browserId}, runId: ${id}`); } From 1d5ccea0fe7eac713b071defdd998b1436ea1234 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 02:17:50 +0530 Subject: [PATCH 20/54] feat: use Authenticated request to pass userId --- server/src/routes/record.ts | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index 51d3ff92..006fff6b 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -76,8 +76,11 @@ router.post('/start', requireSignIn, (req: AuthenticatedRequest, res:Response) = * GET endpoint for terminating the remote browser recording session. * returns whether the termination was successful */ -router.get('/stop/:browserId', requireSignIn, async (req, res) => { - const success = await destroyRemoteBrowser(req.params.browserId); +router.get('/stop/:browserId', requireSignIn, async (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + const success = await destroyRemoteBrowser(req.user?.id, req.params.browserId); return res.send(success); }); @@ -92,10 +95,13 @@ router.get('/active', requireSignIn, (req, res) => { /** * GET endpoint for getting the current url of the active remote browser. */ -router.get('/active/url', requireSignIn, (req, res) => { +router.get('/active/url', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } const id = getActiveBrowserId(); if (id) { - const url = getRemoteBrowserCurrentUrl(id); + const url = getRemoteBrowserCurrentUrl(id, req.user?.id); return res.send(url); } return res.send(null); @@ -104,10 +110,13 @@ router.get('/active/url', requireSignIn, (req, res) => { /** * GET endpoint for getting the current tabs of the active remote browser. */ -router.get('/active/tabs', requireSignIn, (req, res) => { +router.get('/active/tabs', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } const id = getActiveBrowserId(); if (id) { - const hosts = getRemoteBrowserCurrentTabs(id); + const hosts = getRemoteBrowserCurrentTabs(id, req.user?.id); return res.send(hosts); } return res.send([]); @@ -116,9 +125,12 @@ router.get('/active/tabs', requireSignIn, (req, res) => { /** * GET endpoint for starting an interpretation of the currently generated workflow. */ -router.get('/interpret', requireSignIn, async (req, res) => { +router.get('/interpret', requireSignIn, async (req: AuthenticatedRequest, res) => { try { - await interpretWholeWorkflow(); + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + await interpretWholeWorkflow(req.user?.id); return res.send('interpretation done'); } catch (e) { return res.send('interpretation failed'); @@ -128,7 +140,10 @@ router.get('/interpret', requireSignIn, async (req, res) => { /** * GET endpoint for stopping an ongoing interpretation of the currently generated workflow. */ -router.get('/interpret/stop', requireSignIn, async (req, res) => { - await stopRunningInterpretation(); +router.get('/interpret/stop', requireSignIn, async (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + await stopRunningInterpretation(req.user?.id); return res.send('interpretation stopped'); }); From e79532c9f7e446586a1e6bc16256f782e1a1e667 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 02:35:38 +0530 Subject: [PATCH 21/54] feat: use Authenticated request to pass userId --- server/src/routes/storage.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 5ab9ef7a..e14d5e97 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -617,7 +617,7 @@ router.post('/runs/run/:id', requireSignIn, async (req: AuthenticatedRequest, re workflow, currentPage, (newPage: Page) => currentPage = newPage, plainRun.interpreterSettings); const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(plainRun.browserId); + await destroyRemoteBrowser(req.user?.id, plainRun.browserId); await run.update({ ...run, status: 'success', @@ -900,8 +900,9 @@ router.delete('/schedule/:id', requireSignIn, async (req: AuthenticatedRequest, /** * POST endpoint for aborting a current interpretation of the run. */ -router.post('/runs/abort/:id', requireSignIn, async (req, res) => { +router.post('/runs/abort/:id', requireSignIn, async (req: AuthenticatedRequest, res) => { try { + if (!req.user) { return res.status(401).send({ error: 'Unauthorized' }); } const run = await Run.findOne({ where: { runId: req.params.id } }); if (!run) { return res.status(404).send(false); From 591318728a65fd9edd03417d7af4bd242a8d0c3d Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 02:39:21 +0530 Subject: [PATCH 22/54] feat: use Authenticated request to pass userId --- server/src/api/record.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/api/record.ts b/server/src/api/record.ts index b826806d..b4014c3a 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -517,14 +517,14 @@ async function readyForRunHandler(browserId: string, id: string, userId: string) return result.interpretationInfo; } else { logger.log('error', `Interpretation of ${id} failed`); - await destroyRemoteBrowser(browserId); + await destroyRemoteBrowser(browserId, userId); resetRecordingState(browserId, id); return null; } } catch (error: any) { logger.error(`Error during readyForRunHandler: ${error.message}`); - await destroyRemoteBrowser(browserId); + await destroyRemoteBrowser(browserId, userId); return null; } } @@ -568,7 +568,7 @@ async function executeRun(id: string, userId: string) { plainRun.status = 'running'; - const browser = browserPool.getRemoteBrowser(plainRun.browserId, userId); + const browser = browserPool.getRemoteBrowser(userId); if (!browser) { throw new Error('Could not access browser'); } @@ -586,7 +586,7 @@ async function executeRun(id: string, userId: string) { const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(plainRun.browserId); + await destroyRemoteBrowser(plainRun.browserId, userId); const updatedRun = await run.update({ ...run, From 36e77d1666dd2b7956bfba2664383b850ac005de Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 02:41:11 +0530 Subject: [PATCH 23/54] feat: pass userId to browser pool methods --- .../workflow-management/scheduler/index.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index ade7d969..c5a9d19c 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -92,7 +92,7 @@ function AddGeneratedFlags(workflow: WorkflowFile) { return copy; }; -async function executeRun(id: string) { +async function executeRun(id: string, userId: string) { try { const run = await Run.findOne({ where: { runId: id } }); if (!run) { @@ -114,7 +114,7 @@ async function executeRun(id: string) { plainRun.status = 'running'; - const browser = browserPool.getRemoteBrowser(plainRun.browserId); + const browser = browserPool.getRemoteBrowser(userId); if (!browser) { throw new Error('Could not access browser'); } @@ -132,7 +132,7 @@ async function executeRun(id: string) { const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(plainRun.browserId); + await destroyRemoteBrowser(userId, plainRun.browserId); await run.update({ ...run, @@ -207,22 +207,22 @@ async function executeRun(id: string) { } } -async function readyForRunHandler(browserId: string, id: string) { +async function readyForRunHandler(browserId: string, id: string, userId: string) { try { - const interpretation = await executeRun(id); + const interpretation = await executeRun(id, userId); if (interpretation) { logger.log('info', `Interpretation of ${id} succeeded`); } else { logger.log('error', `Interpretation of ${id} failed`); - await destroyRemoteBrowser(browserId); + await destroyRemoteBrowser(userId, browserId); } resetRecordingState(browserId, id); } catch (error: any) { logger.error(`Error during readyForRunHandler: ${error.message}`); - await destroyRemoteBrowser(browserId); + await destroyRemoteBrowser(userId, browserId); } } @@ -245,12 +245,12 @@ export async function handleRunRecording(id: string, userId: string) { rejectUnauthorized: false }); - socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId)); + socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId, userId)); logger.log('info', `Running robot: ${id}`); socket.on('disconnect', () => { - cleanupSocketListeners(socket, browserId, newRunId); + cleanupSocketListeners(socket, browserId, newRunId, userId); }); } catch (error: any) { @@ -258,8 +258,8 @@ export async function handleRunRecording(id: string, userId: string) { } } -function cleanupSocketListeners(socket: Socket, browserId: string, id: string) { - socket.off('ready-for-run', () => readyForRunHandler(browserId, id)); +function cleanupSocketListeners(socket: Socket, browserId: string, id: string, userId: string) { + socket.off('ready-for-run', () => readyForRunHandler(browserId, id, userId)); logger.log('info', `Cleaned up listeners for browserId: ${browserId}, runId: ${id}`); } From 4059e8bd2671bb843126d3a02648c831ad8b96f7 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 02:50:14 +0530 Subject: [PATCH 24/54] feat: pass userId to browser pool methods --- server/src/browser-management/controller.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index 24a677ce..329ffa1a 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -37,7 +37,7 @@ export const initializeRemoteBrowserForRecording = (userId: string): string => { await browserSession.initialize(userId); await browserSession.registerEditorEvents(); await browserSession.subscribeToScreencast(); - browserPool.addRemoteBrowser(id, browserSession, true); + browserPool.addRemoteBrowser(id, browserSession, userId); } socket.emit('loaded'); }); @@ -59,7 +59,7 @@ export const createRemoteBrowserForRun = (userId: string): string => { async (socket: Socket) => { const browserSession = new RemoteBrowser(socket); await browserSession.initialize(userId); - browserPool.addRemoteBrowser(id, browserSession, true); + browserPool.addRemoteBrowser(id, browserSession, userId); socket.emit('ready-for-run'); }); return id; @@ -72,14 +72,14 @@ export const createRemoteBrowserForRun = (userId: string): string => { * @returns {Promise} * @category BrowserManagement-Controller */ -export const destroyRemoteBrowser = async (id: string): Promise => { +export const destroyRemoteBrowser = async (id: string, userId: string): Promise => { const browserSession = browserPool.getRemoteBrowser(id); if (browserSession) { logger.log('debug', `Switching off the browser with id: ${id}`); await browserSession.stopCurrentInterpretation(); await browserSession.switchOff(); } - return browserPool.deleteRemoteBrowser(id); + return browserPool.deleteRemoteBrowser(userId); }; /** @@ -98,7 +98,7 @@ export const getActiveBrowserId = (): string | null => { * @returns {string | undefined} * @category BrowserManagement-Controller */ -export const getRemoteBrowserCurrentUrl = (id: string): string | undefined => { +export const getRemoteBrowserCurrentUrl = (id: string, userId: string): string | undefined => { return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.url(); }; @@ -108,7 +108,7 @@ export const getRemoteBrowserCurrentUrl = (id: string): string | undefined => { * @return {string[] | undefined} * @category BrowserManagement-Controller */ -export const getRemoteBrowserCurrentTabs = (id: string): string[] | undefined => { +export const getRemoteBrowserCurrentTabs = (id: string, userId: string): string[] | undefined => { return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.context().pages() .map((page) => { const parsedUrl = new URL(page.url()); @@ -126,7 +126,7 @@ export const getRemoteBrowserCurrentTabs = (id: string): string[] | undefined => * @returns {Promise} * @category BrowserManagement-Controller */ -export const interpretWholeWorkflow = async () => { +export const interpretWholeWorkflow = async (userId: string) => { const id = getActiveBrowserId(); if (id) { const browser = browserPool.getRemoteBrowser(id); @@ -146,7 +146,7 @@ export const interpretWholeWorkflow = async () => { * @returns {Promise} * @category BrowserManagement-Controller */ -export const stopRunningInterpretation = async () => { +export const stopRunningInterpretation = async (userId: string) => { const id = getActiveBrowserId(); if (id) { const browser = browserPool.getRemoteBrowser(id); From 57e75a5ce7224f81cdd225aceb327da7e65577e9 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 03:54:44 +0530 Subject: [PATCH 25/54] feat: pass userId to getActiveBrowserId --- server/src/browser-management/controller.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index 329ffa1a..16bbb66d 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -21,12 +21,12 @@ import logger from "../logger"; * @category BrowserManagement-Controller */ export const initializeRemoteBrowserForRecording = (userId: string): string => { - const id = getActiveBrowserId() || uuid(); + const id = getActiveBrowserId(userId) || uuid(); createSocketConnection( io.of(id), async (socket: Socket) => { // browser is already active - const activeId = getActiveBrowserId(); + const activeId = getActiveBrowserId(userId); if (activeId) { const remoteBrowser = browserPool.getRemoteBrowser(activeId); remoteBrowser?.updateSocket(socket); @@ -88,8 +88,8 @@ export const destroyRemoteBrowser = async (id: string, userId: string): Promise< * @returns {string | null} * @category BrowserManagement-Controller */ -export const getActiveBrowserId = (): string | null => { - return browserPool.getActiveBrowserId(); +export const getActiveBrowserId = (userId: string): string | null => { + return browserPool.getActiveBrowserId(userId); }; /** @@ -127,7 +127,7 @@ export const getRemoteBrowserCurrentTabs = (id: string, userId: string): string[ * @category BrowserManagement-Controller */ export const interpretWholeWorkflow = async (userId: string) => { - const id = getActiveBrowserId(); + const id = getActiveBrowserId(userId); if (id) { const browser = browserPool.getRemoteBrowser(id); if (browser) { @@ -147,7 +147,7 @@ export const interpretWholeWorkflow = async (userId: string) => { * @category BrowserManagement-Controller */ export const stopRunningInterpretation = async (userId: string) => { - const id = getActiveBrowserId(); + const id = getActiveBrowserId(userId); if (id) { const browser = browserPool.getRemoteBrowser(id); await browser?.stopCurrentInterpretation(); From da8adcd7ad89e23cbdaae34edbcf5bf61255ac94 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 03:55:48 +0530 Subject: [PATCH 26/54] feat: pass userId to getActiveBrowserId --- server/src/routes/record.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index 006fff6b..5e386468 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -87,8 +87,11 @@ router.get('/stop/:browserId', requireSignIn, async (req: AuthenticatedRequest, /** * GET endpoint for getting the id of the active remote browser. */ -router.get('/active', requireSignIn, (req, res) => { - const id = getActiveBrowserId(); +router.get('/active', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + const id = getActiveBrowserId(req.user?.id); return res.send(id); }); @@ -99,7 +102,7 @@ router.get('/active/url', requireSignIn, (req: AuthenticatedRequest, res) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = getActiveBrowserId(); + const id = getActiveBrowserId(req.user?.id); if (id) { const url = getRemoteBrowserCurrentUrl(id, req.user?.id); return res.send(url); @@ -114,7 +117,7 @@ router.get('/active/tabs', requireSignIn, (req: AuthenticatedRequest, res) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = getActiveBrowserId(); + const id = getActiveBrowserId(req.user?.id); if (id) { const hosts = getRemoteBrowserCurrentTabs(id, req.user?.id); return res.send(hosts); From 0385d29deddae87527b3951fce3b7ebac517a02a Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 03:56:18 +0530 Subject: [PATCH 27/54] feat: pass userId to getActiveBrowserId --- server/src/routes/workflow.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/server/src/routes/workflow.ts b/server/src/routes/workflow.ts index 3c0e356b..3a117ce1 100644 --- a/server/src/routes/workflow.ts +++ b/server/src/routes/workflow.ts @@ -7,6 +7,7 @@ import logger from "../logger"; import { browserPool } from "../server"; import { requireSignIn } from '../middlewares/auth'; import Robot from '../models/Robot'; +import { AuthenticatedRequest } from './record'; export const router = Router(); @@ -46,8 +47,9 @@ router.get('/params/:browserId', requireSignIn, (req, res) => { /** * DELETE endpoint for deleting a pair from the generated workflow. */ -router.delete('/pair/:index', requireSignIn, (req, res) => { - const id = browserPool.getActiveBrowserId(); +router.delete('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { return res.status(401).send('User not authenticated'); } + const id = browserPool.getActiveBrowserId(req.user?.id); if (id) { const browser = browserPool.getRemoteBrowser(id); if (browser) { @@ -62,8 +64,9 @@ router.delete('/pair/:index', requireSignIn, (req, res) => { /** * POST endpoint for adding a pair to the generated workflow. */ -router.post('/pair/:index', requireSignIn, (req, res) => { - const id = browserPool.getActiveBrowserId(); +router.post('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { return res.status(401).send('User not authenticated'); } + const id = browserPool.getActiveBrowserId(req.user?.id); if (id) { const browser = browserPool.getRemoteBrowser(id); logger.log('debug', `Adding pair to workflow`); @@ -82,8 +85,9 @@ router.post('/pair/:index', requireSignIn, (req, res) => { /** * PUT endpoint for updating a pair in the generated workflow. */ -router.put('/pair/:index', requireSignIn, (req, res) => { - const id = browserPool.getActiveBrowserId(); +router.put('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { return res.status(401).send('User not authenticated'); } + const id = browserPool.getActiveBrowserId(req.user?.id); if (id) { const browser = browserPool.getRemoteBrowser(id); logger.log('debug', `Updating pair in workflow`); From 86be39a609dc19f877da4858d8e671dfec134116 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 04:07:33 +0530 Subject: [PATCH 28/54] feat: pass userId to getActiveBrowserId --- server/src/workflow-management/classes/Generator.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index b4c72e96..8cdc233c 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -151,8 +151,8 @@ export class WorkflowGenerator { workflow: [], }); socket.on('activeIndex', (data) => this.generatedData.lastIndex = parseInt(data)); - socket.on('decision', async ({ pair, actionType, decision }) => { - const id = browserPool.getActiveBrowserId(); + socket.on('decision', async ({ pair, actionType, decision, userId }) => { + const id = browserPool.getActiveBrowserId(userId); if (id) { // const activeBrowser = browserPool.getRemoteBrowser(id); // const currentPage = activeBrowser?.getCurrentPage(); From 0dfbcd3c170c3e5c4acbfd4487a6d37afcca399b Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 04:08:11 +0530 Subject: [PATCH 29/54] feat: pass req to handleWrapper --- server/src/browser-management/inputHandlers.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index bf365053..298a95ce 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -12,6 +12,7 @@ import { WorkflowGenerator } from "../workflow-management/classes/Generator"; import { Page } from "playwright"; import { throttle } from "../../../src/helpers/inputHelpers"; import { CustomActions } from "../../../src/shared/types"; +import { AuthenticatedRequest } from '../routes/record'; /** * A wrapper function for handling user input. @@ -31,9 +32,21 @@ const handleWrapper = async ( page: Page, args?: any ) => Promise, - args?: any + args?: any, + req?: AuthenticatedRequest, ) => { - const id = browserPool.getActiveBrowserId(); + if (!req || !req.user || typeof req.user === 'string') { + logger.log('warn', `User not authenticated or invalid JWT payload`); + return; + } + + const userId = req.user.id; + if (!userId) { + logger.log('warn', `User ID is missing in JWT payload`); + return; + } + + const id = browserPool.getActiveBrowserId(userId); if (id) { const activeBrowser = browserPool.getRemoteBrowser(id); if (activeBrowser?.interpreter.interpretationInProgress() && !activeBrowser.interpreter.interpretationIsPaused) { From 9798631c1696a244a2164af58f0ced266e3a059b Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 04:41:35 +0530 Subject: [PATCH 30/54] feat: accept user id that owns the browser instance --- server/src/browser-management/classes/RemoteBrowser.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 8ff4f601..8bf8380c 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -90,6 +90,12 @@ export class RemoteBrowser { maxRepeats: 1, }; + /** + * The user ID that owns this browser instance + * @private + */ + private userId: string; + private lastEmittedUrl: string | null = null; /** From c8a10c5230bb96d295bb7e46ef4db28fcc40b2f6 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 05:06:58 +0530 Subject: [PATCH 31/54] feat: is screencast active --- server/src/browser-management/classes/RemoteBrowser.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 8bf8380c..2d161d14 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -112,6 +112,7 @@ export class RemoteBrowser { private screenshotQueue: Buffer[] = []; private isProcessingScreenshot = false; private screencastInterval: NodeJS.Timeout | null = null + private isScreencastActive: boolean = false; /** * Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and From 61914a562a5c1c1682ac16d46dd8d660bb9a6e25 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 05:11:06 +0530 Subject: [PATCH 32/54] feat: instantiate userId --- server/src/browser-management/classes/RemoteBrowser.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 2d161d14..8ae755bd 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -120,8 +120,9 @@ export class RemoteBrowser { * @param socket socket.io socket instance used to communicate with the client side * @constructor */ - public constructor(socket: Socket) { + public constructor(socket: Socket, userId: string) { this.socket = socket; + this.userId = userId; this.interpreter = new WorkflowInterpreter(socket); this.generator = new WorkflowGenerator(socket); } From 242529c3fdedf15ab54b67e7c59d26721f7962fb Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 05:16:44 +0530 Subject: [PATCH 33/54] feat: include userId in event data to ensure client can match it to the correct browser --- server/src/browser-management/classes/RemoteBrowser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 8ae755bd..855c5596 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -201,7 +201,7 @@ export class RemoteBrowser { const currentUrl = page.url(); if (this.shouldEmitUrlChange(currentUrl)) { this.lastEmittedUrl = currentUrl; - this.socket.emit('urlChanged', currentUrl); + this.socket.emit('urlChanged', {url: currentUrl, userId: this.userId}); } } }); From aa51b23f999c21c0425e5fcfa0b56ce0108cbc37 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 16:44:19 +0530 Subject: [PATCH 34/54] feat: pass userId --- server/src/browser-management/controller.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index 16bbb66d..92fb6499 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -32,7 +32,7 @@ export const initializeRemoteBrowserForRecording = (userId: string): string => { remoteBrowser?.updateSocket(socket); await remoteBrowser?.makeAndEmitScreenshot(); } else { - const browserSession = new RemoteBrowser(socket); + const browserSession = new RemoteBrowser(socket, userId); browserSession.interpreter.subscribeToPausing(); await browserSession.initialize(userId); await browserSession.registerEditorEvents(); @@ -57,7 +57,7 @@ export const createRemoteBrowserForRun = (userId: string): string => { createSocketConnectionForRun( io.of(id), async (socket: Socket) => { - const browserSession = new RemoteBrowser(socket); + const browserSession = new RemoteBrowser(socket, userId); await browserSession.initialize(userId); browserPool.addRemoteBrowser(id, browserSession, userId); socket.emit('ready-for-run'); From ea43d9d604fdc62cb15dfd12a926d98c230eeb46 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:40:07 +0530 Subject: [PATCH 35/54] feat: browser pool info interface --- .../browser-management/classes/BrowserPool.ts | 85 +------------------ 1 file changed, 4 insertions(+), 81 deletions(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index cd4962a1..21fc4b00 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -15,86 +15,9 @@ interface BrowserPoolInfo { * @default false */ active: boolean, + /** + * The user ID that owns this browser instance. + */ + userId: string, } -/** - * Dictionary of all the active remote browser's instances indexed by their id. - * The value in this dictionary is of type BrowserPoolInfo, - * which provides additional information about the browser's usage. - * @category Types - */ -interface PoolDictionary { - [key: string]: BrowserPoolInfo, -} - -/** - * A browser pool is a collection of remote browsers that are initialized and ready to be used. - * Adds the possibility to add, remove and retrieve remote browsers from the pool. - * It is possible to manage multiple browsers for creating or running a recording. - * @category BrowserManagement - */ -export class BrowserPool { - - /** - * Holds all the instances of remote browsers. - */ - private pool: PoolDictionary = {}; - - /** - * Adds a remote browser instance to the pool indexed by the id. - * @param id remote browser instance's id - * @param browser remote browser instance - * @param active states if the browser's instance is being actively used - */ - public addRemoteBrowser = (id: string, browser: RemoteBrowser, active: boolean = false): void => { - this.pool = { - ...this.pool, - [id]: { - browser, - active, - }, - } - logger.log('debug', `Remote browser with id: ${id} added to the pool`); - }; - - /** - * Removes the remote browser instance from the pool. - * @param id remote browser instance's id - * @returns true if the browser was removed successfully, false otherwise - */ - public deleteRemoteBrowser = (id: string): boolean => { - if (!this.pool[id]) { - logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); - return false; - } - delete (this.pool[id]); - logger.log('debug', `Remote browser with id: ${id} deleted from the pool`); - return true; - }; - - /** - * Returns the remote browser instance from the pool. - * @param id remote browser instance's id - * @returns remote browser instance or undefined if it does not exist in the pool - */ - public getRemoteBrowser = (id: string): RemoteBrowser | undefined => { - logger.log('debug', `Remote browser with id: ${id} retrieved from the pool`); - return this.pool[id]?.browser; - }; - - /** - * Returns the active browser's instance id from the pool. - * If there is no active browser, it returns undefined. - * If there are multiple active browsers, it returns the first one. - * @returns the first remote active browser instance's id from the pool - */ - public getActiveBrowserId = (): string | null => { - for (const id of Object.keys(this.pool)) { - if (this.pool[id].active) { - return id; - } - } - logger.log('warn', `No active browser in the pool`); - return null; - }; -} From ffc030d58b89b281959e11eae7d78b4c98fbf48f Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:40:34 +0530 Subject: [PATCH 36/54] feat: pool dictionary --- server/src/browser-management/classes/BrowserPool.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 21fc4b00..5244fcea 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -21,3 +21,13 @@ interface BrowserPoolInfo { userId: string, } +/** + * Dictionary of all the active remote browser's instances indexed by their id. + * The value in this dictionary is of type BrowserPoolInfo, + * which provides additional information about the browser's usage. + * @category Types + */ +interface PoolDictionary { + [key: string]: BrowserPoolInfo, +} + From 4fad9634e0a939f8af1ba0331641498d426a685b Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:41:03 +0530 Subject: [PATCH 37/54] feat: init new browser pool --- .../browser-management/classes/BrowserPool.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 5244fcea..bac06d2f 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -31,3 +31,32 @@ interface PoolDictionary { [key: string]: BrowserPoolInfo, } +/** + * A browser pool is a collection of remote browsers that are initialized and ready to be used. + * Enforces a "1 User - 1 Browser" policy, while allowing multiple users to have their own browser instances. + * Adds the possibility to add, remove and retrieve remote browsers from the pool. + * @category BrowserManagement + */ +export class BrowserPool { + /** + * Holds all the instances of remote browsers. + */ + private pool: PoolDictionary = {}; + + /** + * Maps user IDs to their browser IDs. + */ + private userToBrowserMap: Map = new Map(); + + /** + * Adds a remote browser instance to the pool for a specific user. + * If the user already has a browser, the existing browser will be closed and replaced. + * + * @param id remote browser instance's id + * @param browser remote browser instance + * @param userId the user ID that owns this browser instance + * @param active states if the browser's instance is being actively used + * @returns true if a new browser was added, false if an existing browser was replaced + */ + +} \ No newline at end of file From 677f05d0da295e77d07863a949a9fea6e3ad5fdc Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:41:25 +0530 Subject: [PATCH 38/54] feat: add new browser --- .../browser-management/classes/BrowserPool.ts | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index bac06d2f..16fa1908 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -58,5 +58,46 @@ export class BrowserPool { * @param active states if the browser's instance is being actively used * @returns true if a new browser was added, false if an existing browser was replaced */ - + public addRemoteBrowser = ( + id: string, + browser: RemoteBrowser, + userId: string, + active: boolean = false + ): boolean => { + // Check if user already has a browser + const existingBrowserId = this.userToBrowserMap.get(userId); + let replaced = false; + + if (existingBrowserId) { + // Close and remove the existing browser + if (existingBrowserId !== id) { + this.closeAndDeleteBrowser(existingBrowserId); + replaced = true; + } else { + // If it's the same browser ID, just update the info + this.pool[id] = { + browser, + active, + userId, + }; + logger.log('debug', `Updated existing browser with id: ${id} for user: ${userId}`); + return false; + } + } + + // Add the new browser to the pool + this.pool[id] = { + browser, + active, + userId, + }; + + // Update the user-to-browser mapping + this.userToBrowserMap.set(userId, id); + + logger.log('debug', `Remote browser with id: ${id} added to the pool for user: ${userId}`); + return !replaced; + }; + + } \ No newline at end of file From cce901962dd807f59ceb98fe0d4437c7ea49470b Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:41:47 +0530 Subject: [PATCH 39/54] feat: close & del browser --- .../browser-management/classes/BrowserPool.ts | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 16fa1908..19719f3e 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -99,5 +99,31 @@ export class BrowserPool { return !replaced; }; - + /** + * Removes the remote browser instance from the pool. + * Note: This doesn't handle browser closing as RemoteBrowser doesn't expose a close method. + * The caller should ensure the browser is properly closed before calling this method. + * + * @param id remote browser instance's id + * @returns true if the browser was removed successfully, false otherwise + */ + public closeAndDeleteBrowser = (id: string): boolean => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return false; + } + + // Remove the user-to-browser mapping + const userId = this.pool[id].userId; + if (this.userToBrowserMap.get(userId) === id) { + this.userToBrowserMap.delete(userId); + } + + // Remove from pool + delete this.pool[id]; + logger.log('debug', `Remote browser with id: ${id} removed from the pool`); + return true; + }; + + } \ No newline at end of file From 2fa75ab8f39efea31fdf7e2ce3e6ed4de27f1563 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:42:07 +0530 Subject: [PATCH 40/54] feat: del browser --- .../browser-management/classes/BrowserPool.ts | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 19719f3e..6f2e2adf 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -125,5 +125,29 @@ export class BrowserPool { return true; }; + /** + * Removes the remote browser instance from the pool without attempting to close it. + * + * @param id remote browser instance's id + * @returns true if the browser was removed successfully, false otherwise + */ + public deleteRemoteBrowser = (id: string): boolean => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return false; + } + + // Remove the user-to-browser mapping + const userId = this.pool[id].userId; + if (this.userToBrowserMap.get(userId) === id) { + this.userToBrowserMap.delete(userId); + } + + // Remove from pool + delete this.pool[id]; + logger.log('debug', `Remote browser with id: ${id} deleted from the pool`); + return true; + }; + } \ No newline at end of file From 63e48cead7f42e899e80c43b10c05d9cdc6e5d40 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:42:23 +0530 Subject: [PATCH 41/54] feat: get browser --- server/src/browser-management/classes/BrowserPool.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 6f2e2adf..d260f16d 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -149,5 +149,16 @@ export class BrowserPool { return true; }; + /** + * Returns the remote browser instance from the pool. + * + * @param id remote browser instance's id + * @returns remote browser instance or undefined if it does not exist in the pool + */ + public getRemoteBrowser = (id: string): RemoteBrowser | undefined => { + logger.log('debug', `Remote browser with id: ${id} retrieved from the pool`); + return this.pool[id]?.browser; + }; + } \ No newline at end of file From e9d3fcebad909aa3aa94ff479537f992f467e853 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:42:49 +0530 Subject: [PATCH 42/54] feat: get active browser id --- .../browser-management/classes/BrowserPool.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index d260f16d..e09f96ef 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -160,5 +160,28 @@ export class BrowserPool { return this.pool[id]?.browser; }; + /** + * Returns the active browser's instance id for a specific user. + * + * @param userId the user ID to find the browser for + * @returns the browser ID for the user, or null if no browser exists + */ + public getActiveBrowserId = (userId: string): string | null => { + const browserId = this.userToBrowserMap.get(userId); + if (!browserId) { + logger.log('debug', `No browser found for user: ${userId}`); + return null; + } + + // Verify the browser still exists in the pool + if (!this.pool[browserId]) { + this.userToBrowserMap.delete(userId); + logger.log('warn', `Browser mapping found for user: ${userId}, but browser doesn't exist in pool`); + return null; + } + console.log(`Browser Id ${browserId} found for user: ${userId}`); + return browserId; + }; + } \ No newline at end of file From 59e494c9be103fef30203258931d41d578493cb5 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:43:06 +0530 Subject: [PATCH 43/54] feat: get user for browser --- .../src/browser-management/classes/BrowserPool.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index e09f96ef..a52c85d6 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -183,5 +183,18 @@ export class BrowserPool { return browserId; }; + /** + * Returns the user ID associated with a browser ID. + * + * @param browserId the browser ID to find the user for + * @returns the user ID for the browser, or null if the browser doesn't exist + */ + public getUserForBrowser = (browserId: string): string | null => { + if (!this.pool[browserId]) { + return null; + } + return this.pool[browserId].userId; + }; + } \ No newline at end of file From 9a1fa3fbce94ff18812b2eb0fccf044bb9d62b69 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:43:45 +0530 Subject: [PATCH 44/54] feat: set active browser --- .../browser-management/classes/BrowserPool.ts | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index a52c85d6..b3002100 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -196,5 +196,23 @@ export class BrowserPool { return this.pool[browserId].userId; }; - + /** + * Sets the active state of a browser. + * + * @param id the browser ID + * @param active the new active state + * @returns true if successful, false if the browser wasn't found + */ + public setActiveBrowser = (id: string, active: boolean): boolean => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return false; + } + + this.pool[id].active = active; + logger.log('debug', `Remote browser with id: ${id} set to ${active ? 'active' : 'inactive'}`); + return true; + }; + + } \ No newline at end of file From c7fd8cb4a9e926545b870e6b0ddc316c2fe23001 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:44:05 +0530 Subject: [PATCH 45/54] feat: get all active browser ids for user --- .../browser-management/classes/BrowserPool.ts | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index b3002100..4d3e54c0 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -214,5 +214,36 @@ export class BrowserPool { return true; }; + /** + * Returns all browser instances for a specific user. + * Should only be one per the "1 User - 1 Browser" policy, but included for flexibility. + * + * @param userId the user ID to find browsers for + * @returns an array of browser IDs belonging to the user + */ + public getAllBrowserIdsForUser = (userId: string): string[] => { + const browserIds: string[] = []; + + // Normally this would just return the one browser from the map + const mappedBrowserId = this.userToBrowserMap.get(userId); + if (mappedBrowserId && this.pool[mappedBrowserId]) { + browserIds.push(mappedBrowserId); + } + + // But as a safeguard, also check the entire pool for any browsers assigned to this user + // This helps detect and fix any inconsistencies in the maps + for (const [id, info] of Object.entries(this.pool)) { + if (info.userId === userId && !browserIds.includes(id)) { + browserIds.push(id); + // Fix the map if it's inconsistent + if (!mappedBrowserId) { + this.userToBrowserMap.set(userId, id); + } + } + } + + return browserIds; + }; + } \ No newline at end of file From b16e7b3834a5dfff2d69e21d86fc362402d49cad Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:44:34 +0530 Subject: [PATCH 46/54] feat: get pool size --- server/src/browser-management/classes/BrowserPool.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 4d3e54c0..68992002 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -245,5 +245,12 @@ export class BrowserPool { return browserIds; }; + /** + * Returns the total number of browsers in the pool. + */ + public getPoolSize = (): number => { + return Object.keys(this.pool).length; + }; + } \ No newline at end of file From 04629c05c447f9b7cdf9bd5bdb5e1eb23a304cfc Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:44:50 +0530 Subject: [PATCH 47/54] feat: get active user count --- server/src/browser-management/classes/BrowserPool.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 68992002..b4e35582 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -252,5 +252,12 @@ export class BrowserPool { return Object.keys(this.pool).length; }; - + /** + * Returns the total number of active users (users with browsers). + */ + public getActiveUserCount = (): number => { + return this.userToBrowserMap.size; + }; + + } \ No newline at end of file From 8e12516985417cd357913ccc7b79f02e8e9ea4c6 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:45:20 +0530 Subject: [PATCH 48/54] feat: getActiveBrowserForMigration <-> backward compatibility --- .../browser-management/classes/BrowserPool.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index b4e35582..3c86a220 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -259,5 +259,39 @@ export class BrowserPool { return this.userToBrowserMap.size; }; + /** + * Gets the current active browser for the system if there's only one active user. + * This is a migration helper to support code that hasn't been updated to the user-browser model yet. + * + * @param currentUserId The ID of the current user, which will be prioritized if multiple browsers exist + * @returns A browser ID if one can be determined, or null + */ + public getActiveBrowserForMigration = (currentUserId?: string): string | null => { + // If a current user ID is provided and they have a browser, return that + if (currentUserId) { + const browserForUser = this.getActiveBrowserId(currentUserId); + if (browserForUser) { + return browserForUser; + } + } + + // If only one user has a browser, return that + if (this.userToBrowserMap.size === 1) { + const userId = Array.from(this.userToBrowserMap.keys())[0]; + return this.userToBrowserMap.get(userId) || null; + } + + // Fall back to the first active browser if any + for (const id of Object.keys(this.pool)) { + if (this.pool[id].active) { + return id; + } + } + + // If all else fails, return the first browser in the pool + const browserIds = Object.keys(this.pool); + return browserIds.length > 0 ? browserIds[0] : null; + }; + } \ No newline at end of file From 6761968f2d4f8a330d6c183aa8f717ce1798007e Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:46:00 +0530 Subject: [PATCH 49/54] feat: get active browser id <-> backward compatibility --- .../browser-management/classes/BrowserPool.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 3c86a220..77c3a637 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -293,5 +293,22 @@ export class BrowserPool { return browserIds.length > 0 ? browserIds[0] : null; }; - + /** + * Returns the first active browser's instance id from the pool. + * If there is no active browser, it returns null. + * If there are multiple active browsers, it returns the first one. + * + * @returns the first remote active browser instance's id from the pool + * @deprecated Use getBrowserIdForUser instead to enforce the 1 User - 1 Browser policy + */ + public getActiveBrowserIdDep = (): string | null => { + for (const id of Object.keys(this.pool)) { + if (this.pool[id].active) { + return id; + } + } + // Don't log a warning since this behavior is expected in the user-browser model + // logger.log('warn', `No active browser in the pool`); + return null; + }; } \ No newline at end of file From 8f95ad82b9b651dbb24c9cc574b7d14d9baed25a Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:46:28 +0530 Subject: [PATCH 50/54] wip: modify socket events based on user --- .../classes/RemoteBrowser.ts | 108 +++++++++++++++--- 1 file changed, 91 insertions(+), 17 deletions(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 855c5596..85d1928c 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -341,15 +341,40 @@ export class RemoteBrowser { * @returns void */ public registerEditorEvents = (): void => { - this.socket.on('rerender', async () => await this.makeAndEmitScreenshot()); - this.socket.on('settings', (settings) => this.interpreterSettings = settings); - this.socket.on('changeTab', async (tabIndex) => await this.changeTab(tabIndex)); - this.socket.on('addTab', async () => { + // For each event, include userId to make sure events are handled for the correct browser + logger.log('debug', `Registering editor events for user: ${this.userId}`); + + // Listen for specific events for this user + this.socket.on(`rerender:${this.userId}`, async () => { + logger.debug(`Rerender event received for user ${this.userId}`); + await this.makeAndEmitScreenshot(); + }); + + // For backward compatibility, also listen to the general event + this.socket.on('rerender', async () => { + logger.debug(`General rerender event received, checking if for user ${this.userId}`); + await this.makeAndEmitScreenshot(); + }); + + this.socket.on(`settings:${this.userId}`, (settings) => { + this.interpreterSettings = settings; + logger.debug(`Settings updated for user ${this.userId}`); + }); + + this.socket.on(`changeTab:${this.userId}`, async (tabIndex) => { + logger.debug(`Tab change to ${tabIndex} requested for user ${this.userId}`); + await this.changeTab(tabIndex); + }); + + this.socket.on(`addTab:${this.userId}`, async () => { + logger.debug(`New tab requested for user ${this.userId}`); await this.currentPage?.context().newPage(); const lastTabIndex = this.currentPage ? this.currentPage.context().pages().length - 1 : 0; await this.changeTab(lastTabIndex); }); - this.socket.on('closeTab', async (tabInfo) => { + + this.socket.on(`closeTab:${this.userId}`, async (tabInfo) => { + logger.debug(`Close tab ${tabInfo.index} requested for user ${this.userId}`); const page = this.currentPage?.context().pages()[tabInfo.index]; if (page) { if (tabInfo.isCurrent) { @@ -364,24 +389,52 @@ export class RemoteBrowser { await page.close(); logger.log( 'debug', - `${tabInfo.index} page was closed, new length of pages: ${this.currentPage?.context().pages().length}` - ) + `Tab ${tabInfo.index} was closed for user ${this.userId}, new tab count: ${this.currentPage?.context().pages().length}` + ); } else { - logger.log('error', `${tabInfo.index} index out of range of pages`) + logger.log('error', `Tab index ${tabInfo.index} out of range for user ${this.userId}`); } }); - this.socket.on('setViewportSize', async (data: { width: number, height: number }) => { + + this.socket.on(`setViewportSize:${this.userId}`, async (data: { width: number, height: number }) => { const { width, height } = data; - logger.log('debug', `Received viewport size: width=${width}, height=${height}`); + logger.log('debug', `Viewport size change to width=${width}, height=${height} requested for user ${this.userId}`); // Update the browser context's viewport dynamically if (this.context && this.browser) { this.context = await this.browser.newContext({ viewport: { width, height } }); - logger.log('debug', `Viewport size updated to width=${width}, height=${height} for the entire browser context`); + logger.log('debug', `Viewport size updated to width=${width}, height=${height} for user ${this.userId}`); } }); - } - + + // For backward compatibility, also register the standard events + this.socket.on('settings', (settings) => this.interpreterSettings = settings); + this.socket.on('changeTab', async (tabIndex) => await this.changeTab(tabIndex)); + this.socket.on('addTab', async () => { + await this.currentPage?.context().newPage(); + const lastTabIndex = this.currentPage ? this.currentPage.context().pages().length - 1 : 0; + await this.changeTab(lastTabIndex); + }); + this.socket.on('closeTab', async (tabInfo) => { + const page = this.currentPage?.context().pages()[tabInfo.index]; + if (page) { + if (tabInfo.isCurrent) { + if (this.currentPage?.context().pages()[tabInfo.index + 1]) { + await this.changeTab(tabInfo.index + 1); + } else { + await this.changeTab(tabInfo.index - 1); + } + } + await page.close(); + } + }); + this.socket.on('setViewportSize', async (data: { width: number, height: number }) => { + const { width, height } = data; + if (this.context && this.browser) { + this.context = await this.browser.newContext({ viewport: { width, height } }); + } + }); + }; /** * Subscribes the remote browser for a screencast session * on [CDP](https://chromedevtools.github.io/devtools-protocol/) level, @@ -390,16 +443,24 @@ export class RemoteBrowser { * @returns {Promise} */ public subscribeToScreencast = async (): Promise => { + logger.log('debug', `Starting screencast for user: ${this.userId}`); await this.startScreencast(); if (!this.client) { logger.log('warn', 'client is not initialized'); return; } + // Set flag to indicate screencast is active + this.isScreencastActive = true; + this.client.on('Page.screencastFrame', ({ data: base64, sessionId }) => { + // Only process if screencast is still active for this user + if (!this.isScreencastActive) { + return; + } this.emitScreenshot(Buffer.from(base64, 'base64')) setTimeout(async () => { try { - if (!this.client) { + if (!this.client || !this.isScreencastActive) { logger.log('warn', 'client is not initialized'); return; } @@ -418,6 +479,8 @@ export class RemoteBrowser { */ public async switchOff(): Promise { try { + this.isScreencastActive = false; + await this.interpreter.stopInterpretation(); if (this.screencastInterval) { @@ -561,7 +624,11 @@ export class RemoteBrowser { //await this.currentPage.setViewportSize({ height: 400, width: 900 }) this.client = await this.currentPage.context().newCDPSession(this.currentPage); - this.socket.emit('urlChanged', this.currentPage.url()); + // Include userId in the URL change event + this.socket.emit('urlChanged', { + url: this.currentPage.url(), + userId: this.userId + }); await this.makeAndEmitScreenshot(); await this.subscribeToScreencast(); } else { @@ -610,6 +677,8 @@ export class RemoteBrowser { await this.client.send('Page.startScreencast', { format: SCREENCAST_CONFIG.format, }); + // Set flag to indicate screencast is active + this.isScreencastActive = true; // Set up screencast frame handler this.client.on('Page.screencastFrame', async ({ data, sessionId }) => { @@ -635,6 +704,8 @@ export class RemoteBrowser { } try { + // Set flag to indicate screencast is active + this.isScreencastActive = false; await this.client.send('Page.stopScreencast'); this.screenshotQueue = []; this.isProcessingScreenshot = false; @@ -665,8 +736,11 @@ export class RemoteBrowser { const base64Data = optimizedScreenshot.toString('base64'); const dataWithMimeType = `data:image/jpeg;base64,${base64Data}`; - this.socket.emit('screencast', dataWithMimeType); - logger.debug('Screenshot emitted'); +// Emit with user context to ensure the frontend can identify which browser's screenshot this is +this.socket.emit('screencast', { + image: dataWithMimeType, + userId: this.userId +}); logger.debug('Screenshot emitted'); } catch (error) { logger.error('Screenshot emission failed:', error); } finally { From a0e7404d25c8675ba29a8b1a438856baa6ba749f Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:46:47 +0530 Subject: [PATCH 51/54] feat: get active browser id <-> backward compatibility --- server/src/browser-management/classes/BrowserPool.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 77c3a637..4bbc97af 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -301,7 +301,7 @@ export class BrowserPool { * @returns the first remote active browser instance's id from the pool * @deprecated Use getBrowserIdForUser instead to enforce the 1 User - 1 Browser policy */ - public getActiveBrowserIdDep = (): string | null => { + public getActiveBrowserIdLegacy = (): string | null => { for (const id of Object.keys(this.pool)) { if (this.pool[id].active) { return id; From 6de4cfcafea7651a2cb17bcc43b51a4478d70f53 Mon Sep 17 00:00:00 2001 From: Rohit Date: Sat, 8 Mar 2025 17:09:33 +0530 Subject: [PATCH 52/54] feat: server socket management for multiple browsers --- server/src/browser-management/controller.ts | 2 +- .../src/browser-management/inputHandlers.ts | 182 ++++++++++++------ server/src/routes/record.ts | 2 +- server/src/routes/storage.ts | 2 +- server/src/socket-connection/connection.ts | 62 +++++- .../workflow-management/scheduler/index.ts | 6 +- 6 files changed, 186 insertions(+), 70 deletions(-) diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index 92fb6499..e6619d77 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -79,7 +79,7 @@ export const destroyRemoteBrowser = async (id: string, userId: string): Promise< await browserSession.stopCurrentInterpretation(); await browserSession.switchOff(); } - return browserPool.deleteRemoteBrowser(userId); + return browserPool.deleteRemoteBrowser(id); }; /** diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 298a95ce..92c75e1d 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -4,6 +4,8 @@ * These functions are called by the client through socket communication. */ import { Socket } from 'socket.io'; +import { IncomingMessage } from 'http'; +import { JwtPayload } from 'jsonwebtoken'; import logger from "../logger"; import { Coordinates, ScrollDeltas, KeyboardInput, DatePickerEventData } from '../types'; @@ -12,7 +14,14 @@ import { WorkflowGenerator } from "../workflow-management/classes/Generator"; import { Page } from "playwright"; import { throttle } from "../../../src/helpers/inputHelpers"; import { CustomActions } from "../../../src/shared/types"; -import { AuthenticatedRequest } from '../routes/record'; + +interface AuthenticatedIncomingMessage extends IncomingMessage { + user?: JwtPayload | string; +} + +interface AuthenticatedSocket extends Socket { + request: AuthenticatedIncomingMessage; +} /** * A wrapper function for handling user input. @@ -24,6 +33,7 @@ import { AuthenticatedRequest } from '../routes/record'; * * @param handleCallback The callback handler to be called * @param args - arguments to be passed to the handler + * @param socket - socket with authenticated request * @category HelperFunctions */ const handleWrapper = async ( @@ -33,14 +43,14 @@ const handleWrapper = async ( args?: any ) => Promise, args?: any, - req?: AuthenticatedRequest, + socket?: AuthenticatedSocket, ) => { - if (!req || !req.user || typeof req.user === 'string') { + if (!socket || !socket.request || !socket.request.user || typeof socket.request.user === 'string') { logger.log('warn', `User not authenticated or invalid JWT payload`); return; } - const userId = req.user.id; + const userId = socket.request.user.id; if (!userId) { logger.log('warn', `User ID is missing in JWT payload`); return; @@ -79,12 +89,13 @@ interface CustomActionEventData { /** * A wrapper function for handling custom actions. + * @param socket The socket connection * @param customActionEventData The custom action event data * @category HelperFunctions */ -const onGenerateAction = async (customActionEventData: CustomActionEventData) => { +const onGenerateAction = async (socket: AuthenticatedSocket, customActionEventData: CustomActionEventData) => { logger.log('debug', `Generating ${customActionEventData.action} action emitted from client`); - await handleWrapper(handleGenerateAction, customActionEventData); + await handleWrapper(handleGenerateAction, customActionEventData, socket); } /** @@ -102,12 +113,13 @@ const handleGenerateAction = /** * A wrapper function for handling mousedown event. + * @param socket The socket connection * @param coordinates - coordinates of the mouse click * @category HelperFunctions */ -const onMousedown = async (coordinates: Coordinates) => { +const onMousedown = async (socket: AuthenticatedSocket, coordinates: Coordinates) => { logger.log('debug', 'Handling mousedown event emitted from client'); - await handleWrapper(handleMousedown, coordinates); + await handleWrapper(handleMousedown, coordinates, socket); } /** @@ -152,12 +164,13 @@ const handleMousedown = async (generator: WorkflowGenerator, page: Page, { x, y /** * A wrapper function for handling the wheel event. + * @param socket The socket connection * @param scrollDeltas - the scroll deltas of the wheel event * @category HelperFunctions */ -const onWheel = async (scrollDeltas: ScrollDeltas) => { +const onWheel = async (socket: AuthenticatedSocket, scrollDeltas: ScrollDeltas) => { logger.log('debug', 'Handling scroll event emitted from client'); - await handleWrapper(handleWheel, scrollDeltas); + await handleWrapper(handleWheel, scrollDeltas, socket); }; /** @@ -178,12 +191,13 @@ const handleWheel = async (generator: WorkflowGenerator, page: Page, { deltaX, d /** * A wrapper function for handling the mousemove event. + * @param socket The socket connection * @param coordinates - the coordinates of the mousemove event * @category HelperFunctions */ -const onMousemove = async (coordinates: Coordinates) => { +const onMousemove = async (socket: AuthenticatedSocket, coordinates: Coordinates) => { logger.log('debug', 'Handling mousemove event emitted from client'); - await handleWrapper(handleMousemove, coordinates); + await handleWrapper(handleMousemove, coordinates, socket); } /** @@ -212,12 +226,13 @@ const handleMousemove = async (generator: WorkflowGenerator, page: Page, { x, y /** * A wrapper function for handling the keydown event. + * @param socket The socket connection * @param keyboardInput - the keyboard input of the keydown event * @category HelperFunctions */ -const onKeydown = async (keyboardInput: KeyboardInput) => { +const onKeydown = async (socket: AuthenticatedSocket, keyboardInput: KeyboardInput) => { logger.log('debug', 'Handling keydown event emitted from client'); - await handleWrapper(handleKeydown, keyboardInput); + await handleWrapper(handleKeydown, keyboardInput, socket); } /** @@ -248,49 +263,95 @@ const handleDateSelection = async (generator: WorkflowGenerator, page: Page, dat logger.log('debug', `Date ${data.value} selected`); } -const onDateSelection = async (data: DatePickerEventData) => { +/** + * A wrapper function for handling the date selection event. + * @param socket The socket connection + * @param data - the data of the date selection event + * @category HelperFunctions + */ +const onDateSelection = async (socket: AuthenticatedSocket, data: DatePickerEventData) => { logger.log('debug', 'Handling date selection event emitted from client'); - await handleWrapper(handleDateSelection, data); + await handleWrapper(handleDateSelection, data, socket); } +/** + * Handles the dropdown selection event. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param data - the data of the dropdown selection event + * @category BrowserManagement + */ const handleDropdownSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { await generator.onDropdownSelection(page, data); logger.log('debug', `Dropdown value ${data.value} selected`); } -const onDropdownSelection = async (data: { selector: string, value: string }) => { +/** + * A wrapper function for handling the dropdown selection event. + * @param socket The socket connection + * @param data - the data of the dropdown selection event + * @category HelperFunctions + */ +const onDropdownSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => { logger.log('debug', 'Handling dropdown selection event emitted from client'); - await handleWrapper(handleDropdownSelection, data); + await handleWrapper(handleDropdownSelection, data, socket); } +/** + * Handles the time selection event. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param data - the data of the time selection event + * @category BrowserManagement + */ const handleTimeSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { await generator.onTimeSelection(page, data); logger.log('debug', `Time value ${data.value} selected`); } -const onTimeSelection = async (data: { selector: string, value: string }) => { +/** + * A wrapper function for handling the time selection event. + * @param socket The socket connection + * @param data - the data of the time selection event + * @category HelperFunctions + */ +const onTimeSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => { logger.log('debug', 'Handling time selection event emitted from client'); - await handleWrapper(handleTimeSelection, data); + await handleWrapper(handleTimeSelection, data, socket); } +/** + * Handles the datetime-local selection event. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param data - the data of the datetime-local selection event + * @category BrowserManagement + */ const handleDateTimeLocalSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { await generator.onDateTimeLocalSelection(page, data); logger.log('debug', `DateTime Local value ${data.value} selected`); } -const onDateTimeLocalSelection = async (data: { selector: string, value: string }) => { +/** + * A wrapper function for handling the datetime-local selection event. + * @param socket The socket connection + * @param data - the data of the datetime-local selection event + * @category HelperFunctions + */ +const onDateTimeLocalSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => { logger.log('debug', 'Handling datetime-local selection event emitted from client'); - await handleWrapper(handleDateTimeLocalSelection, data); + await handleWrapper(handleDateTimeLocalSelection, data, socket); } /** * A wrapper function for handling the keyup event. + * @param socket The socket connection * @param keyboardInput - the keyboard input of the keyup event * @category HelperFunctions */ -const onKeyup = async (keyboardInput: KeyboardInput) => { +const onKeyup = async (socket: AuthenticatedSocket, keyboardInput: KeyboardInput) => { logger.log('debug', 'Handling keyup event emitted from client'); - await handleWrapper(handleKeyup, keyboardInput); + await handleWrapper(handleKeyup, keyboardInput, socket); } /** @@ -309,12 +370,13 @@ const handleKeyup = async (generator: WorkflowGenerator, page: Page, key: string /** * A wrapper function for handling the url change event. + * @param socket The socket connection * @param url - the new url of the page * @category HelperFunctions */ -const onChangeUrl = async (url: string) => { +const onChangeUrl = async (socket: AuthenticatedSocket, url: string) => { logger.log('debug', 'Handling change url event emitted from client'); - await handleWrapper(handleChangeUrl, url); + await handleWrapper(handleChangeUrl, url, socket); } /** @@ -342,11 +404,12 @@ const handleChangeUrl = async (generator: WorkflowGenerator, page: Page, url: st /** * A wrapper function for handling the refresh event. + * @param socket The socket connection * @category HelperFunctions */ -const onRefresh = async () => { +const onRefresh = async (socket: AuthenticatedSocket) => { logger.log('debug', 'Handling refresh event emitted from client'); - await handleWrapper(handleRefresh); + await handleWrapper(handleRefresh, undefined, socket); } /** @@ -363,11 +426,12 @@ const handleRefresh = async (generator: WorkflowGenerator, page: Page) => { /** * A wrapper function for handling the go back event. + * @param socket The socket connection * @category HelperFunctions */ -const onGoBack = async () => { - logger.log('debug', 'Handling refresh event emitted from client'); - await handleWrapper(handleGoBack); +const onGoBack = async (socket: AuthenticatedSocket) => { + logger.log('debug', 'Handling go back event emitted from client'); + await handleWrapper(handleGoBack, undefined, socket); } /** @@ -385,11 +449,12 @@ const handleGoBack = async (generator: WorkflowGenerator, page: Page) => { /** * A wrapper function for handling the go forward event. + * @param socket The socket connection * @category HelperFunctions */ -const onGoForward = async () => { - logger.log('debug', 'Handling refresh event emitted from client'); - await handleWrapper(handleGoForward); +const onGoForward = async (socket: AuthenticatedSocket) => { + logger.log('debug', 'Handling go forward event emitted from client'); + await handleWrapper(handleGoForward, undefined, socket); } /** @@ -407,18 +472,7 @@ const handleGoForward = async (generator: WorkflowGenerator, page: Page) => { /** * Helper function for registering the handlers onto established websocket connection. - * Registers: - * - mousedownHandler - * - wheelHandler - * - mousemoveHandler - * - keydownHandler - * - keyupHandler - * - changeUrlHandler - * - refreshHandler - * - goBackHandler - * - goForwardHandler - * - onGenerateAction - * input handlers. + * Registers various input handlers. * * All these handlers first generates the workflow pair data * and then calls the corresponding playwright's function to emulate the input. @@ -428,21 +482,25 @@ const handleGoForward = async (generator: WorkflowGenerator, page: Page) => { * @returns void * @category BrowserManagement */ -const registerInputHandlers = (socket: Socket) => { - socket.on("input:mousedown", onMousedown); - socket.on("input:wheel", onWheel); - socket.on("input:mousemove", onMousemove); - socket.on("input:keydown", onKeydown); - socket.on("input:keyup", onKeyup); - socket.on("input:url", onChangeUrl); - socket.on("input:refresh", onRefresh); - socket.on("input:back", onGoBack); - socket.on("input:forward", onGoForward); - socket.on("input:date", onDateSelection); - socket.on("input:dropdown", onDropdownSelection); - socket.on("input:time", onTimeSelection); - socket.on("input:datetime-local", onDateTimeLocalSelection); - socket.on("action", onGenerateAction); +const registerInputHandlers = (socket: Socket) => { + // Cast to our authenticated socket type + const authSocket = socket as AuthenticatedSocket; + + // Register handlers with the socket + socket.on("input:mousedown", (data) => onMousedown(authSocket, data)); + socket.on("input:wheel", (data) => onWheel(authSocket, data)); + socket.on("input:mousemove", (data) => onMousemove(authSocket, data)); + socket.on("input:keydown", (data) => onKeydown(authSocket, data)); + socket.on("input:keyup", (data) => onKeyup(authSocket, data)); + socket.on("input:url", (data) => onChangeUrl(authSocket, data)); + socket.on("input:refresh", () => onRefresh(authSocket)); + socket.on("input:back", () => onGoBack(authSocket)); + socket.on("input:forward", () => onGoForward(authSocket)); + socket.on("input:date", (data) => onDateSelection(authSocket, data)); + socket.on("input:dropdown", (data) => onDropdownSelection(authSocket, data)); + socket.on("input:time", (data) => onTimeSelection(authSocket, data)); + socket.on("input:datetime-local", (data) => onDateTimeLocalSelection(authSocket, data)); + socket.on("action", (data) => onGenerateAction(authSocket, data)); }; -export default registerInputHandlers; +export default registerInputHandlers; \ No newline at end of file diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index 5e386468..495dda5d 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -80,7 +80,7 @@ router.get('/stop/:browserId', requireSignIn, async (req: AuthenticatedRequest, if (!req.user) { return res.status(401).send('User not authenticated'); } - const success = await destroyRemoteBrowser(req.user?.id, req.params.browserId); + const success = await destroyRemoteBrowser(req.params.browserId, req.user?.id); return res.send(success); }); diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index e14d5e97..04c0341c 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -617,7 +617,7 @@ router.post('/runs/run/:id', requireSignIn, async (req: AuthenticatedRequest, re workflow, currentPage, (newPage: Page) => currentPage = newPage, plainRun.interpreterSettings); const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(req.user?.id, plainRun.browserId); + await destroyRemoteBrowser(plainRun.browserId, req.user?.id); await run.update({ ...run, status: 'success', diff --git a/server/src/socket-connection/connection.ts b/server/src/socket-connection/connection.ts index dac7a88e..9fad861b 100644 --- a/server/src/socket-connection/connection.ts +++ b/server/src/socket-connection/connection.ts @@ -1,6 +1,60 @@ import { Namespace, Socket } from 'socket.io'; +import { IncomingMessage } from 'http'; +import { verify, JwtPayload } from 'jsonwebtoken'; import logger from "../logger"; -import registerInputHandlers from '../browser-management/inputHandlers' +import registerInputHandlers from '../browser-management/inputHandlers'; + +interface AuthenticatedIncomingMessage extends IncomingMessage { + user?: JwtPayload | string; +} + +interface AuthenticatedSocket extends Socket { + request: AuthenticatedIncomingMessage; +} + +/** + * Socket.io middleware for authentication + * This is a socket.io specific auth handler that doesn't rely on Express middleware + */ +const socketAuthMiddleware = (socket: Socket, next: (err?: Error) => void) => { + const cookies = socket.handshake.headers.cookie; + if (!cookies) { + return next(new Error('Authentication required')); + } + + const tokenMatch = cookies.split(';').find(c => c.trim().startsWith('token=')); + if (!tokenMatch) { + return next(new Error('Authentication required')); + } + + const token = tokenMatch.split('=')[1]; + if (!token) { + return next(new Error('Authentication required')); + } + + const secret = process.env.JWT_SECRET; + if (!secret) { + return next(new Error('Server configuration error')); + } + + verify(token, secret, (err: any, user: any) => { + if (err) { + logger.log('warn', 'JWT verification error:', err); + return next(new Error('Authentication failed')); + } + + // Normalize payload key + if (user.userId && !user.id) { + user.id = user.userId; + delete user.userId; // temporary: del the old key for clarity + } + + // Attach user to socket request + const authSocket = socket as AuthenticatedSocket; + authSocket.request.user = user; + next(); + }); +}; /** * Opens a websocket canal for duplex data transfer and registers all handlers for this data for the recording session. @@ -13,6 +67,8 @@ export const createSocketConnection = ( io: Namespace, callback: (socket: Socket) => void, ) => { + io.use(socketAuthMiddleware); + const onConnection = async (socket: Socket) => { logger.log('info', "Client connected " + socket.id); registerInputHandlers(socket); @@ -34,6 +90,8 @@ export const createSocketConnectionForRun = ( io: Namespace, callback: (socket: Socket) => void, ) => { + io.use(socketAuthMiddleware); + const onConnection = async (socket: Socket) => { logger.log('info', "Client connected " + socket.id); socket.on('disconnect', () => logger.log('info', "Client disconnected " + socket.id)); @@ -41,4 +99,4 @@ export const createSocketConnectionForRun = ( } io.on('connection', onConnection); -}; +}; \ No newline at end of file diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index c5a9d19c..b67e1ca0 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -132,7 +132,7 @@ async function executeRun(id: string, userId: string) { const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(userId, plainRun.browserId); + await destroyRemoteBrowser(plainRun.browserId, userId); await run.update({ ...run, @@ -215,14 +215,14 @@ async function readyForRunHandler(browserId: string, id: string, userId: string) logger.log('info', `Interpretation of ${id} succeeded`); } else { logger.log('error', `Interpretation of ${id} failed`); - await destroyRemoteBrowser(userId, browserId); + await destroyRemoteBrowser(browserId, userId); } resetRecordingState(browserId, id); } catch (error: any) { logger.error(`Error during readyForRunHandler: ${error.message}`); - await destroyRemoteBrowser(userId, browserId); + await destroyRemoteBrowser(browserId, userId); } } From 0220632aef8f27ba29095ffb8c0a0d4f26e79814 Mon Sep 17 00:00:00 2001 From: Rohit Date: Sat, 8 Mar 2025 17:10:30 +0530 Subject: [PATCH 53/54] feat: emit screenshot for unique user --- src/components/browser/BrowserWindow.tsx | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/components/browser/BrowserWindow.tsx b/src/components/browser/BrowserWindow.tsx index ffeb0df5..05c93c1e 100644 --- a/src/components/browser/BrowserWindow.tsx +++ b/src/components/browser/BrowserWindow.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useEffect, useState } from 'react'; +import React, { useCallback, useContext, useEffect, useState } from 'react'; import { useSocketStore } from '../../context/socket'; import { Button } from '@mui/material'; import Canvas from "../recorder/canvas"; @@ -8,6 +8,7 @@ import { useActionContext } from '../../context/browserActions'; import { useBrowserSteps, TextStep } from '../../context/browserSteps'; import { useGlobalInfoStore } from '../../context/globalInfo'; import { useTranslation } from 'react-i18next'; +import { AuthContext } from '../../context/auth'; interface ElementInfo { tagName: string; @@ -27,6 +28,12 @@ interface AttributeOption { value: string; } +interface ScreencastData { + image: string; + userId: string; +} + + const getAttributeOptions = (tagName: string, elementInfo: ElementInfo | null): AttributeOption[] => { if (!elementInfo) return []; switch (tagName.toLowerCase()) { @@ -72,6 +79,9 @@ export const BrowserWindow = () => { const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext(); const { addTextStep, addListStep } = useBrowserSteps(); + const { state } = useContext(AuthContext); + const { user } = state; + const onMouseMove = (e: MouseEvent) => { if (canvasRef && canvasRef.current && highlighterData) { const canvasRect = canvasRef.current.getBoundingClientRect(); @@ -99,9 +109,15 @@ export const BrowserWindow = () => { } }, [getList, resetListState]); - const screencastHandler = useCallback((data: string) => { - setScreenShot(data); - }, [screenShot]); + const screencastHandler = useCallback((data: string | ScreencastData) => { + if (typeof data === 'string') { + setScreenShot(data); + } else if (data && typeof data === 'object' && 'image' in data) { + if (!data.userId || data.userId === user?.id) { + setScreenShot(data.image); + } + } + }, [screenShot, user?.id]); useEffect(() => { if (socket) { From 4271b18a457cd6ce4466b69350fe0aac7363b838 Mon Sep 17 00:00:00 2001 From: Karishma Shukla Date: Sun, 9 Mar 2025 00:21:52 +0530 Subject: [PATCH 54/54] fix: use runByUserId --- server/src/routes/storage.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 04c0341c..aa3f4e12 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -903,7 +903,10 @@ router.delete('/schedule/:id', requireSignIn, async (req: AuthenticatedRequest, router.post('/runs/abort/:id', requireSignIn, async (req: AuthenticatedRequest, res) => { try { if (!req.user) { return res.status(401).send({ error: 'Unauthorized' }); } - const run = await Run.findOne({ where: { runId: req.params.id } }); + const run = await Run.findOne({ where: { + runId: req.params.id, + runByUserId: req.user.id, + } }); if (!run) { return res.status(404).send(false); } @@ -938,4 +941,4 @@ router.post('/runs/abort/:id', requireSignIn, async (req: AuthenticatedRequest, logger.log('info', `Error while running a robot with name: ${req.params.fileName}_${req.params.runId}.json`); return res.send(false); } -}); \ No newline at end of file +});