diff --git a/README.md b/README.md index 13c994a0..3470a4ee 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,8 @@ You can access the frontend at http://localhost:5173/ and backend at http://loca | `GOOGLE_CLIENT_ID` | No | Client ID for Google OAuth, used for Google Sheet integration authentication. | Google login will not work. | | `GOOGLE_CLIENT_SECRET`| No | Client Secret for Google OAuth. | Google login will not work. | | `GOOGLE_REDIRECT_URI` | No | Redirect URI for handling Google OAuth responses. | Google login will not work. | +| `AIRTABLE_CLIENT_ID` | No | Client ID for Airtable, used for Airtable integration authentication. | Airtable login will not work. | +| `AIRTABLE_REDIRECT_URI` | No | Redirect URI for handling Airtable OAuth responses. | Airtable login will not work. | | `REDIS_HOST` | Yes | Host address of the Redis server, used by BullMQ for scheduling robots. | Redis connection will fail. | | `REDIS_PORT` | Yes | Port number for the Redis server. | Redis connection will fail. | | `MAXUN_TELEMETRY` | No | Disables telemetry to stop sending anonymous usage data. Keeping it enabled helps us understand how the product is used and assess the impact of any new changes. Please keep it enabled. | Telemetry data will not be collected. | diff --git a/maxun-core/src/browserSide/scraper.js b/maxun-core/src/browserSide/scraper.js index f6b53da2..c04a9993 100644 --- a/maxun-core/src/browserSide/scraper.js +++ b/maxun-core/src/browserSide/scraper.js @@ -210,7 +210,6 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, return Array.from(document.querySelectorAll(config.selector)); } - // First handle iframe traversal if present if (config.selector.includes(':>>')) { const parts = config.selector.split(':>>').map(s => s.trim()); let currentElements = [document]; @@ -223,23 +222,44 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, for (const element of currentElements) { try { - // For document or iframe document const doc = element.contentDocument || element || element.contentWindow?.document; if (!doc) continue; - // Query elements in current context + if (part.startsWith('frame[name=') || part.startsWith('iframe[name=')) { + const nameMatch = part.match(/\[name=['"]([^'"]+)['"]\]/); + if (nameMatch && nameMatch[1]) { + const frameName = nameMatch[1]; + let foundFrames = []; + + if (doc.getElementsByName && typeof doc.getElementsByName === 'function') { + foundFrames = Array.from(doc.getElementsByName(frameName)) + .filter(el => el.tagName === 'FRAME' || el.tagName === 'IFRAME'); + } + + if (foundFrames.length === 0) { + const framesBySelector = Array.from(doc.querySelectorAll(`frame[name="${frameName}"], iframe[name="${frameName}"]`)); + foundFrames = framesBySelector; + } + + if (isLast) { + nextElements.push(...foundFrames); + } else { + nextElements.push(...foundFrames); + } + continue; + } + } + const found = Array.from(doc.querySelectorAll(part)); if (isLast) { - // If it's the last part, keep all matching elements nextElements.push(...found); } else { - // If not last, only keep iframes for next iteration - const iframes = found.filter(el => el.tagName === 'IFRAME'); - nextElements.push(...iframes); + const frames = found.filter(el => el.tagName === 'IFRAME' || el.tagName === 'FRAME'); + nextElements.push(...frames); } } catch (error) { - console.warn('Cannot access iframe content:', error, { + console.warn('Cannot access iframe/frame content:', error, { part, element, index: i @@ -285,12 +305,17 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, return []; } - // Modified to handle iframe context for URL resolution function getElementValue(element, attribute) { if (!element) return null; - // Get the base URL for resolving relative URLs - const baseURL = element.ownerDocument?.location?.href || window.location.origin; + let baseURL; + try { + baseURL = element.ownerDocument?.location?.href || + element.ownerDocument?.baseURI || + window.location.origin; + } catch (e) { + baseURL = window.location.origin; + } switch (attribute) { case 'href': { @@ -305,6 +330,10 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, return element.innerText?.trim(); case 'textContent': return element.textContent?.trim(); + case 'innerHTML': + return element.innerHTML; + case 'outerHTML': + return element.outerHTML; default: return element.getAttribute(attribute) || element.innerText?.trim(); } @@ -394,7 +423,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, * @returns {Array.>} Array of arrays of scraped items, one sub-array per list */ window.scrapeList = async function ({ listSelector, fields, limit = 10 }) { - // Enhanced query function to handle both iframe and shadow DOM + // Enhanced query function to handle iframe, frame and shadow DOM const queryElement = (rootElement, selector) => { if (!selector.includes('>>') && !selector.includes(':>>')) { return rootElement.querySelector(selector); @@ -406,14 +435,14 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, for (let i = 0; i < parts.length; i++) { if (!currentElement) return null; - // Handle iframe traversal - if (currentElement.tagName === 'IFRAME') { + // Handle iframe and frame traversal + if (currentElement.tagName === 'IFRAME' || currentElement.tagName === 'FRAME') { try { - const iframeDoc = currentElement.contentDocument || currentElement.contentWindow.document; - currentElement = iframeDoc.querySelector(parts[i]); + const frameDoc = currentElement.contentDocument || currentElement.contentWindow.document; + currentElement = frameDoc.querySelector(parts[i]); continue; } catch (e) { - console.warn('Cannot access iframe content:', e); + console.warn(`Cannot access ${currentElement.tagName.toLowerCase()} content:`, e); return null; } } @@ -456,13 +485,13 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, const nextElements = []; for (const element of currentElements) { - // Handle iframe traversal - if (element.tagName === 'IFRAME') { + // Handle iframe and frame traversal + if (element.tagName === 'IFRAME' || element.tagName === 'FRAME') { try { - const iframeDoc = element.contentDocument || element.contentWindow.document; - nextElements.push(...iframeDoc.querySelectorAll(part)); + const frameDoc = element.contentDocument || element.contentWindow.document; + nextElements.push(...frameDoc.querySelectorAll(part)); } catch (e) { - console.warn('Cannot access iframe content:', e); + console.warn(`Cannot access ${element.tagName.toLowerCase()} content:`, e); continue; } } else { @@ -537,8 +566,8 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, return { type: 'TR', element: currentElement }; } - // Handle iframe crossing - if (currentElement.tagName === 'IFRAME') { + // Handle iframe and frame crossing + if (currentElement.tagName === 'IFRAME' || currentElement.tagName === 'FRAME') { try { currentElement = currentElement.contentDocument.body; } catch (e) { @@ -582,7 +611,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, if (current.tagName === 'TH') return true; - if (current.tagName === 'IFRAME') { + if (current.tagName === 'IFRAME' || current.tagName === 'FRAME') { try { current = current.contentDocument.body; } catch (e) { @@ -638,14 +667,18 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, allElements.push(...shadowHost.getElementsByTagName(baseElement.tagName)); } - // Get elements from iframes - const iframes = document.getElementsByTagName('iframe'); - for (const iframe of iframes) { + // Get elements from iframes and frames + const frames = [ + ...Array.from(document.getElementsByTagName('iframe')), + ...Array.from(document.getElementsByTagName('frame')) + ]; + + for (const frame of frames) { try { - const iframeDoc = iframe.contentDocument || iframe.contentWindow.document; - allElements.push(...iframeDoc.getElementsByTagName(baseElement.tagName)); + const frameDoc = frame.contentDocument || frame.contentWindow.document; + allElements.push(...frameDoc.getElementsByTagName(baseElement.tagName)); } catch (e) { - console.warn('Cannot access iframe content:', e); + console.warn(`Cannot access ${frame.tagName.toLowerCase()} content:`, e); } } @@ -707,7 +740,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, const tableData = []; const nonTableData = []; - // Process table data with both iframe and shadow DOM support + // Process table data with support for iframes, frames, and shadow DOM for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) { const container = containers[containerIndex]; const { tableFields } = containerFields[containerIndex]; @@ -717,14 +750,14 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, const firstElement = queryElement(container, firstField.selector); let tableContext = firstElement; - // Find table context including both iframe and shadow DOM + // Find table context including iframe, frame and shadow DOM while (tableContext && tableContext.tagName !== 'TABLE' && tableContext !== container) { if (tableContext.getRootNode() instanceof ShadowRoot) { tableContext = tableContext.getRootNode().host; continue; } - if (tableContext.tagName === 'IFRAME') { + if (tableContext.tagName === 'IFRAME' || tableContext.tagName === 'FRAME') { try { tableContext = tableContext.contentDocument.body; } catch (e) { @@ -747,13 +780,13 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, rows.push(...tableContext.shadowRoot.getElementsByTagName('TR')); } - // Get rows from iframes - if (tableContext.tagName === 'IFRAME') { + // Get rows from iframes and frames + if (tableContext.tagName === 'IFRAME' || tableContext.tagName === 'FRAME') { try { - const iframeDoc = tableContext.contentDocument || tableContext.contentWindow.document; - rows.push(...iframeDoc.getElementsByTagName('TR')); + const frameDoc = tableContext.contentDocument || tableContext.contentWindow.document; + rows.push(...frameDoc.getElementsByTagName('TR')); } catch (e) { - console.warn('Cannot access iframe rows:', e); + console.warn(`Cannot access ${tableContext.tagName.toLowerCase()} rows:`, e); } } @@ -823,7 +856,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, } } - // Process non-table data with both contexts support + // Process non-table data with all contexts support for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) { if (nonTableData.length >= limit) break; diff --git a/maxun-core/src/interpret.ts b/maxun-core/src/interpret.ts index 88047128..2887b634 100644 --- a/maxun-core/src/interpret.ts +++ b/maxun-core/src/interpret.ts @@ -286,6 +286,12 @@ export default class Interpreter extends EventEmitter { ? arrayToObject(superset[key]) : superset[key]; + if ((key === 'url' || key === 'selectors') && + Array.isArray(value) && Array.isArray(superset[key]) && + value.length === 0 && (superset[key] as any[]).length === 0) { + return true; + } + if (key === 'selectors' && Array.isArray(value) && Array.isArray(superset[key])) { return value.some(selector => (superset[key] as any[]).includes(selector) @@ -592,33 +598,52 @@ export default class Interpreter extends EventEmitter { }; // Enhanced button finder with retry mechanism - const findWorkingButton = async (selectors: string[], retryCount = 0): Promise<{ - button: ElementHandle | null, - workingSelector: string | null + const findWorkingButton = async (selectors: string[]): Promise<{ + button: ElementHandle | null, + workingSelector: string | null, + updatedSelectors: string[] }> => { - for (const selector of selectors) { - try { - const button = await page.waitForSelector(selector, { - state: 'attached', - timeout: 10000 // Reduced timeout for faster checks - }); - if (button) { - debugLog('Found working selector:', selector); - return { button, workingSelector: selector }; + let updatedSelectors = [...selectors]; + + for (let i = 0; i < selectors.length; i++) { + const selector = selectors[i]; + let retryCount = 0; + let selectorSuccess = false; + + while (retryCount < MAX_RETRIES && !selectorSuccess) { + try { + const button = await page.waitForSelector(selector, { + state: 'attached', + timeout: 10000 + }); + + if (button) { + debugLog('Found working selector:', selector); + return { + button, + workingSelector: selector, + updatedSelectors + }; + } + } catch (error) { + retryCount++; + debugLog(`Selector "${selector}" failed: attempt ${retryCount}/${MAX_RETRIES}`); + + if (retryCount < MAX_RETRIES) { + await page.waitForTimeout(RETRY_DELAY); + } else { + debugLog(`Removing failed selector "${selector}" after ${MAX_RETRIES} attempts`); + updatedSelectors = updatedSelectors.filter(s => s !== selector); + } } - } catch (error) { - debugLog(`Selector failed: ${selector}`); } } - - // Implement retry mechanism when no selectors work - if (selectors.length > 0 && retryCount < MAX_RETRIES) { - debugLog(`Retry attempt ${retryCount + 1} of ${MAX_RETRIES}`); - await page.waitForTimeout(RETRY_DELAY); - return findWorkingButton(selectors, retryCount + 1); - } - - return { button: null, workingSelector: null }; + + return { + button: null, + workingSelector: null, + updatedSelectors + }; }; const retryOperation = async (operation: () => Promise, retryCount = 0): Promise => { @@ -680,7 +705,10 @@ export default class Interpreter extends EventEmitter { await scrapeCurrentPage(); if (checkLimit()) return allResults; - const { button, workingSelector } = await findWorkingButton(availableSelectors); + const { button, workingSelector, updatedSelectors } = await findWorkingButton(availableSelectors); + + availableSelectors = updatedSelectors; + if (!button || !workingSelector) { // Final retry for navigation when no selectors work const success = await retryOperation(async () => { @@ -697,10 +725,6 @@ export default class Interpreter extends EventEmitter { break; } - availableSelectors = availableSelectors.slice( - availableSelectors.indexOf(workingSelector) - ); - let retryCount = 0; let navigationSuccess = false; @@ -768,22 +792,25 @@ export default class Interpreter extends EventEmitter { } case 'clickLoadMore': { + await scrapeCurrentPage(); + if (checkLimit()) return allResults; + + let loadMoreCounter = 0; + let previousResultCount = allResults.length; + let noNewItemsCounter = 0; + const MAX_NO_NEW_ITEMS = 2; + while (true) { - // Find working button with retry mechanism, consistent with clickNext - const { button: loadMoreButton, workingSelector } = await findWorkingButton(availableSelectors); + // Find working button with retry mechanism + const { button: loadMoreButton, workingSelector, updatedSelectors } = await findWorkingButton(availableSelectors); + + availableSelectors = updatedSelectors; if (!workingSelector || !loadMoreButton) { debugLog('No working Load More selector found after retries'); - const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config); - allResults = allResults.concat(finalResults); return allResults; } - // Update available selectors to start from the working one - availableSelectors = availableSelectors.slice( - availableSelectors.indexOf(workingSelector) - ); - // Implement retry mechanism for clicking the button let retryCount = 0; let clickSuccess = false; @@ -808,6 +835,8 @@ export default class Interpreter extends EventEmitter { if (clickSuccess) { await page.waitForTimeout(1000); + loadMoreCounter++; + debugLog(`Successfully clicked Load More button (${loadMoreCounter} times)`); } } catch (error) { debugLog(`Click attempt ${retryCount + 1} failed completely.`); @@ -822,8 +851,6 @@ export default class Interpreter extends EventEmitter { if (!clickSuccess) { debugLog(`Load More clicking failed after ${MAX_RETRIES} attempts`); - const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config); - allResults = allResults.concat(finalResults); return allResults; } @@ -833,20 +860,34 @@ export default class Interpreter extends EventEmitter { await page.waitForTimeout(2000); const currentHeight = await page.evaluate(() => document.body.scrollHeight); - if (currentHeight === previousHeight) { - debugLog('No more items loaded after Load More'); - const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config); - allResults = allResults.concat(finalResults); - return allResults; - } + const heightChanged = currentHeight !== previousHeight; previousHeight = currentHeight; - if (config.limit && allResults.length >= config.limit) { - allResults = allResults.slice(0, config.limit); - break; + await scrapeCurrentPage(); + + const currentResultCount = allResults.length; + const newItemsAdded = currentResultCount > previousResultCount; + + if (!newItemsAdded) { + noNewItemsCounter++; + debugLog(`No new items added after click (${noNewItemsCounter}/${MAX_NO_NEW_ITEMS})`); + + if (noNewItemsCounter >= MAX_NO_NEW_ITEMS) { + debugLog(`Stopping after ${MAX_NO_NEW_ITEMS} clicks with no new items`); + return allResults; + } + } else { + noNewItemsCounter = 0; + previousResultCount = currentResultCount; + } + + if (checkLimit()) return allResults; + + if (!heightChanged) { + debugLog('No more items loaded after Load More'); + return allResults; } } - break; } default: { diff --git a/server/src/api/record.ts b/server/src/api/record.ts index 05560487..b4014c3a 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -507,9 +507,9 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) { } } -async function readyForRunHandler(browserId: string, id: string) { +async function readyForRunHandler(browserId: string, id: string, userId: string){ try { - const result = await executeRun(id); + const result = await executeRun(id, userId); if (result && result.success) { logger.log('info', `Interpretation of ${id} succeeded`); @@ -517,14 +517,14 @@ async function readyForRunHandler(browserId: string, id: string) { return result.interpretationInfo; } else { logger.log('error', `Interpretation of ${id} failed`); - await destroyRemoteBrowser(browserId); + await destroyRemoteBrowser(browserId, userId); resetRecordingState(browserId, id); return null; } } catch (error: any) { logger.error(`Error during readyForRunHandler: ${error.message}`); - await destroyRemoteBrowser(browserId); + await destroyRemoteBrowser(browserId, userId); return null; } } @@ -546,7 +546,7 @@ function AddGeneratedFlags(workflow: WorkflowFile) { return copy; }; -async function executeRun(id: string) { +async function executeRun(id: string, userId: string) { try { const run = await Run.findOne({ where: { runId: id } }); if (!run) { @@ -568,7 +568,7 @@ async function executeRun(id: string) { plainRun.status = 'running'; - const browser = browserPool.getRemoteBrowser(plainRun.browserId); + const browser = browserPool.getRemoteBrowser(userId); if (!browser) { throw new Error('Could not access browser'); } @@ -586,7 +586,7 @@ async function executeRun(id: string) { const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(plainRun.browserId); + await destroyRemoteBrowser(plainRun.browserId, userId); const updatedRun = await run.update({ ...run, @@ -672,12 +672,12 @@ export async function handleRunRecording(id: string, userId: string) { rejectUnauthorized: false }); - socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId)); + socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId, userId)); logger.log('info', `Running Robot: ${id}`); socket.on('disconnect', () => { - cleanupSocketListeners(socket, browserId, newRunId); + cleanupSocketListeners(socket, browserId, newRunId, userId); }); // Return the runId immediately, so the client knows the run is started @@ -688,8 +688,8 @@ export async function handleRunRecording(id: string, userId: string) { } } -function cleanupSocketListeners(socket: Socket, browserId: string, id: string) { - socket.off('ready-for-run', () => readyForRunHandler(browserId, id)); +function cleanupSocketListeners(socket: Socket, browserId: string, id: string, userId: string) { + socket.off('ready-for-run', () => readyForRunHandler(browserId, id, userId)); logger.log('info', `Cleaned up listeners for browserId: ${browserId}, runId: ${id}`); } diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index cd4962a1..4bbc97af 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -15,6 +15,10 @@ interface BrowserPoolInfo { * @default false */ active: boolean, + /** + * The user ID that owns this browser instance. + */ + userId: string, } /** @@ -29,36 +33,101 @@ interface PoolDictionary { /** * A browser pool is a collection of remote browsers that are initialized and ready to be used. + * Enforces a "1 User - 1 Browser" policy, while allowing multiple users to have their own browser instances. * Adds the possibility to add, remove and retrieve remote browsers from the pool. - * It is possible to manage multiple browsers for creating or running a recording. * @category BrowserManagement */ export class BrowserPool { - /** * Holds all the instances of remote browsers. */ private pool: PoolDictionary = {}; /** - * Adds a remote browser instance to the pool indexed by the id. + * Maps user IDs to their browser IDs. + */ + private userToBrowserMap: Map = new Map(); + + /** + * Adds a remote browser instance to the pool for a specific user. + * If the user already has a browser, the existing browser will be closed and replaced. + * * @param id remote browser instance's id * @param browser remote browser instance + * @param userId the user ID that owns this browser instance * @param active states if the browser's instance is being actively used + * @returns true if a new browser was added, false if an existing browser was replaced */ - public addRemoteBrowser = (id: string, browser: RemoteBrowser, active: boolean = false): void => { - this.pool = { - ...this.pool, - [id]: { - browser, - active, - }, + public addRemoteBrowser = ( + id: string, + browser: RemoteBrowser, + userId: string, + active: boolean = false + ): boolean => { + // Check if user already has a browser + const existingBrowserId = this.userToBrowserMap.get(userId); + let replaced = false; + + if (existingBrowserId) { + // Close and remove the existing browser + if (existingBrowserId !== id) { + this.closeAndDeleteBrowser(existingBrowserId); + replaced = true; + } else { + // If it's the same browser ID, just update the info + this.pool[id] = { + browser, + active, + userId, + }; + logger.log('debug', `Updated existing browser with id: ${id} for user: ${userId}`); + return false; + } } - logger.log('debug', `Remote browser with id: ${id} added to the pool`); + + // Add the new browser to the pool + this.pool[id] = { + browser, + active, + userId, + }; + + // Update the user-to-browser mapping + this.userToBrowserMap.set(userId, id); + + logger.log('debug', `Remote browser with id: ${id} added to the pool for user: ${userId}`); + return !replaced; }; /** * Removes the remote browser instance from the pool. + * Note: This doesn't handle browser closing as RemoteBrowser doesn't expose a close method. + * The caller should ensure the browser is properly closed before calling this method. + * + * @param id remote browser instance's id + * @returns true if the browser was removed successfully, false otherwise + */ + public closeAndDeleteBrowser = (id: string): boolean => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return false; + } + + // Remove the user-to-browser mapping + const userId = this.pool[id].userId; + if (this.userToBrowserMap.get(userId) === id) { + this.userToBrowserMap.delete(userId); + } + + // Remove from pool + delete this.pool[id]; + logger.log('debug', `Remote browser with id: ${id} removed from the pool`); + return true; + }; + + /** + * Removes the remote browser instance from the pool without attempting to close it. + * * @param id remote browser instance's id * @returns true if the browser was removed successfully, false otherwise */ @@ -67,13 +136,22 @@ export class BrowserPool { logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); return false; } - delete (this.pool[id]); + + // Remove the user-to-browser mapping + const userId = this.pool[id].userId; + if (this.userToBrowserMap.get(userId) === id) { + this.userToBrowserMap.delete(userId); + } + + // Remove from pool + delete this.pool[id]; logger.log('debug', `Remote browser with id: ${id} deleted from the pool`); return true; }; /** * Returns the remote browser instance from the pool. + * * @param id remote browser instance's id * @returns remote browser instance or undefined if it does not exist in the pool */ @@ -83,18 +161,154 @@ export class BrowserPool { }; /** - * Returns the active browser's instance id from the pool. - * If there is no active browser, it returns undefined. - * If there are multiple active browsers, it returns the first one. - * @returns the first remote active browser instance's id from the pool + * Returns the active browser's instance id for a specific user. + * + * @param userId the user ID to find the browser for + * @returns the browser ID for the user, or null if no browser exists */ - public getActiveBrowserId = (): string | null => { + public getActiveBrowserId = (userId: string): string | null => { + const browserId = this.userToBrowserMap.get(userId); + if (!browserId) { + logger.log('debug', `No browser found for user: ${userId}`); + return null; + } + + // Verify the browser still exists in the pool + if (!this.pool[browserId]) { + this.userToBrowserMap.delete(userId); + logger.log('warn', `Browser mapping found for user: ${userId}, but browser doesn't exist in pool`); + return null; + } + console.log(`Browser Id ${browserId} found for user: ${userId}`); + return browserId; + }; + + /** + * Returns the user ID associated with a browser ID. + * + * @param browserId the browser ID to find the user for + * @returns the user ID for the browser, or null if the browser doesn't exist + */ + public getUserForBrowser = (browserId: string): string | null => { + if (!this.pool[browserId]) { + return null; + } + return this.pool[browserId].userId; + }; + + /** + * Sets the active state of a browser. + * + * @param id the browser ID + * @param active the new active state + * @returns true if successful, false if the browser wasn't found + */ + public setActiveBrowser = (id: string, active: boolean): boolean => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return false; + } + + this.pool[id].active = active; + logger.log('debug', `Remote browser with id: ${id} set to ${active ? 'active' : 'inactive'}`); + return true; + }; + + /** + * Returns all browser instances for a specific user. + * Should only be one per the "1 User - 1 Browser" policy, but included for flexibility. + * + * @param userId the user ID to find browsers for + * @returns an array of browser IDs belonging to the user + */ + public getAllBrowserIdsForUser = (userId: string): string[] => { + const browserIds: string[] = []; + + // Normally this would just return the one browser from the map + const mappedBrowserId = this.userToBrowserMap.get(userId); + if (mappedBrowserId && this.pool[mappedBrowserId]) { + browserIds.push(mappedBrowserId); + } + + // But as a safeguard, also check the entire pool for any browsers assigned to this user + // This helps detect and fix any inconsistencies in the maps + for (const [id, info] of Object.entries(this.pool)) { + if (info.userId === userId && !browserIds.includes(id)) { + browserIds.push(id); + // Fix the map if it's inconsistent + if (!mappedBrowserId) { + this.userToBrowserMap.set(userId, id); + } + } + } + + return browserIds; + }; + + /** + * Returns the total number of browsers in the pool. + */ + public getPoolSize = (): number => { + return Object.keys(this.pool).length; + }; + + /** + * Returns the total number of active users (users with browsers). + */ + public getActiveUserCount = (): number => { + return this.userToBrowserMap.size; + }; + + /** + * Gets the current active browser for the system if there's only one active user. + * This is a migration helper to support code that hasn't been updated to the user-browser model yet. + * + * @param currentUserId The ID of the current user, which will be prioritized if multiple browsers exist + * @returns A browser ID if one can be determined, or null + */ + public getActiveBrowserForMigration = (currentUserId?: string): string | null => { + // If a current user ID is provided and they have a browser, return that + if (currentUserId) { + const browserForUser = this.getActiveBrowserId(currentUserId); + if (browserForUser) { + return browserForUser; + } + } + + // If only one user has a browser, return that + if (this.userToBrowserMap.size === 1) { + const userId = Array.from(this.userToBrowserMap.keys())[0]; + return this.userToBrowserMap.get(userId) || null; + } + + // Fall back to the first active browser if any for (const id of Object.keys(this.pool)) { if (this.pool[id].active) { return id; } } - logger.log('warn', `No active browser in the pool`); + + // If all else fails, return the first browser in the pool + const browserIds = Object.keys(this.pool); + return browserIds.length > 0 ? browserIds[0] : null; + }; + + /** + * Returns the first active browser's instance id from the pool. + * If there is no active browser, it returns null. + * If there are multiple active browsers, it returns the first one. + * + * @returns the first remote active browser instance's id from the pool + * @deprecated Use getBrowserIdForUser instead to enforce the 1 User - 1 Browser policy + */ + public getActiveBrowserIdLegacy = (): string | null => { + for (const id of Object.keys(this.pool)) { + if (this.pool[id].active) { + return id; + } + } + // Don't log a warning since this behavior is expected in the user-browser model + // logger.log('warn', `No active browser in the pool`); return null; }; -} +} \ No newline at end of file diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 8ff4f601..85d1928c 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -90,6 +90,12 @@ export class RemoteBrowser { maxRepeats: 1, }; + /** + * The user ID that owns this browser instance + * @private + */ + private userId: string; + private lastEmittedUrl: string | null = null; /** @@ -106,6 +112,7 @@ export class RemoteBrowser { private screenshotQueue: Buffer[] = []; private isProcessingScreenshot = false; private screencastInterval: NodeJS.Timeout | null = null + private isScreencastActive: boolean = false; /** * Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and @@ -113,8 +120,9 @@ export class RemoteBrowser { * @param socket socket.io socket instance used to communicate with the client side * @constructor */ - public constructor(socket: Socket) { + public constructor(socket: Socket, userId: string) { this.socket = socket; + this.userId = userId; this.interpreter = new WorkflowInterpreter(socket); this.generator = new WorkflowGenerator(socket); } @@ -193,7 +201,7 @@ export class RemoteBrowser { const currentUrl = page.url(); if (this.shouldEmitUrlChange(currentUrl)) { this.lastEmittedUrl = currentUrl; - this.socket.emit('urlChanged', currentUrl); + this.socket.emit('urlChanged', {url: currentUrl, userId: this.userId}); } } }); @@ -333,15 +341,40 @@ export class RemoteBrowser { * @returns void */ public registerEditorEvents = (): void => { - this.socket.on('rerender', async () => await this.makeAndEmitScreenshot()); - this.socket.on('settings', (settings) => this.interpreterSettings = settings); - this.socket.on('changeTab', async (tabIndex) => await this.changeTab(tabIndex)); - this.socket.on('addTab', async () => { + // For each event, include userId to make sure events are handled for the correct browser + logger.log('debug', `Registering editor events for user: ${this.userId}`); + + // Listen for specific events for this user + this.socket.on(`rerender:${this.userId}`, async () => { + logger.debug(`Rerender event received for user ${this.userId}`); + await this.makeAndEmitScreenshot(); + }); + + // For backward compatibility, also listen to the general event + this.socket.on('rerender', async () => { + logger.debug(`General rerender event received, checking if for user ${this.userId}`); + await this.makeAndEmitScreenshot(); + }); + + this.socket.on(`settings:${this.userId}`, (settings) => { + this.interpreterSettings = settings; + logger.debug(`Settings updated for user ${this.userId}`); + }); + + this.socket.on(`changeTab:${this.userId}`, async (tabIndex) => { + logger.debug(`Tab change to ${tabIndex} requested for user ${this.userId}`); + await this.changeTab(tabIndex); + }); + + this.socket.on(`addTab:${this.userId}`, async () => { + logger.debug(`New tab requested for user ${this.userId}`); await this.currentPage?.context().newPage(); const lastTabIndex = this.currentPage ? this.currentPage.context().pages().length - 1 : 0; await this.changeTab(lastTabIndex); }); - this.socket.on('closeTab', async (tabInfo) => { + + this.socket.on(`closeTab:${this.userId}`, async (tabInfo) => { + logger.debug(`Close tab ${tabInfo.index} requested for user ${this.userId}`); const page = this.currentPage?.context().pages()[tabInfo.index]; if (page) { if (tabInfo.isCurrent) { @@ -356,24 +389,52 @@ export class RemoteBrowser { await page.close(); logger.log( 'debug', - `${tabInfo.index} page was closed, new length of pages: ${this.currentPage?.context().pages().length}` - ) + `Tab ${tabInfo.index} was closed for user ${this.userId}, new tab count: ${this.currentPage?.context().pages().length}` + ); } else { - logger.log('error', `${tabInfo.index} index out of range of pages`) + logger.log('error', `Tab index ${tabInfo.index} out of range for user ${this.userId}`); } }); - this.socket.on('setViewportSize', async (data: { width: number, height: number }) => { + + this.socket.on(`setViewportSize:${this.userId}`, async (data: { width: number, height: number }) => { const { width, height } = data; - logger.log('debug', `Received viewport size: width=${width}, height=${height}`); + logger.log('debug', `Viewport size change to width=${width}, height=${height} requested for user ${this.userId}`); // Update the browser context's viewport dynamically if (this.context && this.browser) { this.context = await this.browser.newContext({ viewport: { width, height } }); - logger.log('debug', `Viewport size updated to width=${width}, height=${height} for the entire browser context`); + logger.log('debug', `Viewport size updated to width=${width}, height=${height} for user ${this.userId}`); } }); - } - + + // For backward compatibility, also register the standard events + this.socket.on('settings', (settings) => this.interpreterSettings = settings); + this.socket.on('changeTab', async (tabIndex) => await this.changeTab(tabIndex)); + this.socket.on('addTab', async () => { + await this.currentPage?.context().newPage(); + const lastTabIndex = this.currentPage ? this.currentPage.context().pages().length - 1 : 0; + await this.changeTab(lastTabIndex); + }); + this.socket.on('closeTab', async (tabInfo) => { + const page = this.currentPage?.context().pages()[tabInfo.index]; + if (page) { + if (tabInfo.isCurrent) { + if (this.currentPage?.context().pages()[tabInfo.index + 1]) { + await this.changeTab(tabInfo.index + 1); + } else { + await this.changeTab(tabInfo.index - 1); + } + } + await page.close(); + } + }); + this.socket.on('setViewportSize', async (data: { width: number, height: number }) => { + const { width, height } = data; + if (this.context && this.browser) { + this.context = await this.browser.newContext({ viewport: { width, height } }); + } + }); + }; /** * Subscribes the remote browser for a screencast session * on [CDP](https://chromedevtools.github.io/devtools-protocol/) level, @@ -382,16 +443,24 @@ export class RemoteBrowser { * @returns {Promise} */ public subscribeToScreencast = async (): Promise => { + logger.log('debug', `Starting screencast for user: ${this.userId}`); await this.startScreencast(); if (!this.client) { logger.log('warn', 'client is not initialized'); return; } + // Set flag to indicate screencast is active + this.isScreencastActive = true; + this.client.on('Page.screencastFrame', ({ data: base64, sessionId }) => { + // Only process if screencast is still active for this user + if (!this.isScreencastActive) { + return; + } this.emitScreenshot(Buffer.from(base64, 'base64')) setTimeout(async () => { try { - if (!this.client) { + if (!this.client || !this.isScreencastActive) { logger.log('warn', 'client is not initialized'); return; } @@ -410,6 +479,8 @@ export class RemoteBrowser { */ public async switchOff(): Promise { try { + this.isScreencastActive = false; + await this.interpreter.stopInterpretation(); if (this.screencastInterval) { @@ -553,7 +624,11 @@ export class RemoteBrowser { //await this.currentPage.setViewportSize({ height: 400, width: 900 }) this.client = await this.currentPage.context().newCDPSession(this.currentPage); - this.socket.emit('urlChanged', this.currentPage.url()); + // Include userId in the URL change event + this.socket.emit('urlChanged', { + url: this.currentPage.url(), + userId: this.userId + }); await this.makeAndEmitScreenshot(); await this.subscribeToScreencast(); } else { @@ -602,6 +677,8 @@ export class RemoteBrowser { await this.client.send('Page.startScreencast', { format: SCREENCAST_CONFIG.format, }); + // Set flag to indicate screencast is active + this.isScreencastActive = true; // Set up screencast frame handler this.client.on('Page.screencastFrame', async ({ data, sessionId }) => { @@ -627,6 +704,8 @@ export class RemoteBrowser { } try { + // Set flag to indicate screencast is active + this.isScreencastActive = false; await this.client.send('Page.stopScreencast'); this.screenshotQueue = []; this.isProcessingScreenshot = false; @@ -657,8 +736,11 @@ export class RemoteBrowser { const base64Data = optimizedScreenshot.toString('base64'); const dataWithMimeType = `data:image/jpeg;base64,${base64Data}`; - this.socket.emit('screencast', dataWithMimeType); - logger.debug('Screenshot emitted'); +// Emit with user context to ensure the frontend can identify which browser's screenshot this is +this.socket.emit('screencast', { + image: dataWithMimeType, + userId: this.userId +}); logger.debug('Screenshot emitted'); } catch (error) { logger.error('Screenshot emission failed:', error); } finally { diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index 24a677ce..e6619d77 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -21,23 +21,23 @@ import logger from "../logger"; * @category BrowserManagement-Controller */ export const initializeRemoteBrowserForRecording = (userId: string): string => { - const id = getActiveBrowserId() || uuid(); + const id = getActiveBrowserId(userId) || uuid(); createSocketConnection( io.of(id), async (socket: Socket) => { // browser is already active - const activeId = getActiveBrowserId(); + const activeId = getActiveBrowserId(userId); if (activeId) { const remoteBrowser = browserPool.getRemoteBrowser(activeId); remoteBrowser?.updateSocket(socket); await remoteBrowser?.makeAndEmitScreenshot(); } else { - const browserSession = new RemoteBrowser(socket); + const browserSession = new RemoteBrowser(socket, userId); browserSession.interpreter.subscribeToPausing(); await browserSession.initialize(userId); await browserSession.registerEditorEvents(); await browserSession.subscribeToScreencast(); - browserPool.addRemoteBrowser(id, browserSession, true); + browserPool.addRemoteBrowser(id, browserSession, userId); } socket.emit('loaded'); }); @@ -57,9 +57,9 @@ export const createRemoteBrowserForRun = (userId: string): string => { createSocketConnectionForRun( io.of(id), async (socket: Socket) => { - const browserSession = new RemoteBrowser(socket); + const browserSession = new RemoteBrowser(socket, userId); await browserSession.initialize(userId); - browserPool.addRemoteBrowser(id, browserSession, true); + browserPool.addRemoteBrowser(id, browserSession, userId); socket.emit('ready-for-run'); }); return id; @@ -72,7 +72,7 @@ export const createRemoteBrowserForRun = (userId: string): string => { * @returns {Promise} * @category BrowserManagement-Controller */ -export const destroyRemoteBrowser = async (id: string): Promise => { +export const destroyRemoteBrowser = async (id: string, userId: string): Promise => { const browserSession = browserPool.getRemoteBrowser(id); if (browserSession) { logger.log('debug', `Switching off the browser with id: ${id}`); @@ -88,8 +88,8 @@ export const destroyRemoteBrowser = async (id: string): Promise => { * @returns {string | null} * @category BrowserManagement-Controller */ -export const getActiveBrowserId = (): string | null => { - return browserPool.getActiveBrowserId(); +export const getActiveBrowserId = (userId: string): string | null => { + return browserPool.getActiveBrowserId(userId); }; /** @@ -98,7 +98,7 @@ export const getActiveBrowserId = (): string | null => { * @returns {string | undefined} * @category BrowserManagement-Controller */ -export const getRemoteBrowserCurrentUrl = (id: string): string | undefined => { +export const getRemoteBrowserCurrentUrl = (id: string, userId: string): string | undefined => { return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.url(); }; @@ -108,7 +108,7 @@ export const getRemoteBrowserCurrentUrl = (id: string): string | undefined => { * @return {string[] | undefined} * @category BrowserManagement-Controller */ -export const getRemoteBrowserCurrentTabs = (id: string): string[] | undefined => { +export const getRemoteBrowserCurrentTabs = (id: string, userId: string): string[] | undefined => { return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.context().pages() .map((page) => { const parsedUrl = new URL(page.url()); @@ -126,8 +126,8 @@ export const getRemoteBrowserCurrentTabs = (id: string): string[] | undefined => * @returns {Promise} * @category BrowserManagement-Controller */ -export const interpretWholeWorkflow = async () => { - const id = getActiveBrowserId(); +export const interpretWholeWorkflow = async (userId: string) => { + const id = getActiveBrowserId(userId); if (id) { const browser = browserPool.getRemoteBrowser(id); if (browser) { @@ -146,8 +146,8 @@ export const interpretWholeWorkflow = async () => { * @returns {Promise} * @category BrowserManagement-Controller */ -export const stopRunningInterpretation = async () => { - const id = getActiveBrowserId(); +export const stopRunningInterpretation = async (userId: string) => { + const id = getActiveBrowserId(userId); if (id) { const browser = browserPool.getRemoteBrowser(id); await browser?.stopCurrentInterpretation(); diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index bf365053..92c75e1d 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -4,6 +4,8 @@ * These functions are called by the client through socket communication. */ import { Socket } from 'socket.io'; +import { IncomingMessage } from 'http'; +import { JwtPayload } from 'jsonwebtoken'; import logger from "../logger"; import { Coordinates, ScrollDeltas, KeyboardInput, DatePickerEventData } from '../types'; @@ -13,6 +15,14 @@ import { Page } from "playwright"; import { throttle } from "../../../src/helpers/inputHelpers"; import { CustomActions } from "../../../src/shared/types"; +interface AuthenticatedIncomingMessage extends IncomingMessage { + user?: JwtPayload | string; +} + +interface AuthenticatedSocket extends Socket { + request: AuthenticatedIncomingMessage; +} + /** * A wrapper function for handling user input. * This function gets the active browser instance from the browser pool @@ -23,6 +33,7 @@ import { CustomActions } from "../../../src/shared/types"; * * @param handleCallback The callback handler to be called * @param args - arguments to be passed to the handler + * @param socket - socket with authenticated request * @category HelperFunctions */ const handleWrapper = async ( @@ -31,9 +42,21 @@ const handleWrapper = async ( page: Page, args?: any ) => Promise, - args?: any + args?: any, + socket?: AuthenticatedSocket, ) => { - const id = browserPool.getActiveBrowserId(); + if (!socket || !socket.request || !socket.request.user || typeof socket.request.user === 'string') { + logger.log('warn', `User not authenticated or invalid JWT payload`); + return; + } + + const userId = socket.request.user.id; + if (!userId) { + logger.log('warn', `User ID is missing in JWT payload`); + return; + } + + const id = browserPool.getActiveBrowserId(userId); if (id) { const activeBrowser = browserPool.getRemoteBrowser(id); if (activeBrowser?.interpreter.interpretationInProgress() && !activeBrowser.interpreter.interpretationIsPaused) { @@ -66,12 +89,13 @@ interface CustomActionEventData { /** * A wrapper function for handling custom actions. + * @param socket The socket connection * @param customActionEventData The custom action event data * @category HelperFunctions */ -const onGenerateAction = async (customActionEventData: CustomActionEventData) => { +const onGenerateAction = async (socket: AuthenticatedSocket, customActionEventData: CustomActionEventData) => { logger.log('debug', `Generating ${customActionEventData.action} action emitted from client`); - await handleWrapper(handleGenerateAction, customActionEventData); + await handleWrapper(handleGenerateAction, customActionEventData, socket); } /** @@ -89,12 +113,13 @@ const handleGenerateAction = /** * A wrapper function for handling mousedown event. + * @param socket The socket connection * @param coordinates - coordinates of the mouse click * @category HelperFunctions */ -const onMousedown = async (coordinates: Coordinates) => { +const onMousedown = async (socket: AuthenticatedSocket, coordinates: Coordinates) => { logger.log('debug', 'Handling mousedown event emitted from client'); - await handleWrapper(handleMousedown, coordinates); + await handleWrapper(handleMousedown, coordinates, socket); } /** @@ -139,12 +164,13 @@ const handleMousedown = async (generator: WorkflowGenerator, page: Page, { x, y /** * A wrapper function for handling the wheel event. + * @param socket The socket connection * @param scrollDeltas - the scroll deltas of the wheel event * @category HelperFunctions */ -const onWheel = async (scrollDeltas: ScrollDeltas) => { +const onWheel = async (socket: AuthenticatedSocket, scrollDeltas: ScrollDeltas) => { logger.log('debug', 'Handling scroll event emitted from client'); - await handleWrapper(handleWheel, scrollDeltas); + await handleWrapper(handleWheel, scrollDeltas, socket); }; /** @@ -165,12 +191,13 @@ const handleWheel = async (generator: WorkflowGenerator, page: Page, { deltaX, d /** * A wrapper function for handling the mousemove event. + * @param socket The socket connection * @param coordinates - the coordinates of the mousemove event * @category HelperFunctions */ -const onMousemove = async (coordinates: Coordinates) => { +const onMousemove = async (socket: AuthenticatedSocket, coordinates: Coordinates) => { logger.log('debug', 'Handling mousemove event emitted from client'); - await handleWrapper(handleMousemove, coordinates); + await handleWrapper(handleMousemove, coordinates, socket); } /** @@ -199,12 +226,13 @@ const handleMousemove = async (generator: WorkflowGenerator, page: Page, { x, y /** * A wrapper function for handling the keydown event. + * @param socket The socket connection * @param keyboardInput - the keyboard input of the keydown event * @category HelperFunctions */ -const onKeydown = async (keyboardInput: KeyboardInput) => { +const onKeydown = async (socket: AuthenticatedSocket, keyboardInput: KeyboardInput) => { logger.log('debug', 'Handling keydown event emitted from client'); - await handleWrapper(handleKeydown, keyboardInput); + await handleWrapper(handleKeydown, keyboardInput, socket); } /** @@ -235,49 +263,95 @@ const handleDateSelection = async (generator: WorkflowGenerator, page: Page, dat logger.log('debug', `Date ${data.value} selected`); } -const onDateSelection = async (data: DatePickerEventData) => { +/** + * A wrapper function for handling the date selection event. + * @param socket The socket connection + * @param data - the data of the date selection event + * @category HelperFunctions + */ +const onDateSelection = async (socket: AuthenticatedSocket, data: DatePickerEventData) => { logger.log('debug', 'Handling date selection event emitted from client'); - await handleWrapper(handleDateSelection, data); + await handleWrapper(handleDateSelection, data, socket); } +/** + * Handles the dropdown selection event. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param data - the data of the dropdown selection event + * @category BrowserManagement + */ const handleDropdownSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { await generator.onDropdownSelection(page, data); logger.log('debug', `Dropdown value ${data.value} selected`); } -const onDropdownSelection = async (data: { selector: string, value: string }) => { +/** + * A wrapper function for handling the dropdown selection event. + * @param socket The socket connection + * @param data - the data of the dropdown selection event + * @category HelperFunctions + */ +const onDropdownSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => { logger.log('debug', 'Handling dropdown selection event emitted from client'); - await handleWrapper(handleDropdownSelection, data); + await handleWrapper(handleDropdownSelection, data, socket); } +/** + * Handles the time selection event. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param data - the data of the time selection event + * @category BrowserManagement + */ const handleTimeSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { await generator.onTimeSelection(page, data); logger.log('debug', `Time value ${data.value} selected`); } -const onTimeSelection = async (data: { selector: string, value: string }) => { +/** + * A wrapper function for handling the time selection event. + * @param socket The socket connection + * @param data - the data of the time selection event + * @category HelperFunctions + */ +const onTimeSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => { logger.log('debug', 'Handling time selection event emitted from client'); - await handleWrapper(handleTimeSelection, data); + await handleWrapper(handleTimeSelection, data, socket); } +/** + * Handles the datetime-local selection event. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param data - the data of the datetime-local selection event + * @category BrowserManagement + */ const handleDateTimeLocalSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { await generator.onDateTimeLocalSelection(page, data); logger.log('debug', `DateTime Local value ${data.value} selected`); } -const onDateTimeLocalSelection = async (data: { selector: string, value: string }) => { +/** + * A wrapper function for handling the datetime-local selection event. + * @param socket The socket connection + * @param data - the data of the datetime-local selection event + * @category HelperFunctions + */ +const onDateTimeLocalSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => { logger.log('debug', 'Handling datetime-local selection event emitted from client'); - await handleWrapper(handleDateTimeLocalSelection, data); + await handleWrapper(handleDateTimeLocalSelection, data, socket); } /** * A wrapper function for handling the keyup event. + * @param socket The socket connection * @param keyboardInput - the keyboard input of the keyup event * @category HelperFunctions */ -const onKeyup = async (keyboardInput: KeyboardInput) => { +const onKeyup = async (socket: AuthenticatedSocket, keyboardInput: KeyboardInput) => { logger.log('debug', 'Handling keyup event emitted from client'); - await handleWrapper(handleKeyup, keyboardInput); + await handleWrapper(handleKeyup, keyboardInput, socket); } /** @@ -296,12 +370,13 @@ const handleKeyup = async (generator: WorkflowGenerator, page: Page, key: string /** * A wrapper function for handling the url change event. + * @param socket The socket connection * @param url - the new url of the page * @category HelperFunctions */ -const onChangeUrl = async (url: string) => { +const onChangeUrl = async (socket: AuthenticatedSocket, url: string) => { logger.log('debug', 'Handling change url event emitted from client'); - await handleWrapper(handleChangeUrl, url); + await handleWrapper(handleChangeUrl, url, socket); } /** @@ -329,11 +404,12 @@ const handleChangeUrl = async (generator: WorkflowGenerator, page: Page, url: st /** * A wrapper function for handling the refresh event. + * @param socket The socket connection * @category HelperFunctions */ -const onRefresh = async () => { +const onRefresh = async (socket: AuthenticatedSocket) => { logger.log('debug', 'Handling refresh event emitted from client'); - await handleWrapper(handleRefresh); + await handleWrapper(handleRefresh, undefined, socket); } /** @@ -350,11 +426,12 @@ const handleRefresh = async (generator: WorkflowGenerator, page: Page) => { /** * A wrapper function for handling the go back event. + * @param socket The socket connection * @category HelperFunctions */ -const onGoBack = async () => { - logger.log('debug', 'Handling refresh event emitted from client'); - await handleWrapper(handleGoBack); +const onGoBack = async (socket: AuthenticatedSocket) => { + logger.log('debug', 'Handling go back event emitted from client'); + await handleWrapper(handleGoBack, undefined, socket); } /** @@ -372,11 +449,12 @@ const handleGoBack = async (generator: WorkflowGenerator, page: Page) => { /** * A wrapper function for handling the go forward event. + * @param socket The socket connection * @category HelperFunctions */ -const onGoForward = async () => { - logger.log('debug', 'Handling refresh event emitted from client'); - await handleWrapper(handleGoForward); +const onGoForward = async (socket: AuthenticatedSocket) => { + logger.log('debug', 'Handling go forward event emitted from client'); + await handleWrapper(handleGoForward, undefined, socket); } /** @@ -394,18 +472,7 @@ const handleGoForward = async (generator: WorkflowGenerator, page: Page) => { /** * Helper function for registering the handlers onto established websocket connection. - * Registers: - * - mousedownHandler - * - wheelHandler - * - mousemoveHandler - * - keydownHandler - * - keyupHandler - * - changeUrlHandler - * - refreshHandler - * - goBackHandler - * - goForwardHandler - * - onGenerateAction - * input handlers. + * Registers various input handlers. * * All these handlers first generates the workflow pair data * and then calls the corresponding playwright's function to emulate the input. @@ -415,21 +482,25 @@ const handleGoForward = async (generator: WorkflowGenerator, page: Page) => { * @returns void * @category BrowserManagement */ -const registerInputHandlers = (socket: Socket) => { - socket.on("input:mousedown", onMousedown); - socket.on("input:wheel", onWheel); - socket.on("input:mousemove", onMousemove); - socket.on("input:keydown", onKeydown); - socket.on("input:keyup", onKeyup); - socket.on("input:url", onChangeUrl); - socket.on("input:refresh", onRefresh); - socket.on("input:back", onGoBack); - socket.on("input:forward", onGoForward); - socket.on("input:date", onDateSelection); - socket.on("input:dropdown", onDropdownSelection); - socket.on("input:time", onTimeSelection); - socket.on("input:datetime-local", onDateTimeLocalSelection); - socket.on("action", onGenerateAction); +const registerInputHandlers = (socket: Socket) => { + // Cast to our authenticated socket type + const authSocket = socket as AuthenticatedSocket; + + // Register handlers with the socket + socket.on("input:mousedown", (data) => onMousedown(authSocket, data)); + socket.on("input:wheel", (data) => onWheel(authSocket, data)); + socket.on("input:mousemove", (data) => onMousemove(authSocket, data)); + socket.on("input:keydown", (data) => onKeydown(authSocket, data)); + socket.on("input:keyup", (data) => onKeyup(authSocket, data)); + socket.on("input:url", (data) => onChangeUrl(authSocket, data)); + socket.on("input:refresh", () => onRefresh(authSocket)); + socket.on("input:back", () => onGoBack(authSocket)); + socket.on("input:forward", () => onGoForward(authSocket)); + socket.on("input:date", (data) => onDateSelection(authSocket, data)); + socket.on("input:dropdown", (data) => onDropdownSelection(authSocket, data)); + socket.on("input:time", (data) => onTimeSelection(authSocket, data)); + socket.on("input:datetime-local", (data) => onDateTimeLocalSelection(authSocket, data)); + socket.on("action", (data) => onGenerateAction(authSocket, data)); }; -export default registerInputHandlers; +export default registerInputHandlers; \ No newline at end of file diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index 676254be..de468ef7 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -182,18 +182,24 @@ router.get('/stop/:browserId', requireSignIn, async (req: AuthenticatedRequest, /** * GET endpoint for getting the id of the active remote browser. */ -router.get('/active', requireSignIn, (req, res) => { - const id = getActiveBrowserId(); +router.get('/active', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + const id = getActiveBrowserId(req.user?.id); return res.send(id); }); /** * GET endpoint for getting the current url of the active remote browser. */ -router.get('/active/url', requireSignIn, (req, res) => { - const id = getActiveBrowserId(); +router.get('/active/url', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + const id = getActiveBrowserId(req.user?.id); if (id) { - const url = getRemoteBrowserCurrentUrl(id); + const url = getRemoteBrowserCurrentUrl(id, req.user?.id); return res.send(url); } return res.send(null); @@ -202,10 +208,13 @@ router.get('/active/url', requireSignIn, (req, res) => { /** * GET endpoint for getting the current tabs of the active remote browser. */ -router.get('/active/tabs', requireSignIn, (req, res) => { - const id = getActiveBrowserId(); +router.get('/active/tabs', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + const id = getActiveBrowserId(req.user?.id); if (id) { - const hosts = getRemoteBrowserCurrentTabs(id); + const hosts = getRemoteBrowserCurrentTabs(id, req.user?.id); return res.send(hosts); } return res.send([]); @@ -219,7 +228,7 @@ router.get('/interpret', requireSignIn, async (req: AuthenticatedRequest, res) = if (!req.user) { return res.status(401).send('User not authenticated'); } - await interpretWholeWorkflow(); + await interpretWholeWorkflow(req.user?.id); return res.send('interpretation done'); } catch (e) { return res.send('interpretation failed'); @@ -233,7 +242,7 @@ router.get('/interpret/stop', requireSignIn, async (req: AuthenticatedRequest, r if (!req.user) { return res.status(401).send('User not authenticated'); } - await stopRunningInterpretation(); + await stopRunningInterpretation(req.user?.id); return res.send('interpretation stopped'); }); diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 5ab9ef7a..aa3f4e12 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -617,7 +617,7 @@ router.post('/runs/run/:id', requireSignIn, async (req: AuthenticatedRequest, re workflow, currentPage, (newPage: Page) => currentPage = newPage, plainRun.interpreterSettings); const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(plainRun.browserId); + await destroyRemoteBrowser(plainRun.browserId, req.user?.id); await run.update({ ...run, status: 'success', @@ -900,9 +900,13 @@ router.delete('/schedule/:id', requireSignIn, async (req: AuthenticatedRequest, /** * POST endpoint for aborting a current interpretation of the run. */ -router.post('/runs/abort/:id', requireSignIn, async (req, res) => { +router.post('/runs/abort/:id', requireSignIn, async (req: AuthenticatedRequest, res) => { try { - const run = await Run.findOne({ where: { runId: req.params.id } }); + if (!req.user) { return res.status(401).send({ error: 'Unauthorized' }); } + const run = await Run.findOne({ where: { + runId: req.params.id, + runByUserId: req.user.id, + } }); if (!run) { return res.status(404).send(false); } @@ -937,4 +941,4 @@ router.post('/runs/abort/:id', requireSignIn, async (req, res) => { logger.log('info', `Error while running a robot with name: ${req.params.fileName}_${req.params.runId}.json`); return res.send(false); } -}); \ No newline at end of file +}); diff --git a/server/src/routes/workflow.ts b/server/src/routes/workflow.ts index 3c0e356b..3a117ce1 100644 --- a/server/src/routes/workflow.ts +++ b/server/src/routes/workflow.ts @@ -7,6 +7,7 @@ import logger from "../logger"; import { browserPool } from "../server"; import { requireSignIn } from '../middlewares/auth'; import Robot from '../models/Robot'; +import { AuthenticatedRequest } from './record'; export const router = Router(); @@ -46,8 +47,9 @@ router.get('/params/:browserId', requireSignIn, (req, res) => { /** * DELETE endpoint for deleting a pair from the generated workflow. */ -router.delete('/pair/:index', requireSignIn, (req, res) => { - const id = browserPool.getActiveBrowserId(); +router.delete('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { return res.status(401).send('User not authenticated'); } + const id = browserPool.getActiveBrowserId(req.user?.id); if (id) { const browser = browserPool.getRemoteBrowser(id); if (browser) { @@ -62,8 +64,9 @@ router.delete('/pair/:index', requireSignIn, (req, res) => { /** * POST endpoint for adding a pair to the generated workflow. */ -router.post('/pair/:index', requireSignIn, (req, res) => { - const id = browserPool.getActiveBrowserId(); +router.post('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { return res.status(401).send('User not authenticated'); } + const id = browserPool.getActiveBrowserId(req.user?.id); if (id) { const browser = browserPool.getRemoteBrowser(id); logger.log('debug', `Adding pair to workflow`); @@ -82,8 +85,9 @@ router.post('/pair/:index', requireSignIn, (req, res) => { /** * PUT endpoint for updating a pair in the generated workflow. */ -router.put('/pair/:index', requireSignIn, (req, res) => { - const id = browserPool.getActiveBrowserId(); +router.put('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { return res.status(401).send('User not authenticated'); } + const id = browserPool.getActiveBrowserId(req.user?.id); if (id) { const browser = browserPool.getRemoteBrowser(id); logger.log('debug', `Updating pair in workflow`); diff --git a/server/src/socket-connection/connection.ts b/server/src/socket-connection/connection.ts index dac7a88e..9fad861b 100644 --- a/server/src/socket-connection/connection.ts +++ b/server/src/socket-connection/connection.ts @@ -1,6 +1,60 @@ import { Namespace, Socket } from 'socket.io'; +import { IncomingMessage } from 'http'; +import { verify, JwtPayload } from 'jsonwebtoken'; import logger from "../logger"; -import registerInputHandlers from '../browser-management/inputHandlers' +import registerInputHandlers from '../browser-management/inputHandlers'; + +interface AuthenticatedIncomingMessage extends IncomingMessage { + user?: JwtPayload | string; +} + +interface AuthenticatedSocket extends Socket { + request: AuthenticatedIncomingMessage; +} + +/** + * Socket.io middleware for authentication + * This is a socket.io specific auth handler that doesn't rely on Express middleware + */ +const socketAuthMiddleware = (socket: Socket, next: (err?: Error) => void) => { + const cookies = socket.handshake.headers.cookie; + if (!cookies) { + return next(new Error('Authentication required')); + } + + const tokenMatch = cookies.split(';').find(c => c.trim().startsWith('token=')); + if (!tokenMatch) { + return next(new Error('Authentication required')); + } + + const token = tokenMatch.split('=')[1]; + if (!token) { + return next(new Error('Authentication required')); + } + + const secret = process.env.JWT_SECRET; + if (!secret) { + return next(new Error('Server configuration error')); + } + + verify(token, secret, (err: any, user: any) => { + if (err) { + logger.log('warn', 'JWT verification error:', err); + return next(new Error('Authentication failed')); + } + + // Normalize payload key + if (user.userId && !user.id) { + user.id = user.userId; + delete user.userId; // temporary: del the old key for clarity + } + + // Attach user to socket request + const authSocket = socket as AuthenticatedSocket; + authSocket.request.user = user; + next(); + }); +}; /** * Opens a websocket canal for duplex data transfer and registers all handlers for this data for the recording session. @@ -13,6 +67,8 @@ export const createSocketConnection = ( io: Namespace, callback: (socket: Socket) => void, ) => { + io.use(socketAuthMiddleware); + const onConnection = async (socket: Socket) => { logger.log('info', "Client connected " + socket.id); registerInputHandlers(socket); @@ -34,6 +90,8 @@ export const createSocketConnectionForRun = ( io: Namespace, callback: (socket: Socket) => void, ) => { + io.use(socketAuthMiddleware); + const onConnection = async (socket: Socket) => { logger.log('info', "Client connected " + socket.id); socket.on('disconnect', () => logger.log('info', "Client disconnected " + socket.id)); @@ -41,4 +99,4 @@ export const createSocketConnectionForRun = ( } io.on('connection', onConnection); -}; +}; \ No newline at end of file diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 107dc500..4899ac3b 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -151,8 +151,8 @@ export class WorkflowGenerator { workflow: [], }); socket.on('activeIndex', (data) => this.generatedData.lastIndex = parseInt(data)); - socket.on('decision', async ({ pair, actionType, decision }) => { - const id = browserPool.getActiveBrowserId(); + socket.on('decision', async ({ pair, actionType, decision, userId }) => { + const id = browserPool.getActiveBrowserId(userId); if (id) { // const activeBrowser = browserPool.getRemoteBrowser(id); // const currentPage = activeBrowser?.getCurrentPage(); @@ -826,6 +826,7 @@ export class WorkflowGenerator { selectors?.testIdSelector, selectors?.id, selectors?.hrefSelector, + selectors?.relSelector, selectors?.accessibilitySelector, selectors?.attrSelector ] diff --git a/server/src/workflow-management/integrations/gsheet.ts b/server/src/workflow-management/integrations/gsheet.ts index 47dda84e..b492a7df 100644 --- a/server/src/workflow-management/integrations/gsheet.ts +++ b/server/src/workflow-management/integrations/gsheet.ts @@ -49,11 +49,7 @@ export async function updateGoogleSheet(robotId: string, runId: string) { if (plainRobot.google_sheet_email && spreadsheetId) { console.log(`Preparing to write data to Google Sheet for robot: ${robotId}, spreadsheetId: ${spreadsheetId}`); - const headers = Object.keys(data[0]); - const rows = data.map((row: { [key: string]: any }) => Object.values(row)); - const outputData = [headers, ...rows]; - - await writeDataToSheet(robotId, spreadsheetId, outputData); + await writeDataToSheet(robotId, spreadsheetId, data); console.log(`Data written to Google Sheet successfully for Robot: ${robotId} and Run: ${runId}`); } else { console.log('Google Sheets integration not configured.'); @@ -102,7 +98,43 @@ export async function writeDataToSheet(robotId: string, spreadsheetId: string, d const sheets = google.sheets({ version: 'v4', auth: oauth2Client }); - const resource = { values: data }; + const checkResponse = await sheets.spreadsheets.values.get({ + spreadsheetId, + range: 'Sheet1!1:1', + }); + + if (!data || data.length === 0) { + console.log('No data to write. Exiting early.'); + return; + } + + const expectedHeaders = Object.keys(data[0]); + + const rows = data.map(item => Object.values(item)); + + const existingHeaders = + checkResponse.data.values && + checkResponse.data.values[0] ? + checkResponse.data.values[0].map(String) : + []; + + const isSheetEmpty = existingHeaders.length === 0; + + const headersMatch = + !isSheetEmpty && + existingHeaders.length === expectedHeaders.length && + expectedHeaders.every((header, index) => existingHeaders[index] === header); + + let resource; + + if (isSheetEmpty || !headersMatch) { + resource = { values: [expectedHeaders, ...rows] }; + console.log('Including headers in the append operation.'); + } else { + resource = { values: rows }; + console.log('Headers already exist and match, only appending data rows.'); + } + console.log('Attempting to write to spreadsheet:', spreadsheetId); const response = await sheets.spreadsheets.values.append({ diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index ade7d969..b67e1ca0 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -92,7 +92,7 @@ function AddGeneratedFlags(workflow: WorkflowFile) { return copy; }; -async function executeRun(id: string) { +async function executeRun(id: string, userId: string) { try { const run = await Run.findOne({ where: { runId: id } }); if (!run) { @@ -114,7 +114,7 @@ async function executeRun(id: string) { plainRun.status = 'running'; - const browser = browserPool.getRemoteBrowser(plainRun.browserId); + const browser = browserPool.getRemoteBrowser(userId); if (!browser) { throw new Error('Could not access browser'); } @@ -132,7 +132,7 @@ async function executeRun(id: string) { const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(plainRun.browserId); + await destroyRemoteBrowser(plainRun.browserId, userId); await run.update({ ...run, @@ -207,22 +207,22 @@ async function executeRun(id: string) { } } -async function readyForRunHandler(browserId: string, id: string) { +async function readyForRunHandler(browserId: string, id: string, userId: string) { try { - const interpretation = await executeRun(id); + const interpretation = await executeRun(id, userId); if (interpretation) { logger.log('info', `Interpretation of ${id} succeeded`); } else { logger.log('error', `Interpretation of ${id} failed`); - await destroyRemoteBrowser(browserId); + await destroyRemoteBrowser(browserId, userId); } resetRecordingState(browserId, id); } catch (error: any) { logger.error(`Error during readyForRunHandler: ${error.message}`); - await destroyRemoteBrowser(browserId); + await destroyRemoteBrowser(browserId, userId); } } @@ -245,12 +245,12 @@ export async function handleRunRecording(id: string, userId: string) { rejectUnauthorized: false }); - socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId)); + socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId, userId)); logger.log('info', `Running robot: ${id}`); socket.on('disconnect', () => { - cleanupSocketListeners(socket, browserId, newRunId); + cleanupSocketListeners(socket, browserId, newRunId, userId); }); } catch (error: any) { @@ -258,8 +258,8 @@ export async function handleRunRecording(id: string, userId: string) { } } -function cleanupSocketListeners(socket: Socket, browserId: string, id: string) { - socket.off('ready-for-run', () => readyForRunHandler(browserId, id)); +function cleanupSocketListeners(socket: Socket, browserId: string, id: string, userId: string) { + socket.off('ready-for-run', () => readyForRunHandler(browserId, id, userId)); logger.log('info', `Cleaned up listeners for browserId: ${browserId}, runId: ${id}`); } diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts index 20a6a529..80ed55f3 100644 --- a/server/src/workflow-management/selector.ts +++ b/server/src/workflow-management/selector.ts @@ -24,54 +24,64 @@ export const getElementInformation = async ( const elementInfo = await page.evaluate( async ({ x, y }) => { const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // First, get the element at the clicked coordinates in the main document let element = document.elementFromPoint(x, y) as HTMLElement; if (!element) return null; - - // Track the deepest element found + let deepestElement = element; - - // Function to traverse shadow DOM + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; let shadowRoot = current.shadowRoot; let deepest = current; - - while (shadowRoot) { + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; shadowRoot = current.shadowRoot; + depth++; } - + return deepest; }; - - // Handle iframe traversal + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + if (element.tagName === 'IFRAME') { let currentIframe = element as HTMLIFrameElement; - - while (currentIframe) { + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; if (!iframeDocument) break; - + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Update deepest element and check for shadow DOM + deepestElement = traverseShadowDOM(iframeElement); - - // Continue traversing if we found another iframe + if (iframeElement.tagName === 'IFRAME') { currentIframe = iframeElement as HTMLIFrameElement; + depth++; } else { break; } @@ -80,28 +90,81 @@ export const getElementInformation = async ( break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); } else { - // If not an iframe, check for shadow DOM deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; - // Get the element and its iframe path const el = getDeepestElementFromPoint(x, y); if (el) { - // Handle potential anchor parent - const { parentElement } = el; - const targetElement = parentElement?.tagName === 'A' ? parentElement : el; + // Prioritize Link (DO NOT REMOVE) + // const { parentElement } = el; + // const targetElement = parentElement?.tagName === 'A' ? parentElement : el; + + const targetElement = el; - // Get containing context information const ownerDocument = targetElement.ownerDocument; const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; const isIframeContent = Boolean(frameElement); + const isFrameContent = frameElement?.tagName === 'FRAME'; - // Get the containing shadow root if any const containingShadowRoot = targetElement.getRootNode() as ShadowRoot; const isShadowRoot = containingShadowRoot instanceof ShadowRoot; @@ -115,8 +178,11 @@ export const getElementInformation = async ( innerHTML?: string; outerHTML?: string; isIframeContent?: boolean; + isFrameContent?: boolean; iframeURL?: string; + frameURL?: string; iframeIndex?: number; + frameIndex?: number; frameHierarchy?: string[]; isShadowRoot?: boolean; shadowRootMode?: string; @@ -124,43 +190,47 @@ export const getElementInformation = async ( } = { tagName: targetElement?.tagName ?? '', isIframeContent, + isFrameContent, isShadowRoot }; - if (isIframeContent) { - // Include iframe specific information - info.iframeURL = frameElement.src; + if (isIframeContent || isFrameContent) { + if (isIframeContent) { + info.iframeURL = (frameElement as HTMLIFrameElement).src; + } else { + info.frameURL = (frameElement).src; + } - // Calculate the frame's position in the hierarchy let currentFrame = frameElement; const frameHierarchy: string[] = []; let frameIndex = 0; while (currentFrame) { - // Store the frame's identifier (src, id, or index) frameHierarchy.unshift( currentFrame.id || + currentFrame.getAttribute('name') || currentFrame.src || - `iframe[${frameIndex}]` + `${currentFrame.tagName.toLowerCase()}[${frameIndex}]` ); - // Move up to parent frame if it exists const parentDoc = currentFrame.ownerDocument; currentFrame = parentDoc?.defaultView?.frameElement as HTMLIFrameElement; frameIndex++; } info.frameHierarchy = frameHierarchy; - info.iframeIndex = frameIndex - 1; // Adjust for 0-based index + if (isIframeContent) { + info.iframeIndex = frameIndex - 1; + } else { + info.frameIndex = frameIndex - 1; + } } if (isShadowRoot) { - // Include shadow root specific information info.shadowRootMode = containingShadowRoot.mode; info.shadowRootContent = containingShadowRoot.innerHTML; } - // Collect element attributes and properties if (targetElement) { info.attributes = Array.from(targetElement.attributes).reduce( (acc, attr) => { @@ -206,56 +276,65 @@ export const getElementInformation = async ( } else { const elementInfo = await page.evaluate( async ({ x, y }) => { - // Enhanced helper function to get element from point including shadow DOM const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // First, get the element at the clicked coordinates in the main document let element = document.elementFromPoint(x, y) as HTMLElement; if (!element) return null; - - // Track the deepest element found + let deepestElement = element; - - // Function to traverse shadow DOM + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; let shadowRoot = current.shadowRoot; let deepest = current; - - while (shadowRoot) { + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; shadowRoot = current.shadowRoot; + depth++; } - + return deepest; }; - - // Handle iframe traversal + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + if (element.tagName === 'IFRAME') { let currentIframe = element as HTMLIFrameElement; - - while (currentIframe) { + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; if (!iframeDocument) break; - + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Update deepest element and check for shadow DOM + deepestElement = traverseShadowDOM(iframeElement); - - // Continue traversing if we found another iframe + if (iframeElement.tagName === 'IFRAME') { currentIframe = iframeElement as HTMLIFrameElement; + depth++; } else { break; } @@ -264,14 +343,67 @@ export const getElementInformation = async ( break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); } else { - // If not an iframe, check for shadow DOM deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; - + const originalEl = getDeepestElementFromPoint(x, y); if (originalEl) { let element = originalEl; @@ -301,7 +433,7 @@ export const getElementInformation = async ( const significantOverlap = (childRect.width * childRect.height) / - (parentRect.width * parentRect.height) > 0.5; + (parentRect.width * parentRect.height) > 0.1; if (fullyContained && significantOverlap) { const nextParent = element.parentElement; @@ -318,8 +450,9 @@ export const getElementInformation = async ( } const ownerDocument = element.ownerDocument; - const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; + const frameElement = ownerDocument?.defaultView?.frameElement; const isIframeContent = Boolean(frameElement); + const isFrameContent = frameElement?.tagName === 'FRAME'; const containingShadowRoot = element.getRootNode() as ShadowRoot; const isShadowRoot = containingShadowRoot instanceof ShadowRoot; @@ -334,8 +467,11 @@ export const getElementInformation = async ( innerHTML?: string; outerHTML?: string; isIframeContent?: boolean; + isFrameContent?: boolean; iframeURL?: string; + frameURL?: string; iframeIndex?: number; + frameIndex?: number; frameHierarchy?: string[]; isShadowRoot?: boolean; shadowRootMode?: string; @@ -343,44 +479,48 @@ export const getElementInformation = async ( } = { tagName: element?.tagName ?? '', isIframeContent, + isFrameContent, isShadowRoot }; - if (isIframeContent) { - // Include iframe specific information - info.iframeURL = frameElement.src; + if (isIframeContent || isFrameContent) { + if (isIframeContent && !isFrameContent) { + info.iframeURL = (frameElement as HTMLIFrameElement).src; + } else if (isFrameContent) { + info.frameURL = (frameElement as HTMLFrameElement).src; + } - // Calculate the frame's position in the hierarchy let currentFrame = frameElement; const frameHierarchy: string[] = []; let frameIndex = 0; while (currentFrame) { - // Store the frame's identifier (src, id, or index) frameHierarchy.unshift( currentFrame.id || - currentFrame.src || - `iframe[${frameIndex}]` + currentFrame.getAttribute('name') || + (currentFrame as HTMLFrameElement).src || + `${currentFrame.tagName.toLowerCase()}[${frameIndex}]` ); - // Move up to parent frame if it exists const parentDoc = currentFrame.ownerDocument; - currentFrame = parentDoc?.defaultView?.frameElement as HTMLIFrameElement; + currentFrame = parentDoc?.defaultView?.frameElement; frameIndex++; } info.frameHierarchy = frameHierarchy; - info.iframeIndex = frameIndex - 1; // Adjust for 0-based index - }; + if (isIframeContent && !isFrameContent) { + info.iframeIndex = frameIndex - 1; + } else if (isFrameContent) { + info.frameIndex = frameIndex - 1; + } + } if (isShadowRoot) { - // Include shadow root specific information info.shadowRootMode = containingShadowRoot.mode; info.shadowRootContent = containingShadowRoot.innerHTML; } if (element) { - // Get attributes including those from shadow DOM context info.attributes = Array.from(element.attributes).reduce( (acc, attr) => { acc[attr.name] = attr.value; @@ -389,14 +529,21 @@ export const getElementInformation = async ( {} as Record ); - // Handle specific element types if (element.tagName === 'A') { info.url = (element as HTMLAnchorElement).href; info.innerText = element.textContent ?? ''; } else if (element.tagName === 'IMG') { info.imageUrl = (element as HTMLImageElement).src; + } else if (element?.tagName === 'SELECT') { + const selectElement = element as HTMLSelectElement; + info.innerText = selectElement.options[selectElement.selectedIndex]?.text ?? ''; + info.attributes = { + ...info.attributes, + selectedValue: selectElement.value, + }; + } else if (element?.tagName === 'INPUT' && ((element as HTMLInputElement).type === 'time' || (element as HTMLInputElement).type === 'date')) { + info.innerText = (element as HTMLInputElement).value; } else { - // Handle text content with proper null checking info.hasOnlyText = element.children.length === 0 && (element.textContent !== null && element.textContent.trim().length > 0); @@ -427,56 +574,65 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector if (!getList || listSelector !== '') { const rect = await page.evaluate( async ({ x, y }) => { - // Enhanced helper function to get element from point including iframes const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // First, get the element at the clicked coordinates in the main document let element = document.elementFromPoint(x, y) as HTMLElement; if (!element) return null; - - // Track the deepest element found + let deepestElement = element; - - // Function to traverse shadow DOM + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; let shadowRoot = current.shadowRoot; let deepest = current; - - while (shadowRoot) { + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; shadowRoot = current.shadowRoot; + depth++; } - + return deepest; }; - - // Handle iframe traversal + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + if (element.tagName === 'IFRAME') { let currentIframe = element as HTMLIFrameElement; - - while (currentIframe) { + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; if (!iframeDocument) break; - + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Update deepest element and check for shadow DOM + deepestElement = traverseShadowDOM(iframeElement); - - // Continue traversing if we found another iframe + if (iframeElement.tagName === 'IFRAME') { currentIframe = iframeElement as HTMLIFrameElement; + depth++; } else { break; } @@ -485,18 +641,74 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); } else { - // If not an iframe, check for shadow DOM deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; const el = getDeepestElementFromPoint(x, y); if (el) { - const { parentElement } = el; - const element = parentElement?.tagName === 'A' ? parentElement : el; + // Prioritize Link (DO NOT REMOVE) + // const { parentElement } = el; + // const element = parentElement?.tagName === 'A' ? parentElement : el; + + const element = el; const rectangle = element?.getBoundingClientRect(); if (rectangle) { const createRectObject = (rect: DOMRect) => ({ @@ -557,54 +769,64 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector const rect = await page.evaluate( async ({ x, y }) => { const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // First, get the element at the clicked coordinates in the main document let element = document.elementFromPoint(x, y) as HTMLElement; if (!element) return null; - - // Track the deepest element found + let deepestElement = element; - - // Function to traverse shadow DOM + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; let shadowRoot = current.shadowRoot; let deepest = current; - - while (shadowRoot) { + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; shadowRoot = current.shadowRoot; + depth++; } - + return deepest; }; - - // Handle iframe traversal + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + if (element.tagName === 'IFRAME') { let currentIframe = element as HTMLIFrameElement; - - while (currentIframe) { + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; if (!iframeDocument) break; - + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Update deepest element and check for shadow DOM + deepestElement = traverseShadowDOM(iframeElement); - - // Continue traversing if we found another iframe + if (iframeElement.tagName === 'IFRAME') { currentIframe = iframeElement as HTMLIFrameElement; + depth++; } else { break; } @@ -613,11 +835,64 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); } else { - // If not an iframe, check for shadow DOM deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; @@ -650,7 +925,7 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector const significantOverlap = (childRect.width * childRect.height) / - (parentRect.width * parentRect.height) > 0.5; + (parentRect.width * parentRect.height) > 0.1; if (fullyContained && significantOverlap) { const nextParent = element.parentElement; @@ -691,12 +966,12 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector } }); - // Same coordinate adjustment for iframe elements as above + // For elements inside iframes or frames, adjust coordinates relative to the top window let adjustedRect = createRectObject(rectangle); let currentWindow = element.ownerDocument.defaultView; while (currentWindow !== window.top) { - const frameElement = currentWindow?.frameElement as HTMLIFrameElement; + const frameElement = currentWindow?.frameElement; if (!frameElement) break; const frameRect = frameElement.getBoundingClientRect(); @@ -1205,65 +1480,63 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { } const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // Helper function to traverse shadow DOM - const traverseShadowDOM = (element: HTMLElement, depth: number = 0): HTMLElement => { - const MAX_SHADOW_DEPTH = 4; - let current = element; - let deepest = current; - - while (current && depth < MAX_SHADOW_DEPTH) { - const shadowRoot = current.shadowRoot; - if (!shadowRoot) break; - - const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; - if (!shadowElement || shadowElement === current) break; - - deepest = shadowElement; - current = shadowElement; - depth++; - } - - return deepest; - }; - - // Start with the element at the specified coordinates let element = document.elementFromPoint(x, y) as HTMLElement; if (!element) return null; - - // Initialize tracking variables + let deepestElement = element; - let depth = 0; - const MAX_IFRAME_DEPTH = 4; - - // First check if the initial element has a shadow root - deepestElement = traverseShadowDOM(element); - - // If it's an iframe, traverse through iframe hierarchy - if (deepestElement.tagName === 'IFRAME') { - let currentIframe = deepestElement as HTMLIFrameElement; - + + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { + let current = element; + let shadowRoot = current.shadowRoot; + let deepest = current; + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { + const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; + if (!shadowElement || shadowElement === current) break; + + deepest = shadowElement; + current = shadowElement; + shadowRoot = current.shadowRoot; + depth++; + } + + return deepest; + }; + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + + if (element.tagName === 'IFRAME') { + let currentIframe = element as HTMLIFrameElement; + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - - // Access iframe's document - const iframeDoc = currentIframe.contentDocument || currentIframe.contentWindow?.document; - if (!iframeDoc) break; - - // Get element at transformed coordinates in iframe - const iframeElement = iframeDoc.elementFromPoint(iframeX, iframeY) as HTMLElement; + + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; + if (!iframeDocument) break; + + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Check for shadow DOM within iframe - const shadowResult = traverseShadowDOM(iframeElement); - deepestElement = shadowResult; - - // If we found another iframe, continue traversing - if (shadowResult.tagName === 'IFRAME') { - currentIframe = shadowResult as HTMLIFrameElement; + + deepestElement = traverseShadowDOM(iframeElement); + + if (iframeElement.tagName === 'IFRAME') { + currentIframe = iframeElement as HTMLIFrameElement; depth++; } else { break; @@ -1273,74 +1546,129 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); + } else { + deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; - const genSelectorForIframe = (element: HTMLElement) => { - // Helper function to get the complete iframe path up to document root - const getIframePath = (el: HTMLElement) => { - const path = []; - let current = el; - let depth = 0; - const MAX_DEPTH = 4; + + const genSelectorForFrame = (element: HTMLElement) => { + const getFramePath = (el: HTMLElement) => { + const path = []; + let current = el; + let depth = 0; + const MAX_DEPTH = 4; + + while (current && depth < MAX_DEPTH) { + const ownerDocument = current.ownerDocument; - while (current && depth < MAX_DEPTH) { - // Get the owner document of the current element - const ownerDocument = current.ownerDocument; - - // Check if this document belongs to an iframe - const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; - - if (frameElement) { - path.unshift({ - frame: frameElement, - document: ownerDocument, - element: current - }); - // Move up to the parent document's element (the iframe) - current = frameElement; - depth++; - } else { - break; - } + const frameElement = + ownerDocument?.defaultView?.frameElement as HTMLIFrameElement | HTMLFrameElement; + + if (frameElement) { + path.unshift({ + frame: frameElement, + document: ownerDocument, + element: current, + isFrame: frameElement.tagName === 'FRAME' + }); + + current = frameElement; + depth++; + } else { + break; } - return path; + } + return path; }; - - const iframePath = getIframePath(element); - if (iframePath.length === 0) return null; - + + const framePath = getFramePath(element); + if (framePath.length === 0) return null; + try { - const selectorParts: string[] = []; + const selectorParts: string[] = []; + + framePath.forEach((context, index) => { + const frameSelector = context.isFrame ? + `frame[name="${context.frame.getAttribute('name')}"]` : + finder(context.frame, { + root: index === 0 ? document.body : + (framePath[index - 1].document.body as Element) + }); - // Generate selector for each iframe boundary - iframePath.forEach((context, index) => { - // Get selector for the iframe element - const frameSelector = finder(context.frame, { - root: index === 0 ? document.body : - (iframePath[index - 1].document.body as Element) - }); - - // For the last context, get selector for target element - if (index === iframePath.length - 1) { - const elementSelector = finder(element, { - root: context.document.body as Element - }); - selectorParts.push(`${frameSelector} :>> ${elementSelector}`); - } else { - selectorParts.push(frameSelector); - } - }); - - return { - fullSelector: selectorParts.join(' :>> '), - isFrameContent: true - }; + if (index === framePath.length - 1) { + const elementSelector = finder(element, { + root: context.document.body as Element + }); + selectorParts.push(`${frameSelector} :>> ${elementSelector}`); + } else { + selectorParts.push(frameSelector); + } + }); + + return { + fullSelector: selectorParts.join(' :>> '), + isFrameContent: true + }; } catch (e) { - console.warn('Error generating iframe selector:', e); - return null; + console.warn('Error generating frame selector:', e); + return null; } }; @@ -1424,9 +1752,25 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { } - const iframeSelector = genSelectorForIframe(element); + let iframeSelector = null; + try { + // Check if element is within frame/iframe + const isInFrame = element.ownerDocument !== document; + const isInFrameset = () => { + let doc = element.ownerDocument; + return doc.querySelectorAll('frameset').length > 0; + }; + + if (isInFrame || isInFrameset()) { + iframeSelector = genSelectorForFrame(element); + } + } catch (e) { + console.warn('Error detecting frames:', e); + } + const shadowSelector = genSelectorForShadowDOM(element); + const relSelector = genSelectorForAttributes(element, ['rel']); const hrefSelector = genSelectorForAttributes(element, ['href']); const formSelector = genSelectorForAttributes(element, [ 'name', @@ -1473,6 +1817,7 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { hrefSelector, accessibilitySelector, formSelector, + relSelector, iframeSelector: iframeSelector ? { full: iframeSelector.fullSelector, isIframe: iframeSelector.isFrameContent, @@ -1509,6 +1854,11 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { function genSelectorForAttributes(element: HTMLElement, attributes: string[]) { let selector = null; try { + if (attributes.includes('rel') && element.hasAttribute('rel')) { + const relValue = element.getAttribute('rel'); + return `[rel="${relValue}"]`; + } + selector = isAttributesDefined(element, attributes) ? finder(element, { idName: () => false, // Don't use the id to generate a selector @@ -1531,9 +1881,12 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { hoveredElement != null && !hoveredElement.closest('#overlay-controls') != null ) { - const { parentElement } = hoveredElement; + // Prioritize Link (DO NOT REMOVE) + // const { parentElement } = hoveredElement; // Match the logic in recorder.ts for link clicks - const element = parentElement?.tagName === 'A' ? parentElement : hoveredElement; + // const element = parentElement?.tagName === 'A' ? parentElement : hoveredElement; + + const element = hoveredElement; const generatedSelectors = genSelectors(element); return generatedSelectors; } @@ -1562,9 +1915,9 @@ interface SelectorResult { export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates, listSelector: string): Promise => { interface DOMContext { - type: 'iframe' | 'shadow'; + type: 'iframe' | 'frame' | 'shadow'; element: HTMLElement; - container: HTMLIFrameElement | ShadowRoot; + container: HTMLIFrameElement | HTMLFrameElement | ShadowRoot; host?: HTMLElement; document?: Document; } @@ -1573,53 +1926,63 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates if (!listSelector) { const selectors = await page.evaluate(({ x, y }: { x: number, y: number }) => { const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // Function to traverse shadow DOM + let element = document.elementFromPoint(x, y) as HTMLElement; + if (!element) return null; + + let deepestElement = element; + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; - let deepest = current; let shadowRoot = current.shadowRoot; - - while (shadowRoot) { + let deepest = current; + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; shadowRoot = current.shadowRoot; + depth++; } - + return deepest; }; - - // Start with the element at coordinates - let element = document.elementFromPoint(x, y) as HTMLElement; - if (!element) return null; - - let deepestElement = element; - let depth = 0; - const MAX_DEPTH = 4; - - // Handle iframe traversal + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + if (element.tagName === 'IFRAME') { let currentIframe = element as HTMLIFrameElement; - - while (currentIframe && depth < MAX_DEPTH) { + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - - const iframeDoc = currentIframe.contentDocument || currentIframe.contentWindow?.document; - if (!iframeDoc) break; - - const iframeElement = iframeDoc.elementFromPoint(iframeX, iframeY) as HTMLElement; + + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; + if (!iframeDocument) break; + + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Check for shadow DOM within iframe + deepestElement = traverseShadowDOM(iframeElement); - - if (deepestElement.tagName === 'IFRAME') { - currentIframe = deepestElement as HTMLIFrameElement; + + if (iframeElement.tagName === 'IFRAME') { + currentIframe = iframeElement as HTMLIFrameElement; depth++; } else { break; @@ -1629,25 +1992,134 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); } else { - // If not an iframe, check for shadow DOM deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; - // Basic selector generation function getNonUniqueSelector(element: HTMLElement): string { let selector = element.tagName.toLowerCase(); - + + if (selector === 'frame' || selector === 'iframe') { + let baseSelector = selector; + + if (element.className) { + const classes = element.className.split(/\s+/).filter(Boolean); + if (classes.length > 0) { + const validClasses = classes.filter(cls => !cls.startsWith('!') && !cls.includes(':')); + if (validClasses.length > 0) { + baseSelector += '.' + validClasses.map(cls => CSS.escape(cls)).join('.'); + } + } + } + + if (element.id) { + return `${selector}#${CSS.escape(element.id)}`; + } + + if (element.getAttribute('name')) { + return `${selector}[name="${CSS.escape(element.getAttribute('name')!)}"]`; + } + + if (element.parentElement && element.parentElement.tagName === 'FRAMESET') { + const frameIndex = Array.from(element.parentElement.children) + .filter(child => child.tagName.toLowerCase() === selector) + .indexOf(element) + 1; + + if (frameIndex > 0) { + return `${selector}:nth-of-type(${frameIndex})`; + } + } + + if (element.parentElement) { + const siblings = Array.from(element.parentElement.children); + const identicalSiblings = siblings.filter(sibling => { + if (sibling === element) return false; + + let siblingSelector = sibling.tagName.toLowerCase(); + const siblingClassName = typeof sibling.className === 'string' ? sibling.className : ''; + if (siblingClassName) { + const siblingClasses = siblingClassName.split(/\s+/).filter(Boolean); + const validSiblingClasses = siblingClasses.filter(cls => !cls.startsWith('!') && !cls.includes(':')); + if (validSiblingClasses.length > 0) { + siblingSelector += '.' + validSiblingClasses.map(cls => CSS.escape(cls)).join('.'); + } + } + + return siblingSelector === baseSelector; + }); + + if (identicalSiblings.length > 0) { + const position = siblings.indexOf(element) + 1; + return `${baseSelector}:nth-child(${position})`; + } + } + + return baseSelector; + } + if (selector === 'td' && element.parentElement) { - // Find position among td siblings const siblings = Array.from(element.parentElement.children); const position = siblings.indexOf(element) + 1; return `${selector}:nth-child(${position})`; } - + if (element.className) { const classes = element.className.split(/\s+/).filter((cls: string) => Boolean(cls)); if (classes.length > 0) { @@ -1657,9 +2129,8 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates } } } - + if (element.parentElement) { - // Look for identical siblings const siblings = Array.from(element.parentElement.children); const identicalSiblings = siblings.filter(sibling => { if (sibling === element) return false; @@ -1682,11 +2153,10 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates selector += `:nth-child(${position})`; } } - + return selector; } - function getContextPath(element: HTMLElement): DOMContext[] { const path: DOMContext[] = []; let current = element; @@ -1708,18 +2178,19 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates continue; } - // Check for iframe + // Check for iframe or frame const ownerDocument = current.ownerDocument; - const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; + const frameElement = ownerDocument?.defaultView?.frameElement; if (frameElement) { + const isFrame = frameElement.tagName === 'FRAME'; path.unshift({ - type: 'iframe', + type: isFrame ? 'frame' : 'iframe', element: current, - container: frameElement, + container: frameElement as (HTMLIFrameElement | HTMLFrameElement), document: ownerDocument }); - current = frameElement; + current = frameElement as HTMLElement; depth++; continue; } @@ -1786,7 +2257,6 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates } } - // if (listSelector === '') { if (element.tagName !== 'TABLE') { while (element.parentElement) { if (element.tagName.toLowerCase() === 'body' || @@ -1805,7 +2275,7 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates const significantOverlap = (childRect.width * childRect.height) / - (parentRect.width * parentRect.height) > 0.5; + (parentRect.width * parentRect.height) > 0.1; if (fullyContained && significantOverlap) { const nextParent = element.parentElement; @@ -1827,68 +2297,66 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates return selectors || { generalSelector: '' }; } else { - // When we have a list selector, we need special handling while maintaining shadow DOM support + // When we have a list selector, we need special handling while maintaining shadow DOM and frame support const selectors = await page.evaluate(({ x, y }: { x: number, y: number }) => { const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // Helper function to traverse shadow DOM - const traverseShadowDOM = (element: HTMLElement, depth: number = 0): HTMLElement => { - const MAX_SHADOW_DEPTH = 4; + let element = document.elementFromPoint(x, y) as HTMLElement; + if (!element) return null; + + let deepestElement = element; + + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; + let shadowRoot = current.shadowRoot; let deepest = current; - - while (current && depth < MAX_SHADOW_DEPTH) { - const shadowRoot = current.shadowRoot; - if (!shadowRoot) break; - + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; + shadowRoot = current.shadowRoot; depth++; } - + return deepest; }; - // Start with the element at the specified coordinates - let element = document.elementFromPoint(x, y) as HTMLElement; - if (!element) return null; + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; - // Initialize tracking variables - let deepestElement = element; - let depth = 0; - const MAX_IFRAME_DEPTH = 4; + if (element.tagName === 'IFRAME') { + let currentIframe = element as HTMLIFrameElement; + let depth = 0; + const MAX_IFRAME_DEPTH = 4; - // First check if the initial element has a shadow root - deepestElement = traverseShadowDOM(element); - - // If it's an iframe, traverse through iframe hierarchy - if (deepestElement.tagName === 'IFRAME') { - let currentIframe = deepestElement as HTMLIFrameElement; - while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - // Access iframe's document - const iframeDoc = currentIframe.contentDocument || currentIframe.contentWindow?.document; - if (!iframeDoc) break; + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; + if (!iframeDocument) break; - // Get element at transformed coordinates in iframe - const iframeElement = iframeDoc.elementFromPoint(iframeX, iframeY) as HTMLElement; + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - // Check for shadow DOM within iframe - const shadowResult = traverseShadowDOM(iframeElement); - deepestElement = shadowResult; + deepestElement = traverseShadowDOM(iframeElement); - // If we found another iframe, continue traversing - if (shadowResult.tagName === 'IFRAME') { - currentIframe = shadowResult as HTMLIFrameElement; + if (iframeElement.tagName === 'IFRAME') { + currentIframe = iframeElement as HTMLIFrameElement; depth++; } else { break; @@ -1898,33 +2366,145 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); + } else { + deepestElement = traverseShadowDOM(element); } return deepestElement; }; - // Generate basic selector from element's tag and classes function getNonUniqueSelector(element: HTMLElement): string { let selector = element.tagName.toLowerCase(); - + + if (selector === 'frame' || selector === 'iframe') { + let baseSelector = selector; + + if (element.className) { + const classes = element.className.split(/\s+/).filter(Boolean); + if (classes.length > 0) { + const validClasses = classes.filter(cls => !cls.startsWith('!') && !cls.includes(':')); + if (validClasses.length > 0) { + baseSelector += '.' + validClasses.map(cls => CSS.escape(cls)).join('.'); + } + } + } + + if (element.id) { + return `${selector}#${CSS.escape(element.id)}`; + } + + if (element.getAttribute('name')) { + return `${selector}[name="${CSS.escape(element.getAttribute('name')!)}"]`; + } + + if (element.parentElement && element.parentElement.tagName === 'FRAMESET') { + const frameIndex = Array.from(element.parentElement.children) + .filter(child => child.tagName.toLowerCase() === selector) + .indexOf(element) + 1; + + if (frameIndex > 0) { + return `${selector}:nth-of-type(${frameIndex})`; + } + } + + if (element.parentElement) { + const siblings = Array.from(element.parentElement.children); + const identicalSiblings = siblings.filter(sibling => { + if (sibling === element) return false; + + let siblingSelector = sibling.tagName.toLowerCase(); + const siblingClassName = typeof sibling.className === 'string' ? sibling.className : ''; + if (siblingClassName) { + const siblingClasses = siblingClassName.split(/\s+/).filter(Boolean); + const validSiblingClasses = siblingClasses.filter(cls => !cls.startsWith('!') && !cls.includes(':')); + if (validSiblingClasses.length > 0) { + siblingSelector += '.' + validSiblingClasses.map(cls => CSS.escape(cls)).join('.'); + } + } + + return siblingSelector === baseSelector; + }); + + if (identicalSiblings.length > 0) { + const position = siblings.indexOf(element) + 1; + return `${baseSelector}:nth-child(${position})`; + } + } + + return baseSelector; + } + if (selector === 'td' && element.parentElement) { const siblings = Array.from(element.parentElement.children); const position = siblings.indexOf(element) + 1; return `${selector}:nth-child(${position})`; } - + if (element.className) { - const classes = element.className.split(/\s+/).filter(Boolean); + const classes = element.className.split(/\s+/).filter((cls: string) => Boolean(cls)); if (classes.length > 0) { - const validClasses = classes.filter(cls => !cls.startsWith('!') && !cls.includes(':')); + const validClasses = classes.filter((cls: string) => !cls.startsWith('!') && !cls.includes(':')); if (validClasses.length > 0) { selector += '.' + validClasses.map(cls => CSS.escape(cls)).join('.'); } } } - + if (element.parentElement) { - // Look for identical siblings const siblings = Array.from(element.parentElement.children); const identicalSiblings = siblings.filter(sibling => { if (sibling === element) return false; @@ -1947,11 +2527,11 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates selector += `:nth-child(${position})`; } } - + return selector; } - // Get complete context path (both iframe and shadow DOM) + // Get complete context path (iframe, frame, and shadow DOM) function getContextPath(element: HTMLElement): DOMContext[] { const path: DOMContext[] = []; let current = element; @@ -1973,18 +2553,19 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates continue; } - // Check for iframe + // Check for iframe or frame const ownerDocument = current.ownerDocument; - const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; + const frameElement = ownerDocument?.defaultView?.frameElement; if (frameElement) { + const isFrame = frameElement.tagName === 'FRAME'; path.unshift({ - type: 'iframe', + type: isFrame ? 'frame' : 'iframe', element: current, - container: frameElement, + container: frameElement as (HTMLIFrameElement | HTMLFrameElement), document: ownerDocument }); - current = frameElement; + current = frameElement as HTMLElement; depth++; continue; } @@ -2045,9 +2626,9 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates const generalSelector = getSelectorPath(element); return { generalSelector }; - }, coordinates); + }, coordinates); - return selectors || { generalSelector: '' }; + return selectors || { generalSelector: '' }; } } catch (error) { console.error('Error in getNonUniqueSelectors:', error); @@ -2120,11 +2701,13 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro return `${hostSelector} >> ${elementSelector}`; } - // Check for iframe context + // Check for iframe/frame context const ownerDocument = element.ownerDocument; - const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; + const frameElement = ownerDocument?.defaultView?.frameElement; if (frameElement) { - const frameSelector = getNonUniqueSelector(frameElement); + const frameSelector = getNonUniqueSelector(frameElement as HTMLElement); + const isFrame = frameElement.tagName === 'FRAME'; + // Use the appropriate delimiter based on whether it's a frame or iframe return `${frameSelector} :>> ${elementSelector}`; } @@ -2133,8 +2716,7 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro return `${parentSelector} > ${elementSelector}`; } - - // Function to get all children from special contexts + // Function to get all children from special contexts including frames function getSpecialContextChildren(element: HTMLElement): HTMLElement[] { const children: HTMLElement[] = []; @@ -2160,10 +2742,43 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro } } + // Get frame children + const frames = Array.from(element.querySelectorAll('frame')) as HTMLFrameElement[]; + for (const frame of frames) { + try { + const frameDoc = frame.contentDocument || frame.contentWindow?.document; + if (frameDoc) { + const frameElements = Array.from(frameDoc.querySelectorAll('*')) as HTMLElement[]; + children.push(...frameElements); + } + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + + // Check for framesets + const framesets = Array.from(element.querySelectorAll('frameset')) as HTMLElement[]; + for (const frameset of framesets) { + const framesToCheck = Array.from(frameset.querySelectorAll('frame')) as HTMLFrameElement[]; + for (const frame of framesToCheck) { + try { + const frameDoc = frame.contentDocument || frame.contentWindow?.document; + if (frameDoc) { + const frameElements = Array.from(frameDoc.querySelectorAll('*')) as HTMLElement[]; + children.push(...frameElements); + } + } catch (error) { + console.warn('Cannot access frameset frame content:', error); + continue; + } + } + } + return children; } - // Function to recursively get all descendant selectors including shadow DOM and iframes + // Function to recursively get all descendant selectors function getAllDescendantSelectors(element: HTMLElement): string[] { let selectors: string[] = []; @@ -2177,7 +2792,7 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro // Process regular descendants selectors = selectors.concat(getAllDescendantSelectors(child)); - // Process special context children (shadow DOM and iframes) + // Process special context children (shadow DOM, iframes, and frames) const specialChildren = getSpecialContextChildren(child); for (const specialChild of specialChildren) { const specialPath = getSelectorPath(specialChild); @@ -2202,7 +2817,7 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro return selectors; } - // Handle both shadow DOM and iframe parent selectors + // Handle both shadow DOM, iframe, and frame parent selectors let parentElements: HTMLElement[] = []; // Check for special context traversal in parent selector @@ -2242,6 +2857,42 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro continue; } } + + // Check for frame + if (element.tagName === 'FRAME') { + try { + const frameDoc = (element as HTMLFrameElement).contentDocument || + (element as HTMLFrameElement).contentWindow?.document; + if (frameDoc) { + const frameChildren = Array.from( + frameDoc.querySelectorAll(selectorParts[i]) + ) as HTMLElement[]; + newParentElements.push(...frameChildren); + } + } catch (error) { + console.warn('Cannot access frame content during traversal:', error); + continue; + } + } + + // Check for frameset + if (element.tagName === 'FRAMESET') { + const frames = Array.from(element.querySelectorAll('frame')) as HTMLFrameElement[]; + for (const frame of frames) { + try { + const frameDoc = frame.contentDocument || frame.contentWindow?.document; + if (frameDoc) { + const frameChildren = Array.from( + frameDoc.querySelectorAll(selectorParts[i]) + ) as HTMLElement[]; + newParentElements.push(...frameChildren); + } + } catch (error) { + console.warn('Cannot access frameset frame during traversal:', error); + continue; + } + } + } } parentElements = newParentElements; diff --git a/src/components/browser/BrowserWindow.tsx b/src/components/browser/BrowserWindow.tsx index cb7e44d2..8edae179 100644 --- a/src/components/browser/BrowserWindow.tsx +++ b/src/components/browser/BrowserWindow.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useEffect, useState } from 'react'; +import React, { useCallback, useContext, useEffect, useState } from 'react'; import { useSocketStore } from '../../context/socket'; import { Button } from '@mui/material'; import Canvas from "../recorder/canvas"; @@ -8,6 +8,7 @@ import { useActionContext } from '../../context/browserActions'; import { useBrowserSteps, TextStep } from '../../context/browserSteps'; import { useGlobalInfoStore } from '../../context/globalInfo'; import { useTranslation } from 'react-i18next'; +import { AuthContext } from '../../context/auth'; interface ElementInfo { tagName: string; @@ -27,6 +28,12 @@ interface AttributeOption { value: string; } +interface ScreencastData { + image: string; + userId: string; +} + + const getAttributeOptions = (tagName: string, elementInfo: ElementInfo | null): AttributeOption[] => { if (!elementInfo) return []; switch (tagName.toLowerCase()) { @@ -71,6 +78,9 @@ export const BrowserWindow = () => { const { notify } = useGlobalInfoStore(); const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext(); const { addTextStep, addListStep } = useBrowserSteps(); + + const { state } = useContext(AuthContext); + const { user } = state; useEffect(() => { if (listSelector) { @@ -85,7 +95,7 @@ export const BrowserWindow = () => { if (storedListSelector && !listSelector) { setListSelector(storedListSelector); } - }, []); + }, []); const onMouseMove = (e: MouseEvent) => { if (canvasRef && canvasRef.current && highlighterData) { @@ -114,9 +124,15 @@ export const BrowserWindow = () => { } }, [getList, resetListState]); - const screencastHandler = useCallback((data: string) => { - setScreenShot(data); - }, [screenShot]); + const screencastHandler = useCallback((data: string | ScreencastData) => { + if (typeof data === 'string') { + setScreenShot(data); + } else if (data && typeof data === 'object' && 'image' in data) { + if (!data.userId || data.userId === user?.id) { + setScreenShot(data.image); + } + } + }, [screenShot, user?.id]); useEffect(() => { if (socket) { diff --git a/src/index.css b/src/index.css index 252b737a..e63d18f7 100644 --- a/src/index.css +++ b/src/index.css @@ -11,7 +11,19 @@ body { padding: 0; scrollbar-gutter: stable; overflow-y: auto; - +} + +input:-webkit-autofill, +input:-webkit-autofill:hover, +input:-webkit-autofill:focus, +textarea:-webkit-autofill, +textarea:-webkit-autofill:hover, +textarea:-webkit-autofill:focus, +select:-webkit-autofill, +select:-webkit-autofill:hover, +select:-webkit-autofill:focus { + -webkit-box-shadow: 0 0 0 1000px transparent inset !important; + transition: background-color 5000s ease-in-out 0s !important; } html { @@ -22,6 +34,7 @@ html { a { color: #ff00c3; + &:hover { color: #ff00c3; } @@ -29,7 +42,7 @@ a { code { font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', - monospace; + monospace; color: #ff00c3; } @@ -44,7 +57,6 @@ code { align-items: center; overflow: hidden; position: relative; - } #browser-content { @@ -52,13 +64,10 @@ code { width: 100%; display: flex; flex-direction: column; - transform: scale(1); /* Ensure no scaling */ - transform-origin: top left; /* Keep the position fixed */ -} - - -#browser { - + transform: scale(1); + /* Ensure no scaling */ + transform-origin: top left; + /* Keep the position fixed */ } #browser-window { @@ -163,4 +172,4 @@ code { height: calc(100vh - 2rem); margin: 1rem 55rem; } -} +} \ No newline at end of file