diff --git a/.gitignore b/.gitignore index 74279fc9..6b2b3f98 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,4 @@ /build -package-lock.json +package-lock.json \ No newline at end of file diff --git a/ENVEXAMPLE b/ENVEXAMPLE index 650c7027..dbe5470c 100644 --- a/ENVEXAMPLE +++ b/ENVEXAMPLE @@ -14,6 +14,7 @@ MINIO_ACCESS_KEY=minio_access_key # MinIO access key MINIO_SECRET_KEY=minio_secret_key # MinIO secret key REDIS_HOST=redis # Redis host in Docker REDIS_PORT=6379 # Redis port (default: 6379) +REDIS_PASSWORD=redis_password # Redis password (This is optional. Needed to authenticate with a password-protected Redis instance; if not set, Redis will connect without authentication.) # Backend and Frontend URLs and Ports BACKEND_PORT=8080 # Port to run backend on. Needed for Docker setup @@ -28,5 +29,10 @@ GOOGLE_CLIENT_ID=your_google_client_id GOOGLE_CLIENT_SECRET=your_google_client_secret GOOGLE_REDIRECT_URI=your_google_redirect_uri +# Optional Airtable OAuth settings for Airtable Integration + +AIRTABLE_CLIENT_ID=your_airtable_client_id +AIRTABLE_REDIRECT_URI=http://localhost:8080/auth/airtable/callback + # Telemetry Settings - Please keep it enabled. Keeping it enabled helps us understand how the product is used and assess the impact of any new changes. MAXUN_TELEMETRY=true diff --git a/README.md b/README.md index 13c994a0..75fee4c5 100644 --- a/README.md +++ b/README.md @@ -103,8 +103,11 @@ You can access the frontend at http://localhost:5173/ and backend at http://loca | `GOOGLE_CLIENT_ID` | No | Client ID for Google OAuth, used for Google Sheet integration authentication. | Google login will not work. | | `GOOGLE_CLIENT_SECRET`| No | Client Secret for Google OAuth. | Google login will not work. | | `GOOGLE_REDIRECT_URI` | No | Redirect URI for handling Google OAuth responses. | Google login will not work. | +| `AIRTABLE_CLIENT_ID` | No | Client ID for Airtable, used for Airtable integration authentication. | Airtable login will not work. | +| `AIRTABLE_REDIRECT_URI` | No | Redirect URI for handling Airtable OAuth responses. | Airtable login will not work. | | `REDIS_HOST` | Yes | Host address of the Redis server, used by BullMQ for scheduling robots. | Redis connection will fail. | -| `REDIS_PORT` | Yes | Port number for the Redis server. | Redis connection will fail. | +| `REDIS_PORT` | Yes | Port number for the Redis server. | Redis connection will fail. | +| `REDIS_PASSWORD` | No | Password for Redis Authentication. | Needed to authenticate with a password-protected Redis instance; if not set, Redis will attempt to connect without authentication. | | `MAXUN_TELEMETRY` | No | Disables telemetry to stop sending anonymous usage data. Keeping it enabled helps us understand how the product is used and assess the impact of any new changes. Please keep it enabled. | Telemetry data will not be collected. | diff --git a/maxun-core/package.json b/maxun-core/package.json index 2f04d42a..50d2c420 100644 --- a/maxun-core/package.json +++ b/maxun-core/package.json @@ -1,6 +1,6 @@ { "name": "maxun-core", - "version": "0.0.12", + "version": "0.0.13", "description": "Core package for Maxun, responsible for data extraction", "main": "build/index.js", "typings": "build/index.d.ts", diff --git a/maxun-core/src/browserSide/scraper.js b/maxun-core/src/browserSide/scraper.js index f6b53da2..c04a9993 100644 --- a/maxun-core/src/browserSide/scraper.js +++ b/maxun-core/src/browserSide/scraper.js @@ -210,7 +210,6 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, return Array.from(document.querySelectorAll(config.selector)); } - // First handle iframe traversal if present if (config.selector.includes(':>>')) { const parts = config.selector.split(':>>').map(s => s.trim()); let currentElements = [document]; @@ -223,23 +222,44 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, for (const element of currentElements) { try { - // For document or iframe document const doc = element.contentDocument || element || element.contentWindow?.document; if (!doc) continue; - // Query elements in current context + if (part.startsWith('frame[name=') || part.startsWith('iframe[name=')) { + const nameMatch = part.match(/\[name=['"]([^'"]+)['"]\]/); + if (nameMatch && nameMatch[1]) { + const frameName = nameMatch[1]; + let foundFrames = []; + + if (doc.getElementsByName && typeof doc.getElementsByName === 'function') { + foundFrames = Array.from(doc.getElementsByName(frameName)) + .filter(el => el.tagName === 'FRAME' || el.tagName === 'IFRAME'); + } + + if (foundFrames.length === 0) { + const framesBySelector = Array.from(doc.querySelectorAll(`frame[name="${frameName}"], iframe[name="${frameName}"]`)); + foundFrames = framesBySelector; + } + + if (isLast) { + nextElements.push(...foundFrames); + } else { + nextElements.push(...foundFrames); + } + continue; + } + } + const found = Array.from(doc.querySelectorAll(part)); if (isLast) { - // If it's the last part, keep all matching elements nextElements.push(...found); } else { - // If not last, only keep iframes for next iteration - const iframes = found.filter(el => el.tagName === 'IFRAME'); - nextElements.push(...iframes); + const frames = found.filter(el => el.tagName === 'IFRAME' || el.tagName === 'FRAME'); + nextElements.push(...frames); } } catch (error) { - console.warn('Cannot access iframe content:', error, { + console.warn('Cannot access iframe/frame content:', error, { part, element, index: i @@ -285,12 +305,17 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, return []; } - // Modified to handle iframe context for URL resolution function getElementValue(element, attribute) { if (!element) return null; - // Get the base URL for resolving relative URLs - const baseURL = element.ownerDocument?.location?.href || window.location.origin; + let baseURL; + try { + baseURL = element.ownerDocument?.location?.href || + element.ownerDocument?.baseURI || + window.location.origin; + } catch (e) { + baseURL = window.location.origin; + } switch (attribute) { case 'href': { @@ -305,6 +330,10 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, return element.innerText?.trim(); case 'textContent': return element.textContent?.trim(); + case 'innerHTML': + return element.innerHTML; + case 'outerHTML': + return element.outerHTML; default: return element.getAttribute(attribute) || element.innerText?.trim(); } @@ -394,7 +423,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, * @returns {Array.>} Array of arrays of scraped items, one sub-array per list */ window.scrapeList = async function ({ listSelector, fields, limit = 10 }) { - // Enhanced query function to handle both iframe and shadow DOM + // Enhanced query function to handle iframe, frame and shadow DOM const queryElement = (rootElement, selector) => { if (!selector.includes('>>') && !selector.includes(':>>')) { return rootElement.querySelector(selector); @@ -406,14 +435,14 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, for (let i = 0; i < parts.length; i++) { if (!currentElement) return null; - // Handle iframe traversal - if (currentElement.tagName === 'IFRAME') { + // Handle iframe and frame traversal + if (currentElement.tagName === 'IFRAME' || currentElement.tagName === 'FRAME') { try { - const iframeDoc = currentElement.contentDocument || currentElement.contentWindow.document; - currentElement = iframeDoc.querySelector(parts[i]); + const frameDoc = currentElement.contentDocument || currentElement.contentWindow.document; + currentElement = frameDoc.querySelector(parts[i]); continue; } catch (e) { - console.warn('Cannot access iframe content:', e); + console.warn(`Cannot access ${currentElement.tagName.toLowerCase()} content:`, e); return null; } } @@ -456,13 +485,13 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, const nextElements = []; for (const element of currentElements) { - // Handle iframe traversal - if (element.tagName === 'IFRAME') { + // Handle iframe and frame traversal + if (element.tagName === 'IFRAME' || element.tagName === 'FRAME') { try { - const iframeDoc = element.contentDocument || element.contentWindow.document; - nextElements.push(...iframeDoc.querySelectorAll(part)); + const frameDoc = element.contentDocument || element.contentWindow.document; + nextElements.push(...frameDoc.querySelectorAll(part)); } catch (e) { - console.warn('Cannot access iframe content:', e); + console.warn(`Cannot access ${element.tagName.toLowerCase()} content:`, e); continue; } } else { @@ -537,8 +566,8 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, return { type: 'TR', element: currentElement }; } - // Handle iframe crossing - if (currentElement.tagName === 'IFRAME') { + // Handle iframe and frame crossing + if (currentElement.tagName === 'IFRAME' || currentElement.tagName === 'FRAME') { try { currentElement = currentElement.contentDocument.body; } catch (e) { @@ -582,7 +611,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, if (current.tagName === 'TH') return true; - if (current.tagName === 'IFRAME') { + if (current.tagName === 'IFRAME' || current.tagName === 'FRAME') { try { current = current.contentDocument.body; } catch (e) { @@ -638,14 +667,18 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, allElements.push(...shadowHost.getElementsByTagName(baseElement.tagName)); } - // Get elements from iframes - const iframes = document.getElementsByTagName('iframe'); - for (const iframe of iframes) { + // Get elements from iframes and frames + const frames = [ + ...Array.from(document.getElementsByTagName('iframe')), + ...Array.from(document.getElementsByTagName('frame')) + ]; + + for (const frame of frames) { try { - const iframeDoc = iframe.contentDocument || iframe.contentWindow.document; - allElements.push(...iframeDoc.getElementsByTagName(baseElement.tagName)); + const frameDoc = frame.contentDocument || frame.contentWindow.document; + allElements.push(...frameDoc.getElementsByTagName(baseElement.tagName)); } catch (e) { - console.warn('Cannot access iframe content:', e); + console.warn(`Cannot access ${frame.tagName.toLowerCase()} content:`, e); } } @@ -707,7 +740,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, const tableData = []; const nonTableData = []; - // Process table data with both iframe and shadow DOM support + // Process table data with support for iframes, frames, and shadow DOM for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) { const container = containers[containerIndex]; const { tableFields } = containerFields[containerIndex]; @@ -717,14 +750,14 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, const firstElement = queryElement(container, firstField.selector); let tableContext = firstElement; - // Find table context including both iframe and shadow DOM + // Find table context including iframe, frame and shadow DOM while (tableContext && tableContext.tagName !== 'TABLE' && tableContext !== container) { if (tableContext.getRootNode() instanceof ShadowRoot) { tableContext = tableContext.getRootNode().host; continue; } - if (tableContext.tagName === 'IFRAME') { + if (tableContext.tagName === 'IFRAME' || tableContext.tagName === 'FRAME') { try { tableContext = tableContext.contentDocument.body; } catch (e) { @@ -747,13 +780,13 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, rows.push(...tableContext.shadowRoot.getElementsByTagName('TR')); } - // Get rows from iframes - if (tableContext.tagName === 'IFRAME') { + // Get rows from iframes and frames + if (tableContext.tagName === 'IFRAME' || tableContext.tagName === 'FRAME') { try { - const iframeDoc = tableContext.contentDocument || tableContext.contentWindow.document; - rows.push(...iframeDoc.getElementsByTagName('TR')); + const frameDoc = tableContext.contentDocument || tableContext.contentWindow.document; + rows.push(...frameDoc.getElementsByTagName('TR')); } catch (e) { - console.warn('Cannot access iframe rows:', e); + console.warn(`Cannot access ${tableContext.tagName.toLowerCase()} rows:`, e); } } @@ -823,7 +856,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, } } - // Process non-table data with both contexts support + // Process non-table data with all contexts support for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) { if (nonTableData.length >= limit) break; diff --git a/maxun-core/src/interpret.ts b/maxun-core/src/interpret.ts index 88047128..2887b634 100644 --- a/maxun-core/src/interpret.ts +++ b/maxun-core/src/interpret.ts @@ -286,6 +286,12 @@ export default class Interpreter extends EventEmitter { ? arrayToObject(superset[key]) : superset[key]; + if ((key === 'url' || key === 'selectors') && + Array.isArray(value) && Array.isArray(superset[key]) && + value.length === 0 && (superset[key] as any[]).length === 0) { + return true; + } + if (key === 'selectors' && Array.isArray(value) && Array.isArray(superset[key])) { return value.some(selector => (superset[key] as any[]).includes(selector) @@ -592,33 +598,52 @@ export default class Interpreter extends EventEmitter { }; // Enhanced button finder with retry mechanism - const findWorkingButton = async (selectors: string[], retryCount = 0): Promise<{ - button: ElementHandle | null, - workingSelector: string | null + const findWorkingButton = async (selectors: string[]): Promise<{ + button: ElementHandle | null, + workingSelector: string | null, + updatedSelectors: string[] }> => { - for (const selector of selectors) { - try { - const button = await page.waitForSelector(selector, { - state: 'attached', - timeout: 10000 // Reduced timeout for faster checks - }); - if (button) { - debugLog('Found working selector:', selector); - return { button, workingSelector: selector }; + let updatedSelectors = [...selectors]; + + for (let i = 0; i < selectors.length; i++) { + const selector = selectors[i]; + let retryCount = 0; + let selectorSuccess = false; + + while (retryCount < MAX_RETRIES && !selectorSuccess) { + try { + const button = await page.waitForSelector(selector, { + state: 'attached', + timeout: 10000 + }); + + if (button) { + debugLog('Found working selector:', selector); + return { + button, + workingSelector: selector, + updatedSelectors + }; + } + } catch (error) { + retryCount++; + debugLog(`Selector "${selector}" failed: attempt ${retryCount}/${MAX_RETRIES}`); + + if (retryCount < MAX_RETRIES) { + await page.waitForTimeout(RETRY_DELAY); + } else { + debugLog(`Removing failed selector "${selector}" after ${MAX_RETRIES} attempts`); + updatedSelectors = updatedSelectors.filter(s => s !== selector); + } } - } catch (error) { - debugLog(`Selector failed: ${selector}`); } } - - // Implement retry mechanism when no selectors work - if (selectors.length > 0 && retryCount < MAX_RETRIES) { - debugLog(`Retry attempt ${retryCount + 1} of ${MAX_RETRIES}`); - await page.waitForTimeout(RETRY_DELAY); - return findWorkingButton(selectors, retryCount + 1); - } - - return { button: null, workingSelector: null }; + + return { + button: null, + workingSelector: null, + updatedSelectors + }; }; const retryOperation = async (operation: () => Promise, retryCount = 0): Promise => { @@ -680,7 +705,10 @@ export default class Interpreter extends EventEmitter { await scrapeCurrentPage(); if (checkLimit()) return allResults; - const { button, workingSelector } = await findWorkingButton(availableSelectors); + const { button, workingSelector, updatedSelectors } = await findWorkingButton(availableSelectors); + + availableSelectors = updatedSelectors; + if (!button || !workingSelector) { // Final retry for navigation when no selectors work const success = await retryOperation(async () => { @@ -697,10 +725,6 @@ export default class Interpreter extends EventEmitter { break; } - availableSelectors = availableSelectors.slice( - availableSelectors.indexOf(workingSelector) - ); - let retryCount = 0; let navigationSuccess = false; @@ -768,22 +792,25 @@ export default class Interpreter extends EventEmitter { } case 'clickLoadMore': { + await scrapeCurrentPage(); + if (checkLimit()) return allResults; + + let loadMoreCounter = 0; + let previousResultCount = allResults.length; + let noNewItemsCounter = 0; + const MAX_NO_NEW_ITEMS = 2; + while (true) { - // Find working button with retry mechanism, consistent with clickNext - const { button: loadMoreButton, workingSelector } = await findWorkingButton(availableSelectors); + // Find working button with retry mechanism + const { button: loadMoreButton, workingSelector, updatedSelectors } = await findWorkingButton(availableSelectors); + + availableSelectors = updatedSelectors; if (!workingSelector || !loadMoreButton) { debugLog('No working Load More selector found after retries'); - const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config); - allResults = allResults.concat(finalResults); return allResults; } - // Update available selectors to start from the working one - availableSelectors = availableSelectors.slice( - availableSelectors.indexOf(workingSelector) - ); - // Implement retry mechanism for clicking the button let retryCount = 0; let clickSuccess = false; @@ -808,6 +835,8 @@ export default class Interpreter extends EventEmitter { if (clickSuccess) { await page.waitForTimeout(1000); + loadMoreCounter++; + debugLog(`Successfully clicked Load More button (${loadMoreCounter} times)`); } } catch (error) { debugLog(`Click attempt ${retryCount + 1} failed completely.`); @@ -822,8 +851,6 @@ export default class Interpreter extends EventEmitter { if (!clickSuccess) { debugLog(`Load More clicking failed after ${MAX_RETRIES} attempts`); - const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config); - allResults = allResults.concat(finalResults); return allResults; } @@ -833,20 +860,34 @@ export default class Interpreter extends EventEmitter { await page.waitForTimeout(2000); const currentHeight = await page.evaluate(() => document.body.scrollHeight); - if (currentHeight === previousHeight) { - debugLog('No more items loaded after Load More'); - const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config); - allResults = allResults.concat(finalResults); - return allResults; - } + const heightChanged = currentHeight !== previousHeight; previousHeight = currentHeight; - if (config.limit && allResults.length >= config.limit) { - allResults = allResults.slice(0, config.limit); - break; + await scrapeCurrentPage(); + + const currentResultCount = allResults.length; + const newItemsAdded = currentResultCount > previousResultCount; + + if (!newItemsAdded) { + noNewItemsCounter++; + debugLog(`No new items added after click (${noNewItemsCounter}/${MAX_NO_NEW_ITEMS})`); + + if (noNewItemsCounter >= MAX_NO_NEW_ITEMS) { + debugLog(`Stopping after ${MAX_NO_NEW_ITEMS} clicks with no new items`); + return allResults; + } + } else { + noNewItemsCounter = 0; + previousResultCount = currentResultCount; + } + + if (checkLimit()) return allResults; + + if (!heightChanged) { + debugLog('No more items loaded after Load More'); + return allResults; } } - break; } default: { diff --git a/package.json b/package.json index 0580d2de..57a22d1d 100644 --- a/package.json +++ b/package.json @@ -16,16 +16,19 @@ "@types/bcrypt": "^5.0.2", "@types/body-parser": "^1.19.5", "@types/csurf": "^1.11.5", + "@types/express-session": "^1.18.1", "@types/jsonwebtoken": "^9.0.7", "@types/node": "22.7.9", "@types/react": "^18.0.5", "@types/react-dom": "^18.0.1", "@types/uuid": "^8.3.4", + "airtable": "^0.12.2", "axios": "^0.26.0", "bcrypt": "^5.1.1", "body-parser": "^1.20.3", "buffer": "^6.0.3", "bullmq": "^5.12.15", + "connect-redis": "^8.0.1", "cookie-parser": "^1.4.6", "cors": "^2.8.5", "cron-parser": "^4.9.0", @@ -33,6 +36,7 @@ "csurf": "^1.11.0", "dotenv": "^16.0.0", "express": "^4.17.2", + "express-session": "^1.18.1", "fortawesome": "^0.0.1-security", "google-auth-library": "^9.14.1", "googleapis": "^144.0.0", @@ -47,11 +51,13 @@ "lodash": "^4.17.21", "loglevel": "^1.8.0", "loglevel-plugin-remote": "^0.6.8", - "maxun-core": "^0.0.12", + "maxun-core": "^0.0.13", "minio": "^8.0.1", "moment-timezone": "^0.5.45", "node-cron": "^3.0.3", "pg": "^8.13.0", + "pg-boss": "^10.1.6", + "pkce-challenge": "^4.1.0", "playwright": "^1.48.2", "playwright-extra": "^4.3.6", "posthog-node": "^4.2.1", @@ -65,6 +71,7 @@ "react-router-dom": "^6.26.1", "react-simple-code-editor": "^0.11.2", "react-transition-group": "^4.4.2", + "redis": "^4.7.0", "sequelize": "^6.37.3", "sequelize-typescript": "^2.1.6", "sharp": "^0.33.5", @@ -106,6 +113,7 @@ "@types/prismjs": "^1.26.0", "@types/react-highlight": "^0.12.5", "@types/react-transition-group": "^4.4.4", + "@types/redis": "^4.0.11", "@types/styled-components": "^5.1.23", "@types/swagger-jsdoc": "^6.0.4", "@types/swagger-ui-express": "^4.1.6", @@ -115,6 +123,7 @@ "cross-env": "^7.0.3", "js-cookie": "^3.0.5", "nodemon": "^2.0.15", + "sequelize-cli": "^6.6.2", "ts-node": "^10.4.0", "vite": "^5.4.10" } diff --git a/perf/performance.ts b/perf/performance.ts index c50ef850..c499ecd2 100644 --- a/perf/performance.ts +++ b/perf/performance.ts @@ -18,7 +18,7 @@ export class FrontendPerformanceMonitor { }; this.lastFrameTime = performance.now(); this.frameCount = 0; - + // Start monitoring this.startMonitoring(); } @@ -93,6 +93,262 @@ export class FrontendPerformanceMonitor { } } +// Enhanced Performance Monitor with Memory Management +export class EnhancedPerformanceMonitor extends FrontendPerformanceMonitor { + private memoryWarningThreshold: number = 100000000; // 100MB + private memoryAlertCallback?: (usage: MemoryInfo) => void; + private frameTimeHistory: number[] = []; + private maxMetricsHistory: number = 100; // Keep only the last 100 readings + private isThrottled: boolean = false; + private rafHandle: number | null = null; + private memoryCheckInterval: NodeJS.Timeout | null = null; + + constructor(options?: { + memoryWarningThreshold?: number, + maxMetricsHistory?: number, + memoryAlertCallback?: (usage: MemoryInfo) => void + }) { + super(); + + if (options) { + if (options.memoryWarningThreshold) { + this.memoryWarningThreshold = options.memoryWarningThreshold; + } + + if (options.maxMetricsHistory) { + this.maxMetricsHistory = options.maxMetricsHistory; + } + + if (options.memoryAlertCallback) { + this.memoryAlertCallback = options.memoryAlertCallback; + } + } + + // Override the parent's monitoring with our enhanced version + this.startEnhancedMonitoring(); + } + + private startEnhancedMonitoring(): void { + // Stop any existing monitoring from parent class + if (this.rafHandle) { + cancelAnimationFrame(this.rafHandle); + } + + if (this.memoryCheckInterval) { + clearInterval(this.memoryCheckInterval); + } + + // Enhanced FPS monitoring with frame time tracking + let lastFrameTime = performance.now(); + let frameCount = 0; + let frameTimes: number[] = []; + + const measureFPS = () => { + const now = performance.now(); + const frameTime = now - lastFrameTime; + lastFrameTime = now; + + // Track individual frame times for jank detection + frameTimes.push(frameTime); + if (frameTimes.length > 60) { // Keep only last 60 frame times + frameTimes.shift(); + } + + frameCount++; + this.frameTimeHistory.push(frameTime); + + // Calculate FPS every second + if (this.frameTimeHistory.length >= 60) { + const totalTime = this.frameTimeHistory.reduce((sum, time) => sum + time, 0); + const fps = Math.round((this.frameTimeHistory.length * 1000) / totalTime); + + // Get metrics from parent class + const metrics = this.getMetrics(); + metrics.fps.push(fps); + + // Limit metrics history + if (metrics.fps.length > this.maxMetricsHistory) { + metrics.fps.shift(); + } + + // Detect jank (long frames) + const jankThreshold = 16.7 * 2; // 2x normal frame time at 60fps + const jankFrames = frameTimes.filter(time => time > jankThreshold); + + if (jankFrames.length > 10) { // If more than 10 out of 60 frames are janky + this.detectPerformanceIssue('jank', { + jankFrames: jankFrames.length, + averageJankTime: jankFrames.reduce((sum, time) => sum + time, 0) / jankFrames.length + }); + } + + // Reset for next measurement + this.frameTimeHistory = []; + frameTimes = []; + } + + this.rafHandle = requestAnimationFrame(measureFPS); + }; + + this.rafHandle = requestAnimationFrame(measureFPS); + + // Enhanced memory monitoring + if (window.performance && (performance as any).memory) { + this.memoryCheckInterval = setInterval(() => { + const memory = (performance as any).memory; + const memoryInfo = { + usedJSHeapSize: memory.usedJSHeapSize, + totalJSHeapSize: memory.totalJSHeapSize, + timestamp: Date.now() + }; + + // Get metrics from parent class + const metrics = this.getMetrics(); + metrics.memoryUsage.push(memoryInfo); + + // Limit metrics history + if (metrics.memoryUsage.length > this.maxMetricsHistory) { + metrics.memoryUsage.shift(); + } + + // Check for memory warnings + if (memoryInfo.usedJSHeapSize > this.memoryWarningThreshold) { + this.detectPerformanceIssue('memory', memoryInfo); + + if (this.memoryAlertCallback) { + this.memoryAlertCallback(memoryInfo); + } + } + + // Check for memory leaks (steady increase) + if (metrics.memoryUsage.length >= 10) { + const recentMemory = metrics.memoryUsage.slice(-10); + let increasingCount = 0; + + for (let i = 1; i < recentMemory.length; i++) { + if (recentMemory[i].usedJSHeapSize > recentMemory[i - 1].usedJSHeapSize) { + increasingCount++; + } + } + + // If memory increased in 8 out of 9 consecutive readings + if (increasingCount >= 8) { + this.detectPerformanceIssue('memoryLeak', { + startMemory: recentMemory[0].usedJSHeapSize, + currentMemory: recentMemory[recentMemory.length - 1].usedJSHeapSize, + increaseRate: (recentMemory[recentMemory.length - 1].usedJSHeapSize - recentMemory[0].usedJSHeapSize) / + (recentMemory[recentMemory.length - 1].timestamp - recentMemory[0].timestamp) * 1000 // bytes per second + }); + } + } + }, 1000); + } + } + + // Method to detect various performance issues + private detectPerformanceIssue(type: 'jank' | 'memory' | 'memoryLeak', data: any): void { + console.warn(`Performance issue detected: ${type}`, data); + + if (type === 'memory' || type === 'memoryLeak') { + // Auto-throttle rendering if memory issues detected + if (!this.isThrottled) { + this.throttleRendering(); + } + + // Suggest garbage collection + this.suggestGarbageCollection(); + } + } + + // Get access to the metrics from parent class + private getMetrics(): any { + return (this as any).metrics; + } + + // Throttle rendering to reduce memory pressure + private throttleRendering(): void { + this.isThrottled = true; + console.info('Throttling rendering due to memory pressure'); + // Application code would implement throttling behavior + } + + // Un-throttle rendering when memory pressure is reduced + public unthrottleRendering(): void { + if (this.isThrottled) { + this.isThrottled = false; + console.info('Resuming normal rendering'); + } + } + + // Suggest garbage collection to the browser + private suggestGarbageCollection(): void { + if (window.gc) { + try { + window.gc(); + } catch (e) { + // gc() might not be available without special flags + } + } + + // Alternative approach to encourage garbage collection + const largeArray = new Array(1000000).fill(0); + largeArray.length = 0; + } + + // Enhanced performance report with more detailed metrics + public getEnhancedPerformanceReport(): EnhancedPerformanceReport { + const baseReport = super.getPerformanceReport(); + const metrics = this.getMetrics(); + + // Calculate 95th percentile render time + const sortedRenderTimes = [...metrics.renderTime].sort((a, b) => a - b); + const idx95 = Math.floor(sortedRenderTimes.length * 0.95); + const renderTime95Percentile = sortedRenderTimes[idx95] || 0; + + // Calculate memory growth rate + let memoryGrowthRate = 0; + if (metrics.memoryUsage.length >= 2) { + const first = metrics.memoryUsage[0]; + const last = metrics.memoryUsage[metrics.memoryUsage.length - 1]; + const timeDiffInSeconds = (last.timestamp - first.timestamp) / 1000; + memoryGrowthRate = timeDiffInSeconds > 0 + ? (last.usedJSHeapSize - first.usedJSHeapSize) / timeDiffInSeconds + : 0; + } + + return { + ...baseReport, + renderTime95Percentile, + memoryGrowthRate, + isThrottled: this.isThrottled, + heapUsagePercentage: baseReport.lastMemoryUsage + ? (baseReport.lastMemoryUsage.usedJSHeapSize / baseReport.lastMemoryUsage.totalJSHeapSize) * 100 + : 0 + }; + } + + // Clean up resources when no longer needed + public dispose(): void { + if (this.rafHandle) { + cancelAnimationFrame(this.rafHandle); + this.rafHandle = null; + } + + if (this.memoryCheckInterval) { + clearInterval(this.memoryCheckInterval); + this.memoryCheckInterval = null; + } + } +} + +// Extended types +interface EnhancedPerformanceReport extends PerformanceReport { + renderTime95Percentile: number; + memoryGrowthRate: number; // bytes per second + isThrottled: boolean; + heapUsagePercentage: number; +} + // Backend Performance Monitoring export class BackendPerformanceMonitor { private metrics: { diff --git a/public/locales/de.json b/public/locales/de.json index 07cf45bd..363aebbf 100644 --- a/public/locales/de.json +++ b/public/locales/de.json @@ -139,41 +139,12 @@ "no_key_message": "Sie haben noch keinen API-Schlüssel generiert.", "generate_button": "API-Schlüssel generieren", "notifications": { - "errors": { - "fetch": { - "network": "Netzwerkfehler beim Abrufen des API-Schlüssels: ${error}", - "unauthorized": "Sie müssen angemeldet sein, um auf den API-Schlüssel zuzugreifen", - "not_found": "API-Schlüssel für Ihr Konto wurde nicht gefunden", - "server": "Serverfehler beim Abrufen des API-Schlüssels. Bitte versuchen Sie es später erneut", - "unknown": "Unbekannter Fehler beim Abrufen des API-Schlüssels: ${error}" - }, - "generate": { - "network": "Netzwerkfehler bei der Generierung des API-Schlüssels: ${error}", - "unauthorized": "Sie müssen angemeldet sein, um einen API-Schlüssel zu generieren", - "key_exists": "Sie haben bereits einen API-Schlüssel. Bitte löschen Sie zuerst den vorhandenen", - "not_found": "Benutzerkonto nicht gefunden", - "server": "Serverfehler bei der Generierung des API-Schlüssels. Bitte versuchen Sie es später erneut", - "unknown": "Unbekannter Fehler bei der Generierung des API-Schlüssels: ${error}" - }, - "delete": { - "network": "Netzwerkfehler beim Löschen des API-Schlüssels: ${error}", - "unauthorized": "Sie müssen angemeldet sein, um den API-Schlüssel zu löschen", - "not_found": "Benutzerkonto nicht gefunden", - "key_not_found": "Kein API-Schlüssel zum Löschen gefunden", - "server": "Serverfehler beim Löschen des API-Schlüssels. Bitte versuchen Sie es später erneut", - "unknown": "Unbekannter Fehler beim Löschen des API-Schlüssels: ${error}" - }, - "copy": { - "failed": "Fehler beim Kopieren des API-Schlüssels in die Zwischenablage", - "no_key": "Kein API-Schlüssel zum Kopieren verfügbar" - } - }, - "success": { - "fetch": "API-Schlüssel erfolgreich abgerufen", - "generate": "Neuer API-Schlüssel erfolgreich generiert", - "delete": "API-Schlüssel erfolgreich gelöscht", - "copy": "API-Schlüssel in die Zwischenablage kopiert" - } + "fetch_error": "API-Schlüssel konnte nicht abgerufen werden - ${error}", + "generate_success": "API-Schlüssel erfolgreich generiert", + "generate_error": "API-Schlüssel konnte nicht generiert werden - ${error}", + "delete_success": "API-Schlüssel erfolgreich gelöscht", + "delete_error": "API-Schlüssel konnte nicht gelöscht werden - ${error}", + "copy_success": "API-Schlüssel erfolgreich kopiert" } }, "action_description": { @@ -334,29 +305,90 @@ } }, "integration_settings": { - "title": "Mit Google Sheet integrieren", + "title": "Integrationseinstellungen", "descriptions": { - "sync_info": "Wenn Sie diese Option aktivieren, werden bei jeder erfolgreichen Ausführung dieses Roboters die erfassten Daten in Ihrem Google Sheet ergänzt.", "authenticated_as": "Authentifiziert als: {{email}}" }, - "alerts": { - "success": { - "title": "Google Sheet erfolgreich integriert.", - "content": "Jedes Mal, wenn dieser Roboter eine erfolgreiche Ausführung erstellt, werden die erfassten Daten Ihrem Google Sheet {{sheetName}} hinzugefügt. Sie können die Datenaktualisierungen", - "here": "hier", - "note": "Hinweis:", - "sync_limitation": "Die vor der Integration mit Google Sheets extrahierten Daten werden nicht im Google Sheet synchronisiert. Nur die nach der Integration extrahierten Daten werden synchronisiert." + "buttons": { + "submit": "Absenden", + "remove_integration": "Integration entfernen" + }, + "google": { + "title": "Mit Google Sheet integrieren", + "descriptions": { + "sync_info": "Wenn Sie diese Option aktivieren, werden bei jeder erfolgreichen Ausführung einer Aufgabe durch diesen Roboter die erfassten Daten an Ihr Google Sheet angehängt.", + "authenticated_as": "Authentifiziert als: {{email}}" + }, + "alerts": { + "success": { + "title": "Google Sheet erfolgreich integriert.", + "content": "Jedes Mal, wenn dieser Roboter einen erfolgreichen Lauf erstellt, werden seine erfassten Daten an Ihr Google Sheet {{sheetName}} angehängt. Sie können die Datenaktualisierungen überprüfen", + "here": "hier", + "note": "Hinweis:", + "sync_limitation": "Die vor der Integration mit Google Sheets extrahierten Daten werden nicht im Google Sheet synchronisiert. Nur die nach der Integration extrahierten Daten werden synchronisiert." + } + }, + "buttons": { + "authenticate": "Mit Google authentifizieren", + "fetch_sheets": "Google Tabellenkalkulationen abrufen", + "remove_integration": "Integration entfernen", + "submit": "Absenden" + }, + "fields": { + "select_sheet": "Google Sheet auswählen", + "selected_sheet": "Ausgewähltes Sheet: {{name}} (ID: {{id}})" + }, + "errors": { + "auth_error": "Fehler bei der Authentifizierung mit Google", + "fetch_error": "Fehler beim Abrufen der Tabellenkalkulationsdateien: {{message}}", + "update_error": "Fehler beim Aktualisieren der Google Sheet-ID: {{message}}", + "remove_error": "Fehler beim Entfernen der Google Sheets-Integration: {{message}}" + }, + "notifications": { + "sheet_selected": "Google Sheet erfolgreich ausgewählt", + "integration_removed": "Google Sheets-Integration erfolgreich entfernt" } }, - "buttons": { - "authenticate": "Mit Google authentifizieren", - "fetch_sheets": "Google Sheets abrufen", - "remove_integration": "Integration entfernen", - "submit": "Absenden" - }, - "fields": { - "select_sheet": "Google Sheet auswählen", - "selected_sheet": "Ausgewähltes Sheet: {{name}} (ID: {{id}})" + "airtable": { + "title": "Mit Airtable integrieren", + "descriptions": { + "sync_info": "Wenn Sie diese Option aktivieren, werden bei jeder erfolgreichen Ausführung einer Aufgabe durch diesen Roboter die erfassten Daten an Ihre Airtable angehängt.", + "authenticated_as": "Erfolgreich bei Airtable authentifiziert. Sie können nun die Basis und den Tisch auswählen, in die Sie integrieren möchten." + }, + "alerts": { + "success": { + "title": "Airtable Base erfolgreich integriert", + "content": "Jedes Mal, wenn dieser Roboter einen erfolgreichen Lauf erstellt, werden seine erfassten Daten an Ihre {{baseName}}-Basis und Ihre {{tableName}}-Tabelle angehängt. Sie können Updates überprüfen", + "here": "hier", + "note": "Hinweis:", + "sync_limitation": "Nur nach der Integration erfasste Daten werden mit Airtable synchronisiert." + } + }, + "buttons": { + "authenticate": "Mit Airtable verbinden", + "fetch_bases": "Airtable Basen abrufen", + "fetch_tables": "Airtable Tabellen abrufen", + "remove_integration": "Integration entfernen", + "submit": "Basis und Tabelle auswählen" + }, + "fields": { + "select_base": "Airtable Basis auswählen", + "select_table": "Airtable Tabelle auswählen", + "selected_base": "Ausgewählte Basis: {{name}}", + "selected_table": "Ausgewählte Tabelle: {{name}}" + }, + "errors": { + "auth_error": "Fehler bei der Authentifizierung mit Airtable", + "fetch_error": "Fehler beim Abrufen von Airtable-Basen: {{message}}", + "fetch_tables_error": "Fehler beim Abrufen von Airtable-Tabellen: {{message}}", + "update_error": "Fehler beim Aktualisieren der Airtable-Basis: {{message}}", + "remove_error": "Fehler beim Entfernen der Airtable-Integration: {{message}}" + }, + "notifications": { + "base_selected": "Airtable-Basis erfolgreich ausgewählt", + "table_selected": "Airtable-Tabelle erfolgreich ausgewählt", + "integration_removed": "Airtable-Integration erfolgreich entfernt" + } } }, "robot_duplication": { @@ -480,6 +512,7 @@ "success": "Erfolg", "running": "Läuft", "scheduled": "Geplant", + "queued": "In Warteschlange", "failed": "Fehlgeschlagen" }, "run_settings_modal": { diff --git a/public/locales/en.json b/public/locales/en.json index e847d832..53ac4184 100644 --- a/public/locales/en.json +++ b/public/locales/en.json @@ -54,6 +54,12 @@ "label":"URL", "button":"Start Recording" }, + "warning_modal":{ + "title":"Active Browser Detected", + "message": "There is already a browser recording session running. Would you like to discard it and create a new recording?", + "discard_and_create":"Discard & Create New", + "cancel":"Cancel" + }, "edit":"Edit", "delete":"Delete", "duplicate":"Duplicate", @@ -140,41 +146,12 @@ "no_key_message": "You haven't generated an API key yet.", "generate_button": "Generate API Key", "notifications": { - "errors": { - "fetch": { - "network": "Network error while fetching API key: ${error}", - "unauthorized": "You must be logged in to access API key", - "not_found": "Unable to find API key for your account", - "server": "Server error while fetching API key. Please try again later", - "unknown": "Unknown error occurred while fetching API key: ${error}" - }, - "generate": { - "network": "Network error while generating API key: ${error}", - "unauthorized": "You must be logged in to generate an API key", - "key_exists": "You already have an API key. Please delete the existing one first", - "not_found": "User account not found", - "server": "Server error while generating API key. Please try again later", - "unknown": "Unknown error occurred while generating API key: ${error}" - }, - "delete": { - "network": "Network error while deleting API key: ${error}", - "unauthorized": "You must be logged in to delete API key", - "not_found": "User account not found", - "key_not_found": "No API key found to delete", - "server": "Server error while deleting API key. Please try again later", - "unknown": "Unknown error occurred while deleting API key: ${error}" - }, - "copy": { - "failed": "Failed to copy API key to clipboard", - "no_key": "No API key available to copy" - } - }, - "success": { - "fetch": "API key retrieved successfully", - "generate": "New API key generated successfully", - "delete": "API key deleted successfully", - "copy": "API key copied to clipboard" - } + "fetch_error": "Failed to fetch API Key - ${error}", + "generate_success": "Generated API Key successfully", + "generate_error": "Failed to generate API Key - ${error}", + "delete_success": "API Key deleted successfully", + "delete_error": "Failed to delete API Key - ${error}", + "copy_success": "Copied API Key successfully" } }, "action_description": { @@ -336,38 +313,90 @@ } }, "integration_settings": { - "title": "Integrate with Google Sheet", + "title": "Integration Settings", "descriptions": { - "sync_info": "If you enable this option, every time this robot runs a task successfully, its captured data will be appended to your Google Sheet.", "authenticated_as": "Authenticated as: {{email}}" }, - "alerts": { - "success": { - "title": "Google Sheet Integrated Successfully.", - "content": "Every time this robot creates a successful run, its captured data is appended to your {{sheetName}} Google Sheet. You can check the data updates", - "here": "here", - "note": "Note:", - "sync_limitation": "The data extracted before integrating with Google Sheets will not be synced in the Google Sheet. Only the data extracted after the integration will be synced." + "buttons": { + "submit": "Submit", + "remove_integration": "Remove Integration" + }, + "google": { + "title": "Integrate with Google Sheet", + "descriptions": { + "sync_info": "If you enable this option, every time this robot runs a task successfully, its captured data will be appended to your Google Sheet.", + "authenticated_as": "Authenticated as: {{email}}" + }, + "alerts": { + "success": { + "title": "Google Sheet Integrated Successfully.", + "content": "Every time this robot creates a successful run, its captured data is appended to your {{sheetName}} Google Sheet. You can check the data updates", + "here": "here", + "note": "Note:", + "sync_limitation": "The data extracted before integrating with Google Sheets will not be synced in the Google Sheet. Only the data extracted after the integration will be synced." + } + }, + "buttons": { + "authenticate": "Authenticate with Google", + "fetch_sheets": "Fetch Google Spreadsheets", + "remove_integration": "Remove Integration", + "submit": "Submit" + }, + "fields": { + "select_sheet": "Select Google Sheet", + "selected_sheet": "Selected Sheet: {{name}} (ID: {{id}})" + }, + "errors": { + "auth_error": "Error authenticating with Google", + "fetch_error": "Error fetching spreadsheet files: {{message}}", + "update_error": "Error updating Google Sheet ID: {{message}}", + "remove_error": "Error removing Google Sheets integration: {{message}}" + }, + "notifications": { + "sheet_selected": "Google Sheet selected successfully", + "integration_removed": "Google Sheets integration removed successfully" } }, - "buttons": { - "authenticate": "Authenticate with Google", - "fetch_sheets": "Fetch Google Spreadsheets", - "remove_integration": "Remove Integration", - "submit": "Submit" - }, - "fields": { - "select_sheet": "Select Google Sheet", - "selected_sheet": "Selected Sheet: {{name}} (ID: {{id}})" - }, - "errors": { - "auth_error": "Error authenticating with Google", - "fetch_error": "Error fetching spreadsheet files: {{message}}", - "update_error": "Error updating Google Sheet ID: {{message}}", - "remove_error": "Error removing Google Sheets integration: {{message}}" - }, - "notifications": { - "sheet_selected": "Google Sheet selected successfully" + "airtable": { + "title": "Integrate with Airtable", + "descriptions": { + "sync_info": "If you enable this option, every time this robot runs a task successfully, its captured data will be appended to your Airtable Base.", + "authenticated_as": "Authenticated with Airtable successfully. You can now select the base and table to integrate with." + }, + "alerts": { + "success": { + "title": "Airtable Base Integrated Successfully", + "content": "Every time this robot creates a successful run, its captured data is appended to your {{baseName}} Base and {{tableName}} Table. You can check updates", + "here": "here", + "note": "Note:", + "sync_limitation": "Only data captured after integration will be synced to Airtable." + } + }, + "buttons": { + "authenticate": "Connect with Airtable", + "fetch_bases": "Fetch Airtable Bases", + "fetch_tables": "Fetch Airtable Tables", + "remove_integration": "Remove Integration", + "submit": "Select Base and Table" + }, + "fields": { + "select_base": "Select Airtable Base", + "select_table": "Select Airtable Table", + "selected_base": "Selected Base: {{name}}", + "selected_table": "Selected Table: {{name}}" + }, + "errors": { + "auth_error": "Error authenticating with Airtable", + "fetch_error": "Error fetching Airtable bases: {{message}}", + "fetch_tables_error": "Error fetching Airtable tables: {{message}}", + "update_error": "Error updating Airtable base: {{message}}", + "remove_error": "Error removing Airtable integration: {{message}}" + }, + "notifications": { + "base_selected": "Airtable base selected successfully", + "table_selected": "Airtable table selected successfully", + "integration_removed": "Airtable integration removed successfully" + } } }, "robot_duplication": { @@ -491,6 +520,7 @@ "success": "Success", "running": "Running", "scheduled": "Scheduled", + "queued": "Queued", "failed": "Failed" }, "run_settings_modal": { diff --git a/public/locales/es.json b/public/locales/es.json index 895ab9bd..14db7b5d 100644 --- a/public/locales/es.json +++ b/public/locales/es.json @@ -140,41 +140,12 @@ "no_key_message": "Aún no has generado una clave API.", "generate_button": "Generar Clave API", "notifications": { - "errors": { - "fetch": { - "network": "Error de red al obtener la clave API: ${error}", - "unauthorized": "Debes iniciar sesión para acceder a la clave API", - "not_found": "No se pudo encontrar la clave API para tu cuenta", - "server": "Error del servidor al obtener la clave API. Por favor, inténtalo más tarde", - "unknown": "Error desconocido al obtener la clave API: ${error}" - }, - "generate": { - "network": "Error de red al generar la clave API: ${error}", - "unauthorized": "Debes iniciar sesión para generar una clave API", - "key_exists": "Ya tienes una clave API. Por favor, elimina la existente primero", - "not_found": "Cuenta de usuario no encontrada", - "server": "Error del servidor al generar la clave API. Por favor, inténtalo más tarde", - "unknown": "Error desconocido al generar la clave API: ${error}" - }, - "delete": { - "network": "Error de red al eliminar la clave API: ${error}", - "unauthorized": "Debes iniciar sesión para eliminar la clave API", - "not_found": "Cuenta de usuario no encontrada", - "key_not_found": "No se encontró ninguna clave API para eliminar", - "server": "Error del servidor al eliminar la clave API. Por favor, inténtalo más tarde", - "unknown": "Error desconocido al eliminar la clave API: ${error}" - }, - "copy": { - "failed": "Error al copiar la clave API al portapapeles", - "no_key": "No hay clave API disponible para copiar" - } - }, - "success": { - "fetch": "Clave API obtenida exitosamente", - "generate": "Nueva clave API generada exitosamente", - "delete": "Clave API eliminada exitosamente", - "copy": "Clave API copiada al portapapeles" - } + "fetch_error": "Error al obtener la clave API - ${error}", + "generate_success": "Clave API generada con éxito", + "generate_error": "Error al generar la clave API - ${error}", + "delete_success": "Clave API eliminada con éxito", + "delete_error": "Error al eliminar la clave API - ${error}", + "copy_success": "Clave API copiada con éxito" } }, "action_description": { @@ -335,29 +306,90 @@ } }, "integration_settings": { - "title": "Integrar con Google Sheet", + "title": "Ajustes de Integración", "descriptions": { - "sync_info": "Si habilitas esta opción, cada vez que este robot ejecute una tarea exitosamente, sus datos capturados se añadirán a tu Google Sheet.", "authenticated_as": "Autenticado como: {{email}}" }, - "alerts": { - "success": { - "title": "Google Sheet integrado exitosamente.", - "content": "Cada vez que este robot crea una ejecución exitosa, sus datos capturados se añaden a tu Google Sheet {{sheetName}}. Puedes verificar las actualizaciones de datos", - "here": "aquí", - "note": "Nota:", - "sync_limitation": "Los datos extraídos antes de la integración con Google Sheets no se sincronizarán en el Google Sheet. Solo se sincronizarán los datos extraídos después de la integración." + "buttons": { + "submit": "Enviar", + "remove_integration": "Eliminar Integración" + }, + "google": { + "title": "Integrar con Google Sheet", + "descriptions": { + "sync_info": "Si habilita esta opción, cada vez que este robot ejecute una tarea correctamente, sus datos capturados se añadirán a su Google Sheet.", + "authenticated_as": "Autenticado como: {{email}}" + }, + "alerts": { + "success": { + "title": "Google Sheet Integrado Correctamente.", + "content": "Cada vez que este robot crea una ejecución exitosa, sus datos capturados se añaden a su Google Sheet {{sheetName}}. Puede comprobar las actualizaciones de datos", + "here": "aquí", + "note": "Nota:", + "sync_limitation": "Los datos extraídos antes de la integración con Google Sheets no se sincronizarán en la Google Sheet. Solo los datos extraídos después de la integración se sincronizarán." + } + }, + "buttons": { + "authenticate": "Autenticar con Google", + "fetch_sheets": "Obtener Hojas de Cálculo de Google", + "remove_integration": "Eliminar Integración", + "submit": "Enviar" + }, + "fields": { + "select_sheet": "Seleccionar Google Sheet", + "selected_sheet": "Hoja Seleccionada: {{name}} (ID: {{id}})" + }, + "errors": { + "auth_error": "Error al autenticar con Google", + "fetch_error": "Error al obtener archivos de hojas de cálculo: {{message}}", + "update_error": "Error al actualizar ID de Google Sheet: {{message}}", + "remove_error": "Error al eliminar la integración de Google Sheets: {{message}}" + }, + "notifications": { + "sheet_selected": "Google Sheet seleccionado correctamente", + "integration_removed": "Integración de Google Sheets eliminada correctamente" } }, - "buttons": { - "authenticate": "Autenticar con Google", - "fetch_sheets": "Obtener Google Sheets", - "remove_integration": "Eliminar integración", - "submit": "Enviar" - }, - "fields": { - "select_sheet": "Seleccionar Google Sheet", - "selected_sheet": "Hoja seleccionada: {{name}} (ID: {{id}})" + "airtable": { + "title": "Integrar con Airtable", + "descriptions": { + "sync_info": "Si habilita esta opción, cada vez que este robot ejecute una tarea correctamente, sus datos capturados se añadirán a su Airtable.", + "authenticated_as": "Autenticado con Airtable exitosamente. Ahora puede seleccionar la base y la mesa con las que desea integrar." + }, + "alerts": { + "success": { + "title": "Base Airtable integrada con éxito", + "content": "Cada vez que este robot crea una ejecución exitosa, los datos capturados se agregan a su base {{baseName}} y a su tabla {{tableName}}. Puedes consultar actualizaciones", + "here": "aquí", + "note": "Nota:", + "sync_limitation": "Solo los datos capturados después de la integración se sincronizarán con Airtable." + } + }, + "buttons": { + "authenticate": "Conectar con Airtable", + "fetch_bases": "Obtener Bases de Airtable", + "fetch_tables": "Obtener Tablas de Airtable", + "remove_integration": "Eliminar Integración", + "submit": "Seleccionar Base y Tabla" + }, + "fields": { + "select_base": "Seleccionar Base de Airtable", + "select_table": "Seleccionar Tabla de Airtable", + "selected_base": "Base Seleccionada: {{name}}", + "selected_table": "Tabla Seleccionada: {{name}}" + }, + "errors": { + "auth_error": "Error al autenticar con Airtable", + "fetch_error": "Error al obtener bases de Airtable: {{message}}", + "fetch_tables_error": "Error al obtener tablas de Airtable: {{message}}", + "update_error": "Error al actualizar base de Airtable: {{message}}", + "remove_error": "Error al eliminar la integración de Airtable: {{message}}" + }, + "notifications": { + "base_selected": "Base de Airtable seleccionada correctamente", + "table_selected": "Tabla de Airtable seleccionada correctamente", + "integration_removed": "Integración de Airtable eliminada correctamente" + } } }, "robot_duplication": { @@ -481,6 +513,7 @@ "success": "Éxito", "running": "Ejecutando", "scheduled": "Programado", + "queued": "En cola", "failed": "Fallido" }, "run_settings_modal": { diff --git a/public/locales/ja.json b/public/locales/ja.json index ee18c96c..54f7e788 100644 --- a/public/locales/ja.json +++ b/public/locales/ja.json @@ -140,41 +140,12 @@ "no_key_message": "APIキーはまだ生成されていません。", "generate_button": "APIキーを生成", "notifications": { - "errors": { - "fetch": { - "network": "APIキーの取得中にネットワークエラーが発生しました：${error}", - "unauthorized": "APIキーにアクセスするにはログインが必要です", - "not_found": "アカウントのAPIキーが見つかりません", - "server": "APIキーの取得中にサーバーエラーが発生しました。後でもう一度お試しください", - "unknown": "APIキーの取得中に不明なエラーが発生しました：${error}" - }, - "generate": { - "network": "APIキーの生成中にネットワークエラーが発生しました：${error}", - "unauthorized": "APIキーを生成するにはログインが必要です", - "key_exists": "APIキーが既に存在します。既存のキーを先に削除してください", - "not_found": "ユーザーアカウントが見つかりません", - "server": "APIキーの生成中にサーバーエラーが発生しました。後でもう一度お試しください", - "unknown": "APIキーの生成中に不明なエラーが発生しました：${error}" - }, - "delete": { - "network": "APIキーの削除中にネットワークエラーが発生しました：${error}", - "unauthorized": "APIキーを削除するにはログインが必要です", - "not_found": "ユーザーアカウントが見つかりません", - "key_not_found": "削除するAPIキーが見つかりません", - "server": "APIキーの削除中にサーバーエラーが発生しました。後でもう一度お試しください", - "unknown": "APIキーの削除中に不明なエラーが発生しました：${error}" - }, - "copy": { - "failed": "APIキーのクリップボードへのコピーに失敗しました", - "no_key": "コピーできるAPIキーがありません" - } - }, - "success": { - "fetch": "APIキーの取得に成功しました", - "generate": "新しいAPIキーの生成に成功しました", - "delete": "APIキーの削除に成功しました", - "copy": "APIキーをクリップボードにコピーしました" - } + "fetch_error": "APIキーの取得に失敗しました - ${error}", + "generate_success": "APIキーの生成に成功しました", + "generate_error": "APIキーの生成に失敗しました - ${error}", + "delete_success": "APIキーの削除に成功しました", + "delete_error": "APIキーの削除に失敗しました - ${error}", + "copy_success": "APIキーのコピーに成功しました" } }, "action_description": { @@ -335,29 +306,90 @@ } }, "integration_settings": { - "title": "Google Sheetと連携", + "title": "連携設定", "descriptions": { - "sync_info": "このオプションを有効にすると、このロボットがタスクを正常に実行するたびに、取得したデータがGoogle Sheetに追加されます。", - "authenticated_as": "認証済みユーザー: {{email}}" - }, - "alerts": { - "success": { - "title": "Google Sheetの連携が完了しました。", - "content": "このロボットが正常に実行を完了するたびに、取得したデータはGoogle Sheet {{sheetName}}に追加されます。データの更新は", - "here": "こちら", - "note": "注意：", - "sync_limitation": "Google Sheetsとの連携前に抽出されたデータは同期されません。連携後に抽出されたデータのみが同期されます。" - } + "authenticated_as": "認証済み: {{email}}" }, "buttons": { - "authenticate": "Googleで認証", - "fetch_sheets": "Google Sheetsを取得", - "remove_integration": "連携を解除", - "submit": "送信" + "submit": "送信", + "remove_integration": "連携を解除" }, - "fields": { - "select_sheet": "Google Sheetを選択", - "selected_sheet": "選択したシート: {{name}} (ID: {{id}})" + "google": { + "title": "Google シートと連携", + "descriptions": { + "sync_info": "このオプションを有効にすると、このロボットがタスクを正常に実行するたびに、取得したデータがGoogle シートに追加されます。", + "authenticated_as": "認証済み: {{email}}" + }, + "alerts": { + "success": { + "title": "Google シートの連携に成功しました。", + "content": "このロボットが正常に実行されるたびに、取得したデータが{{sheetName}}という名前のGoogle シートに追加されます。データの更新を確認できます", + "here": "こちら", + "note": "注意:", + "sync_limitation": "Google シートとの連携前に抽出されたデータはGoogle シートに同期されません。連携後に抽出されたデータのみが同期されます。" + } + }, + "buttons": { + "authenticate": "Googleで認証", + "fetch_sheets": "Google スプレッドシートを取得", + "remove_integration": "連携を解除", + "submit": "送信" + }, + "fields": { + "select_sheet": "Google シートを選択", + "selected_sheet": "選択したシート: {{name}} (ID: {{id}})" + }, + "errors": { + "auth_error": "Googleでの認証エラー", + "fetch_error": "スプレッドシートファイルの取得エラー: {{message}}", + "update_error": "Google シートIDの更新エラー: {{message}}", + "remove_error": "Google シート連携の解除エラー: {{message}}" + }, + "notifications": { + "sheet_selected": "Google シートが正常に選択されました", + "integration_removed": "Google シートの連携が正常に解除されました" + } + }, + "airtable": { + "title": "Airtableと連携", + "descriptions": { + "sync_info": "このオプションを有効にすると、このロボットがタスクを正常に実行するたびに、取得したデータがAirtableに追加されます。", + "authenticated_as": "Airtableで認証に成功しました。統合するベースとテーブルを選択できるようになりました。" + }, + "alerts": { + "success": { + "title": "Airtable ベースの統合に成功", + "content": "このロボットが正常な実行を作成するたびに、キャプチャされたデータが {{baseName}} ベースと {{tableName}} テーブルに追加されます。更新情報を確認できます", + "here": "こちら", + "note": "注意:", + "sync_limitation": "連携後に取得されたデータのみがAirtableに同期されます。" + } + }, + "buttons": { + "authenticate": "Airtableと接続", + "fetch_bases": "Airtableベースを取得", + "fetch_tables": "Airtableテーブルを取得", + "remove_integration": "連携を解除", + "submit": "ベースとテーブルを選択" + }, + "fields": { + "select_base": "Airtableベースを選択", + "select_table": "Airtableテーブルを選択", + "selected_base": "選択したベース: {{name}}", + "selected_table": "選択したテーブル: {{name}}" + }, + "errors": { + "auth_error": "Airtableでの認証エラー", + "fetch_error": "Airtableベースの取得エラー: {{message}}", + "fetch_tables_error": "Airtableテーブルの取得エラー: {{message}}", + "update_error": "Airtableベースの更新エラー: {{message}}", + "remove_error": "Airtable連携の解除エラー: {{message}}" + }, + "notifications": { + "base_selected": "Airtableベースが正常に選択されました", + "table_selected": "Airtableテーブルが正常に選択されました", + "integration_removed": "Airtable連携が正常に解除されました" + } } }, "robot_duplication": { @@ -481,6 +513,7 @@ "success": "成功", "running": "実行中", "scheduled": "スケジュール済み", + "queued": "キューに入れました", "failed": "失敗" }, "run_settings_modal": { diff --git a/public/locales/zh.json b/public/locales/zh.json index 5cec8268..cc23556e 100644 --- a/public/locales/zh.json +++ b/public/locales/zh.json @@ -140,41 +140,12 @@ "no_key_message": "您还未生成API密钥。", "generate_button": "生成API密钥", "notifications": { - "errors": { - "fetch": { - "network": "获取API密钥时发生网络错误：${error}", - "unauthorized": "您必须登录才能访问API密钥", - "not_found": "找不到您账户的API密钥", - "server": "获取API密钥时发生服务器错误。请稍后重试", - "unknown": "获取API密钥时发生未知错误：${error}" - }, - "generate": { - "network": "生成API密钥时发生网络错误：${error}", - "unauthorized": "您必须登录才能生成API密钥", - "key_exists": "您已经有一个API密钥。请先删除现有的密钥", - "not_found": "找不到用户账户", - "server": "生成API密钥时发生服务器错误。请稍后重试", - "unknown": "生成API密钥时发生未知错误：${error}" - }, - "delete": { - "network": "删除API密钥时发生网络错误：${error}", - "unauthorized": "您必须登录才能删除API密钥", - "not_found": "找不到用户账户", - "key_not_found": "找不到要删除的API密钥", - "server": "删除API密钥时发生服务器错误。请稍后重试", - "unknown": "删除API密钥时发生未知错误：${error}" - }, - "copy": { - "failed": "复制API密钥到剪贴板失败", - "no_key": "没有可复制的API密钥" - } - }, - "success": { - "fetch": "成功获取API密钥", - "generate": "成功生成新的API密钥", - "delete": "成功删除API密钥", - "copy": "已将API密钥复制到剪贴板" - } + "fetch_error": "获取API密钥失败 - ${error}", + "generate_success": "成功生成API密钥", + "generate_error": "生成API密钥失败 - ${error}", + "delete_success": "成功删除API密钥", + "delete_error": "删除API密钥失败 - ${error}", + "copy_success": "成功复制API密钥" } }, "action_description": { @@ -335,29 +306,90 @@ } }, "integration_settings": { - "title": "与Google Sheet集成", + "title": "集成设置", "descriptions": { - "sync_info": "如果启用此选项，每次机器人成功运行任务时，捕获的数据都会追加到您的Google Sheet中。", - "authenticated_as": "已验证身份: {{email}}" - }, - "alerts": { - "success": { - "title": "Google Sheet集成成功。", - "content": "每次此机器人创建成功运行时，捕获的数据都会追加到您的Google Sheet {{sheetName}}中。您可以查看数据更新", - "here": "在此处", - "note": "注意：", - "sync_limitation": "与Google Sheets集成之前提取的数据将不会同步到Google Sheet中。只有集成后提取的数据才会同步。" - } + "authenticated_as": "已认证为：{{email}}" }, "buttons": { - "authenticate": "使用Google验证", - "fetch_sheets": "获取Google Sheets", - "remove_integration": "移除集成", - "submit": "提交" + "submit": "提交", + "remove_integration": "移除集成" }, - "fields": { - "select_sheet": "选择Google Sheet", - "selected_sheet": "已选择表格: {{name}} (ID: {{id}})" + "google": { + "title": "与Google表格集成", + "descriptions": { + "sync_info": "如果启用此选项，每次机器人成功运行任务时，其捕获的数据将附加到您的Google表格中。", + "authenticated_as": "已认证为：{{email}}" + }, + "alerts": { + "success": { + "title": "Google表格集成成功。", + "content": "每次此机器人创建成功运行时，其捕获的数据将附加到您的{{sheetName}} Google表格中。您可以检查数据更新", + "here": "在这里", + "note": "注意：", + "sync_limitation": "在与Google表格集成之前提取的数据将不会在Google表格中同步。只有集成后提取的数据才会同步。" + } + }, + "buttons": { + "authenticate": "使用Google认证", + "fetch_sheets": "获取Google电子表格", + "remove_integration": "移除集成", + "submit": "提交" + }, + "fields": { + "select_sheet": "选择Google表格", + "selected_sheet": "已选择表格：{{name}}（ID：{{id}}）" + }, + "errors": { + "auth_error": "使用Google认证时出错", + "fetch_error": "获取电子表格文件时出错：{{message}}", + "update_error": "更新Google表格ID时出错：{{message}}", + "remove_error": "移除Google表格集成时出错：{{message}}" + }, + "notifications": { + "sheet_selected": "Google表格选择成功", + "integration_removed": "Google表格集成已成功移除" + } + }, + "airtable": { + "title": "与Airtable集成", + "descriptions": { + "sync_info": "如果启用此选项，每次机器人成功运行任务时，其捕获的数据将附加到您的Airtable中。", + "authenticated_as": "已成功通过 Airtable 进行身份验证。您现在可以选择要集成的底座和桌子。" + }, + "alerts": { + "success": { + "title": "Airtable 基地成功集成", + "content": "每次此机器人成功运行时，其捕获的数据都会附加到您的 {{baseName}} 基础和 {{tableName}} 表中。您可以检查更新情况", + "here": "在这里", + "note": "注意：", + "sync_limitation": "只有集成后捕获的数据才会同步到Airtable。" + } + }, + "buttons": { + "authenticate": "连接Airtable", + "fetch_bases": "获取Airtable基础", + "fetch_tables": "获取Airtable表格", + "remove_integration": "移除集成", + "submit": "选择基础和表格" + }, + "fields": { + "select_base": "选择Airtable基础", + "select_table": "选择Airtable表格", + "selected_base": "已选择基础：{{name}}", + "selected_table": "已选择表格：{{name}}" + }, + "errors": { + "auth_error": "使用Airtable认证时出错", + "fetch_error": "获取Airtable基础时出错：{{message}}", + "fetch_tables_error": "获取Airtable表格时出错：{{message}}", + "update_error": "更新Airtable基础时出错：{{message}}", + "remove_error": "移除Airtable集成时出错：{{message}}" + }, + "notifications": { + "base_selected": "Airtable基础选择成功", + "table_selected": "Airtable表格选择成功", + "integration_removed": "Airtable集成已成功移除" + } } }, "robot_duplication": { @@ -481,6 +513,7 @@ "success": "成功", "running": "运行中", "scheduled": "已计划", + "queued": "排队", "failed": "失败" }, "run_settings_modal": { diff --git a/public/svg/airtable.svg b/public/svg/airtable.svg new file mode 100644 index 00000000..e3883c37 --- /dev/null +++ b/public/svg/airtable.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/svg/gsheet.svg b/public/svg/gsheet.svg new file mode 100644 index 00000000..84d26be4 --- /dev/null +++ b/public/svg/gsheet.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/server/config/config.json b/server/config/config.json new file mode 100644 index 00000000..395914f9 --- /dev/null +++ b/server/config/config.json @@ -0,0 +1,26 @@ +{ + "development": { + "username": "postgres", + "password": "postgres", + "database": "maxun", + "host": "localhost", + "port": 5432, + "dialect": "postgres" + }, + "test": { + "username": "postgres", + "password": "postgres", + "database": "maxun_test", + "host": "localhost", + "port": 5432, + "dialect": "postgres" + }, + "production": { + "username": "postgres", + "password": "postgres", + "database": "maxun_prod", + "host": "localhost", + "port": 5432, + "dialect": "postgres" + } + } \ No newline at end of file diff --git a/server/src/api/record.ts b/server/src/api/record.ts index 05560487..b4014c3a 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -507,9 +507,9 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) { } } -async function readyForRunHandler(browserId: string, id: string) { +async function readyForRunHandler(browserId: string, id: string, userId: string){ try { - const result = await executeRun(id); + const result = await executeRun(id, userId); if (result && result.success) { logger.log('info', `Interpretation of ${id} succeeded`); @@ -517,14 +517,14 @@ async function readyForRunHandler(browserId: string, id: string) { return result.interpretationInfo; } else { logger.log('error', `Interpretation of ${id} failed`); - await destroyRemoteBrowser(browserId); + await destroyRemoteBrowser(browserId, userId); resetRecordingState(browserId, id); return null; } } catch (error: any) { logger.error(`Error during readyForRunHandler: ${error.message}`); - await destroyRemoteBrowser(browserId); + await destroyRemoteBrowser(browserId, userId); return null; } } @@ -546,7 +546,7 @@ function AddGeneratedFlags(workflow: WorkflowFile) { return copy; }; -async function executeRun(id: string) { +async function executeRun(id: string, userId: string) { try { const run = await Run.findOne({ where: { runId: id } }); if (!run) { @@ -568,7 +568,7 @@ async function executeRun(id: string) { plainRun.status = 'running'; - const browser = browserPool.getRemoteBrowser(plainRun.browserId); + const browser = browserPool.getRemoteBrowser(userId); if (!browser) { throw new Error('Could not access browser'); } @@ -586,7 +586,7 @@ async function executeRun(id: string) { const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(plainRun.browserId); + await destroyRemoteBrowser(plainRun.browserId, userId); const updatedRun = await run.update({ ...run, @@ -672,12 +672,12 @@ export async function handleRunRecording(id: string, userId: string) { rejectUnauthorized: false }); - socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId)); + socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId, userId)); logger.log('info', `Running Robot: ${id}`); socket.on('disconnect', () => { - cleanupSocketListeners(socket, browserId, newRunId); + cleanupSocketListeners(socket, browserId, newRunId, userId); }); // Return the runId immediately, so the client knows the run is started @@ -688,8 +688,8 @@ export async function handleRunRecording(id: string, userId: string) { } } -function cleanupSocketListeners(socket: Socket, browserId: string, id: string) { - socket.off('ready-for-run', () => readyForRunHandler(browserId, id)); +function cleanupSocketListeners(socket: Socket, browserId: string, id: string, userId: string) { + socket.off('ready-for-run', () => readyForRunHandler(browserId, id, userId)); logger.log('info', `Cleaned up listeners for browserId: ${browserId}, runId: ${id}`); } diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index cd4962a1..c1f0f557 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -4,6 +4,12 @@ import logger from "../../logger"; /** * @category Types */ +/** + * Represents the possible states of a remote browser. + * @category Types + */ +type BrowserState = "recording" | "run"; + interface BrowserPoolInfo { /** * The instance of remote browser. @@ -15,6 +21,16 @@ interface BrowserPoolInfo { * @default false */ active: boolean, + /** + * The user ID that owns this browser instance. + */ + userId: string, + /** + * The current state of the browser. + * Can be "recording" or "run". + * @default "recording" + */ + state: BrowserState, } /** @@ -29,36 +45,130 @@ interface PoolDictionary { /** * A browser pool is a collection of remote browsers that are initialized and ready to be used. + * Enforces a "1 User - 2 Browser" policy, while allowing multiple users to have their own browser instances. * Adds the possibility to add, remove and retrieve remote browsers from the pool. - * It is possible to manage multiple browsers for creating or running a recording. * @category BrowserManagement */ export class BrowserPool { - /** * Holds all the instances of remote browsers. */ private pool: PoolDictionary = {}; /** - * Adds a remote browser instance to the pool indexed by the id. + * Maps user IDs to their browser IDs. + * A user can have up to 2 browsers. + */ + private userToBrowserMap: Map = new Map(); + + /** + * Adds a remote browser instance to the pool for a specific user. + * If the user already has two browsers, the oldest browser will be closed and replaced. + * * @param id remote browser instance's id * @param browser remote browser instance + * @param userId the user ID that owns this browser instance * @param active states if the browser's instance is being actively used + * @returns true if a new browser was added, false if an existing browser was replaced */ - public addRemoteBrowser = (id: string, browser: RemoteBrowser, active: boolean = false): void => { - this.pool = { - ...this.pool, - [id]: { + public addRemoteBrowser = ( + id: string, + browser: RemoteBrowser, + userId: string, + active: boolean = false, + state: BrowserState = "recording" + ): boolean => { + // Check if browser with this ID already exists and belongs to this user + if (this.pool[id] && this.pool[id].userId === userId) { + // Just update the existing browser + this.pool[id] = { browser, active, - }, + userId, + state: this.pool[id].state || state, + }; + logger.log('debug', `Updated existing browser with id: ${id} for user: ${userId}`); + return false; } - logger.log('debug', `Remote browser with id: ${id} added to the pool`); + + // Get existing browsers for this user + let userBrowserIds = this.userToBrowserMap.get(userId) || []; + let replaced = false; + + // If trying to add a "recording" browser, check if one already exists + if (state === "recording") { + // Check if user already has a recording browser + const hasRecordingBrowser = userBrowserIds.some(browserId => + this.pool[browserId] && this.pool[browserId].state === "recording" + ); + + if (hasRecordingBrowser) { + logger.log('debug', `User ${userId} already has a browser in "recording" state`); + return false; + } + } + + // For "run" state, check if the user already has the maximum number of browsers (2) + if (userBrowserIds.length >= 2 && !userBrowserIds.includes(id)) { + logger.log('debug', "User already has the maximum number of browsers (2)"); + return false; + } + + // Add the new browser to the pool + this.pool[id] = { + browser, + active, + userId, + state, + }; + + // Update the user-to-browser mapping + if (!userBrowserIds.includes(id)) { + userBrowserIds.push(id); + } + this.userToBrowserMap.set(userId, userBrowserIds); + + logger.log('debug', `Remote browser with id: ${id} added to the pool for user: ${userId}`); + return !replaced; }; /** * Removes the remote browser instance from the pool. + * Note: This doesn't handle browser closing as RemoteBrowser doesn't expose a close method. + * The caller should ensure the browser is properly closed before calling this method. + * + * @param id remote browser instance's id + * @returns true if the browser was removed successfully, false otherwise + */ + public closeAndDeleteBrowser = (id: string): boolean => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return false; + } + + // Remove the user-to-browser mapping + const userId = this.pool[id].userId; + const userBrowserIds = this.userToBrowserMap.get(userId) || []; + + if (userBrowserIds.includes(id)) { + const updatedBrowserIds = userBrowserIds.filter(bid => bid !== id); + + if (updatedBrowserIds.length === 0) { + this.userToBrowserMap.delete(userId); + } else { + this.userToBrowserMap.set(userId, updatedBrowserIds); + } + } + + // Remove from pool + delete this.pool[id]; + logger.log('debug', `Remote browser with id: ${id} removed from the pool`); + return true; + }; + + /** + * Removes the remote browser instance from the pool without attempting to close it. + * * @param id remote browser instance's id * @returns true if the browser was removed successfully, false otherwise */ @@ -67,13 +177,30 @@ export class BrowserPool { logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); return false; } - delete (this.pool[id]); + + // Remove the user-to-browser mapping + const userId = this.pool[id].userId; + const userBrowserIds = this.userToBrowserMap.get(userId) || []; + + if (userBrowserIds.includes(id)) { + const updatedBrowserIds = userBrowserIds.filter(bid => bid !== id); + + if (updatedBrowserIds.length === 0) { + this.userToBrowserMap.delete(userId); + } else { + this.userToBrowserMap.set(userId, updatedBrowserIds); + } + } + + // Remove from pool + delete this.pool[id]; logger.log('debug', `Remote browser with id: ${id} deleted from the pool`); return true; }; /** * Returns the remote browser instance from the pool. + * * @param id remote browser instance's id * @returns remote browser instance or undefined if it does not exist in the pool */ @@ -83,18 +210,318 @@ export class BrowserPool { }; /** - * Returns the active browser's instance id from the pool. - * If there is no active browser, it returns undefined. - * If there are multiple active browsers, it returns the first one. - * @returns the first remote active browser instance's id from the pool + * Returns the active browser's instance id for a specific user. + * If state is specified, only returns a browser with that exact state. + * + * @param userId the user ID to find the browser for + * @param state optional browser state filter ("recording" or "run") + * @returns the browser ID for the user, or null if no browser exists with the required state */ - public getActiveBrowserId = (): string | null => { + public getActiveBrowserId = (userId: string, state?: BrowserState): string | null => { + const browserIds = this.userToBrowserMap.get(userId); + if (!browserIds || browserIds.length === 0) { + logger.log('debug', `No browser found for user: ${userId}`); + return null; + } + + // If state is specified, only return browsers with that exact state + if (state) { + // Check browsers in reverse order (newest first) to find one with the specified state + for (let i = browserIds.length - 1; i >= 0; i--) { + const browserId = browserIds[i]; + + // Verify the browser still exists in the pool + if (!this.pool[browserId]) { + browserIds.splice(i, 1); + continue; + } + + // Check if browser matches state filter + if (this.pool[browserId].state === state) { + return browserId; + } + } + + // If no browser with matching state, return null + logger.log('debug', `No browser with state ${state} found for user: ${userId}`); + return null; + } + + // If no state specified, return any browser + for (let i = browserIds.length - 1; i >= 0; i--) { + const browserId = browserIds[i]; + + // Verify the browser still exists in the pool + if (!this.pool[browserId]) { + browserIds.splice(i, 1); + continue; + } + + // Return the first browser found + if (this.pool[browserId]) { + console.log(`Active browser Id ${browserId} found for user: ${userId}`); + return browserId; + } + } + + // If no active browser, return the most recent one + if (browserIds.length > 0) { + const mostRecentId = browserIds[browserIds.length - 1]; + console.log(`No active browser found, returning most recent browser Id ${mostRecentId} for user: ${userId}`); + return mostRecentId; + } + + // Clean up the mapping if all browsers were invalid + if (browserIds.length === 0) { + this.userToBrowserMap.delete(userId); + } + + logger.log('warn', `Browser mapping found for user: ${userId}, but no valid browsers exist in pool`); + return null; + }; + + /** + * Returns the user ID associated with a browser ID. + * + * @param browserId the browser ID to find the user for + * @returns the user ID for the browser, or null if the browser doesn't exist + */ + public getUserForBrowser = (browserId: string): string | null => { + if (!this.pool[browserId]) { + return null; + } + return this.pool[browserId].userId; + }; + + /** + * Sets the active state of a browser. + * + * @param id the browser ID + * @param active the new active state + * @returns true if successful, false if the browser wasn't found + */ + public setActiveBrowser = (id: string, active: boolean): boolean => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return false; + } + + this.pool[id].active = active; + logger.log('debug', `Remote browser with id: ${id} set to ${active ? 'active' : 'inactive'}`); + return true; + }; + + /** + * Sets the state of a browser. + * Only allows one browser in "recording" state per user. + * + * @param id the browser ID + * @param state the new state ("recording" or "run") + * @returns true if successful, false if the browser wasn't found or state change not allowed + */ + public setBrowserState = (id: string, state: BrowserState): boolean => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return false; + } + + // If trying to set to "recording" state, check if another browser is already recording + if (state === "recording") { + const userId = this.pool[id].userId; + const userBrowserIds = this.userToBrowserMap.get(userId) || []; + + // Check if any other browser for this user is already in recording state + const hasAnotherRecordingBrowser = userBrowserIds.some(browserId => + browserId !== id && + this.pool[browserId] && + this.pool[browserId].state === "recording" + ); + + if (hasAnotherRecordingBrowser) { + logger.log('warn', `Cannot set browser ${id} to "recording" state: User ${userId} already has a browser in recording state`); + return false; + } + } + + this.pool[id].state = state; + logger.log('debug', `Remote browser with id: ${id} state set to ${state}`); + return true; + }; + + /** + * Gets the current state of a browser. + * + * @param id the browser ID + * @returns the current state or null if the browser wasn't found + */ + public getBrowserState = (id: string): BrowserState | null => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return null; + } + + return this.pool[id].state; + }; + + /** + * Returns all browser instances for a specific user. + * With the "1 User - 2 Browser" policy, this can return up to 2 browsers. + * + * @param userId the user ID to find browsers for + * @returns an array of browser IDs belonging to the user + */ + public getAllBrowserIdsForUser = (userId: string): string[] => { + const browserIds: string[] = []; + + // Get browser IDs from the map + const mappedBrowserIds = this.userToBrowserMap.get(userId) || []; + + // Filter to only include IDs that exist in the pool + for (const id of mappedBrowserIds) { + if (this.pool[id]) { + browserIds.push(id); + } + } + + // As a safeguard, also check the entire pool for any browsers assigned to this user + // This helps detect and fix any inconsistencies in the maps + for (const [id, info] of Object.entries(this.pool)) { + if (info.userId === userId && !browserIds.includes(id)) { + browserIds.push(id); + } + } + + // Update the map if inconsistencies were found + if (browserIds.length > 0 && JSON.stringify(browserIds) !== JSON.stringify(mappedBrowserIds)) { + // Limit to 2 browsers if more were found + const limitedBrowserIds = browserIds.slice(-2); + this.userToBrowserMap.set(userId, limitedBrowserIds); + } + + return browserIds; + }; + + /** + * Returns the total number of browsers in the pool. + */ + public getPoolSize = (): number => { + return Object.keys(this.pool).length; + }; + + /** + * Returns the total number of active users (users with browsers). + */ + public getActiveUserCount = (): number => { + return this.userToBrowserMap.size; + }; + + /** + * Gets the current active browser for the system if there's only one active user. + * This is a migration helper to support code that hasn't been updated to the user-browser model yet. + * + * @param currentUserId The ID of the current user, which will be prioritized if multiple browsers exist + * @param state Optional state filter to find browsers in a specific state + * @returns A browser ID if one can be determined, or null + */ + public getActiveBrowserForMigration = (currentUserId?: string, state?: BrowserState): string | null => { + // If a current user ID is provided and they have a browser, return that + if (currentUserId) { + const browserForUser = this.getActiveBrowserId(currentUserId, state); + if (browserForUser) { + return browserForUser; + } + + // If state is specified and no matching browser was found, return null + if (state) { + return null; + } + } + + // If only one user has a browser, try to find a matching browser + if (this.userToBrowserMap.size === 1) { + const userId = Array.from(this.userToBrowserMap.keys())[0]; + const browserIds = this.userToBrowserMap.get(userId) || []; + + // If state is specified, only look for that state + if (state) { + // Return the active browser that matches the state + for (let i = browserIds.length - 1; i >= 0; i--) { + const bid = browserIds[i]; + if (this.pool[bid]?.active && this.pool[bid].state === state) { + return bid; + } + } + + // If no active browser with matching state, try to find any browser with matching state + for (let i = browserIds.length - 1; i >= 0; i--) { + const bid = browserIds[i]; + if (this.pool[bid] && this.pool[bid].state === state) { + return bid; + } + } + + // If still no matching browser, return null + return null; + } + + // If no state filter, find any active browser + for (let i = browserIds.length - 1; i >= 0; i--) { + if (this.pool[browserIds[i]]?.active) { + return browserIds[i]; + } + } + + return browserIds.length > 0 ? browserIds[browserIds.length - 1] : null; + } + + // Fall back to checking all browsers if no user was specified + if (state) { + // Look for active browsers with the specific state + for (const id of Object.keys(this.pool)) { + if (this.pool[id].active && this.pool[id].state === state) { + return id; + } + } + + // Then look for any browser with the specific state + for (const id of Object.keys(this.pool)) { + if (this.pool[id].state === state) { + return id; + } + } + + // If no browser with the requested state is found, return null + return null; + } + + // If no state filter, find any active browser for (const id of Object.keys(this.pool)) { if (this.pool[id].active) { return id; } } - logger.log('warn', `No active browser in the pool`); + + // If all else fails, return the first browser in the pool + const browserIds = Object.keys(this.pool); + return browserIds.length > 0 ? browserIds[0] : null; + }; + + /** + * Returns the first active browser's instance id from the pool. + * If there is no active browser, it returns null. + * If there are multiple active browsers, it returns the first one. + * + * @returns the first remote active browser instance's id from the pool + * @deprecated Use getBrowserIdForUser instead to enforce the 1 User - 2 Browser policy + */ + public getActiveBrowserIdLegacy = (): string | null => { + for (const id of Object.keys(this.pool)) { + if (this.pool[id].active) { + return id; + } + } + // Don't log a warning since this behavior is expected in the user-browser model + // logger.log('warn', `No active browser in the pool`); return null; }; -} +} \ No newline at end of file diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 8ff4f601..85d1928c 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -90,6 +90,12 @@ export class RemoteBrowser { maxRepeats: 1, }; + /** + * The user ID that owns this browser instance + * @private + */ + private userId: string; + private lastEmittedUrl: string | null = null; /** @@ -106,6 +112,7 @@ export class RemoteBrowser { private screenshotQueue: Buffer[] = []; private isProcessingScreenshot = false; private screencastInterval: NodeJS.Timeout | null = null + private isScreencastActive: boolean = false; /** * Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and @@ -113,8 +120,9 @@ export class RemoteBrowser { * @param socket socket.io socket instance used to communicate with the client side * @constructor */ - public constructor(socket: Socket) { + public constructor(socket: Socket, userId: string) { this.socket = socket; + this.userId = userId; this.interpreter = new WorkflowInterpreter(socket); this.generator = new WorkflowGenerator(socket); } @@ -193,7 +201,7 @@ export class RemoteBrowser { const currentUrl = page.url(); if (this.shouldEmitUrlChange(currentUrl)) { this.lastEmittedUrl = currentUrl; - this.socket.emit('urlChanged', currentUrl); + this.socket.emit('urlChanged', {url: currentUrl, userId: this.userId}); } } }); @@ -333,15 +341,40 @@ export class RemoteBrowser { * @returns void */ public registerEditorEvents = (): void => { - this.socket.on('rerender', async () => await this.makeAndEmitScreenshot()); - this.socket.on('settings', (settings) => this.interpreterSettings = settings); - this.socket.on('changeTab', async (tabIndex) => await this.changeTab(tabIndex)); - this.socket.on('addTab', async () => { + // For each event, include userId to make sure events are handled for the correct browser + logger.log('debug', `Registering editor events for user: ${this.userId}`); + + // Listen for specific events for this user + this.socket.on(`rerender:${this.userId}`, async () => { + logger.debug(`Rerender event received for user ${this.userId}`); + await this.makeAndEmitScreenshot(); + }); + + // For backward compatibility, also listen to the general event + this.socket.on('rerender', async () => { + logger.debug(`General rerender event received, checking if for user ${this.userId}`); + await this.makeAndEmitScreenshot(); + }); + + this.socket.on(`settings:${this.userId}`, (settings) => { + this.interpreterSettings = settings; + logger.debug(`Settings updated for user ${this.userId}`); + }); + + this.socket.on(`changeTab:${this.userId}`, async (tabIndex) => { + logger.debug(`Tab change to ${tabIndex} requested for user ${this.userId}`); + await this.changeTab(tabIndex); + }); + + this.socket.on(`addTab:${this.userId}`, async () => { + logger.debug(`New tab requested for user ${this.userId}`); await this.currentPage?.context().newPage(); const lastTabIndex = this.currentPage ? this.currentPage.context().pages().length - 1 : 0; await this.changeTab(lastTabIndex); }); - this.socket.on('closeTab', async (tabInfo) => { + + this.socket.on(`closeTab:${this.userId}`, async (tabInfo) => { + logger.debug(`Close tab ${tabInfo.index} requested for user ${this.userId}`); const page = this.currentPage?.context().pages()[tabInfo.index]; if (page) { if (tabInfo.isCurrent) { @@ -356,24 +389,52 @@ export class RemoteBrowser { await page.close(); logger.log( 'debug', - `${tabInfo.index} page was closed, new length of pages: ${this.currentPage?.context().pages().length}` - ) + `Tab ${tabInfo.index} was closed for user ${this.userId}, new tab count: ${this.currentPage?.context().pages().length}` + ); } else { - logger.log('error', `${tabInfo.index} index out of range of pages`) + logger.log('error', `Tab index ${tabInfo.index} out of range for user ${this.userId}`); } }); - this.socket.on('setViewportSize', async (data: { width: number, height: number }) => { + + this.socket.on(`setViewportSize:${this.userId}`, async (data: { width: number, height: number }) => { const { width, height } = data; - logger.log('debug', `Received viewport size: width=${width}, height=${height}`); + logger.log('debug', `Viewport size change to width=${width}, height=${height} requested for user ${this.userId}`); // Update the browser context's viewport dynamically if (this.context && this.browser) { this.context = await this.browser.newContext({ viewport: { width, height } }); - logger.log('debug', `Viewport size updated to width=${width}, height=${height} for the entire browser context`); + logger.log('debug', `Viewport size updated to width=${width}, height=${height} for user ${this.userId}`); } }); - } - + + // For backward compatibility, also register the standard events + this.socket.on('settings', (settings) => this.interpreterSettings = settings); + this.socket.on('changeTab', async (tabIndex) => await this.changeTab(tabIndex)); + this.socket.on('addTab', async () => { + await this.currentPage?.context().newPage(); + const lastTabIndex = this.currentPage ? this.currentPage.context().pages().length - 1 : 0; + await this.changeTab(lastTabIndex); + }); + this.socket.on('closeTab', async (tabInfo) => { + const page = this.currentPage?.context().pages()[tabInfo.index]; + if (page) { + if (tabInfo.isCurrent) { + if (this.currentPage?.context().pages()[tabInfo.index + 1]) { + await this.changeTab(tabInfo.index + 1); + } else { + await this.changeTab(tabInfo.index - 1); + } + } + await page.close(); + } + }); + this.socket.on('setViewportSize', async (data: { width: number, height: number }) => { + const { width, height } = data; + if (this.context && this.browser) { + this.context = await this.browser.newContext({ viewport: { width, height } }); + } + }); + }; /** * Subscribes the remote browser for a screencast session * on [CDP](https://chromedevtools.github.io/devtools-protocol/) level, @@ -382,16 +443,24 @@ export class RemoteBrowser { * @returns {Promise} */ public subscribeToScreencast = async (): Promise => { + logger.log('debug', `Starting screencast for user: ${this.userId}`); await this.startScreencast(); if (!this.client) { logger.log('warn', 'client is not initialized'); return; } + // Set flag to indicate screencast is active + this.isScreencastActive = true; + this.client.on('Page.screencastFrame', ({ data: base64, sessionId }) => { + // Only process if screencast is still active for this user + if (!this.isScreencastActive) { + return; + } this.emitScreenshot(Buffer.from(base64, 'base64')) setTimeout(async () => { try { - if (!this.client) { + if (!this.client || !this.isScreencastActive) { logger.log('warn', 'client is not initialized'); return; } @@ -410,6 +479,8 @@ export class RemoteBrowser { */ public async switchOff(): Promise { try { + this.isScreencastActive = false; + await this.interpreter.stopInterpretation(); if (this.screencastInterval) { @@ -553,7 +624,11 @@ export class RemoteBrowser { //await this.currentPage.setViewportSize({ height: 400, width: 900 }) this.client = await this.currentPage.context().newCDPSession(this.currentPage); - this.socket.emit('urlChanged', this.currentPage.url()); + // Include userId in the URL change event + this.socket.emit('urlChanged', { + url: this.currentPage.url(), + userId: this.userId + }); await this.makeAndEmitScreenshot(); await this.subscribeToScreencast(); } else { @@ -602,6 +677,8 @@ export class RemoteBrowser { await this.client.send('Page.startScreencast', { format: SCREENCAST_CONFIG.format, }); + // Set flag to indicate screencast is active + this.isScreencastActive = true; // Set up screencast frame handler this.client.on('Page.screencastFrame', async ({ data, sessionId }) => { @@ -627,6 +704,8 @@ export class RemoteBrowser { } try { + // Set flag to indicate screencast is active + this.isScreencastActive = false; await this.client.send('Page.stopScreencast'); this.screenshotQueue = []; this.isProcessingScreenshot = false; @@ -657,8 +736,11 @@ export class RemoteBrowser { const base64Data = optimizedScreenshot.toString('base64'); const dataWithMimeType = `data:image/jpeg;base64,${base64Data}`; - this.socket.emit('screencast', dataWithMimeType); - logger.debug('Screenshot emitted'); +// Emit with user context to ensure the frontend can identify which browser's screenshot this is +this.socket.emit('screencast', { + image: dataWithMimeType, + userId: this.userId +}); logger.debug('Screenshot emitted'); } catch (error) { logger.error('Screenshot emission failed:', error); } finally { diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index 24a677ce..2cd9498d 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -21,23 +21,23 @@ import logger from "../logger"; * @category BrowserManagement-Controller */ export const initializeRemoteBrowserForRecording = (userId: string): string => { - const id = getActiveBrowserId() || uuid(); + const id = getActiveBrowserIdByState(userId, "recording") || uuid(); createSocketConnection( io.of(id), async (socket: Socket) => { // browser is already active - const activeId = getActiveBrowserId(); + const activeId = getActiveBrowserIdByState(userId, "recording"); if (activeId) { const remoteBrowser = browserPool.getRemoteBrowser(activeId); remoteBrowser?.updateSocket(socket); await remoteBrowser?.makeAndEmitScreenshot(); } else { - const browserSession = new RemoteBrowser(socket); + const browserSession = new RemoteBrowser(socket, userId); browserSession.interpreter.subscribeToPausing(); await browserSession.initialize(userId); await browserSession.registerEditorEvents(); await browserSession.subscribeToScreencast(); - browserPool.addRemoteBrowser(id, browserSession, true); + browserPool.addRemoteBrowser(id, browserSession, userId, false, "recording"); } socket.emit('loaded'); }); @@ -57,9 +57,9 @@ export const createRemoteBrowserForRun = (userId: string): string => { createSocketConnectionForRun( io.of(id), async (socket: Socket) => { - const browserSession = new RemoteBrowser(socket); + const browserSession = new RemoteBrowser(socket, userId); await browserSession.initialize(userId); - browserPool.addRemoteBrowser(id, browserSession, true); + browserPool.addRemoteBrowser(id, browserSession, userId, false, "run"); socket.emit('ready-for-run'); }); return id; @@ -72,7 +72,7 @@ export const createRemoteBrowserForRun = (userId: string): string => { * @returns {Promise} * @category BrowserManagement-Controller */ -export const destroyRemoteBrowser = async (id: string): Promise => { +export const destroyRemoteBrowser = async (id: string, userId: string): Promise => { const browserSession = browserPool.getRemoteBrowser(id); if (browserSession) { logger.log('debug', `Switching off the browser with id: ${id}`); @@ -88,8 +88,19 @@ export const destroyRemoteBrowser = async (id: string): Promise => { * @returns {string | null} * @category BrowserManagement-Controller */ -export const getActiveBrowserId = (): string | null => { - return browserPool.getActiveBrowserId(); +export const getActiveBrowserId = (userId: string): string | null => { + return browserPool.getActiveBrowserId(userId); +}; + +/** + * Returns the id of an active browser with the specified state or null. + * @param userId the user ID to find the browser for + * @param state the browser state to filter by ("recording" or "run") + * @returns {string | null} + * @category BrowserManagement-Controller + */ +export const getActiveBrowserIdByState = (userId: string, state: "recording" | "run"): string | null => { + return browserPool.getActiveBrowserId(userId, state); }; /** @@ -98,7 +109,7 @@ export const getActiveBrowserId = (): string | null => { * @returns {string | undefined} * @category BrowserManagement-Controller */ -export const getRemoteBrowserCurrentUrl = (id: string): string | undefined => { +export const getRemoteBrowserCurrentUrl = (id: string, userId: string): string | undefined => { return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.url(); }; @@ -108,7 +119,7 @@ export const getRemoteBrowserCurrentUrl = (id: string): string | undefined => { * @return {string[] | undefined} * @category BrowserManagement-Controller */ -export const getRemoteBrowserCurrentTabs = (id: string): string[] | undefined => { +export const getRemoteBrowserCurrentTabs = (id: string, userId: string): string[] | undefined => { return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.context().pages() .map((page) => { const parsedUrl = new URL(page.url()); @@ -126,8 +137,8 @@ export const getRemoteBrowserCurrentTabs = (id: string): string[] | undefined => * @returns {Promise} * @category BrowserManagement-Controller */ -export const interpretWholeWorkflow = async () => { - const id = getActiveBrowserId(); +export const interpretWholeWorkflow = async (userId: string) => { + const id = getActiveBrowserIdByState(userId, "recording"); if (id) { const browser = browserPool.getRemoteBrowser(id); if (browser) { @@ -146,8 +157,8 @@ export const interpretWholeWorkflow = async () => { * @returns {Promise} * @category BrowserManagement-Controller */ -export const stopRunningInterpretation = async () => { - const id = getActiveBrowserId(); +export const stopRunningInterpretation = async (userId: string) => { + const id = getActiveBrowserIdByState(userId, "recording"); if (id) { const browser = browserPool.getRemoteBrowser(id); await browser?.stopCurrentInterpretation(); diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index bf365053..8f4161b5 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -4,6 +4,8 @@ * These functions are called by the client through socket communication. */ import { Socket } from 'socket.io'; +import { IncomingMessage } from 'http'; +import { JwtPayload } from 'jsonwebtoken'; import logger from "../logger"; import { Coordinates, ScrollDeltas, KeyboardInput, DatePickerEventData } from '../types'; @@ -13,6 +15,14 @@ import { Page } from "playwright"; import { throttle } from "../../../src/helpers/inputHelpers"; import { CustomActions } from "../../../src/shared/types"; +interface AuthenticatedIncomingMessage extends IncomingMessage { + user?: JwtPayload | string; +} + +interface AuthenticatedSocket extends Socket { + request: AuthenticatedIncomingMessage; +} + /** * A wrapper function for handling user input. * This function gets the active browser instance from the browser pool @@ -23,6 +33,7 @@ import { CustomActions } from "../../../src/shared/types"; * * @param handleCallback The callback handler to be called * @param args - arguments to be passed to the handler + * @param socket - socket with authenticated request * @category HelperFunctions */ const handleWrapper = async ( @@ -31,9 +42,21 @@ const handleWrapper = async ( page: Page, args?: any ) => Promise, - args?: any + args?: any, + socket?: AuthenticatedSocket, ) => { - const id = browserPool.getActiveBrowserId(); + if (!socket || !socket.request || !socket.request.user || typeof socket.request.user === 'string') { + logger.log('warn', `User not authenticated or invalid JWT payload`); + return; + } + + const userId = socket.request.user.id; + if (!userId) { + logger.log('warn', `User ID is missing in JWT payload`); + return; + } + + const id = browserPool.getActiveBrowserId(userId, "recording"); if (id) { const activeBrowser = browserPool.getRemoteBrowser(id); if (activeBrowser?.interpreter.interpretationInProgress() && !activeBrowser.interpreter.interpretationIsPaused) { @@ -66,12 +89,13 @@ interface CustomActionEventData { /** * A wrapper function for handling custom actions. + * @param socket The socket connection * @param customActionEventData The custom action event data * @category HelperFunctions */ -const onGenerateAction = async (customActionEventData: CustomActionEventData) => { +const onGenerateAction = async (socket: AuthenticatedSocket, customActionEventData: CustomActionEventData) => { logger.log('debug', `Generating ${customActionEventData.action} action emitted from client`); - await handleWrapper(handleGenerateAction, customActionEventData); + await handleWrapper(handleGenerateAction, customActionEventData, socket); } /** @@ -89,12 +113,13 @@ const handleGenerateAction = /** * A wrapper function for handling mousedown event. + * @param socket The socket connection * @param coordinates - coordinates of the mouse click * @category HelperFunctions */ -const onMousedown = async (coordinates: Coordinates) => { +const onMousedown = async (socket: AuthenticatedSocket, coordinates: Coordinates) => { logger.log('debug', 'Handling mousedown event emitted from client'); - await handleWrapper(handleMousedown, coordinates); + await handleWrapper(handleMousedown, coordinates, socket); } /** @@ -139,12 +164,13 @@ const handleMousedown = async (generator: WorkflowGenerator, page: Page, { x, y /** * A wrapper function for handling the wheel event. + * @param socket The socket connection * @param scrollDeltas - the scroll deltas of the wheel event * @category HelperFunctions */ -const onWheel = async (scrollDeltas: ScrollDeltas) => { +const onWheel = async (socket: AuthenticatedSocket, scrollDeltas: ScrollDeltas) => { logger.log('debug', 'Handling scroll event emitted from client'); - await handleWrapper(handleWheel, scrollDeltas); + await handleWrapper(handleWheel, scrollDeltas, socket); }; /** @@ -165,12 +191,13 @@ const handleWheel = async (generator: WorkflowGenerator, page: Page, { deltaX, d /** * A wrapper function for handling the mousemove event. + * @param socket The socket connection * @param coordinates - the coordinates of the mousemove event * @category HelperFunctions */ -const onMousemove = async (coordinates: Coordinates) => { +const onMousemove = async (socket: AuthenticatedSocket, coordinates: Coordinates) => { logger.log('debug', 'Handling mousemove event emitted from client'); - await handleWrapper(handleMousemove, coordinates); + await handleWrapper(handleMousemove, coordinates, socket); } /** @@ -199,12 +226,13 @@ const handleMousemove = async (generator: WorkflowGenerator, page: Page, { x, y /** * A wrapper function for handling the keydown event. + * @param socket The socket connection * @param keyboardInput - the keyboard input of the keydown event * @category HelperFunctions */ -const onKeydown = async (keyboardInput: KeyboardInput) => { +const onKeydown = async (socket: AuthenticatedSocket, keyboardInput: KeyboardInput) => { logger.log('debug', 'Handling keydown event emitted from client'); - await handleWrapper(handleKeydown, keyboardInput); + await handleWrapper(handleKeydown, keyboardInput, socket); } /** @@ -235,49 +263,95 @@ const handleDateSelection = async (generator: WorkflowGenerator, page: Page, dat logger.log('debug', `Date ${data.value} selected`); } -const onDateSelection = async (data: DatePickerEventData) => { +/** + * A wrapper function for handling the date selection event. + * @param socket The socket connection + * @param data - the data of the date selection event + * @category HelperFunctions + */ +const onDateSelection = async (socket: AuthenticatedSocket, data: DatePickerEventData) => { logger.log('debug', 'Handling date selection event emitted from client'); - await handleWrapper(handleDateSelection, data); + await handleWrapper(handleDateSelection, data, socket); } +/** + * Handles the dropdown selection event. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param data - the data of the dropdown selection event + * @category BrowserManagement + */ const handleDropdownSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { await generator.onDropdownSelection(page, data); logger.log('debug', `Dropdown value ${data.value} selected`); } -const onDropdownSelection = async (data: { selector: string, value: string }) => { +/** + * A wrapper function for handling the dropdown selection event. + * @param socket The socket connection + * @param data - the data of the dropdown selection event + * @category HelperFunctions + */ +const onDropdownSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => { logger.log('debug', 'Handling dropdown selection event emitted from client'); - await handleWrapper(handleDropdownSelection, data); + await handleWrapper(handleDropdownSelection, data, socket); } +/** + * Handles the time selection event. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param data - the data of the time selection event + * @category BrowserManagement + */ const handleTimeSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { await generator.onTimeSelection(page, data); logger.log('debug', `Time value ${data.value} selected`); } -const onTimeSelection = async (data: { selector: string, value: string }) => { +/** + * A wrapper function for handling the time selection event. + * @param socket The socket connection + * @param data - the data of the time selection event + * @category HelperFunctions + */ +const onTimeSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => { logger.log('debug', 'Handling time selection event emitted from client'); - await handleWrapper(handleTimeSelection, data); + await handleWrapper(handleTimeSelection, data, socket); } +/** + * Handles the datetime-local selection event. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param data - the data of the datetime-local selection event + * @category BrowserManagement + */ const handleDateTimeLocalSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { await generator.onDateTimeLocalSelection(page, data); logger.log('debug', `DateTime Local value ${data.value} selected`); } -const onDateTimeLocalSelection = async (data: { selector: string, value: string }) => { +/** + * A wrapper function for handling the datetime-local selection event. + * @param socket The socket connection + * @param data - the data of the datetime-local selection event + * @category HelperFunctions + */ +const onDateTimeLocalSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => { logger.log('debug', 'Handling datetime-local selection event emitted from client'); - await handleWrapper(handleDateTimeLocalSelection, data); + await handleWrapper(handleDateTimeLocalSelection, data, socket); } /** * A wrapper function for handling the keyup event. + * @param socket The socket connection * @param keyboardInput - the keyboard input of the keyup event * @category HelperFunctions */ -const onKeyup = async (keyboardInput: KeyboardInput) => { +const onKeyup = async (socket: AuthenticatedSocket, keyboardInput: KeyboardInput) => { logger.log('debug', 'Handling keyup event emitted from client'); - await handleWrapper(handleKeyup, keyboardInput); + await handleWrapper(handleKeyup, keyboardInput, socket); } /** @@ -296,12 +370,13 @@ const handleKeyup = async (generator: WorkflowGenerator, page: Page, key: string /** * A wrapper function for handling the url change event. + * @param socket The socket connection * @param url - the new url of the page * @category HelperFunctions */ -const onChangeUrl = async (url: string) => { +const onChangeUrl = async (socket: AuthenticatedSocket, url: string) => { logger.log('debug', 'Handling change url event emitted from client'); - await handleWrapper(handleChangeUrl, url); + await handleWrapper(handleChangeUrl, url, socket); } /** @@ -329,11 +404,12 @@ const handleChangeUrl = async (generator: WorkflowGenerator, page: Page, url: st /** * A wrapper function for handling the refresh event. + * @param socket The socket connection * @category HelperFunctions */ -const onRefresh = async () => { +const onRefresh = async (socket: AuthenticatedSocket) => { logger.log('debug', 'Handling refresh event emitted from client'); - await handleWrapper(handleRefresh); + await handleWrapper(handleRefresh, undefined, socket); } /** @@ -350,11 +426,12 @@ const handleRefresh = async (generator: WorkflowGenerator, page: Page) => { /** * A wrapper function for handling the go back event. + * @param socket The socket connection * @category HelperFunctions */ -const onGoBack = async () => { - logger.log('debug', 'Handling refresh event emitted from client'); - await handleWrapper(handleGoBack); +const onGoBack = async (socket: AuthenticatedSocket) => { + logger.log('debug', 'Handling go back event emitted from client'); + await handleWrapper(handleGoBack, undefined, socket); } /** @@ -372,11 +449,12 @@ const handleGoBack = async (generator: WorkflowGenerator, page: Page) => { /** * A wrapper function for handling the go forward event. + * @param socket The socket connection * @category HelperFunctions */ -const onGoForward = async () => { - logger.log('debug', 'Handling refresh event emitted from client'); - await handleWrapper(handleGoForward); +const onGoForward = async (socket: AuthenticatedSocket) => { + logger.log('debug', 'Handling go forward event emitted from client'); + await handleWrapper(handleGoForward, undefined, socket); } /** @@ -394,18 +472,7 @@ const handleGoForward = async (generator: WorkflowGenerator, page: Page) => { /** * Helper function for registering the handlers onto established websocket connection. - * Registers: - * - mousedownHandler - * - wheelHandler - * - mousemoveHandler - * - keydownHandler - * - keyupHandler - * - changeUrlHandler - * - refreshHandler - * - goBackHandler - * - goForwardHandler - * - onGenerateAction - * input handlers. + * Registers various input handlers. * * All these handlers first generates the workflow pair data * and then calls the corresponding playwright's function to emulate the input. @@ -415,21 +482,25 @@ const handleGoForward = async (generator: WorkflowGenerator, page: Page) => { * @returns void * @category BrowserManagement */ -const registerInputHandlers = (socket: Socket) => { - socket.on("input:mousedown", onMousedown); - socket.on("input:wheel", onWheel); - socket.on("input:mousemove", onMousemove); - socket.on("input:keydown", onKeydown); - socket.on("input:keyup", onKeyup); - socket.on("input:url", onChangeUrl); - socket.on("input:refresh", onRefresh); - socket.on("input:back", onGoBack); - socket.on("input:forward", onGoForward); - socket.on("input:date", onDateSelection); - socket.on("input:dropdown", onDropdownSelection); - socket.on("input:time", onTimeSelection); - socket.on("input:datetime-local", onDateTimeLocalSelection); - socket.on("action", onGenerateAction); +const registerInputHandlers = (socket: Socket) => { + // Cast to our authenticated socket type + const authSocket = socket as AuthenticatedSocket; + + // Register handlers with the socket + socket.on("input:mousedown", (data) => onMousedown(authSocket, data)); + socket.on("input:wheel", (data) => onWheel(authSocket, data)); + socket.on("input:mousemove", (data) => onMousemove(authSocket, data)); + socket.on("input:keydown", (data) => onKeydown(authSocket, data)); + socket.on("input:keyup", (data) => onKeyup(authSocket, data)); + socket.on("input:url", (data) => onChangeUrl(authSocket, data)); + socket.on("input:refresh", () => onRefresh(authSocket)); + socket.on("input:back", () => onGoBack(authSocket)); + socket.on("input:forward", () => onGoForward(authSocket)); + socket.on("input:date", (data) => onDateSelection(authSocket, data)); + socket.on("input:dropdown", (data) => onDropdownSelection(authSocket, data)); + socket.on("input:time", (data) => onTimeSelection(authSocket, data)); + socket.on("input:datetime-local", (data) => onDateTimeLocalSelection(authSocket, data)); + socket.on("action", (data) => onGenerateAction(authSocket, data)); }; -export default registerInputHandlers; +export default registerInputHandlers; \ No newline at end of file diff --git a/server/src/models/Robot.ts b/server/src/models/Robot.ts index 3b2717d6..1d03999e 100644 --- a/server/src/models/Robot.ts +++ b/server/src/models/Robot.ts @@ -25,7 +25,13 @@ interface RobotAttributes { google_sheet_id?: string | null; google_access_token?: string | null; google_refresh_token?: string | null; + airtable_base_id?: string | null; + airtable_base_name?: string | null; + airtable_table_name?: string | null; + airtable_access_token?: string | null; + airtable_refresh_token?: string | null; schedule?: ScheduleConfig | null; + airtable_table_id?: string | null; } interface ScheduleConfig { @@ -49,10 +55,16 @@ class Robot extends Model implements R public recording_meta!: RobotMeta; public recording!: RobotWorkflow; public google_sheet_email!: string | null; - public google_sheet_name?: string | null; - public google_sheet_id?: string | null; + public google_sheet_name!: string | null; + public google_sheet_id!: string | null; public google_access_token!: string | null; public google_refresh_token!: string | null; + public airtable_base_id!: string | null; + public airtable_base_name!: string | null; + public airtable_table_name!: string | null; + public airtable_access_token!: string | null; + public airtable_refresh_token!: string | null; + public airtable_table_id!: string | null; public schedule!: ScheduleConfig | null; } @@ -95,6 +107,30 @@ Robot.init( type: DataTypes.STRING, allowNull: true, }, + airtable_base_id: { + type: DataTypes.STRING, + allowNull: true, + }, + airtable_base_name: { + type: DataTypes.STRING, + allowNull: true, + }, + airtable_table_name: { + type: DataTypes.STRING, + allowNull: true, + }, + airtable_table_id: { + type: DataTypes.STRING, + allowNull: true, + }, + airtable_access_token: { + type: DataTypes.TEXT, + allowNull: true, + }, + airtable_refresh_token: { + type: DataTypes.TEXT, + allowNull: true, + }, schedule: { type: DataTypes.JSONB, allowNull: true, diff --git a/server/src/pgboss-worker.ts b/server/src/pgboss-worker.ts new file mode 100644 index 00000000..3566b4f8 --- /dev/null +++ b/server/src/pgboss-worker.ts @@ -0,0 +1,488 @@ +/** + * Recording worker using PgBoss for asynchronous browser recording operations + */ +import PgBoss, { Job } from 'pg-boss'; +import logger from './logger'; +import { + initializeRemoteBrowserForRecording, + destroyRemoteBrowser, + interpretWholeWorkflow, + stopRunningInterpretation, + createRemoteBrowserForRun +} from './browser-management/controller'; +import { WorkflowFile } from 'maxun-core'; +import Run from './models/Run'; +import Robot from './models/Robot'; +import { browserPool } from './server'; +import { Page } from 'playwright'; +import { BinaryOutputService } from './storage/mino'; +import { capture } from './utils/analytics'; +import { googleSheetUpdateTasks, processGoogleSheetUpdates } from './workflow-management/integrations/gsheet'; +import { airtableUpdateTasks, processAirtableUpdates } from './workflow-management/integrations/airtable'; +import { RemoteBrowser } from './browser-management/classes/RemoteBrowser'; +import { io as serverIo } from "./server"; + +const pgBossConnectionString = `postgres://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`; + +interface InitializeBrowserData { + userId: string; +} + +interface InterpretWorkflow { + userId: string; +} + +interface StopInterpretWorkflow { + userId: string; +} + +interface DestroyBrowserData { + browserId: string; + userId: string; +} + +interface ExecuteRunData { + userId: string; + runId: string; + browserId: string; +} + +const pgBoss = new PgBoss({connectionString: pgBossConnectionString, schema: 'public'}); + +/** + * Extract data safely from a job (single job or job array) + */ +function extractJobData(job: Job | Job[]): T { + if (Array.isArray(job)) { + if (job.length === 0) { + throw new Error('Empty job array received'); + } + return job[0].data; + } + return job.data; +} + +function AddGeneratedFlags(workflow: WorkflowFile) { + const copy = JSON.parse(JSON.stringify(workflow)); + for (let i = 0; i < workflow.workflow.length; i++) { + copy.workflow[i].what.unshift({ + action: 'flag', + args: ['generated'], + }); + } + return copy; +}; + +/** + * Function to reset browser state without creating a new browser + */ +async function resetBrowserState(browser: RemoteBrowser): Promise { + try { + const currentPage = browser.getCurrentPage(); + if (!currentPage) { + logger.log('error', 'No current page available to reset browser state'); + return false; + } + + // Navigate to blank page to reset state + await currentPage.goto('about:blank'); + + // Clear browser storage + await currentPage.evaluate(() => { + try { + localStorage.clear(); + sessionStorage.clear(); + } catch (e) { + // Ignore errors in cleanup + } + }); + + // Clear cookies + const context = currentPage.context(); + await context.clearCookies(); + + return true; + } catch (error) { + logger.log('error', `Failed to reset browser state`); + return false; + } +} + +/** + * Modified checkAndProcessQueuedRun function - only changes browser reset logic + */ +async function checkAndProcessQueuedRun(userId: string, browserId: string): Promise { + try { + // Find the oldest queued run for this specific browser + const queuedRun = await Run.findOne({ + where: { + browserId: browserId, + status: 'queued' + }, + order: [['startedAt', 'ASC']] + }); + + if (!queuedRun) { + logger.log('info', `No queued runs found for browser ${browserId}`); + return false; + } + + // Reset the browser state before next run + const browser = browserPool.getRemoteBrowser(browserId); + if (browser) { + logger.log('info', `Resetting browser state for browser ${browserId} before next run`); + await resetBrowserState(browser); + } + + // Update the queued run to running status + await queuedRun.update({ + status: 'running', + log: 'Run started - using browser from previous run' + }); + + // Schedule the run execution + await pgBoss.createQueue('execute-run'); + const executeJobId = await pgBoss.send('execute-run', { + userId: userId, + runId: queuedRun.runId, + browserId: browserId + }); + + logger.log('info', `Scheduled queued run ${queuedRun.runId} to use browser ${browserId}, job ID: ${executeJobId}`); + return true; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Error checking for queued runs: ${errorMessage}`); + return false; + } +} + +/** + * Modified processRunExecution function - only add browser reset + */ +async function processRunExecution(job: Job) { + try { + const data = job.data; + logger.log('info', `Processing run execution job for runId: ${data.runId}, browserId: ${data.browserId}`); + + // Find the run + const run = await Run.findOne({ where: { runId: data.runId } }); + if (!run) { + logger.log('error', `Run ${data.runId} not found in database`); + return { success: false }; + } + + const plainRun = run.toJSON(); + + // Find the recording + const recording = await Robot.findOne({ where: { 'recording_meta.id': plainRun.robotMetaId }, raw: true }); + if (!recording) { + logger.log('error', `Recording for run ${data.runId} not found`); + + // Update run status to failed + await run.update({ + status: 'failed', + finishedAt: new Date().toLocaleString(), + log: 'Failed: Recording not found', + }); + + // Check for queued runs even if this one failed + await checkAndProcessQueuedRun(data.userId, data.browserId); + + return { success: false }; + } + + // Get the browser and execute the run + const browser = browserPool.getRemoteBrowser(plainRun.browserId); + let currentPage = browser?.getCurrentPage(); + + if (!browser || !currentPage) { + logger.log('error', `Browser or page not available for run ${data.runId}`); + + await pgBoss.fail(job.id, "Failed to get browser or page for run"); + + // Even if this run failed, check for queued runs + await checkAndProcessQueuedRun(data.userId, data.browserId); + + return { success: false }; + } + + try { + // Reset the browser state before executing this run + await resetBrowserState(browser); + + // Execute the workflow + const workflow = AddGeneratedFlags(recording.recording); + const interpretationInfo = await browser.interpreter.InterpretRecording( + workflow, + currentPage, + (newPage: Page) => currentPage = newPage, + plainRun.interpreterSettings + ); + + // Process the results + const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); + const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); + + // Update the run record with results + await run.update({ + ...run, + status: 'success', + finishedAt: new Date().toLocaleString(), + browserId: plainRun.browserId, + log: interpretationInfo.log.join('\n'), + serializableOutput: interpretationInfo.serializableOutput, + binaryOutput: uploadedBinaryOutput, + }); + + // Track extraction metrics + let totalRowsExtracted = 0; + let extractedScreenshotsCount = 0; + let extractedItemsCount = 0; + + if (run.dataValues.binaryOutput && run.dataValues.binaryOutput["item-0"]) { + extractedScreenshotsCount = 1; + } + + if (run.dataValues.serializableOutput && run.dataValues.serializableOutput["item-0"]) { + const itemsArray = run.dataValues.serializableOutput["item-0"]; + extractedItemsCount = itemsArray.length; + + totalRowsExtracted = itemsArray.reduce((total, item) => { + return total + Object.keys(item).length; + }, 0); + } + + console.log(`Extracted Items Count: ${extractedItemsCount}`); + console.log(`Extracted Screenshots Count: ${extractedScreenshotsCount}`); + console.log(`Total Rows Extracted: ${totalRowsExtracted}`); + + // Capture metrics + capture( + 'maxun-oss-run-created-manual', + { + runId: data.runId, + user_id: data.userId, + created_at: new Date().toISOString(), + status: 'success', + totalRowsExtracted, + extractedItemsCount, + extractedScreenshotsCount, + } + ); + + // Schedule updates for Google Sheets and Airtable + try { + googleSheetUpdateTasks[plainRun.runId] = { + robotId: plainRun.robotMetaId, + runId: plainRun.runId, + status: 'pending', + retries: 5, + }; + + airtableUpdateTasks[plainRun.runId] = { + robotId: plainRun.robotMetaId, + runId: plainRun.runId, + status: 'pending', + retries: 5, + }; + + processAirtableUpdates(); + processGoogleSheetUpdates(); + } catch (err: any) { + logger.log('error', `Failed to update Google Sheet for run: ${plainRun.runId}: ${err.message}`); + } + + serverIo.of(plainRun.browserId).emit('run-completed', { + runId: data.runId, + robotMetaId: plainRun.robotMetaId, + robotName: recording.recording_meta.name, + status: 'success', + finishedAt: new Date().toLocaleString() + });; + + // Check for and process queued runs before destroying the browser + const queuedRunProcessed = await checkAndProcessQueuedRun(data.userId, plainRun.browserId); + + // Only destroy the browser if no queued run was found + if (!queuedRunProcessed) { + await destroyRemoteBrowser(plainRun.browserId, data.userId); + logger.log('info', `No queued runs found for browser ${plainRun.browserId}, browser destroyed`); + } + + return { success: true }; + } catch (executionError: any) { + logger.log('error', `Run execution failed for run ${data.runId}: ${executionError.message}`); + + await run.update({ + status: 'failed', + finishedAt: new Date().toLocaleString(), + log: `Failed: ${executionError.message}`, + }); + + // Check for queued runs before destroying the browser + const queuedRunProcessed = await checkAndProcessQueuedRun(data.userId, plainRun.browserId); + + // Only destroy the browser if no queued run was found + if (!queuedRunProcessed) { + try { + await destroyRemoteBrowser(plainRun.browserId, data.userId); + logger.log('info', `No queued runs found for browser ${plainRun.browserId}, browser destroyed`); + } catch (cleanupError: any) { + logger.log('warn', `Failed to clean up browser for failed run ${data.runId}: ${cleanupError.message}`); + } + } + + // Capture failure metrics + capture( + 'maxun-oss-run-created-manual', + { + runId: data.runId, + user_id: data.userId, + created_at: new Date().toISOString(), + status: 'failed', + error_message: executionError.message, + } + ); + + return { success: false }; + } + + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Failed to process run execution job: ${errorMessage}`); + return { success: false }; + } +} + + +async function registerRunExecutionWorker() { + try { + // Worker for executing runs + await pgBoss.work('execute-run', async (job: Job | Job[]) => { + try { + const singleJob = Array.isArray(job) ? job[0] : job; + return await processRunExecution(singleJob); + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Run execution job failed: ${errorMessage}`); + throw error; + } + }); + + // setInterval(checkForStuckQueuedRuns, 30000); + + logger.log('info', 'Run execution worker registered successfully'); + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Failed to register run execution worker: ${errorMessage}`); + } +} + + +/** + * Initialize PgBoss and register all workers + */ +async function startWorkers() { + try { + logger.log('info', 'Starting PgBoss worker...'); + await pgBoss.start(); + logger.log('info', 'PgBoss worker started successfully'); + + // Worker for initializing browser recording + await pgBoss.work('initialize-browser-recording', async (job: Job | Job[]) => { + try { + const data = extractJobData(job); + const userId = data.userId; + + logger.log('info', `Starting browser initialization job for user: ${userId}`); + const browserId = initializeRemoteBrowserForRecording(userId); + logger.log('info', `Browser recording job completed with browserId: ${browserId}`); + return { browserId }; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Browser recording job failed: ${errorMessage}`); + throw error; + } + }); + + // Worker for stopping a browser + await pgBoss.work('destroy-browser', async (job: Job | Job[]) => { + try { + const data = extractJobData(job); + const { browserId, userId } = data; + + logger.log('info', `Starting browser destruction job for browser: ${browserId}`); + const success = await destroyRemoteBrowser(browserId, userId); + logger.log('info', `Browser destruction job completed with result: ${success}`); + return { success }; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Destroy browser job failed: ${errorMessage}`); + throw error; + } + }); + + // Worker for interpreting workflow + await pgBoss.work('interpret-workflow', async (job: Job | Job[]) => { + try { + const data = extractJobData(job); + const userId = data.userId; + + logger.log('info', 'Starting workflow interpretation job'); + await interpretWholeWorkflow(userId); + logger.log('info', 'Workflow interpretation job completed'); + return { success: true }; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Interpret workflow job failed: ${errorMessage}`); + throw error; + } + }); + + // Worker for stopping workflow interpretation + await pgBoss.work('stop-interpretation', async (job: Job | Job[]) => { + try { + const data = extractJobData(job); + const userId = data.userId; + + logger.log('info', 'Starting stop interpretation job'); + await stopRunningInterpretation(userId); + logger.log('info', 'Stop interpretation job completed'); + return { success: true }; + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Stop interpretation job failed: ${errorMessage}`); + throw error; + } + }); + + // Register the run execution worker + await registerRunExecutionWorker(); + + logger.log('info', 'All recording workers registered successfully'); + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.log('error', `Failed to start PgBoss workers: ${errorMessage}`); + process.exit(1); + } +} + +// Start all workers +startWorkers(); + +// Handle graceful shutdown +process.on('SIGTERM', async () => { + logger.log('info', 'SIGTERM received, shutting down PgBoss...'); + await pgBoss.stop(); + process.exit(0); +}); + +process.on('SIGINT', async () => { + logger.log('info', 'SIGINT received, shutting down PgBoss...'); + await pgBoss.stop(); + process.exit(0); +}); + +// For use in other files +export { pgBoss }; \ No newline at end of file diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index 728cc42c..4a435cd6 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -1,4 +1,5 @@ import { Router, Request, Response } from "express"; + import User from "../models/User"; import Robot from "../models/Robot"; import jwt from "jsonwebtoken"; @@ -7,6 +8,16 @@ import { requireSignIn } from "../middlewares/auth"; import { genAPIKey } from "../utils/api"; import { google } from "googleapis"; import { capture } from "../utils/analytics"; +import crypto from 'crypto'; + + +declare module "express-session" { + interface SessionData { + code_verifier: string; + robotId: string; + } +} + export const router = Router(); interface AuthenticatedRequest extends Request { @@ -263,7 +274,11 @@ router.get( async (req: AuthenticatedRequest, res) => { try { if (!req.user) { - return res.status(401).json({ ok: false, error: "Unauthorized" }); + return res.status(401).json({ + ok: false, + error: "Unauthorized", + code: "unauthorized" + }); } const user = await User.findByPk(req.user.id, { @@ -272,15 +287,25 @@ router.get( }); if (!user) { - return res.status(404).json({ message: "User not found" }); + return res.status(404).json({ + ok: false, + error: "User not found", + code: "not_found" + }); } return res.status(200).json({ + ok: true, message: "API key fetched successfully", api_key: user.api_key || null, }); } catch (error) { - return res.status(500).json({ message: "Error fetching API key", error }); + console.error('API Key fetch error:', error); + return res.status(500).json({ + ok: false, + error: "Error fetching API key", + code: "server", + }); } } ); @@ -618,4 +643,292 @@ router.post( }); } } -); \ No newline at end of file +); + + +// Airtable OAuth Routes +router.get("/airtable", requireSignIn, (req: Request, res) => { + const authenticatedReq = req as AuthenticatedRequest; + const { robotId } = authenticatedReq.query; + if (!robotId) { + return res.status(400).json({ message: "Robot ID is required" }); + } + + // Generate PKCE codes + const code_verifier = crypto.randomBytes(64).toString('base64url'); + const code_challenge = crypto.createHash('sha256') + .update(code_verifier) + .digest('base64url'); + + // Store in session + authenticatedReq.session.code_verifier = code_verifier; + authenticatedReq.session.robotId = robotId.toString(); + + const params = new URLSearchParams({ + client_id: process.env.AIRTABLE_CLIENT_ID!, + redirect_uri: process.env.AIRTABLE_REDIRECT_URI!, + response_type: 'code', + state: robotId.toString(), + scope: 'data.records:read data.records:write schema.bases:read schema.bases:write', + code_challenge: code_challenge, + code_challenge_method: 'S256' + }); + + res.redirect(`https://airtable.com/oauth2/v1/authorize?${params}`); +}); + +router.get("/airtable/callback", requireSignIn, async (req: Request, res) => { + const authenticatedReq = req as AuthenticatedRequest; + const baseUrl = process.env.PUBLIC_URL || "http://localhost:5173"; + + try { + const { code, state, error } = authenticatedReq.query; + + if (error) { + return res.redirect( + `${baseUrl}/robots/${state}/integrate?error=${encodeURIComponent(error.toString())}` + ); + } + + if (!code || !state) { + return res.status(400).json({ message: "Missing authorization code or state" }); + } + + // Verify session data + if (!authenticatedReq.session?.code_verifier || authenticatedReq.session.robotId !== state.toString()) { + return res.status(400).json({ + message: "Session expired - please restart the OAuth flow" + }); + } + + // Exchange code for tokens + const tokenResponse = await fetch("https://airtable.com/oauth2/v1/token", { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams({ + grant_type: "authorization_code", + code: code.toString(), + client_id: process.env.AIRTABLE_CLIENT_ID!, + redirect_uri: process.env.AIRTABLE_REDIRECT_URI!, + code_verifier: authenticatedReq.session.code_verifier + }), + }); + + if (!tokenResponse.ok) { + const errorData = await tokenResponse.json(); + console.error('Token exchange failed:', errorData); + return res.redirect( + `${baseUrl}/robots/${state}/integrate?error=${encodeURIComponent(errorData.error_description || 'Authentication failed')}` + ); + } + + const tokens = await tokenResponse.json(); + + // Update robot with credentials + const robot = await Robot.findOne({ + where: { "recording_meta.id": req.session.robotId } + }); + + if (!robot) { + return res.status(404).json({ message: "Robot not found" }); + } + + await robot.update({ + airtable_access_token: tokens.access_token, + airtable_refresh_token: tokens.refresh_token, + }); + + res.cookie("airtable_auth_status", "success", { + httpOnly: false, + maxAge: 60000, + }); // 1-minute expiration + // res.cookie("airtable_auth_message", "Robot successfully authenticated", { + // httpOnly: false, + // maxAge: 60000, + // }); + + res.cookie('robot_auth_robotId', req.session.robotId, { + httpOnly: false, + maxAge: 60000, + }); + + // Clear session data + authenticatedReq.session.destroy((err) => { + if (err) console.error('Session cleanup error:', err); + }); + + const redirectUrl = `${baseUrl}/robots/`; + + res.redirect(redirectUrl); + } catch (error: any) { + console.error('Airtable callback error:', error); + res.redirect( + `${baseUrl}/robots/${req.session.robotId}/integrate?error=${encodeURIComponent(error.message)}` + ); + } +}); + +// Get Airtable bases +router.get("/airtable/bases", requireSignIn, async (req: Request, res) => { + const authenticatedReq = req as AuthenticatedRequest; + try { + const { robotId } = authenticatedReq.query; + if (!robotId) { + return res.status(400).json({ message: "Robot ID is required" }); + } + + const robot = await Robot.findOne({ + where: { "recording_meta.id": robotId.toString() }, + raw: true, + }); + + if (!robot?.airtable_access_token) { + return res.status(400).json({ message: "Robot not authenticated with Airtable" }); + } + + const response = await fetch('https://api.airtable.com/v0/meta/bases', { + headers: { + 'Authorization': `Bearer ${robot.airtable_access_token}` + } + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.error.message || 'Failed to fetch bases'); + } + + const data = await response.json(); + res.json(data.bases.map((base: any) => ({ + id: base.id, + name: base.name + }))); + + } catch (error: any) { + res.status(500).json({ message: error.message }); + } +}); + +// Update robot with selected base +router.post("/airtable/update", requireSignIn, async (req: Request, res) => { + const authenticatedReq = req as AuthenticatedRequest; + const { baseId, robotId , baseName, tableName, tableId} = req.body; + + if (!baseId || !robotId) { + return res.status(400).json({ message: "Base ID and Robot ID are required" }); + } + + try { + const robot = await Robot.findOne({ + where: { "recording_meta.id": robotId } + }); + + if (!robot) { + return res.status(404).json({ message: "Robot not found" }); + } + + await robot.update({ + airtable_base_id: baseId, + airtable_table_name: tableName, + airtable_table_id: tableId, + airtable_base_name: baseName, + }); + + capture("maxun-oss-airtable-integration-created", { + user_id: authenticatedReq.user?.id, + robot_id: robotId, + created_at: new Date().toISOString(), + }); + + res.json({ message: "Airtable base updated successfully" }); + + } catch (error: any) { + res.status(500).json({ message: error.message }); + } +}); + +// Remove Airtable integration +router.post("/airtable/remove", requireSignIn, async (req: Request, res) => { + const authenticatedReq = req as AuthenticatedRequest; + const { robotId } = authenticatedReq.body; + if (!robotId) { + return res.status(400).json({ message: "Robot ID is required" }); + } + + try { + const robot = await Robot.findOne({ + where: { "recording_meta.id": robotId } + }); + + if (!robot) { + return res.status(404).json({ message: "Robot not found" }); + } + + await robot.update({ + airtable_access_token: null, + airtable_refresh_token: null, + airtable_base_id: null, + airtable_base_name: null, + airtable_table_name: null, + airtable_table_id: null, + }); + + capture("maxun-oss-airtable-integration-removed", { + user_id: authenticatedReq.user?.id, + robot_id: robotId, + deleted_at: new Date().toISOString(), + }); + + res.json({ message: "Airtable integration removed successfully" }); + + } catch (error: any) { + res.status(500).json({ message: error.message }); + } +}); + + + +// Fetch tables from an Airtable base +router.get("/airtable/tables", requireSignIn, async (req: Request, res) => { + const authenticatedReq = req as AuthenticatedRequest; + try { + const { baseId, robotId } = authenticatedReq.query; + + if (!baseId || !robotId) { + return res.status(400).json({ message: "Base ID and Robot ID are required" }); + } + + const robot = await Robot.findOne({ + where: { "recording_meta.id": robotId.toString() }, + raw: true, + }); + + if (!robot?.airtable_access_token) { + return res.status(400).json({ message: "Robot not authenticated with Airtable" }); + } + + const response = await fetch(`https://api.airtable.com/v0/meta/bases/${baseId}/tables`, { + headers: { + 'Authorization': `Bearer ${robot.airtable_access_token}` + } + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.error.message || 'Failed to fetch tables'); + } + + const data = await response.json(); + res.json(data.tables.map((table: any) => ({ + id: table.id, + name: table.name, + fields: table.fields + }))); + + } catch (error: any) { + res.status(500).json({ message: error.message }); + } +}); + + diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index 51d3ff92..fee5039a 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -9,22 +9,58 @@ import { getActiveBrowserId, interpretWholeWorkflow, stopRunningInterpretation, - getRemoteBrowserCurrentUrl, getRemoteBrowserCurrentTabs, -} from '../browser-management/controller' + getRemoteBrowserCurrentUrl, + getRemoteBrowserCurrentTabs, + getActiveBrowserIdByState, +} from '../browser-management/controller'; import { chromium } from 'playwright-extra'; import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import logger from "../logger"; import { getDecryptedProxyConfig } from './proxy'; import { requireSignIn } from '../middlewares/auth'; +import { pgBoss } from '../pgboss-worker'; export const router = Router(); chromium.use(stealthPlugin()); - export interface AuthenticatedRequest extends Request { user?: any; } +async function waitForJobCompletion(jobId: string, queueName: string, timeout = 15000): Promise { + return new Promise((resolve, reject) => { + const startTime = Date.now(); + + const checkJobStatus = async () => { + if (Date.now() - startTime > timeout) { + return reject(new Error(`Timeout waiting for job ${jobId} to complete`)); + } + + try { + const job = await pgBoss.getJobById(queueName, jobId); + + if (!job) { + return reject(new Error(`Job ${jobId} not found`)); + } + + if (job.state === 'completed') { + return resolve(job.output); + } + + if (job.state === 'failed') { + return reject(new Error(`Job ${jobId} failed.`)); + } + + setTimeout(checkJobStatus, 200); + } catch (error) { + reject(error); + } + }; + + checkJobStatus(); + }); +} + /** * Logs information about remote browser recording session. */ @@ -33,31 +69,53 @@ router.all('/', requireSignIn, (req, res, next) => { next() // pass control to the next handler }) + /** - * GET endpoint for starting the remote browser recording session. - * returns session's id + * GET endpoint for starting the remote browser recording session + * Waits for job completion */ router.get('/start', requireSignIn, async (req: AuthenticatedRequest, res: Response) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const proxyConfig = await getDecryptedProxyConfig(req.user.id); - // Prepare the proxy options dynamically based on the user's proxy configuration - let proxyOptions: any = {}; // Default to no proxy - - if (proxyConfig.proxy_url) { - // Set the server, and if username & password exist, set those as well - proxyOptions = { - server: proxyConfig.proxy_url, - ...(proxyConfig.proxy_username && proxyConfig.proxy_password && { - username: proxyConfig.proxy_username, - password: proxyConfig.proxy_password, - }), - }; + + try { + await pgBoss.createQueue('initialize-browser-recording'); + + const jobId = await pgBoss.send('initialize-browser-recording', { + userId: req.user.id, + timestamp: new Date().toISOString() + }); + + if (!jobId) { + const browserId = initializeRemoteBrowserForRecording(req.user.id); + return res.send(browserId); + } + + logger.log('info', `Queued browser initialization job: ${jobId}, waiting for completion...`); + + try { + const result = await waitForJobCompletion(jobId, 'initialize-browser-recording', 15000); + + if (result && result.browserId) { + return res.send(result.browserId); + } else { + return res.send(jobId); + } + } catch (waitError: any) { + return res.send(jobId); + } + } catch (error: any) { + logger.log('error', `Failed to queue browser initialization job: ${error.message}`); + + try { + const browserId = initializeRemoteBrowserForRecording(req.user.id); + return res.send( browserId ); + } catch (directError: any) { + logger.log('error', `Direct initialization also failed: ${directError.message}`); + return res.status(500).send('Failed to start recording'); + } } - - const id = initializeRemoteBrowserForRecording(req.user.id); - return res.send(id); }); /** @@ -76,26 +134,65 @@ router.post('/start', requireSignIn, (req: AuthenticatedRequest, res:Response) = * GET endpoint for terminating the remote browser recording session. * returns whether the termination was successful */ -router.get('/stop/:browserId', requireSignIn, async (req, res) => { - const success = await destroyRemoteBrowser(req.params.browserId); - return res.send(success); +router.get('/stop/:browserId', requireSignIn, async (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + + try { + await pgBoss.createQueue('destroy-browser'); + + const jobId = await pgBoss.send('destroy-browser', { + browserId: req.params.browserId, + userId: req.user.id, + timestamp: new Date().toISOString() + }); + + if (!jobId) { + const browserId = initializeRemoteBrowserForRecording(req.user.id); + return res.send( browserId ); + } + + logger.log('info', `Queued browser destruction job: ${jobId}, waiting for completion...`); + + try { + const result = await waitForJobCompletion(jobId, 'destroy-browser', 15000); + + if (result) { + return res.send(result.success); + } else { + return res.send(false); + } + } catch (waitError: any) { + return res.send(false); + } + } catch (error: any) { + logger.log('error', `Failed to stop browser: ${error.message}`); + return res.status(500).send(false); + } }); /** * GET endpoint for getting the id of the active remote browser. */ -router.get('/active', requireSignIn, (req, res) => { - const id = getActiveBrowserId(); +router.get('/active', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + const id = getActiveBrowserIdByState(req.user?.id, "recording"); return res.send(id); }); /** * GET endpoint for getting the current url of the active remote browser. */ -router.get('/active/url', requireSignIn, (req, res) => { - const id = getActiveBrowserId(); +router.get('/active/url', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + const id = getActiveBrowserIdByState(req.user?.id, "recording"); if (id) { - const url = getRemoteBrowserCurrentUrl(id); + const url = getRemoteBrowserCurrentUrl(id, req.user?.id); return res.send(url); } return res.send(null); @@ -104,10 +201,13 @@ router.get('/active/url', requireSignIn, (req, res) => { /** * GET endpoint for getting the current tabs of the active remote browser. */ -router.get('/active/tabs', requireSignIn, (req, res) => { - const id = getActiveBrowserId(); +router.get('/active/tabs', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + const id = getActiveBrowserIdByState(req.user?.id, "recording"); if (id) { - const hosts = getRemoteBrowserCurrentTabs(id); + const hosts = getRemoteBrowserCurrentTabs(id, req.user?.id); return res.send(hosts); } return res.send([]); @@ -116,19 +216,78 @@ router.get('/active/tabs', requireSignIn, (req, res) => { /** * GET endpoint for starting an interpretation of the currently generated workflow. */ -router.get('/interpret', requireSignIn, async (req, res) => { +router.get('/interpret', requireSignIn, async (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + try { - await interpretWholeWorkflow(); - return res.send('interpretation done'); - } catch (e) { - return res.send('interpretation failed'); + await pgBoss.createQueue('interpret-workflow'); + + const jobId = await pgBoss.send('interpret-workflow', { + userId: req.user.id, + timestamp: new Date().toISOString() + }); + + if (!jobId) { + await interpretWholeWorkflow(req.user?.id); + return res.send('interpretation done'); + } + + logger.log('info', `Queued interpret workflow job: ${jobId}, waiting for completion...`); + + try { + const result = await waitForJobCompletion(jobId, 'interpret-workflow', 15000); + + if (result) { + return res.send('interpretation done'); + } else { + return res.send('interpretation failed'); + } + } catch (waitError: any) { + return res.send('interpretation failed'); + } + } catch (error: any) { + logger.log('error', `Failed to stop interpret workflow: ${error.message}`); + return res.status(500).send('interpretation failed'); } }); -/** - * GET endpoint for stopping an ongoing interpretation of the currently generated workflow. - */ -router.get('/interpret/stop', requireSignIn, async (req, res) => { - await stopRunningInterpretation(); - return res.send('interpretation stopped'); +router.get('/interpret/stop', requireSignIn, async (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + + try { + await pgBoss.createQueue('stop-interpretation'); + + const jobId = await pgBoss.send('stop-interpretation', { + userId: req.user.id, + timestamp: new Date().toISOString() + }); + + if (!jobId) { + await stopRunningInterpretation(req.user?.id); + return res.send('interpretation stopped'); + } + + logger.log('info', `Queued stop interpret workflow job: ${jobId}, waiting for completion...`); + + try { + const result = await waitForJobCompletion(jobId, 'stop-interpretation', 15000); + + if (result) { + return res.send('interpretation stopped'); + } else { + return res.send('interpretation failed to stop'); + } + } catch (waitError: any) { + return res.send('interpretation failed to stop'); + } + } catch (error: any) { + logger.log('error', `Failed to stop interpretation: ${error.message}`); + return res.status(500).send('interpretation failed to stop'); + } }); + +export default router; \ No newline at end of file diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 5a46dedd..ffdf0149 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -1,6 +1,6 @@ import { Router } from 'express'; import logger from "../logger"; -import { createRemoteBrowserForRun, destroyRemoteBrowser } from "../browser-management/controller"; +import { createRemoteBrowserForRun, destroyRemoteBrowser, getActiveBrowserIdByState } from "../browser-management/controller"; import { chromium } from 'playwright-extra'; import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import { browserPool } from "../server"; @@ -21,6 +21,8 @@ import { tryCatch } from 'bullmq'; import { encrypt, decrypt } from '../utils/auth'; import { WorkflowFile } from 'maxun-core'; import { Page } from 'playwright'; +import { airtableUpdateTasks, processAirtableUpdates } from '../workflow-management/integrations/airtable'; +import { pgBoss } from '../pgboss-worker'; chromium.use(stealthPlugin()); export const router = Router(); @@ -28,7 +30,7 @@ export const router = Router(); export const processWorkflowActions = async (workflow: any[], checkLimit: boolean = false): Promise => { const processedWorkflow = JSON.parse(JSON.stringify(workflow)); - processedWorkflow.workflow.forEach((pair: any) => { + processedWorkflow.forEach((pair: any) => { pair.what.forEach((action: any) => { // Handle limit validation for scrapeList action if (action.action === 'scrapeList' && checkLimit && Array.isArray(action.args) && action.args.length > 0) { @@ -493,6 +495,8 @@ router.delete('/runs/:id', requireSignIn, async (req: AuthenticatedRequest, res) /** * PUT endpoint for starting a remote browser instance and saving run metadata to the storage. * Making it ready for interpretation and returning a runId. + * + * If the user has reached their browser limit, the run will be queued using PgBoss. */ router.put('/runs/:id', requireSignIn, async (req: AuthenticatedRequest, res) => { try { @@ -524,35 +528,81 @@ router.put('/runs/:id', requireSignIn, async (req: AuthenticatedRequest, res) => }; } - console.log(`Proxy config for run: ${JSON.stringify(proxyOptions)}`) - - const id = createRemoteBrowserForRun(req.user.id); + console.log(`Proxy config for run: ${JSON.stringify(proxyOptions)}`); + // Generate runId first const runId = uuid(); + + // Check if user has reached browser limit + const userBrowserIds = browserPool.getAllBrowserIdsForUser(req.user.id); + const canCreateBrowser = userBrowserIds.length < 2; + + if (canCreateBrowser) { + // User has available browser slots, create it directly + const id = createRemoteBrowserForRun(req.user.id); - const run = await Run.create({ - status: 'running', - name: recording.recording_meta.name, - robotId: recording.id, - robotMetaId: recording.recording_meta.id, - startedAt: new Date().toLocaleString(), - finishedAt: '', - browserId: id, - interpreterSettings: req.body, - log: '', - runId, - runByUserId: req.user.id, - serializableOutput: {}, - binaryOutput: {}, - }); + const run = await Run.create({ + status: 'running', + name: recording.recording_meta.name, + robotId: recording.id, + robotMetaId: recording.recording_meta.id, + startedAt: new Date().toLocaleString(), + finishedAt: '', + browserId: id, + interpreterSettings: req.body, + log: '', + runId, + runByUserId: req.user.id, + serializableOutput: {}, + binaryOutput: {}, + }); - const plainRun = run.toJSON(); + const plainRun = run.toJSON(); - return res.send({ - browserId: id, - runId: plainRun.runId, - robotMetaId: recording.recording_meta.id, - }); + return res.send({ + browserId: id, + runId: plainRun.runId, + robotMetaId: recording.recording_meta.id, + queued: false + }); + } else { + const browserId = getActiveBrowserIdByState(req.user.id, "run") + + if (browserId) { + // User has reached the browser limit, queue the run + try { + // Create the run record with 'queued' status + await Run.create({ + status: 'queued', + name: recording.recording_meta.name, + robotId: recording.id, + robotMetaId: recording.recording_meta.id, + startedAt: new Date().toLocaleString(), + finishedAt: '', + browserId: browserId, // Random will be updated later + interpreterSettings: req.body, + log: 'Run queued - waiting for available browser slot', + runId, + runByUserId: req.user.id, + serializableOutput: {}, + binaryOutput: {}, + }); + + return res.send({ + browserId: browserId, + runId: runId, + robotMetaId: recording.recording_meta.id, + queued: true, + }); + } catch (queueError: any) { + logger.log('error', `Failed to queue run job: ${queueError.message}`); + return res.status(503).send({ error: 'Unable to queue run, please try again later' }); + } + } else { + logger.log('info', "Browser id does not exist"); + return res.send(''); + } + } } catch (e) { const { message } = e as Error; logger.log('info', `Error while creating a run with robot id: ${req.params.id} - ${message}`); @@ -607,73 +657,20 @@ router.post('/runs/run/:id', requireSignIn, async (req: AuthenticatedRequest, re return res.status(404).send(false); } - // interpret the run in active browser - const browser = browserPool.getRemoteBrowser(plainRun.browserId); - let currentPage = browser?.getCurrentPage(); - if (browser && currentPage) { - const workflow = AddGeneratedFlags(recording.recording); - const interpretationInfo = await browser.interpreter.InterpretRecording( - workflow, currentPage, (newPage: Page) => currentPage = newPage, plainRun.interpreterSettings); - const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); - const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(plainRun.browserId); - await run.update({ - ...run, - status: 'success', - finishedAt: new Date().toLocaleString(), - browserId: plainRun.browserId, - log: interpretationInfo.log.join('\n'), - serializableOutput: interpretationInfo.serializableOutput, - binaryOutput: uploadedBinaryOutput, + try { + // Queue the execution job + await pgBoss.createQueue('execute-run'); + + const jobId = await pgBoss.send('execute-run', { + userId: req.user.id, + runId: req.params.id, + browserId: plainRun.browserId }); - - let totalRowsExtracted = 0; - let extractedScreenshotsCount = 0; - let extractedItemsCount = 0; - - if (run.dataValues.binaryOutput && run.dataValues.binaryOutput["item-0"]) { - extractedScreenshotsCount = 1; - } - - if (run.dataValues.serializableOutput && run.dataValues.serializableOutput["item-0"]) { - const itemsArray = run.dataValues.serializableOutput["item-0"]; - extractedItemsCount = itemsArray.length; - - totalRowsExtracted = itemsArray.reduce((total, item) => { - return total + Object.keys(item).length; - }, 0); - } - - console.log(`Extracted Items Count: ${extractedItemsCount}`); - console.log(`Extracted Screenshots Count: ${extractedScreenshotsCount}`); - console.log(`Total Rows Extracted: ${totalRowsExtracted}`); - - capture( - 'maxun-oss-run-created-manual', - { - runId: req.params.id, - user_id: req.user?.id, - created_at: new Date().toISOString(), - status: 'success', - totalRowsExtracted, - extractedItemsCount, - extractedScreenshotsCount, - } - ) - try { - googleSheetUpdateTasks[plainRun.runId] = { - robotId: plainRun.robotMetaId, - runId: plainRun.runId, - status: 'pending', - retries: 5, - }; - processGoogleSheetUpdates(); - } catch (err: any) { - logger.log('error', `Failed to update Google Sheet for run: ${plainRun.runId}: ${err.message}`); - } - return res.send(true); - } else { - throw new Error('Could not destroy browser'); + + logger.log('info', `Queued run execution job with ID: ${jobId} for run: ${req.params.id}`); + } catch (queueError: any) { + logger.log('error', `Failed to queue run execution`); + } } catch (e) { const { message } = e as Error; @@ -890,9 +887,13 @@ router.delete('/schedule/:id', requireSignIn, async (req: AuthenticatedRequest, /** * POST endpoint for aborting a current interpretation of the run. */ -router.post('/runs/abort/:id', requireSignIn, async (req, res) => { +router.post('/runs/abort/:id', requireSignIn, async (req: AuthenticatedRequest, res) => { try { - const run = await Run.findOne({ where: { runId: req.params.id } }); + if (!req.user) { return res.status(401).send({ error: 'Unauthorized' }); } + const run = await Run.findOne({ where: { + runId: req.params.id, + runByUserId: req.user.id, + } }); if (!run) { return res.status(404).send(false); } @@ -927,4 +928,4 @@ router.post('/runs/abort/:id', requireSignIn, async (req, res) => { logger.log('info', `Error while running a robot with name: ${req.params.fileName}_${req.params.runId}.json`); return res.send(false); } -}); \ No newline at end of file +}); diff --git a/server/src/routes/workflow.ts b/server/src/routes/workflow.ts index 3c0e356b..c8ad92d5 100644 --- a/server/src/routes/workflow.ts +++ b/server/src/routes/workflow.ts @@ -7,6 +7,7 @@ import logger from "../logger"; import { browserPool } from "../server"; import { requireSignIn } from '../middlewares/auth'; import Robot from '../models/Robot'; +import { AuthenticatedRequest } from './record'; export const router = Router(); @@ -46,8 +47,9 @@ router.get('/params/:browserId', requireSignIn, (req, res) => { /** * DELETE endpoint for deleting a pair from the generated workflow. */ -router.delete('/pair/:index', requireSignIn, (req, res) => { - const id = browserPool.getActiveBrowserId(); +router.delete('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { return res.status(401).send('User not authenticated'); } + const id = browserPool.getActiveBrowserId(req.user?.id, "recording"); if (id) { const browser = browserPool.getRemoteBrowser(id); if (browser) { @@ -62,8 +64,9 @@ router.delete('/pair/:index', requireSignIn, (req, res) => { /** * POST endpoint for adding a pair to the generated workflow. */ -router.post('/pair/:index', requireSignIn, (req, res) => { - const id = browserPool.getActiveBrowserId(); +router.post('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { return res.status(401).send('User not authenticated'); } + const id = browserPool.getActiveBrowserId(req.user?.id, "recording"); if (id) { const browser = browserPool.getRemoteBrowser(id); logger.log('debug', `Adding pair to workflow`); @@ -82,8 +85,9 @@ router.post('/pair/:index', requireSignIn, (req, res) => { /** * PUT endpoint for updating a pair in the generated workflow. */ -router.put('/pair/:index', requireSignIn, (req, res) => { - const id = browserPool.getActiveBrowserId(); +router.put('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { return res.status(401).send('User not authenticated'); } + const id = browserPool.getActiveBrowserId(req.user?.id, "recording"); if (id) { const browser = browserPool.getRemoteBrowser(id); logger.log('debug', `Updating pair in workflow`); diff --git a/server/src/server.ts b/server/src/server.ts index 581ecadc..0bd46b38 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -18,6 +18,9 @@ import { fork } from 'child_process'; import { capture } from "./utils/analytics"; import swaggerUi from 'swagger-ui-express'; import swaggerSpec from './swagger/config'; + +import session from 'express-session'; + import Run from './models/Run'; const app = express(); @@ -27,6 +30,16 @@ app.use(cors({ })); app.use(express.json()); + +app.use( + session({ + secret: 'your_secret_key', // Replace with a secure secret key + resave: false, // Do not resave the session if it hasn't changed + saveUninitialized: true, // Save new sessions + cookie: { secure: false }, // Set to true if using HTTPS + }) +); + const server = http.createServer(app); /** @@ -65,8 +78,11 @@ readdirSync(path.join(__dirname, 'api')).forEach((r) => { const isProduction = process.env.NODE_ENV === 'production'; const workerPath = path.resolve(__dirname, isProduction ? './worker.js' : './worker.ts'); +const recordingWorkerPath = path.resolve(__dirname, isProduction ? './pgboss-worker.js' : './pgboss-worker.ts'); let workerProcess: any; +let recordingWorkerProcess: any; + if (!isProduction) { workerProcess = fork(workerPath, [], { execArgv: ['--inspect=5859'], @@ -80,6 +96,19 @@ if (!isProduction) { workerProcess.on('exit', (code: any) => { console.log(`Worker exited with code: ${code}`); }); + + recordingWorkerProcess = fork(recordingWorkerPath, [], { + execArgv: ['--inspect=5860'], + }); + recordingWorkerProcess.on('message', (message: any) => { + console.log(`Message from recording worker: ${message}`); + }); + recordingWorkerProcess.on('error', (error: any) => { + console.error(`Error in recording worker: ${error}`); + }); + recordingWorkerProcess.on('exit', (code: any) => { + console.log(`Recording worker exited with code: ${code}`); + }); } app.get('/', function (req, res) { @@ -132,7 +161,8 @@ process.on('SIGINT', async () => { } if (!isProduction) { - workerProcess.kill(); + if (workerProcess) workerProcess.kill(); + if (recordingWorkerProcess) recordingWorkerProcess.kill(); } process.exit(); }); diff --git a/server/src/socket-connection/connection.ts b/server/src/socket-connection/connection.ts index dac7a88e..9fad861b 100644 --- a/server/src/socket-connection/connection.ts +++ b/server/src/socket-connection/connection.ts @@ -1,6 +1,60 @@ import { Namespace, Socket } from 'socket.io'; +import { IncomingMessage } from 'http'; +import { verify, JwtPayload } from 'jsonwebtoken'; import logger from "../logger"; -import registerInputHandlers from '../browser-management/inputHandlers' +import registerInputHandlers from '../browser-management/inputHandlers'; + +interface AuthenticatedIncomingMessage extends IncomingMessage { + user?: JwtPayload | string; +} + +interface AuthenticatedSocket extends Socket { + request: AuthenticatedIncomingMessage; +} + +/** + * Socket.io middleware for authentication + * This is a socket.io specific auth handler that doesn't rely on Express middleware + */ +const socketAuthMiddleware = (socket: Socket, next: (err?: Error) => void) => { + const cookies = socket.handshake.headers.cookie; + if (!cookies) { + return next(new Error('Authentication required')); + } + + const tokenMatch = cookies.split(';').find(c => c.trim().startsWith('token=')); + if (!tokenMatch) { + return next(new Error('Authentication required')); + } + + const token = tokenMatch.split('=')[1]; + if (!token) { + return next(new Error('Authentication required')); + } + + const secret = process.env.JWT_SECRET; + if (!secret) { + return next(new Error('Server configuration error')); + } + + verify(token, secret, (err: any, user: any) => { + if (err) { + logger.log('warn', 'JWT verification error:', err); + return next(new Error('Authentication failed')); + } + + // Normalize payload key + if (user.userId && !user.id) { + user.id = user.userId; + delete user.userId; // temporary: del the old key for clarity + } + + // Attach user to socket request + const authSocket = socket as AuthenticatedSocket; + authSocket.request.user = user; + next(); + }); +}; /** * Opens a websocket canal for duplex data transfer and registers all handlers for this data for the recording session. @@ -13,6 +67,8 @@ export const createSocketConnection = ( io: Namespace, callback: (socket: Socket) => void, ) => { + io.use(socketAuthMiddleware); + const onConnection = async (socket: Socket) => { logger.log('info', "Client connected " + socket.id); registerInputHandlers(socket); @@ -34,6 +90,8 @@ export const createSocketConnectionForRun = ( io: Namespace, callback: (socket: Socket) => void, ) => { + io.use(socketAuthMiddleware); + const onConnection = async (socket: Socket) => { logger.log('info', "Client connected " + socket.id); socket.on('disconnect', () => logger.log('info', "Client disconnected " + socket.id)); @@ -41,4 +99,4 @@ export const createSocketConnectionForRun = ( } io.on('connection', onConnection); -}; +}; \ No newline at end of file diff --git a/server/src/worker.ts b/server/src/worker.ts index 3010a6b2..352fcb13 100644 --- a/server/src/worker.ts +++ b/server/src/worker.ts @@ -9,6 +9,7 @@ const connection = new IORedis({ host: process.env.REDIS_HOST, port: process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT, 10) : 6379, maxRetriesPerRequest: null, + password: process.env.REDIS_PASSWORD ? process.env.REDIS_PASSWORD : undefined, }); connection.on('connect', () => { diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 2eb5f130..aea37126 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -151,8 +151,8 @@ export class WorkflowGenerator { workflow: [], }); socket.on('activeIndex', (data) => this.generatedData.lastIndex = parseInt(data)); - socket.on('decision', async ({ pair, actionType, decision }) => { - const id = browserPool.getActiveBrowserId(); + socket.on('decision', async ({ pair, actionType, decision, userId }) => { + const id = browserPool.getActiveBrowserId(userId, "recording"); if (id) { // const activeBrowser = browserPool.getRemoteBrowser(id); // const currentPage = activeBrowser?.getCurrentPage(); @@ -708,6 +708,7 @@ export class WorkflowGenerator { public updateSocket = (socket: Socket): void => { this.socket = socket; this.registerEventHandlers(socket); + this.initializeSocketListeners(); }; /** @@ -825,6 +826,7 @@ export class WorkflowGenerator { selectors?.testIdSelector, selectors?.id, selectors?.hrefSelector, + selectors?.relSelector, selectors?.accessibilitySelector, selectors?.attrSelector ] diff --git a/server/src/workflow-management/integrations/airtable.ts b/server/src/workflow-management/integrations/airtable.ts new file mode 100644 index 00000000..93474721 --- /dev/null +++ b/server/src/workflow-management/integrations/airtable.ts @@ -0,0 +1,324 @@ +import Airtable from "airtable"; +import axios from "axios"; +import logger from "../../logger"; +import Run from "../../models/Run"; +import Robot from "../../models/Robot"; + +interface AirtableUpdateTask { + robotId: string; + runId: string; + status: 'pending' | 'completed' | 'failed'; + retries: number; +} + +const MAX_RETRIES = 3; +const BASE_API_DELAY = 2000; + +export let airtableUpdateTasks: { [runId: string]: AirtableUpdateTask } = {}; + +async function refreshAirtableToken(refreshToken: string) { + try { + const response = await axios.post( + "https://airtable.com/oauth2/v1/token", + new URLSearchParams({ + grant_type: "refresh_token", + refresh_token: refreshToken, + client_id: process.env.AIRTABLE_CLIENT_ID!, + }), + { + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + } + ); + + return response.data; + } catch (error: any) { + logger.log("error", `Failed to refresh Airtable token: ${error.message}`); + throw new Error(`Token refresh failed: ${error.response?.data?.error_description || error.message}`); + } +} + +export async function updateAirtable(robotId: string, runId: string) { + try { + const run = await Run.findOne({ where: { runId } }); + if (!run) throw new Error(`Run not found for runId: ${runId}`); + + const plainRun = run.toJSON(); + if (plainRun.status !== 'success') { + console.log('Run status is not success'); + return; + } + + let data: { [key: string]: any }[] = []; + if (plainRun.serializableOutput?.['item-0']) { + data = plainRun.serializableOutput['item-0'] as { [key: string]: any }[]; + } else if (plainRun.binaryOutput?.['item-0']) { + data = [{ "File URL": plainRun.binaryOutput['item-0'] }]; + } + + const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + if (!robot) throw new Error(`Robot not found for robotId: ${robotId}`); + + const plainRobot = robot.toJSON(); + if (plainRobot.airtable_base_id && plainRobot.airtable_table_name && plainRobot.airtable_table_id) { + console.log(`Writing to Airtable base ${plainRobot.airtable_base_id}`); + await writeDataToAirtable( + robotId, + plainRobot.airtable_base_id, + plainRobot.airtable_table_name, + plainRobot.airtable_table_id, + data + ); + console.log(`Data written to Airtable for ${robotId}`); + } + } catch (error: any) { + console.error(`Airtable update failed: ${error.message}`); + throw error; + } +} + +async function withTokenRefresh(robotId: string, apiCall: (accessToken: string) => Promise): Promise { + const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + if (!robot) throw new Error(`Robot not found for robotId: ${robotId}`); + + let accessToken = robot.get('airtable_access_token') as string; + let refreshToken = robot.get('airtable_refresh_token') as string; + + if (!accessToken || !refreshToken) { + throw new Error('Airtable credentials not configured'); + } + + try { + return await apiCall(accessToken); + } catch (error: any) { + if (error.response?.status === 401 || + (error.statusCode === 401) || + error.message.includes('unauthorized') || + error.message.includes('expired')) { + + logger.log("info", `Refreshing expired Airtable token for robot: ${robotId}`); + + try { + const tokens = await refreshAirtableToken(refreshToken); + + await robot.update({ + airtable_access_token: tokens.access_token, + airtable_refresh_token: tokens.refresh_token || refreshToken + }); + + return await apiCall(tokens.access_token); + } catch (refreshError: any) { + logger.log("error", `Failed to refresh token: ${refreshError.message}`); + throw new Error(`Token refresh failed: ${refreshError.message}`); + } + } + + throw error; + } +} + +export async function writeDataToAirtable( + robotId: string, + baseId: string, + tableName: string, + tableId: string, + data: any[] +) { + try { + return await withTokenRefresh(robotId, async (accessToken: string) => { + const airtable = new Airtable({ apiKey: accessToken }); + const base = airtable.base(baseId); + + const existingFields = await getExistingFields(base, tableName); + console.log(`Found ${existingFields.length} existing fields in Airtable`); + + const dataFields = [...new Set(data.flatMap(row => Object.keys(row)))]; + console.log(`Found ${dataFields.length} fields in data: ${dataFields.join(', ')}`); + + const missingFields = dataFields.filter(field => !existingFields.includes(field)); + console.log(`Found ${missingFields.length} missing fields: ${missingFields.join(', ')}`); + + for (const field of missingFields) { + const sampleRow = data.find(row => field in row); + if (sampleRow) { + const sampleValue = sampleRow[field]; + try { + await createAirtableField(baseId, tableName, field, sampleValue, accessToken, tableId); + console.log(`Successfully created field: ${field}`); + } catch (fieldError: any) { + console.warn(`Warning: Could not create field "${field}": ${fieldError.message}`); + } + } + } + + await deleteEmptyRecords(base, tableName); + + const BATCH_SIZE = 10; + for (let i = 0; i < data.length; i += BATCH_SIZE) { + const batch = data.slice(i, i + BATCH_SIZE); + await retryableAirtableWrite(base, tableName, batch); + } + + logger.log('info', `Successfully wrote ${data.length} records to Airtable`); + }); + } catch (error: any) { + logger.log('error', `Airtable write failed: ${error.message}`); + throw error; + } +} + +async function deleteEmptyRecords(base: Airtable.Base, tableName: string): Promise { + console.log('Checking for empty records to clear...'); + + try { + const existingRecords = await base(tableName).select().all(); + console.log(`Found ${existingRecords.length} total records`); + + const emptyRecords = existingRecords.filter(record => { + const fields = record.fields; + return !fields || Object.keys(fields).length === 0 || + Object.values(fields).every(value => + value === null || value === undefined || value === ''); + }); + + if (emptyRecords.length > 0) { + const BATCH_SIZE = 10; + for (let i = 0; i < emptyRecords.length; i += BATCH_SIZE) { + const batch = emptyRecords.slice(i, i + BATCH_SIZE); + const recordIds = batch.map(record => record.id); + await base(tableName).destroy(recordIds); + } + } + } catch (error: any) { + console.warn(`Warning: Could not clear empty records: ${error.message}`); + console.warn('Will continue without deleting empty records'); + } +} + +async function retryableAirtableWrite( + base: Airtable.Base, + tableName: string, + batch: any[], + retries = MAX_RETRIES +): Promise { + try { + await base(tableName).create(batch.map(row => ({ fields: row }))); + } catch (error) { + if (retries > 0) { + await new Promise(resolve => setTimeout(resolve, BASE_API_DELAY)); + return retryableAirtableWrite(base, tableName, batch, retries - 1); + } + throw error; + } +} + +// Helper functions +async function getExistingFields(base: Airtable.Base, tableName: string): Promise { + try { + const records = await base(tableName).select({ pageSize: 5 }).firstPage(); + if (records.length > 0) { + const fieldNames = new Set(); + records.forEach(record => { + Object.keys(record.fields).forEach(field => fieldNames.add(field)); + }); + + const headers = Array.from(fieldNames); + console.log(`Found ${headers.length} headers from records: ${headers.join(', ')}`); + return headers; + } + return []; + } catch (error) { + return []; + } +} + +async function createAirtableField( + baseId: string, + tableName: string, + fieldName: string, + sampleValue: any, + accessToken: string, + tableId: string, + retries = MAX_RETRIES +): Promise { + try { + const fieldType = inferFieldType(sampleValue); + + console.log(`Creating field ${fieldName} with type ${fieldType}`); + + const response = await axios.post( + `https://api.airtable.com/v0/meta/bases/${baseId}/tables/${tableId}/fields`, + { name: fieldName, type: fieldType }, + { headers: { Authorization: `Bearer ${accessToken}` } } + ); + + logger.log('info', `Created field: ${fieldName} (${fieldType})`); + return response.data; + } catch (error: any) { + if (retries > 0 && error.response?.status === 429) { + await new Promise(resolve => setTimeout(resolve, BASE_API_DELAY)); + return createAirtableField(baseId, tableName, fieldName, sampleValue, accessToken, tableId, retries - 1); + } + + if (error.response?.status === 422) { + console.log(`Field ${fieldName} may already exist or has validation issues`); + return; + } + + const errorMessage = error.response?.data?.error?.message || error.message; + const statusCode = error.response?.status || 'No Status Code'; + console.warn(`Field creation issue (${statusCode}): ${errorMessage}`); + } +} + +function inferFieldType(value: any): string { + if (value === null || value === undefined) return 'singleLineText'; + if (typeof value === 'number') return 'number'; + if (typeof value === 'boolean') return 'checkbox'; + if (value instanceof Date) return 'dateTime'; + if (Array.isArray(value)) { + return value.length > 0 && typeof value[0] === 'object' ? 'multipleRecordLinks' : 'multipleSelects'; + } + if (typeof value === 'string' && isValidUrl(value)) return 'url'; + return 'singleLineText'; +} + +function isValidUrl(str: string): boolean { + try { + new URL(str); + return true; + } catch (_) { + return false; + } +} + +export const processAirtableUpdates = async () => { + while (true) { + let hasPendingTasks = false; + + for (const runId in airtableUpdateTasks) { + const task = airtableUpdateTasks[runId]; + if (task.status !== 'pending') continue; + + hasPendingTasks = true; + try { + await updateAirtable(task.robotId, task.runId); + delete airtableUpdateTasks[runId]; + } catch (error: any) { + task.retries += 1; + if (task.retries >= MAX_RETRIES) { + task.status = 'failed'; + logger.log('error', `Permanent failure for run ${runId}: ${error.message}`); + } + } + } + + if (!hasPendingTasks) { + console.log('No pending Airtable update tasks, exiting processor'); + break; + } + + await new Promise(resolve => setTimeout(resolve, 5000)); + } +}; \ No newline at end of file diff --git a/server/src/workflow-management/integrations/gsheet.ts b/server/src/workflow-management/integrations/gsheet.ts index 47dda84e..b492a7df 100644 --- a/server/src/workflow-management/integrations/gsheet.ts +++ b/server/src/workflow-management/integrations/gsheet.ts @@ -49,11 +49,7 @@ export async function updateGoogleSheet(robotId: string, runId: string) { if (plainRobot.google_sheet_email && spreadsheetId) { console.log(`Preparing to write data to Google Sheet for robot: ${robotId}, spreadsheetId: ${spreadsheetId}`); - const headers = Object.keys(data[0]); - const rows = data.map((row: { [key: string]: any }) => Object.values(row)); - const outputData = [headers, ...rows]; - - await writeDataToSheet(robotId, spreadsheetId, outputData); + await writeDataToSheet(robotId, spreadsheetId, data); console.log(`Data written to Google Sheet successfully for Robot: ${robotId} and Run: ${runId}`); } else { console.log('Google Sheets integration not configured.'); @@ -102,7 +98,43 @@ export async function writeDataToSheet(robotId: string, spreadsheetId: string, d const sheets = google.sheets({ version: 'v4', auth: oauth2Client }); - const resource = { values: data }; + const checkResponse = await sheets.spreadsheets.values.get({ + spreadsheetId, + range: 'Sheet1!1:1', + }); + + if (!data || data.length === 0) { + console.log('No data to write. Exiting early.'); + return; + } + + const expectedHeaders = Object.keys(data[0]); + + const rows = data.map(item => Object.values(item)); + + const existingHeaders = + checkResponse.data.values && + checkResponse.data.values[0] ? + checkResponse.data.values[0].map(String) : + []; + + const isSheetEmpty = existingHeaders.length === 0; + + const headersMatch = + !isSheetEmpty && + existingHeaders.length === expectedHeaders.length && + expectedHeaders.every((header, index) => existingHeaders[index] === header); + + let resource; + + if (isSheetEmpty || !headersMatch) { + resource = { values: [expectedHeaders, ...rows] }; + console.log('Including headers in the append operation.'); + } else { + resource = { values: rows }; + console.log('Headers already exist and match, only appending data rows.'); + } + console.log('Attempting to write to spreadsheet:', spreadsheetId); const response = await sheets.spreadsheets.values.append({ diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index ade7d969..b67e1ca0 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -92,7 +92,7 @@ function AddGeneratedFlags(workflow: WorkflowFile) { return copy; }; -async function executeRun(id: string) { +async function executeRun(id: string, userId: string) { try { const run = await Run.findOne({ where: { runId: id } }); if (!run) { @@ -114,7 +114,7 @@ async function executeRun(id: string) { plainRun.status = 'running'; - const browser = browserPool.getRemoteBrowser(plainRun.browserId); + const browser = browserPool.getRemoteBrowser(userId); if (!browser) { throw new Error('Could not access browser'); } @@ -132,7 +132,7 @@ async function executeRun(id: string) { const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(plainRun.browserId); + await destroyRemoteBrowser(plainRun.browserId, userId); await run.update({ ...run, @@ -207,22 +207,22 @@ async function executeRun(id: string) { } } -async function readyForRunHandler(browserId: string, id: string) { +async function readyForRunHandler(browserId: string, id: string, userId: string) { try { - const interpretation = await executeRun(id); + const interpretation = await executeRun(id, userId); if (interpretation) { logger.log('info', `Interpretation of ${id} succeeded`); } else { logger.log('error', `Interpretation of ${id} failed`); - await destroyRemoteBrowser(browserId); + await destroyRemoteBrowser(browserId, userId); } resetRecordingState(browserId, id); } catch (error: any) { logger.error(`Error during readyForRunHandler: ${error.message}`); - await destroyRemoteBrowser(browserId); + await destroyRemoteBrowser(browserId, userId); } } @@ -245,12 +245,12 @@ export async function handleRunRecording(id: string, userId: string) { rejectUnauthorized: false }); - socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId)); + socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId, userId)); logger.log('info', `Running robot: ${id}`); socket.on('disconnect', () => { - cleanupSocketListeners(socket, browserId, newRunId); + cleanupSocketListeners(socket, browserId, newRunId, userId); }); } catch (error: any) { @@ -258,8 +258,8 @@ export async function handleRunRecording(id: string, userId: string) { } } -function cleanupSocketListeners(socket: Socket, browserId: string, id: string) { - socket.off('ready-for-run', () => readyForRunHandler(browserId, id)); +function cleanupSocketListeners(socket: Socket, browserId: string, id: string, userId: string) { + socket.off('ready-for-run', () => readyForRunHandler(browserId, id, userId)); logger.log('info', `Cleaned up listeners for browserId: ${browserId}, runId: ${id}`); } diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts index 20a6a529..80ed55f3 100644 --- a/server/src/workflow-management/selector.ts +++ b/server/src/workflow-management/selector.ts @@ -24,54 +24,64 @@ export const getElementInformation = async ( const elementInfo = await page.evaluate( async ({ x, y }) => { const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // First, get the element at the clicked coordinates in the main document let element = document.elementFromPoint(x, y) as HTMLElement; if (!element) return null; - - // Track the deepest element found + let deepestElement = element; - - // Function to traverse shadow DOM + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; let shadowRoot = current.shadowRoot; let deepest = current; - - while (shadowRoot) { + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; shadowRoot = current.shadowRoot; + depth++; } - + return deepest; }; - - // Handle iframe traversal + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + if (element.tagName === 'IFRAME') { let currentIframe = element as HTMLIFrameElement; - - while (currentIframe) { + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; if (!iframeDocument) break; - + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Update deepest element and check for shadow DOM + deepestElement = traverseShadowDOM(iframeElement); - - // Continue traversing if we found another iframe + if (iframeElement.tagName === 'IFRAME') { currentIframe = iframeElement as HTMLIFrameElement; + depth++; } else { break; } @@ -80,28 +90,81 @@ export const getElementInformation = async ( break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); } else { - // If not an iframe, check for shadow DOM deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; - // Get the element and its iframe path const el = getDeepestElementFromPoint(x, y); if (el) { - // Handle potential anchor parent - const { parentElement } = el; - const targetElement = parentElement?.tagName === 'A' ? parentElement : el; + // Prioritize Link (DO NOT REMOVE) + // const { parentElement } = el; + // const targetElement = parentElement?.tagName === 'A' ? parentElement : el; + + const targetElement = el; - // Get containing context information const ownerDocument = targetElement.ownerDocument; const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; const isIframeContent = Boolean(frameElement); + const isFrameContent = frameElement?.tagName === 'FRAME'; - // Get the containing shadow root if any const containingShadowRoot = targetElement.getRootNode() as ShadowRoot; const isShadowRoot = containingShadowRoot instanceof ShadowRoot; @@ -115,8 +178,11 @@ export const getElementInformation = async ( innerHTML?: string; outerHTML?: string; isIframeContent?: boolean; + isFrameContent?: boolean; iframeURL?: string; + frameURL?: string; iframeIndex?: number; + frameIndex?: number; frameHierarchy?: string[]; isShadowRoot?: boolean; shadowRootMode?: string; @@ -124,43 +190,47 @@ export const getElementInformation = async ( } = { tagName: targetElement?.tagName ?? '', isIframeContent, + isFrameContent, isShadowRoot }; - if (isIframeContent) { - // Include iframe specific information - info.iframeURL = frameElement.src; + if (isIframeContent || isFrameContent) { + if (isIframeContent) { + info.iframeURL = (frameElement as HTMLIFrameElement).src; + } else { + info.frameURL = (frameElement).src; + } - // Calculate the frame's position in the hierarchy let currentFrame = frameElement; const frameHierarchy: string[] = []; let frameIndex = 0; while (currentFrame) { - // Store the frame's identifier (src, id, or index) frameHierarchy.unshift( currentFrame.id || + currentFrame.getAttribute('name') || currentFrame.src || - `iframe[${frameIndex}]` + `${currentFrame.tagName.toLowerCase()}[${frameIndex}]` ); - // Move up to parent frame if it exists const parentDoc = currentFrame.ownerDocument; currentFrame = parentDoc?.defaultView?.frameElement as HTMLIFrameElement; frameIndex++; } info.frameHierarchy = frameHierarchy; - info.iframeIndex = frameIndex - 1; // Adjust for 0-based index + if (isIframeContent) { + info.iframeIndex = frameIndex - 1; + } else { + info.frameIndex = frameIndex - 1; + } } if (isShadowRoot) { - // Include shadow root specific information info.shadowRootMode = containingShadowRoot.mode; info.shadowRootContent = containingShadowRoot.innerHTML; } - // Collect element attributes and properties if (targetElement) { info.attributes = Array.from(targetElement.attributes).reduce( (acc, attr) => { @@ -206,56 +276,65 @@ export const getElementInformation = async ( } else { const elementInfo = await page.evaluate( async ({ x, y }) => { - // Enhanced helper function to get element from point including shadow DOM const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // First, get the element at the clicked coordinates in the main document let element = document.elementFromPoint(x, y) as HTMLElement; if (!element) return null; - - // Track the deepest element found + let deepestElement = element; - - // Function to traverse shadow DOM + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; let shadowRoot = current.shadowRoot; let deepest = current; - - while (shadowRoot) { + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; shadowRoot = current.shadowRoot; + depth++; } - + return deepest; }; - - // Handle iframe traversal + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + if (element.tagName === 'IFRAME') { let currentIframe = element as HTMLIFrameElement; - - while (currentIframe) { + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; if (!iframeDocument) break; - + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Update deepest element and check for shadow DOM + deepestElement = traverseShadowDOM(iframeElement); - - // Continue traversing if we found another iframe + if (iframeElement.tagName === 'IFRAME') { currentIframe = iframeElement as HTMLIFrameElement; + depth++; } else { break; } @@ -264,14 +343,67 @@ export const getElementInformation = async ( break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); } else { - // If not an iframe, check for shadow DOM deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; - + const originalEl = getDeepestElementFromPoint(x, y); if (originalEl) { let element = originalEl; @@ -301,7 +433,7 @@ export const getElementInformation = async ( const significantOverlap = (childRect.width * childRect.height) / - (parentRect.width * parentRect.height) > 0.5; + (parentRect.width * parentRect.height) > 0.1; if (fullyContained && significantOverlap) { const nextParent = element.parentElement; @@ -318,8 +450,9 @@ export const getElementInformation = async ( } const ownerDocument = element.ownerDocument; - const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; + const frameElement = ownerDocument?.defaultView?.frameElement; const isIframeContent = Boolean(frameElement); + const isFrameContent = frameElement?.tagName === 'FRAME'; const containingShadowRoot = element.getRootNode() as ShadowRoot; const isShadowRoot = containingShadowRoot instanceof ShadowRoot; @@ -334,8 +467,11 @@ export const getElementInformation = async ( innerHTML?: string; outerHTML?: string; isIframeContent?: boolean; + isFrameContent?: boolean; iframeURL?: string; + frameURL?: string; iframeIndex?: number; + frameIndex?: number; frameHierarchy?: string[]; isShadowRoot?: boolean; shadowRootMode?: string; @@ -343,44 +479,48 @@ export const getElementInformation = async ( } = { tagName: element?.tagName ?? '', isIframeContent, + isFrameContent, isShadowRoot }; - if (isIframeContent) { - // Include iframe specific information - info.iframeURL = frameElement.src; + if (isIframeContent || isFrameContent) { + if (isIframeContent && !isFrameContent) { + info.iframeURL = (frameElement as HTMLIFrameElement).src; + } else if (isFrameContent) { + info.frameURL = (frameElement as HTMLFrameElement).src; + } - // Calculate the frame's position in the hierarchy let currentFrame = frameElement; const frameHierarchy: string[] = []; let frameIndex = 0; while (currentFrame) { - // Store the frame's identifier (src, id, or index) frameHierarchy.unshift( currentFrame.id || - currentFrame.src || - `iframe[${frameIndex}]` + currentFrame.getAttribute('name') || + (currentFrame as HTMLFrameElement).src || + `${currentFrame.tagName.toLowerCase()}[${frameIndex}]` ); - // Move up to parent frame if it exists const parentDoc = currentFrame.ownerDocument; - currentFrame = parentDoc?.defaultView?.frameElement as HTMLIFrameElement; + currentFrame = parentDoc?.defaultView?.frameElement; frameIndex++; } info.frameHierarchy = frameHierarchy; - info.iframeIndex = frameIndex - 1; // Adjust for 0-based index - }; + if (isIframeContent && !isFrameContent) { + info.iframeIndex = frameIndex - 1; + } else if (isFrameContent) { + info.frameIndex = frameIndex - 1; + } + } if (isShadowRoot) { - // Include shadow root specific information info.shadowRootMode = containingShadowRoot.mode; info.shadowRootContent = containingShadowRoot.innerHTML; } if (element) { - // Get attributes including those from shadow DOM context info.attributes = Array.from(element.attributes).reduce( (acc, attr) => { acc[attr.name] = attr.value; @@ -389,14 +529,21 @@ export const getElementInformation = async ( {} as Record ); - // Handle specific element types if (element.tagName === 'A') { info.url = (element as HTMLAnchorElement).href; info.innerText = element.textContent ?? ''; } else if (element.tagName === 'IMG') { info.imageUrl = (element as HTMLImageElement).src; + } else if (element?.tagName === 'SELECT') { + const selectElement = element as HTMLSelectElement; + info.innerText = selectElement.options[selectElement.selectedIndex]?.text ?? ''; + info.attributes = { + ...info.attributes, + selectedValue: selectElement.value, + }; + } else if (element?.tagName === 'INPUT' && ((element as HTMLInputElement).type === 'time' || (element as HTMLInputElement).type === 'date')) { + info.innerText = (element as HTMLInputElement).value; } else { - // Handle text content with proper null checking info.hasOnlyText = element.children.length === 0 && (element.textContent !== null && element.textContent.trim().length > 0); @@ -427,56 +574,65 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector if (!getList || listSelector !== '') { const rect = await page.evaluate( async ({ x, y }) => { - // Enhanced helper function to get element from point including iframes const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // First, get the element at the clicked coordinates in the main document let element = document.elementFromPoint(x, y) as HTMLElement; if (!element) return null; - - // Track the deepest element found + let deepestElement = element; - - // Function to traverse shadow DOM + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; let shadowRoot = current.shadowRoot; let deepest = current; - - while (shadowRoot) { + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; shadowRoot = current.shadowRoot; + depth++; } - + return deepest; }; - - // Handle iframe traversal + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + if (element.tagName === 'IFRAME') { let currentIframe = element as HTMLIFrameElement; - - while (currentIframe) { + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; if (!iframeDocument) break; - + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Update deepest element and check for shadow DOM + deepestElement = traverseShadowDOM(iframeElement); - - // Continue traversing if we found another iframe + if (iframeElement.tagName === 'IFRAME') { currentIframe = iframeElement as HTMLIFrameElement; + depth++; } else { break; } @@ -485,18 +641,74 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); } else { - // If not an iframe, check for shadow DOM deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; const el = getDeepestElementFromPoint(x, y); if (el) { - const { parentElement } = el; - const element = parentElement?.tagName === 'A' ? parentElement : el; + // Prioritize Link (DO NOT REMOVE) + // const { parentElement } = el; + // const element = parentElement?.tagName === 'A' ? parentElement : el; + + const element = el; const rectangle = element?.getBoundingClientRect(); if (rectangle) { const createRectObject = (rect: DOMRect) => ({ @@ -557,54 +769,64 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector const rect = await page.evaluate( async ({ x, y }) => { const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // First, get the element at the clicked coordinates in the main document let element = document.elementFromPoint(x, y) as HTMLElement; if (!element) return null; - - // Track the deepest element found + let deepestElement = element; - - // Function to traverse shadow DOM + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; let shadowRoot = current.shadowRoot; let deepest = current; - - while (shadowRoot) { + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; shadowRoot = current.shadowRoot; + depth++; } - + return deepest; }; - - // Handle iframe traversal + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + if (element.tagName === 'IFRAME') { let currentIframe = element as HTMLIFrameElement; - - while (currentIframe) { + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; if (!iframeDocument) break; - + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Update deepest element and check for shadow DOM + deepestElement = traverseShadowDOM(iframeElement); - - // Continue traversing if we found another iframe + if (iframeElement.tagName === 'IFRAME') { currentIframe = iframeElement as HTMLIFrameElement; + depth++; } else { break; } @@ -613,11 +835,64 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); } else { - // If not an iframe, check for shadow DOM deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; @@ -650,7 +925,7 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector const significantOverlap = (childRect.width * childRect.height) / - (parentRect.width * parentRect.height) > 0.5; + (parentRect.width * parentRect.height) > 0.1; if (fullyContained && significantOverlap) { const nextParent = element.parentElement; @@ -691,12 +966,12 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector } }); - // Same coordinate adjustment for iframe elements as above + // For elements inside iframes or frames, adjust coordinates relative to the top window let adjustedRect = createRectObject(rectangle); let currentWindow = element.ownerDocument.defaultView; while (currentWindow !== window.top) { - const frameElement = currentWindow?.frameElement as HTMLIFrameElement; + const frameElement = currentWindow?.frameElement; if (!frameElement) break; const frameRect = frameElement.getBoundingClientRect(); @@ -1205,65 +1480,63 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { } const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // Helper function to traverse shadow DOM - const traverseShadowDOM = (element: HTMLElement, depth: number = 0): HTMLElement => { - const MAX_SHADOW_DEPTH = 4; - let current = element; - let deepest = current; - - while (current && depth < MAX_SHADOW_DEPTH) { - const shadowRoot = current.shadowRoot; - if (!shadowRoot) break; - - const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; - if (!shadowElement || shadowElement === current) break; - - deepest = shadowElement; - current = shadowElement; - depth++; - } - - return deepest; - }; - - // Start with the element at the specified coordinates let element = document.elementFromPoint(x, y) as HTMLElement; if (!element) return null; - - // Initialize tracking variables + let deepestElement = element; - let depth = 0; - const MAX_IFRAME_DEPTH = 4; - - // First check if the initial element has a shadow root - deepestElement = traverseShadowDOM(element); - - // If it's an iframe, traverse through iframe hierarchy - if (deepestElement.tagName === 'IFRAME') { - let currentIframe = deepestElement as HTMLIFrameElement; - + + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { + let current = element; + let shadowRoot = current.shadowRoot; + let deepest = current; + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { + const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; + if (!shadowElement || shadowElement === current) break; + + deepest = shadowElement; + current = shadowElement; + shadowRoot = current.shadowRoot; + depth++; + } + + return deepest; + }; + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + + if (element.tagName === 'IFRAME') { + let currentIframe = element as HTMLIFrameElement; + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - - // Access iframe's document - const iframeDoc = currentIframe.contentDocument || currentIframe.contentWindow?.document; - if (!iframeDoc) break; - - // Get element at transformed coordinates in iframe - const iframeElement = iframeDoc.elementFromPoint(iframeX, iframeY) as HTMLElement; + + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; + if (!iframeDocument) break; + + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Check for shadow DOM within iframe - const shadowResult = traverseShadowDOM(iframeElement); - deepestElement = shadowResult; - - // If we found another iframe, continue traversing - if (shadowResult.tagName === 'IFRAME') { - currentIframe = shadowResult as HTMLIFrameElement; + + deepestElement = traverseShadowDOM(iframeElement); + + if (iframeElement.tagName === 'IFRAME') { + currentIframe = iframeElement as HTMLIFrameElement; depth++; } else { break; @@ -1273,74 +1546,129 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); + } else { + deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; - const genSelectorForIframe = (element: HTMLElement) => { - // Helper function to get the complete iframe path up to document root - const getIframePath = (el: HTMLElement) => { - const path = []; - let current = el; - let depth = 0; - const MAX_DEPTH = 4; + + const genSelectorForFrame = (element: HTMLElement) => { + const getFramePath = (el: HTMLElement) => { + const path = []; + let current = el; + let depth = 0; + const MAX_DEPTH = 4; + + while (current && depth < MAX_DEPTH) { + const ownerDocument = current.ownerDocument; - while (current && depth < MAX_DEPTH) { - // Get the owner document of the current element - const ownerDocument = current.ownerDocument; - - // Check if this document belongs to an iframe - const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; - - if (frameElement) { - path.unshift({ - frame: frameElement, - document: ownerDocument, - element: current - }); - // Move up to the parent document's element (the iframe) - current = frameElement; - depth++; - } else { - break; - } + const frameElement = + ownerDocument?.defaultView?.frameElement as HTMLIFrameElement | HTMLFrameElement; + + if (frameElement) { + path.unshift({ + frame: frameElement, + document: ownerDocument, + element: current, + isFrame: frameElement.tagName === 'FRAME' + }); + + current = frameElement; + depth++; + } else { + break; } - return path; + } + return path; }; - - const iframePath = getIframePath(element); - if (iframePath.length === 0) return null; - + + const framePath = getFramePath(element); + if (framePath.length === 0) return null; + try { - const selectorParts: string[] = []; + const selectorParts: string[] = []; + + framePath.forEach((context, index) => { + const frameSelector = context.isFrame ? + `frame[name="${context.frame.getAttribute('name')}"]` : + finder(context.frame, { + root: index === 0 ? document.body : + (framePath[index - 1].document.body as Element) + }); - // Generate selector for each iframe boundary - iframePath.forEach((context, index) => { - // Get selector for the iframe element - const frameSelector = finder(context.frame, { - root: index === 0 ? document.body : - (iframePath[index - 1].document.body as Element) - }); - - // For the last context, get selector for target element - if (index === iframePath.length - 1) { - const elementSelector = finder(element, { - root: context.document.body as Element - }); - selectorParts.push(`${frameSelector} :>> ${elementSelector}`); - } else { - selectorParts.push(frameSelector); - } - }); - - return { - fullSelector: selectorParts.join(' :>> '), - isFrameContent: true - }; + if (index === framePath.length - 1) { + const elementSelector = finder(element, { + root: context.document.body as Element + }); + selectorParts.push(`${frameSelector} :>> ${elementSelector}`); + } else { + selectorParts.push(frameSelector); + } + }); + + return { + fullSelector: selectorParts.join(' :>> '), + isFrameContent: true + }; } catch (e) { - console.warn('Error generating iframe selector:', e); - return null; + console.warn('Error generating frame selector:', e); + return null; } }; @@ -1424,9 +1752,25 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { } - const iframeSelector = genSelectorForIframe(element); + let iframeSelector = null; + try { + // Check if element is within frame/iframe + const isInFrame = element.ownerDocument !== document; + const isInFrameset = () => { + let doc = element.ownerDocument; + return doc.querySelectorAll('frameset').length > 0; + }; + + if (isInFrame || isInFrameset()) { + iframeSelector = genSelectorForFrame(element); + } + } catch (e) { + console.warn('Error detecting frames:', e); + } + const shadowSelector = genSelectorForShadowDOM(element); + const relSelector = genSelectorForAttributes(element, ['rel']); const hrefSelector = genSelectorForAttributes(element, ['href']); const formSelector = genSelectorForAttributes(element, [ 'name', @@ -1473,6 +1817,7 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { hrefSelector, accessibilitySelector, formSelector, + relSelector, iframeSelector: iframeSelector ? { full: iframeSelector.fullSelector, isIframe: iframeSelector.isFrameContent, @@ -1509,6 +1854,11 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { function genSelectorForAttributes(element: HTMLElement, attributes: string[]) { let selector = null; try { + if (attributes.includes('rel') && element.hasAttribute('rel')) { + const relValue = element.getAttribute('rel'); + return `[rel="${relValue}"]`; + } + selector = isAttributesDefined(element, attributes) ? finder(element, { idName: () => false, // Don't use the id to generate a selector @@ -1531,9 +1881,12 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => { hoveredElement != null && !hoveredElement.closest('#overlay-controls') != null ) { - const { parentElement } = hoveredElement; + // Prioritize Link (DO NOT REMOVE) + // const { parentElement } = hoveredElement; // Match the logic in recorder.ts for link clicks - const element = parentElement?.tagName === 'A' ? parentElement : hoveredElement; + // const element = parentElement?.tagName === 'A' ? parentElement : hoveredElement; + + const element = hoveredElement; const generatedSelectors = genSelectors(element); return generatedSelectors; } @@ -1562,9 +1915,9 @@ interface SelectorResult { export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates, listSelector: string): Promise => { interface DOMContext { - type: 'iframe' | 'shadow'; + type: 'iframe' | 'frame' | 'shadow'; element: HTMLElement; - container: HTMLIFrameElement | ShadowRoot; + container: HTMLIFrameElement | HTMLFrameElement | ShadowRoot; host?: HTMLElement; document?: Document; } @@ -1573,53 +1926,63 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates if (!listSelector) { const selectors = await page.evaluate(({ x, y }: { x: number, y: number }) => { const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // Function to traverse shadow DOM + let element = document.elementFromPoint(x, y) as HTMLElement; + if (!element) return null; + + let deepestElement = element; + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; - let deepest = current; let shadowRoot = current.shadowRoot; - - while (shadowRoot) { + let deepest = current; + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; shadowRoot = current.shadowRoot; + depth++; } - + return deepest; }; - - // Start with the element at coordinates - let element = document.elementFromPoint(x, y) as HTMLElement; - if (!element) return null; - - let deepestElement = element; - let depth = 0; - const MAX_DEPTH = 4; - - // Handle iframe traversal + + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; + if (element.tagName === 'IFRAME') { let currentIframe = element as HTMLIFrameElement; - - while (currentIframe && depth < MAX_DEPTH) { + let depth = 0; + const MAX_IFRAME_DEPTH = 4; + + while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - - const iframeDoc = currentIframe.contentDocument || currentIframe.contentWindow?.document; - if (!iframeDoc) break; - - const iframeElement = iframeDoc.elementFromPoint(iframeX, iframeY) as HTMLElement; + + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; + if (!iframeDocument) break; + + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - - // Check for shadow DOM within iframe + deepestElement = traverseShadowDOM(iframeElement); - - if (deepestElement.tagName === 'IFRAME') { - currentIframe = deepestElement as HTMLIFrameElement; + + if (iframeElement.tagName === 'IFRAME') { + currentIframe = iframeElement as HTMLIFrameElement; depth++; } else { break; @@ -1629,25 +1992,134 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); } else { - // If not an iframe, check for shadow DOM deepestElement = traverseShadowDOM(element); } - + return deepestElement; }; - // Basic selector generation function getNonUniqueSelector(element: HTMLElement): string { let selector = element.tagName.toLowerCase(); - + + if (selector === 'frame' || selector === 'iframe') { + let baseSelector = selector; + + if (element.className) { + const classes = element.className.split(/\s+/).filter(Boolean); + if (classes.length > 0) { + const validClasses = classes.filter(cls => !cls.startsWith('!') && !cls.includes(':')); + if (validClasses.length > 0) { + baseSelector += '.' + validClasses.map(cls => CSS.escape(cls)).join('.'); + } + } + } + + if (element.id) { + return `${selector}#${CSS.escape(element.id)}`; + } + + if (element.getAttribute('name')) { + return `${selector}[name="${CSS.escape(element.getAttribute('name')!)}"]`; + } + + if (element.parentElement && element.parentElement.tagName === 'FRAMESET') { + const frameIndex = Array.from(element.parentElement.children) + .filter(child => child.tagName.toLowerCase() === selector) + .indexOf(element) + 1; + + if (frameIndex > 0) { + return `${selector}:nth-of-type(${frameIndex})`; + } + } + + if (element.parentElement) { + const siblings = Array.from(element.parentElement.children); + const identicalSiblings = siblings.filter(sibling => { + if (sibling === element) return false; + + let siblingSelector = sibling.tagName.toLowerCase(); + const siblingClassName = typeof sibling.className === 'string' ? sibling.className : ''; + if (siblingClassName) { + const siblingClasses = siblingClassName.split(/\s+/).filter(Boolean); + const validSiblingClasses = siblingClasses.filter(cls => !cls.startsWith('!') && !cls.includes(':')); + if (validSiblingClasses.length > 0) { + siblingSelector += '.' + validSiblingClasses.map(cls => CSS.escape(cls)).join('.'); + } + } + + return siblingSelector === baseSelector; + }); + + if (identicalSiblings.length > 0) { + const position = siblings.indexOf(element) + 1; + return `${baseSelector}:nth-child(${position})`; + } + } + + return baseSelector; + } + if (selector === 'td' && element.parentElement) { - // Find position among td siblings const siblings = Array.from(element.parentElement.children); const position = siblings.indexOf(element) + 1; return `${selector}:nth-child(${position})`; } - + if (element.className) { const classes = element.className.split(/\s+/).filter((cls: string) => Boolean(cls)); if (classes.length > 0) { @@ -1657,9 +2129,8 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates } } } - + if (element.parentElement) { - // Look for identical siblings const siblings = Array.from(element.parentElement.children); const identicalSiblings = siblings.filter(sibling => { if (sibling === element) return false; @@ -1682,11 +2153,10 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates selector += `:nth-child(${position})`; } } - + return selector; } - function getContextPath(element: HTMLElement): DOMContext[] { const path: DOMContext[] = []; let current = element; @@ -1708,18 +2178,19 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates continue; } - // Check for iframe + // Check for iframe or frame const ownerDocument = current.ownerDocument; - const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; + const frameElement = ownerDocument?.defaultView?.frameElement; if (frameElement) { + const isFrame = frameElement.tagName === 'FRAME'; path.unshift({ - type: 'iframe', + type: isFrame ? 'frame' : 'iframe', element: current, - container: frameElement, + container: frameElement as (HTMLIFrameElement | HTMLFrameElement), document: ownerDocument }); - current = frameElement; + current = frameElement as HTMLElement; depth++; continue; } @@ -1786,7 +2257,6 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates } } - // if (listSelector === '') { if (element.tagName !== 'TABLE') { while (element.parentElement) { if (element.tagName.toLowerCase() === 'body' || @@ -1805,7 +2275,7 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates const significantOverlap = (childRect.width * childRect.height) / - (parentRect.width * parentRect.height) > 0.5; + (parentRect.width * parentRect.height) > 0.1; if (fullyContained && significantOverlap) { const nextParent = element.parentElement; @@ -1827,68 +2297,66 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates return selectors || { generalSelector: '' }; } else { - // When we have a list selector, we need special handling while maintaining shadow DOM support + // When we have a list selector, we need special handling while maintaining shadow DOM and frame support const selectors = await page.evaluate(({ x, y }: { x: number, y: number }) => { const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { - // Helper function to traverse shadow DOM - const traverseShadowDOM = (element: HTMLElement, depth: number = 0): HTMLElement => { - const MAX_SHADOW_DEPTH = 4; + let element = document.elementFromPoint(x, y) as HTMLElement; + if (!element) return null; + + let deepestElement = element; + + const traverseShadowDOM = (element: HTMLElement): HTMLElement => { let current = element; + let shadowRoot = current.shadowRoot; let deepest = current; - - while (current && depth < MAX_SHADOW_DEPTH) { - const shadowRoot = current.shadowRoot; - if (!shadowRoot) break; - + let depth = 0; + const MAX_SHADOW_DEPTH = 4; + + while (shadowRoot && depth < MAX_SHADOW_DEPTH) { const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; if (!shadowElement || shadowElement === current) break; deepest = shadowElement; current = shadowElement; + shadowRoot = current.shadowRoot; depth++; } - + return deepest; }; - // Start with the element at the specified coordinates - let element = document.elementFromPoint(x, y) as HTMLElement; - if (!element) return null; + const isInFrameset = () => { + let node = element; + while (node && node.parentElement) { + if (node.tagName === 'FRAMESET' || node.tagName === 'FRAME') { + return true; + } + node = node.parentElement; + } + return false; + }; - // Initialize tracking variables - let deepestElement = element; - let depth = 0; - const MAX_IFRAME_DEPTH = 4; + if (element.tagName === 'IFRAME') { + let currentIframe = element as HTMLIFrameElement; + let depth = 0; + const MAX_IFRAME_DEPTH = 4; - // First check if the initial element has a shadow root - deepestElement = traverseShadowDOM(element); - - // If it's an iframe, traverse through iframe hierarchy - if (deepestElement.tagName === 'IFRAME') { - let currentIframe = deepestElement as HTMLIFrameElement; - while (currentIframe && depth < MAX_IFRAME_DEPTH) { try { - // Convert coordinates to iframe's local space const iframeRect = currentIframe.getBoundingClientRect(); const iframeX = x - iframeRect.left; const iframeY = y - iframeRect.top; - // Access iframe's document - const iframeDoc = currentIframe.contentDocument || currentIframe.contentWindow?.document; - if (!iframeDoc) break; + const iframeDocument = currentIframe.contentDocument || currentIframe.contentWindow?.document; + if (!iframeDocument) break; - // Get element at transformed coordinates in iframe - const iframeElement = iframeDoc.elementFromPoint(iframeX, iframeY) as HTMLElement; + const iframeElement = iframeDocument.elementFromPoint(iframeX, iframeY) as HTMLElement; if (!iframeElement) break; - // Check for shadow DOM within iframe - const shadowResult = traverseShadowDOM(iframeElement); - deepestElement = shadowResult; + deepestElement = traverseShadowDOM(iframeElement); - // If we found another iframe, continue traversing - if (shadowResult.tagName === 'IFRAME') { - currentIframe = shadowResult as HTMLIFrameElement; + if (iframeElement.tagName === 'IFRAME') { + currentIframe = iframeElement as HTMLIFrameElement; depth++; } else { break; @@ -1898,33 +2366,145 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates break; } } + } + else if (element.tagName === 'FRAME' || isInFrameset()) { + const framesToCheck = []; + + if (element.tagName === 'FRAME') { + framesToCheck.push(element as HTMLFrameElement); + } + + if (isInFrameset()) { + document.querySelectorAll('frame').forEach(frame => { + framesToCheck.push(frame as HTMLFrameElement); + }); + } + + let frameDepth = 0; + const MAX_FRAME_DEPTH = 4; + + const processFrames = (frames: HTMLFrameElement[], currentDepth: number) => { + if (currentDepth >= MAX_FRAME_DEPTH) return; + + for (const frameElement of frames) { + try { + const frameRect = frameElement.getBoundingClientRect(); + const frameX = x - frameRect.left; + const frameY = y - frameRect.top; + + if (frameX < 0 || frameY < 0 || frameX > frameRect.width || frameY > frameRect.height) { + continue; + } + + const frameDocument = + frameElement.contentDocument || + frameElement.contentWindow?.document; + + if (!frameDocument) continue; + + const frameElementAtPoint = frameDocument.elementFromPoint(frameX, frameY) as HTMLElement; + if (!frameElementAtPoint) continue; + + deepestElement = traverseShadowDOM(frameElementAtPoint); + + if (frameElementAtPoint.tagName === 'FRAME') { + processFrames([frameElementAtPoint as HTMLFrameElement], currentDepth + 1); + } + + break; + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + }; + + processFrames(framesToCheck, frameDepth); + } else { + deepestElement = traverseShadowDOM(element); } return deepestElement; }; - // Generate basic selector from element's tag and classes function getNonUniqueSelector(element: HTMLElement): string { let selector = element.tagName.toLowerCase(); - + + if (selector === 'frame' || selector === 'iframe') { + let baseSelector = selector; + + if (element.className) { + const classes = element.className.split(/\s+/).filter(Boolean); + if (classes.length > 0) { + const validClasses = classes.filter(cls => !cls.startsWith('!') && !cls.includes(':')); + if (validClasses.length > 0) { + baseSelector += '.' + validClasses.map(cls => CSS.escape(cls)).join('.'); + } + } + } + + if (element.id) { + return `${selector}#${CSS.escape(element.id)}`; + } + + if (element.getAttribute('name')) { + return `${selector}[name="${CSS.escape(element.getAttribute('name')!)}"]`; + } + + if (element.parentElement && element.parentElement.tagName === 'FRAMESET') { + const frameIndex = Array.from(element.parentElement.children) + .filter(child => child.tagName.toLowerCase() === selector) + .indexOf(element) + 1; + + if (frameIndex > 0) { + return `${selector}:nth-of-type(${frameIndex})`; + } + } + + if (element.parentElement) { + const siblings = Array.from(element.parentElement.children); + const identicalSiblings = siblings.filter(sibling => { + if (sibling === element) return false; + + let siblingSelector = sibling.tagName.toLowerCase(); + const siblingClassName = typeof sibling.className === 'string' ? sibling.className : ''; + if (siblingClassName) { + const siblingClasses = siblingClassName.split(/\s+/).filter(Boolean); + const validSiblingClasses = siblingClasses.filter(cls => !cls.startsWith('!') && !cls.includes(':')); + if (validSiblingClasses.length > 0) { + siblingSelector += '.' + validSiblingClasses.map(cls => CSS.escape(cls)).join('.'); + } + } + + return siblingSelector === baseSelector; + }); + + if (identicalSiblings.length > 0) { + const position = siblings.indexOf(element) + 1; + return `${baseSelector}:nth-child(${position})`; + } + } + + return baseSelector; + } + if (selector === 'td' && element.parentElement) { const siblings = Array.from(element.parentElement.children); const position = siblings.indexOf(element) + 1; return `${selector}:nth-child(${position})`; } - + if (element.className) { - const classes = element.className.split(/\s+/).filter(Boolean); + const classes = element.className.split(/\s+/).filter((cls: string) => Boolean(cls)); if (classes.length > 0) { - const validClasses = classes.filter(cls => !cls.startsWith('!') && !cls.includes(':')); + const validClasses = classes.filter((cls: string) => !cls.startsWith('!') && !cls.includes(':')); if (validClasses.length > 0) { selector += '.' + validClasses.map(cls => CSS.escape(cls)).join('.'); } } } - + if (element.parentElement) { - // Look for identical siblings const siblings = Array.from(element.parentElement.children); const identicalSiblings = siblings.filter(sibling => { if (sibling === element) return false; @@ -1947,11 +2527,11 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates selector += `:nth-child(${position})`; } } - + return selector; } - // Get complete context path (both iframe and shadow DOM) + // Get complete context path (iframe, frame, and shadow DOM) function getContextPath(element: HTMLElement): DOMContext[] { const path: DOMContext[] = []; let current = element; @@ -1973,18 +2553,19 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates continue; } - // Check for iframe + // Check for iframe or frame const ownerDocument = current.ownerDocument; - const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; + const frameElement = ownerDocument?.defaultView?.frameElement; if (frameElement) { + const isFrame = frameElement.tagName === 'FRAME'; path.unshift({ - type: 'iframe', + type: isFrame ? 'frame' : 'iframe', element: current, - container: frameElement, + container: frameElement as (HTMLIFrameElement | HTMLFrameElement), document: ownerDocument }); - current = frameElement; + current = frameElement as HTMLElement; depth++; continue; } @@ -2045,9 +2626,9 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates const generalSelector = getSelectorPath(element); return { generalSelector }; - }, coordinates); + }, coordinates); - return selectors || { generalSelector: '' }; + return selectors || { generalSelector: '' }; } } catch (error) { console.error('Error in getNonUniqueSelectors:', error); @@ -2120,11 +2701,13 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro return `${hostSelector} >> ${elementSelector}`; } - // Check for iframe context + // Check for iframe/frame context const ownerDocument = element.ownerDocument; - const frameElement = ownerDocument?.defaultView?.frameElement as HTMLIFrameElement; + const frameElement = ownerDocument?.defaultView?.frameElement; if (frameElement) { - const frameSelector = getNonUniqueSelector(frameElement); + const frameSelector = getNonUniqueSelector(frameElement as HTMLElement); + const isFrame = frameElement.tagName === 'FRAME'; + // Use the appropriate delimiter based on whether it's a frame or iframe return `${frameSelector} :>> ${elementSelector}`; } @@ -2133,8 +2716,7 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro return `${parentSelector} > ${elementSelector}`; } - - // Function to get all children from special contexts + // Function to get all children from special contexts including frames function getSpecialContextChildren(element: HTMLElement): HTMLElement[] { const children: HTMLElement[] = []; @@ -2160,10 +2742,43 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro } } + // Get frame children + const frames = Array.from(element.querySelectorAll('frame')) as HTMLFrameElement[]; + for (const frame of frames) { + try { + const frameDoc = frame.contentDocument || frame.contentWindow?.document; + if (frameDoc) { + const frameElements = Array.from(frameDoc.querySelectorAll('*')) as HTMLElement[]; + children.push(...frameElements); + } + } catch (error) { + console.warn('Cannot access frame content:', error); + continue; + } + } + + // Check for framesets + const framesets = Array.from(element.querySelectorAll('frameset')) as HTMLElement[]; + for (const frameset of framesets) { + const framesToCheck = Array.from(frameset.querySelectorAll('frame')) as HTMLFrameElement[]; + for (const frame of framesToCheck) { + try { + const frameDoc = frame.contentDocument || frame.contentWindow?.document; + if (frameDoc) { + const frameElements = Array.from(frameDoc.querySelectorAll('*')) as HTMLElement[]; + children.push(...frameElements); + } + } catch (error) { + console.warn('Cannot access frameset frame content:', error); + continue; + } + } + } + return children; } - // Function to recursively get all descendant selectors including shadow DOM and iframes + // Function to recursively get all descendant selectors function getAllDescendantSelectors(element: HTMLElement): string[] { let selectors: string[] = []; @@ -2177,7 +2792,7 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro // Process regular descendants selectors = selectors.concat(getAllDescendantSelectors(child)); - // Process special context children (shadow DOM and iframes) + // Process special context children (shadow DOM, iframes, and frames) const specialChildren = getSpecialContextChildren(child); for (const specialChild of specialChildren) { const specialPath = getSelectorPath(specialChild); @@ -2202,7 +2817,7 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro return selectors; } - // Handle both shadow DOM and iframe parent selectors + // Handle both shadow DOM, iframe, and frame parent selectors let parentElements: HTMLElement[] = []; // Check for special context traversal in parent selector @@ -2242,6 +2857,42 @@ export const getChildSelectors = async (page: Page, parentSelector: string): Pro continue; } } + + // Check for frame + if (element.tagName === 'FRAME') { + try { + const frameDoc = (element as HTMLFrameElement).contentDocument || + (element as HTMLFrameElement).contentWindow?.document; + if (frameDoc) { + const frameChildren = Array.from( + frameDoc.querySelectorAll(selectorParts[i]) + ) as HTMLElement[]; + newParentElements.push(...frameChildren); + } + } catch (error) { + console.warn('Cannot access frame content during traversal:', error); + continue; + } + } + + // Check for frameset + if (element.tagName === 'FRAMESET') { + const frames = Array.from(element.querySelectorAll('frame')) as HTMLFrameElement[]; + for (const frame of frames) { + try { + const frameDoc = frame.contentDocument || frame.contentWindow?.document; + if (frameDoc) { + const frameChildren = Array.from( + frameDoc.querySelectorAll(selectorParts[i]) + ) as HTMLElement[]; + newParentElements.push(...frameChildren); + } + } catch (error) { + console.warn('Cannot access frameset frame during traversal:', error); + continue; + } + } + } } parentElements = newParentElements; diff --git a/src/components/api/ApiKey.tsx b/src/components/api/ApiKey.tsx index f4904574..f84f7a55 100644 --- a/src/components/api/ApiKey.tsx +++ b/src/components/api/ApiKey.tsx @@ -43,33 +43,8 @@ const ApiKeyManager = () => { try { const { data } = await axios.get(`${apiUrl}/auth/api-key`); setApiKey(data.api_key); - notify('success', t('apikey.notifications.success.fetch')); } catch (error: any) { - const status = error.response?.status; - let errorKey = 'unknown'; - - switch (status) { - case 401: - errorKey = 'unauthorized'; - break; - case 404: - errorKey = 'not_found'; - break; - case 500: - errorKey = 'server'; - break; - default: - if (error.message?.includes('Network Error')) { - errorKey = 'network'; - } - } - - notify( - 'error', - t(`apikey.notifications.errors.fetch.${errorKey}`, { - error: error.response?.data?.message || error.message - }) - ); + notify('error', t('apikey.notifications.fetch_error', { error: error.message })); } finally { setLoading(false); } @@ -83,36 +58,11 @@ const ApiKeyManager = () => { setLoading(true); try { const { data } = await axios.post(`${apiUrl}/auth/generate-api-key`); - if (data.ok && data.api_key) { - setApiKey(data.api_key); - notify('success', t('apikey.notifications.success.generate')); - } + setApiKey(data.api_key); + + notify('success', t('apikey.notifications.generate_success')); } catch (error: any) { - const status = error.response?.status; - let errorKey = 'unknown'; - - switch (status) { - case 401: - errorKey = 'unauthorized'; - break; - case 403: - errorKey = 'limit_reached'; - break; - case 500: - errorKey = 'server'; - break; - default: - if (error.message?.includes('Network Error')) { - errorKey = 'network'; - } - } - - notify( - 'error', - t(`apikey.notifications.errors.generate.${errorKey}`, { - error: error.response?.data?.message || error.message - }) - ); + notify('error', t('apikey.notifications.generate_error', { error: error.message })); } finally { setLoading(false); } @@ -121,54 +71,22 @@ const ApiKeyManager = () => { const deleteApiKey = async () => { setLoading(true); try { - const response = await axios.delete(`${apiUrl}/auth/delete-api-key`); - if (response.data.ok) { - setApiKey(null); - notify('success', t('apikey.notifications.success.delete')); - } + await axios.delete(`${apiUrl}/auth/delete-api-key`); + setApiKey(null); + notify('success', t('apikey.notifications.delete_success')); } catch (error: any) { - const status = error.response?.status; - let errorKey = 'unknown'; - - switch (status) { - case 401: - errorKey = 'unauthorized'; - break; - case 404: - errorKey = 'not_found'; - break; - case 500: - errorKey = 'server'; - break; - default: - if (error.message?.includes('Network Error')) { - errorKey = 'network'; - } - } - - notify( - 'error', - t(`apikey.notifications.errors.delete.${errorKey}`, { - error: error.response?.data?.message || error.message - }) - ); + notify('error', t('apikey.notifications.delete_error', { error: error.message })); } finally { setLoading(false); } }; - const copyToClipboard = async () => { - if (!apiKey) return; - - try { - await navigator.clipboard.writeText(apiKey); + const copyToClipboard = () => { + if (apiKey) { + navigator.clipboard.writeText(apiKey); setCopySuccess(true); - notify('success', t('apikey.notifications.success.copy')); - - // Reset copy success state after 2 seconds setTimeout(() => setCopySuccess(false), 2000); - } catch (error) { - notify('error', t('apikey.notifications.errors.copy.failed')); + notify('info', t('apikey.notifications.copy_success')); } }; diff --git a/src/components/browser/BrowserNavBar.tsx b/src/components/browser/BrowserNavBar.tsx index c3db51c1..64759d62 100644 --- a/src/components/browser/BrowserNavBar.tsx +++ b/src/components/browser/BrowserNavBar.tsx @@ -51,9 +51,10 @@ const BrowserNavBar: FC = ({ socket?.emit('input:url', address); }, [socket]); - const handleCurrentUrlChange = useCallback((url: string) => { - handleUrlChanged(url); - setRecordingUrl(url); + const handleCurrentUrlChange = useCallback((data: { url: string, userId: string }) => { + handleUrlChanged(data.url); + setRecordingUrl(data.url); + window.sessionStorage.setItem('recordingUrl', data.url); }, [handleUrlChanged, recordingUrl]); useEffect(() => { diff --git a/src/components/browser/BrowserRecordingSave.tsx b/src/components/browser/BrowserRecordingSave.tsx index 81719251..d4fd54fb 100644 --- a/src/components/browser/BrowserRecordingSave.tsx +++ b/src/components/browser/BrowserRecordingSave.tsx @@ -14,9 +14,9 @@ import { MoreHoriz } from '@mui/icons-material'; const BrowserRecordingSave = () => { const { t } = useTranslation(); - const [openDiscardModal, setOpenDiscardModal] = useState(false); - const [openResetModal, setOpenResetModal] = useState(false); - const [anchorEl, setAnchorEl] = React.useState(null); + const [openDiscardModal, setOpenDiscardModal] = useState(false); + const [openResetModal, setOpenResetModal] = useState(false); + const [anchorEl, setAnchorEl] = React.useState(null); const { recordingName, browserId, initialUrl, setRecordingUrl, setBrowserId, notify, setCurrentWorkflowActionsState, resetInterpretationLog } = useGlobalInfoStore(); const navigate = useNavigate(); @@ -42,10 +42,25 @@ const BrowserRecordingSave = () => { const goToMainMenu = async () => { if (browserId) { await stopRecording(browserId); - notify('warning', t('browser_recording.notifications.terminated')); + + const notificationData = { + type: 'warning', + message: t('browser_recording.notifications.terminated'), + timestamp: Date.now() + }; + window.sessionStorage.setItem('pendingNotification', JSON.stringify(notificationData)); + + if (window.opener) { + window.opener.postMessage({ + type: 'recording-notification', + notification: notificationData + }, '*'); + } + setBrowserId(null); + + window.close(); } - navigate('/'); }; const performReset = () => { @@ -82,7 +97,13 @@ const BrowserRecordingSave = () => { socket?.emit('new-recording'); socket.emit('input:url', initialUrl); // Update the URL in the navbar to match - setRecordingUrl(initialUrl); + let sessionInitialUrl = window.sessionStorage.getItem('initialUrl'); + if (sessionInitialUrl) { + setRecordingUrl(sessionInitialUrl); + window.sessionStorage.setItem('recordingUrl', sessionInitialUrl); + } else { + setRecordingUrl(initialUrl); + } } // Close the reset confirmation modal @@ -92,7 +113,7 @@ const BrowserRecordingSave = () => { notify('info', t('browser_recording.notifications.environment_reset')); }; - const handleClick = (event: React.MouseEvent) => { + const handleClick = (event: any) => { setAnchorEl(event.currentTarget); }; diff --git a/src/components/browser/BrowserWindow.tsx b/src/components/browser/BrowserWindow.tsx index ffeb0df5..8edae179 100644 --- a/src/components/browser/BrowserWindow.tsx +++ b/src/components/browser/BrowserWindow.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useEffect, useState } from 'react'; +import React, { useCallback, useContext, useEffect, useState } from 'react'; import { useSocketStore } from '../../context/socket'; import { Button } from '@mui/material'; import Canvas from "../recorder/canvas"; @@ -8,6 +8,7 @@ import { useActionContext } from '../../context/browserActions'; import { useBrowserSteps, TextStep } from '../../context/browserSteps'; import { useGlobalInfoStore } from '../../context/globalInfo'; import { useTranslation } from 'react-i18next'; +import { AuthContext } from '../../context/auth'; interface ElementInfo { tagName: string; @@ -27,6 +28,12 @@ interface AttributeOption { value: string; } +interface ScreencastData { + image: string; + userId: string; +} + + const getAttributeOptions = (tagName: string, elementInfo: ElementInfo | null): AttributeOption[] => { if (!elementInfo) return []; switch (tagName.toLowerCase()) { @@ -71,6 +78,24 @@ export const BrowserWindow = () => { const { notify } = useGlobalInfoStore(); const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext(); const { addTextStep, addListStep } = useBrowserSteps(); + + const { state } = useContext(AuthContext); + const { user } = state; + + useEffect(() => { + if (listSelector) { + window.sessionStorage.setItem('recordingListSelector', listSelector); + } + }, [listSelector]); + + useEffect(() => { + const storedListSelector = window.sessionStorage.getItem('recordingListSelector'); + + // Only restore state if it exists in sessionStorage + if (storedListSelector && !listSelector) { + setListSelector(storedListSelector); + } + }, []); const onMouseMove = (e: MouseEvent) => { if (canvasRef && canvasRef.current && highlighterData) { @@ -99,9 +124,15 @@ export const BrowserWindow = () => { } }, [getList, resetListState]); - const screencastHandler = useCallback((data: string) => { - setScreenShot(data); - }, [screenShot]); + const screencastHandler = useCallback((data: string | ScreencastData) => { + if (typeof data === 'string') { + setScreenShot(data); + } else if (data && typeof data === 'object' && 'image' in data) { + if (!data.userId || data.userId === user?.id) { + setScreenShot(data.image); + } + } + }, [screenShot, user?.id]); useEffect(() => { if (socket) { @@ -195,13 +226,25 @@ export const BrowserWindow = () => { useEffect(() => { document.addEventListener('mousemove', onMouseMove, false); if (socket) { - socket.on("highlighter", highlighterHandler); + socket.off("highlighter", highlighterHandler); + + socket.on("highlighter", highlighterHandler); } return () => { - document.removeEventListener('mousemove', onMouseMove); - socket?.off("highlighter", highlighterHandler); + document.removeEventListener('mousemove', onMouseMove); + if (socket) { + socket.off("highlighter", highlighterHandler); + } }; - }, [socket, onMouseMove]); + }, [socket, highlighterHandler, onMouseMove, getList, listSelector]); + + useEffect(() => { + if (socket && listSelector) { + console.log('Syncing list selector with server:', listSelector); + socket.emit('setGetList', { getList: true }); + socket.emit('listSelector', { selector: listSelector }); + } + }, [socket, listSelector]); useEffect(() => { if (captureStage === 'initial' && listSelector) { diff --git a/src/components/integration/IntegrationSettings.tsx b/src/components/integration/IntegrationSettings.tsx index 31c93368..5ca71840 100644 --- a/src/components/integration/IntegrationSettings.tsx +++ b/src/components/integration/IntegrationSettings.tsx @@ -6,90 +6,150 @@ import { CircularProgress, Alert, AlertTitle, - Chip, + Button, + TextField, } from "@mui/material"; -import Button from "@mui/material/Button"; -import TextField from "@mui/material/TextField"; import axios from "axios"; import { useGlobalInfoStore } from "../../context/globalInfo"; import { getStoredRecording } from "../../api/storage"; import { apiUrl } from "../../apiConfig.js"; -import { useTranslation } from "react-i18next"; +import Cookies from "js-cookie"; + +import { useTranslation } from "react-i18next"; +import { useNavigate } from "react-router-dom"; interface IntegrationProps { isOpen: boolean; handleStart: (data: IntegrationSettings) => void; handleClose: () => void; + preSelectedIntegrationType?: "googleSheets" | "airtable" | null; } export interface IntegrationSettings { - spreadsheetId: string; - spreadsheetName: string; + spreadsheetId?: string; + spreadsheetName?: string; + airtableBaseId?: string; + airtableBaseName?: string; + airtableTableName?: string, + airtableTableId?: string, data: string; + integrationType: "googleSheets" | "airtable"; } +const getCookie = (name: string): string | null => { + const value = `; ${document.cookie}`; + const parts = value.split(`; ${name}=`); + if (parts.length === 2) { + return parts.pop()?.split(";").shift() || null; + } + return null; +}; + +const removeCookie = (name: string): void => { + document.cookie = `${name}=; expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/`; +}; + export const IntegrationSettingsModal = ({ isOpen, handleStart, handleClose, + preSelectedIntegrationType = null, }: IntegrationProps) => { const { t } = useTranslation(); const [settings, setSettings] = useState({ spreadsheetId: "", spreadsheetName: "", + airtableBaseId: "", + airtableBaseName: "", + airtableTableName: "", + airtableTableId: "", data: "", + integrationType: preSelectedIntegrationType || "googleSheets", }); - const [spreadsheets, setSpreadsheets] = useState< - { id: string; name: string }[] - >([]); + const [spreadsheets, setSpreadsheets] = useState<{ id: string; name: string }[]>([]); + const [airtableBases, setAirtableBases] = useState<{ id: string; name: string }[]>([]); + const [airtableTables, setAirtableTables] = useState<{ id: string; name: string }[]>([]); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); - const { recordingId, notify } = useGlobalInfoStore(); + const { + recordingId, + notify, + setRerenderRobots + } = useGlobalInfoStore(); + const [recording, setRecording] = useState(null); + const navigate = useNavigate(); + + const [selectedIntegrationType, setSelectedIntegrationType] = useState< + "googleSheets" | "airtable" | null + >(preSelectedIntegrationType); const authenticateWithGoogle = () => { window.location.href = `${apiUrl}/auth/google?robotId=${recordingId}`; }; - const handleOAuthCallback = async () => { - try { - const response = await axios.get(`${apiUrl}/auth/google/callback`); - const { google_sheet_email, files } = response.data; - } catch (error) { - setError("Error authenticating with Google"); - } + // Authenticate with Airtable + const authenticateWithAirtable = () => { + window.location.href = `${apiUrl}/auth/airtable?robotId=${recordingId}`; }; + // Fetch Google Sheets files const fetchSpreadsheetFiles = async () => { try { const response = await axios.get( `${apiUrl}/auth/gsheets/files?robotId=${recordingId}`, - { - withCredentials: true, - } + { withCredentials: true } ); setSpreadsheets(response.data); } catch (error: any) { - console.error( - "Error fetching spreadsheet files:", - error.response?.data?.message || error.message - ); - notify( - "error", - t('integration_settings.errors.fetch_error', { - message: error.response?.data?.message || error.message - }) - ); + setLoading(false); + console.error("Error fetching spreadsheet files:", error); + notify("error", t("integration_settings.google.errors.fetch_error", { + message: error.response?.data?.message || error.message, + })); } }; + // Fetch Airtable bases + const fetchAirtableBases = async () => { + try { + const response = await axios.get( + `${apiUrl}/auth/airtable/bases?robotId=${recordingId}`, + { withCredentials: true } + ); + setAirtableBases(response.data); + } catch (error: any) { + setLoading(false); + console.error("Error fetching Airtable bases:", error); + notify("error", t("integration_settings.airtable.errors.fetch_error", { + message: error.response?.data?.message || error.message, + })); + } + }; + + const fetchAirtableTables = async (baseId: string, recordingId: string) => { + try { + const response = await axios.get( + `${apiUrl}/auth/airtable/tables?robotId=${recordingId}&baseId=${baseId}`, + { withCredentials: true } + ); + setAirtableTables(response.data); + } + catch (error: any) { + setLoading(false); + console.error("Error fetching Airtable tables:", error); + notify("error", t("integration_settings.airtable.errors.fetch_tables_error", { + message: error.response?.data?.message || error.message, + })); + } + } + + // Handle Google Sheets selection const handleSpreadsheetSelect = (e: React.ChangeEvent) => { - const selectedSheet = spreadsheets.find( - (sheet) => sheet.id === e.target.value - ); + const selectedSheet = spreadsheets.find((sheet) => sheet.id === e.target.value); if (selectedSheet) { setSettings({ ...settings, @@ -99,9 +159,48 @@ export const IntegrationSettingsModal = ({ } }; + // Handle Airtable base selection + const handleAirtableBaseSelect = async (e: React.ChangeEvent) => { + const selectedBase = airtableBases.find((base) => base.id === e.target.value); + + if (selectedBase) { + setSettings((prevSettings) => ({ + ...prevSettings, + airtableBaseId: selectedBase.id, + airtableBaseName: selectedBase.name, + })); + + if (recordingId) { + await fetchAirtableTables(selectedBase.id, recordingId); + } else { + console.error("Recording ID is null"); + } + } + }; + + const handleAirtabletableSelect = (e: React.ChangeEvent) => { + const selectedTable = airtableTables.find((table) => table.id === e.target.value); + if (selectedTable) { + setSettings((prevSettings) => ({ + ...prevSettings, + airtableTableId: e.target.value, + airtableTableName: selectedTable?.name || "", + })); + } + }; + + const refreshRecordingData = async () => { + if (!recordingId) return null; + const updatedRecording = await getStoredRecording(recordingId); + setRecording(updatedRecording); + setRerenderRobots(true); + return updatedRecording; + }; + const updateGoogleSheetId = async () => { try { - const response = await axios.post( + setLoading(true); + await axios.post( `${apiUrl}/auth/gsheets/update`, { spreadsheetId: settings.spreadsheetId, @@ -110,53 +209,212 @@ export const IntegrationSettingsModal = ({ }, { withCredentials: true } ); - notify(`success`, t('integration_settings.notifications.sheet_selected')); - console.log("Google Sheet ID updated:", response.data); + + // Refresh recording data immediately + await refreshRecordingData(); + + notify("success", t("integration_settings.google.notifications.sheet_selected")); + setLoading(false); } catch (error: any) { - console.error( - "Error updating Google Sheet ID:", - error.response?.data?.message || error.message - ); + setLoading(false); + console.error("Error updating Google Sheet ID:", error); + notify("error", t("integration_settings.google.errors.update_error", { + message: error.response?.data?.message || error.message, + })); } }; - const removeIntegration = async () => { + // Update Airtable integration + const updateAirtableBase = async () => { try { + setLoading(true); + await axios.post( + `${apiUrl}/auth/airtable/update`, + { + baseId: settings.airtableBaseId, + baseName: settings.airtableBaseName, + robotId: recordingId, + tableName: settings.airtableTableName, + tableId: settings.airtableTableId, + }, + { withCredentials: true } + ); + + await refreshRecordingData(); + + notify("success", t("integration_settings.airtable.notifications.base_selected")); + setLoading(false); + } catch (error: any) { + setLoading(false); + console.error("Error updating Airtable base:", error); + notify("error", t("integration_settings.airtable.errors.update_error", { + message: error.response?.data?.message || error.message, + })); + } + }; + + // Remove Google Sheets integration + const removeGoogleSheetsIntegration = async () => { + try { + setLoading(true); await axios.post( `${apiUrl}/auth/gsheets/remove`, { robotId: recordingId }, { withCredentials: true } ); - setRecording(null); + // Clear UI state setSpreadsheets([]); - setSettings({ spreadsheetId: "", spreadsheetName: "", data: "" }); + setSettings({ ...settings, spreadsheetId: "", spreadsheetName: "" }); + + // Refresh recording data + await refreshRecordingData(); + + notify("success", t("integration_settings.google.notifications.integration_removed")); + setLoading(false); } catch (error: any) { - console.error( - "Error removing Google Sheets integration:", - error.response?.data?.message || error.message + setLoading(false); + console.error("Error removing Google Sheets integration:", error); + notify("error", t("integration_settings.google.errors.remove_error", { + message: error.response?.data?.message || error.message, + })); + } + }; + + // Remove Airtable integration + const removeAirtableIntegration = async () => { + try { + setLoading(true); + await axios.post( + `${apiUrl}/auth/airtable/remove`, + { robotId: recordingId }, + { withCredentials: true } ); + + setAirtableBases([]); + setAirtableTables([]); + setSettings({ ...settings, airtableBaseId: "", airtableBaseName: "", airtableTableName: "", airtableTableId: "" }); + + await refreshRecordingData(); + + notify("success", t("integration_settings.airtable.notifications.integration_removed")); + setLoading(false); + } catch (error: any) { + setLoading(false); + console.error("Error removing Airtable integration:", error); + notify("error", t("integration_settings.airtable.errors.remove_error", { + message: error.response?.data?.message || error.message, + })); + } + }; + + const handleAirtableOAuthCallback = async () => { + try { + const response = await axios.get(`${apiUrl}/auth/airtable/callback`); + if (response.data.success) { + await refreshRecordingData(); + } + } catch (error) { + setError(t("integration_settings.airtable.errors.auth_error")); } }; useEffect(() => { - // Check if we're on the callback URL - const urlParams = new URLSearchParams(window.location.search); - const code = urlParams.get("code"); - if (code) { - handleOAuthCallback(); - } - const fetchRecordingInfo = async () => { if (!recordingId) return; + + setLoading(true); + const recording = await getStoredRecording(recordingId); if (recording) { setRecording(recording); + + if (preSelectedIntegrationType) { + setSettings(prev => ({ ...prev, integrationType: preSelectedIntegrationType })); + } + else if (recording.google_sheet_id) { + setSettings(prev => ({ ...prev, integrationType: "googleSheets" })); + } else if (recording.airtable_base_id) { + setSettings(prev => ({ + ...prev, + airtableBaseId: recording.airtable_base_id || "", + airtableBaseName: recording.airtable_base_name || "", + airtableTableName: recording.airtable_table_name || "", + airtableTableId: recording.airtable_table_id || "", + integrationType: recording.airtable_base_id ? "airtable" : "googleSheets" + })); + } } + + setLoading(false); }; fetchRecordingInfo(); - }, [recordingId]); + }, [recordingId, preSelectedIntegrationType]); + + useEffect(() => { + const status = getCookie("airtable_auth_status"); + const message = getCookie("airtable_auth_message"); + + if (status === "success") { + notify("success", message || t("integration_settings.airtable.notifications.auth_success")); + removeCookie("airtable_auth_status"); + removeCookie("airtable_auth_message"); + refreshRecordingData(); + } + + const urlParams = new URLSearchParams(window.location.search); + const code = urlParams.get("code"); + if (code) { + handleAirtableOAuthCallback(); + } + }, []); + + // Add this UI at the top of the modal return statement + if (!selectedIntegrationType) { + return ( + +
+
+ + + +
+
+ + ); + } return ( @@ -166,118 +424,217 @@ export const IntegrationSettingsModal = ({ alignItems: "flex-start", marginLeft: "65px", }}> - - {t('integration_settings.title')} - - {recording && recording.google_sheet_id ? ( + {settings.integrationType === "googleSheets" && ( <> - - {t('integration_settings.alerts.success.title')} - {t('integration_settings.alerts.success.content', { sheetName: recording.google_sheet_name })} - - {t('integration_settings.alerts.success.here')} - . -
- {t('integration_settings.alerts.success.note')} {t('integration_settings.alerts.success.sync_limitation')} - - - - ) : ( - <> - {!recording?.google_sheet_email ? ( + + {t("integration_settings.google.title")} + + + {recording?.google_sheet_id ? ( <> -

{t('integration_settings.descriptions.sync_info')}

+ + {t("integration_settings.google.alerts.success.title")} + {t("integration_settings.google.alerts.success.content", { + sheetName: recording.google_sheet_name, + })} + + {t("integration_settings.google.alerts.success.here")} + + ) : ( <> - {recording.google_sheet_email && ( - - {t('integration_settings.descriptions.authenticated_as', { - email: recording.google_sheet_email - })} - - )} - - {loading ? ( - - ) : error ? ( - {error} - ) : spreadsheets.length === 0 ? ( + {!recording?.google_sheet_email ? ( <> -
+

{t("integration_settings.google.descriptions.sync_info")}

+ + + ) : ( + <> + + {t("integration_settings.google.descriptions.authenticated_as", { + email: recording.google_sheet_email, + })} + + {loading ? ( + + ) : error ? ( + {error} + ) : spreadsheets.length === 0 ? ( - -
- - ) : ( - <> - - {spreadsheets.map((sheet) => ( - - {sheet.name} - - ))} - - - {settings.spreadsheetId && ( - - {t('integration_settings.fields.selected_sheet', { - name: spreadsheets.find((s) => s.id === settings.spreadsheetId)?.name, - id: settings.spreadsheetId - })} - + ) : ( + <> + + {spreadsheets.map((sheet) => ( + + {sheet.name} + + ))} + + + )} + + )} + + )} + + )} + {settings.integrationType === "airtable" && ( + <> + + {t("integration_settings.airtable.title")} + + + {recording?.airtable_base_id ? ( + <> + + {t("integration_settings.airtable.alerts.success.title")} + {t("integration_settings.airtable.alerts.success.content", { + baseName: recording.airtable_base_name, + tableName: recording.airtable_table_name + })} + + {t("integration_settings.airtable.alerts.success.here")} + + + + + ) : ( + <> + {!recording?.airtable_access_token ? ( + <> +

{t("integration_settings.airtable.descriptions.sync_info")}

+ ) : ( + <> + + {t("integration_settings.airtable.descriptions.authenticated_as")} + + {loading ? ( + + ) : error ? ( + {error} + ) : airtableBases.length === 0 ? ( + + ) : ( + <> + + {airtableBases.map((base) => ( + + {base.name} + + ))} + + + {airtableTables.map((table) => ( + + {table.name} + + ))} + + + + )} + )} )} diff --git a/src/components/proxy/ProxyForm.tsx b/src/components/proxy/ProxyForm.tsx index 08a4ce8b..36dfa0b6 100644 --- a/src/components/proxy/ProxyForm.tsx +++ b/src/components/proxy/ProxyForm.tsx @@ -100,7 +100,10 @@ const ProxyForm: React.FC = () => { try { const response = await sendProxyConfig(proxyConfigForm); if (response) { + setIsProxyConfigured(true); + setProxy({ proxy_url: proxyConfigForm.server_url, auth: requiresAuth }); notify('success', t('proxy.notifications.config_success')); + fetchProxyConfig(); } else { notify('error', t('proxy.notifications.config_error')); console.log(`${t('proxy.notifications.config_error')} ${response}`) diff --git a/src/components/recorder/Renderer.tsx b/src/components/recorder/Renderer.tsx new file mode 100644 index 00000000..bff63b08 --- /dev/null +++ b/src/components/recorder/Renderer.tsx @@ -0,0 +1,241 @@ +export class CanvasRenderer { + private canvas: HTMLCanvasElement; + private ctx: CanvasRenderingContext2D; + private offscreenCanvas: OffscreenCanvas | null = null; + private offscreenCtx: CanvasRenderingContext2D | null = null; + private lastFrameRequest: number | null = null; + private imageCache: Map = new Map(); + private consecutiveFrameCount: number = 0; + private lastDrawTime: number = 0; + private memoryCheckCounter: number = 0; + private lastMemoryCheck: number = 0; + private memoryThreshold: number = 100000000; // 100MB + + constructor(canvas: HTMLCanvasElement) { + this.canvas = canvas; + + // Get 2D context with optimized settings + const ctx = canvas.getContext('2d', { + alpha: false, // Disable alpha for better performance + desynchronized: true, // Reduce latency when possible + }); + + if (!ctx) { + throw new Error('Could not get 2D context from canvas'); + } + + this.ctx = ctx; + + // Apply performance optimizations + this.ctx.imageSmoothingEnabled = false; + + // Set up offscreen canvas if supported + if (typeof OffscreenCanvas !== 'undefined') { + this.offscreenCanvas = new OffscreenCanvas(canvas.width, canvas.height); + const offCtx = this.offscreenCanvas.getContext('2d', { + alpha: false + }); + + if (offCtx) { + this.offscreenCtx = offCtx as unknown as CanvasRenderingContext2D; + this.offscreenCtx.imageSmoothingEnabled = false; + } + } + + // Initial timestamp + this.lastDrawTime = performance.now(); + this.lastMemoryCheck = performance.now(); + } + + /** + * Renders a screenshot to the canvas, optimized for performance + */ + public drawScreenshot( + screenshot: string | ImageBitmap | HTMLImageElement, + x: number = 0, + y: number = 0, + width?: number, + height?: number + ): void { + // Cancel any pending frame request + if (this.lastFrameRequest !== null) { + cancelAnimationFrame(this.lastFrameRequest); + } + + // Check memory usage periodically + this.memoryCheckCounter++; + const now = performance.now(); + + if (this.memoryCheckCounter >= 30 || now - this.lastMemoryCheck > 5000) { + this.checkMemoryUsage(); + this.memoryCheckCounter = 0; + this.lastMemoryCheck = now; + } + + // Request a new frame + this.lastFrameRequest = requestAnimationFrame(() => { + this.renderFrame(screenshot, x, y, width, height); + }); + } + + private renderFrame( + screenshot: string | ImageBitmap | HTMLImageElement, + x: number, + y: number, + width?: number, + height?: number + ): void { + // Target context (offscreen if available, otherwise main) + const targetCtx = this.offscreenCtx || this.ctx; + + // Start timing the render + const startTime = performance.now(); + const timeSinceLastDraw = startTime - this.lastDrawTime; + + // Adaptive frame skipping for high-frequency updates + // If we're getting updates faster than 60fps and this isn't the first frame + if (timeSinceLastDraw < 16 && this.consecutiveFrameCount > 5) { + this.consecutiveFrameCount++; + + // Skip some frames when we're getting excessive updates + if (this.consecutiveFrameCount % 2 !== 0) { + return; + } + } else { + this.consecutiveFrameCount = 0; + } + + try { + if (typeof screenshot === 'string') { + // Check if we have this image in cache + let img = this.imageCache.get(screenshot); + + if (!img) { + img = new Image(); + img.src = screenshot; + this.imageCache.set(screenshot, img); + + // If image isn't loaded yet, draw when it loads + if (!img.complete) { + img.onload = () => { + if (img) { + this.drawScreenshot(img, x, y, width, height); + } + }; + return; + } + } + + targetCtx.drawImage( + img, + x, y, + width || img.width, + height || img.height + ); + } else { + // Draw ImageBitmap or HTMLImageElement directly + targetCtx.drawImage( + screenshot, + x, y, + width || screenshot.width, + height || screenshot.height + ); + } + + // If using offscreen canvas, copy to main canvas + if (this.offscreenCanvas && this.offscreenCtx) { + if ('transferToImageBitmap' in this.offscreenCanvas) { + // Use more efficient transfer when available + const bitmap = this.offscreenCanvas.transferToImageBitmap(); + this.ctx.drawImage(bitmap, 0, 0); + } else { + // Fallback to drawImage + this.ctx.drawImage(this.offscreenCanvas, 0, 0); + } + } + + // Update timestamp + this.lastDrawTime = performance.now(); + } catch (error) { + console.error('Error rendering frame:', error); + } + } + + /** + * Checks current memory usage and cleans up if necessary + */ + private checkMemoryUsage(): void { + if (window.performance && (performance as any).memory) { + const memory = (performance as any).memory; + + if (memory.usedJSHeapSize > this.memoryThreshold) { + this.cleanupMemory(); + } + } + } + + /** + * Cleans up resources to reduce memory usage + */ + private cleanupMemory(): void { + // Limit image cache size + if (this.imageCache.size > 20) { + // Keep only the most recent 10 images + const keysToDelete = Array.from(this.imageCache.keys()).slice(0, this.imageCache.size - 10); + keysToDelete.forEach(key => { + this.imageCache.delete(key); + }); + } + + // Suggest garbage collection + if (window.gc) { + try { + window.gc(); + } catch (e) { + // GC not available, ignore + } + } + } + + /** + * Update canvas dimensions + */ + public updateCanvasSize(width: number, height: number): void { + this.canvas.width = width; + this.canvas.height = height; + + // Re-apply context settings + this.ctx.imageSmoothingEnabled = false; + + // Update offscreen canvas if available + if (this.offscreenCanvas) { + this.offscreenCanvas.width = width; + this.offscreenCanvas.height = height; + + if (this.offscreenCtx) { + this.offscreenCtx.imageSmoothingEnabled = false; + } + } + } + + /** + * Clean up resources + */ + public dispose(): void { + // Cancel any pending frame requests + if (this.lastFrameRequest !== null) { + cancelAnimationFrame(this.lastFrameRequest); + this.lastFrameRequest = null; + } + + // Clear the image cache + this.imageCache.clear(); + + // Clear canvases + this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height); + + if (this.offscreenCtx && this.offscreenCanvas) { + this.offscreenCtx.clearRect(0, 0, this.offscreenCanvas.width, this.offscreenCanvas.height); + } + } +} \ No newline at end of file diff --git a/src/components/recorder/SaveRecording.tsx b/src/components/recorder/SaveRecording.tsx index c6b5687b..f7020b44 100644 --- a/src/components/recorder/SaveRecording.tsx +++ b/src/components/recorder/SaveRecording.tsx @@ -47,13 +47,27 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => { }; const exitRecording = useCallback(async () => { - notify('success', t('save_recording.notifications.save_success')); + const notificationData = { + type: 'success', + message: t('save_recording.notifications.save_success'), + timestamp: Date.now() + }; + window.sessionStorage.setItem('pendingNotification', JSON.stringify(notificationData)); + + if (window.opener) { + window.opener.postMessage({ + type: 'recording-notification', + notification: notificationData + }, '*'); + } + if (browserId) { await stopRecording(browserId); } setBrowserId(null); - navigate('/'); - }, [setBrowserId, browserId, notify]); + + window.close(); + }, [setBrowserId, browserId]); // notifies backed to save the recording in progress, // releases resources and changes the view for main page by clearing the global browserId @@ -142,4 +156,4 @@ const modalStyle = { height: 'fit-content', display: 'block', padding: '20px', -}; +}; \ No newline at end of file diff --git a/src/components/recorder/canvas.tsx b/src/components/recorder/canvas.tsx index cc75a6ba..9640c061 100644 --- a/src/components/recorder/canvas.tsx +++ b/src/components/recorder/canvas.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useEffect, useRef } from 'react'; +import React, { memo, useCallback, useEffect, useRef } from 'react'; import { useSocketStore } from '../../context/socket'; import { useGlobalInfoStore } from "../../context/globalInfo"; import { useActionContext } from '../../context/browserActions'; @@ -34,6 +34,9 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { const getTextRef = useRef(getText); const getListRef = useRef(getList); + const MOUSE_MOVE_THROTTLE = 16; // ~60fps + const lastMouseMoveTime = useRef(0); + const [datePickerInfo, setDatePickerInfo] = React.useState<{ coordinates: Coordinates; selector: string; @@ -129,29 +132,38 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { } notifyLastAction('click'); break; - case 'mousemove': - if (lastMousePosition.current.x !== clickCoordinates.x || - lastMousePosition.current.y !== clickCoordinates.y) { + case 'mousemove': { + const now = performance.now(); + if (now - lastMouseMoveTime.current < MOUSE_MOVE_THROTTLE) { + return; + } + lastMouseMoveTime.current = now; + + const dx = Math.abs(lastMousePosition.current.x - clickCoordinates.x); + const dy = Math.abs(lastMousePosition.current.y - clickCoordinates.y); + if (dx > 1 || dy > 1) { lastMousePosition.current = { x: clickCoordinates.x, y: clickCoordinates.y, }; - socket.emit('input:mousemove', { - x: clickCoordinates.x, - y: clickCoordinates.y, - }); + socket.emit('input:mousemove', clickCoordinates); notifyLastAction('move'); } break; - case 'wheel': + } + + // Optimize wheel events + case 'wheel': { const wheelEvent = event as WheelEvent; - const deltas = { - deltaX: Math.round(wheelEvent.deltaX), - deltaY: Math.round(wheelEvent.deltaY), - }; - socket.emit('input:wheel', deltas); - notifyLastAction('scroll'); + const deltaX = Math.round(wheelEvent.deltaX / 10) * 10; + const deltaY = Math.round(wheelEvent.deltaY / 10) * 10; + + if (Math.abs(deltaX) > 5 || Math.abs(deltaY) > 5) { + socket.emit('input:wheel', { deltaX, deltaY }); + notifyLastAction('scroll'); + } break; + } default: console.log('Default mouseEvent registered'); return; @@ -207,9 +219,14 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { {datePickerInfo && ( { }; -export default Canvas; \ No newline at end of file +export default memo(Canvas); \ No newline at end of file diff --git a/src/components/robot/Recordings.tsx b/src/components/robot/Recordings.tsx index e63f5e1c..780d01cf 100644 --- a/src/components/robot/Recordings.tsx +++ b/src/components/robot/Recordings.tsx @@ -59,12 +59,18 @@ export const Recordings = ({ }; const authStatus = getAndClearCookie('robot_auth_status'); + const airtableAuthStatus = getAndClearCookie('airtable_auth_status'); const robotId = getAndClearCookie('robot_auth_robotId'); - if (authStatus === 'success' && robotId) { + if (airtableAuthStatus === 'success' && robotId) { + console.log("Airtable Auth Status:", airtableAuthStatus); + notify(airtableAuthStatus, t("recordingtable.notifications.auth_success")); + handleNavigate(`/robots/${robotId}/integrate/airtable`, robotId, "", []); + } + else if (authStatus === 'success' && robotId) { + console.log("Google Auth Status:", authStatus); notify(authStatus, t("recordingtable.notifications.auth_success")); - - handleNavigate(`/robots/${robotId}/integrate`, robotId, "", []);'' + handleNavigate(`/robots/${robotId}/integrate/google`, robotId, "", []); } }, []); @@ -90,6 +96,24 @@ export const Recordings = ({ handleStart={handleScheduleRecording} /> ); + } else if (currentPath.endsWith("/integrate/google")) { + return ( + {}} + preSelectedIntegrationType="googleSheets" + /> + ); + } else if (currentPath.endsWith("/integrate/airtable")) { + return ( + {}} + preSelectedIntegrationType="airtable" + /> + ); } else if (currentPath.endsWith("/integrate")) { return ( ([]); const [isModalOpen, setModalOpen] = React.useState(false); const [searchTerm, setSearchTerm] = React.useState(''); + const [isWarningModalOpen, setWarningModalOpen] = React.useState(false); + const [activeBrowserId, setActiveBrowserId] = React.useState(''); const columns = useMemo(() => [ { id: 'interpret', label: t('recordingtable.run'), minWidth: 80 }, @@ -176,6 +183,30 @@ export const RecordingsTable = ({ setRecordingId } = useGlobalInfoStore(); const navigate = useNavigate(); + useEffect(() => { + const handleMessage = (event: any) => { + if (event.data && event.data.type === 'recording-notification') { + const notificationData = event.data.notification; + if (notificationData) { + notify(notificationData.type, notificationData.message); + + if ((notificationData.type === 'success' && + notificationData.message.includes('saved')) || + (notificationData.type === 'warning' && + notificationData.message.includes('terminated'))) { + setRerenderRobots(true); + } + } + } + }; + + window.addEventListener('message', handleMessage); + + return () => { + window.removeEventListener('message', handleMessage); + }; + }, [notify, setRerenderRobots]); + const handleChangePage = useCallback((event: unknown, newPage: number) => { setPage(newPage); }, []); @@ -233,28 +264,69 @@ export const RecordingsTable = ({ }, [setRecordings, notify, t]); const handleNewRecording = useCallback(async () => { - if (browserId) { - setBrowserId(null); - await stopRecording(browserId); + const activeBrowserId = await getActiveBrowserId(); + + if (activeBrowserId) { + setActiveBrowserId(activeBrowserId); + setWarningModalOpen(true); + } else { + setModalOpen(true); } - setModalOpen(true); - }, [browserId]); + }, []); - const handleStartRecording = useCallback(() => { - setBrowserId('new-recording'); - setRecordingName(''); - setRecordingId(''); - navigate('/recording'); - }, [navigate]); + const notifyRecordingTabsToClose = (browserId: string) => { + const closeMessage = { + action: 'close-recording-tab', + browserId: browserId, + timestamp: Date.now() + }; + window.sessionStorage.setItem('recordingTabCloseMessage', JSON.stringify(closeMessage)); + + if (window.openedRecordingWindow && !window.openedRecordingWindow.closed) { + try { + window.openedRecordingWindow.close(); + } catch (e) { + console.log('Could not directly close recording window:', e); + } + } + }; + + const handleDiscardAndCreate = async () => { + if (activeBrowserId) { + await stopRecording(activeBrowserId); + notify('warning', t('browser_recording.notifications.terminated')); + + notifyRecordingTabsToClose(activeBrowserId); + } + + setWarningModalOpen(false); + setModalOpen(true); + }; const startRecording = () => { setModalOpen(false); - handleStartRecording(); + + // Set local state + setBrowserId('new-recording'); + setRecordingName(''); + setRecordingId(''); + + window.sessionStorage.setItem('browserId', 'new-recording'); + + const sessionId = Date.now().toString(); + window.sessionStorage.setItem('recordingSessionId', sessionId); + window.sessionStorage.setItem('recordingUrl', recordingUrl); + + window.openedRecordingWindow = window.open(`/recording-setup?session=${sessionId}`, '_blank'); + + window.sessionStorage.setItem('nextTabIsRecording', 'true'); }; const setBrowserRecordingUrl = (event: React.ChangeEvent) => { setInitialUrl(event.target.value); setRecordingUrl(event.target.value); + + window.sessionStorage.setItem('initialUrl', event.target.value); } useEffect(() => { @@ -401,6 +473,30 @@ export const RecordingsTable = ({ onPageChange={handleChangePage} onRowsPerPageChange={handleChangeRowsPerPage} /> + setWarningModalOpen(false)} modalStyle={modalStyle}> +
+ {t('recordingtable.warning_modal.title')} + + {t('recordingtable.warning_modal.message')} + + + + + + +
+ setModalOpen(false)} modalStyle={modalStyle}>
{t('recordingtable.modal.title')} diff --git a/src/components/run/ColapsibleRow.tsx b/src/components/run/ColapsibleRow.tsx index 87e38684..4d5fe822 100644 --- a/src/components/run/ColapsibleRow.tsx +++ b/src/components/run/ColapsibleRow.tsx @@ -123,6 +123,7 @@ export const CollapsibleRow = ({ row, handleDelete, isOpen, currentLog, abortRun {row.status === 'success' && } {row.status === 'running' && } {row.status === 'scheduled' && } + {row.status === 'queued' && } {row.status === 'failed' && } ) diff --git a/src/components/run/RunContent.tsx b/src/components/run/RunContent.tsx index cb3b63a6..77bce544 100644 --- a/src/components/run/RunContent.tsx +++ b/src/components/run/RunContent.tsx @@ -134,13 +134,13 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe }}>
- {interpretationInProgress ? currentLog : row.log} + {row.status === 'running' ? currentLog : row.log}
- {interpretationInProgress ? : null} - {interpretationInProgress ? ( + {row.status === 'running' || row.status === 'queued' ? ( {t('run_content.loading')} @@ -248,4 +248,4 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe ); -}; +}; \ No newline at end of file diff --git a/src/index.css b/src/index.css index 252b737a..e63d18f7 100644 --- a/src/index.css +++ b/src/index.css @@ -11,7 +11,19 @@ body { padding: 0; scrollbar-gutter: stable; overflow-y: auto; - +} + +input:-webkit-autofill, +input:-webkit-autofill:hover, +input:-webkit-autofill:focus, +textarea:-webkit-autofill, +textarea:-webkit-autofill:hover, +textarea:-webkit-autofill:focus, +select:-webkit-autofill, +select:-webkit-autofill:hover, +select:-webkit-autofill:focus { + -webkit-box-shadow: 0 0 0 1000px transparent inset !important; + transition: background-color 5000s ease-in-out 0s !important; } html { @@ -22,6 +34,7 @@ html { a { color: #ff00c3; + &:hover { color: #ff00c3; } @@ -29,7 +42,7 @@ a { code { font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', - monospace; + monospace; color: #ff00c3; } @@ -44,7 +57,6 @@ code { align-items: center; overflow: hidden; position: relative; - } #browser-content { @@ -52,13 +64,10 @@ code { width: 100%; display: flex; flex-direction: column; - transform: scale(1); /* Ensure no scaling */ - transform-origin: top left; /* Keep the position fixed */ -} - - -#browser { - + transform: scale(1); + /* Ensure no scaling */ + transform-origin: top left; + /* Keep the position fixed */ } #browser-window { @@ -163,4 +172,4 @@ code { height: calc(100vh - 2rem); margin: 1rem 55rem; } -} +} \ No newline at end of file diff --git a/src/pages/MainPage.tsx b/src/pages/MainPage.tsx index 0801a933..60990291 100644 --- a/src/pages/MainPage.tsx +++ b/src/pages/MainPage.tsx @@ -103,6 +103,17 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps) setSockets(sockets => [...sockets, socket]); socket.on('ready-for-run', () => readyForRunHandler(browserId, runId)); socket.on('debugMessage', debugMessageHandler); + socket.on('run-completed', (data) => { + setRerenderRuns(true); + + const robotName = data.robotName; + + if (data.status === 'success') { + notify('success', t('main_page.notifications.interpretation_success', { name: robotName })); + } else { + notify('error', t('main_page.notifications.interpretation_failed', { name: robotName })); + } + }); setContent('runs'); if (browserId) { notify('info', t('main_page.notifications.run_started', { name: runningRecordingName })); diff --git a/src/pages/PageWrapper.tsx b/src/pages/PageWrapper.tsx index 1e73f26a..16f2e50d 100644 --- a/src/pages/PageWrapper.tsx +++ b/src/pages/PageWrapper.tsx @@ -16,10 +16,11 @@ import { NotFoundPage } from '../components/dashboard/NotFound'; export const PageWrapper = () => { const [open, setOpen] = useState(false); + const [isRecordingMode, setIsRecordingMode] = useState(false); const navigate = useNavigate(); - const { browserId, setBrowserId, notification, recordingName, setRecordingName, recordingId, setRecordingId } = useGlobalInfoStore(); + const { browserId, setBrowserId, notification, recordingName, setRecordingName, recordingId, setRecordingId, setRecordingUrl } = useGlobalInfoStore(); const handleEditRecording = (recordingId: string, fileName: string) => { setRecordingName(fileName); @@ -35,23 +36,62 @@ export const PageWrapper = () => { return notification.isOpen; } + /** + * Get the current tab's state from session storage + */ + const getTabState = (key: string): string | null => { + try { + const value = window.sessionStorage.getItem(key); + return value; + } catch (error) { + return null; + } + }; + useEffect(() => { - const isRecordingInProgress = async () => { - const id = await getActiveBrowserId(); - if (id) { - setBrowserId(id); + const tabMode = getTabState('tabMode'); + const urlParams = new URLSearchParams(window.location.search); + const sessionParam = urlParams.get('session'); + const storedSessionId = getTabState('recordingSessionId'); + const storedRecordingUrl = getTabState('recordingUrl'); + + if (location.pathname === '/recording-setup' && sessionParam && sessionParam === storedSessionId) { + setBrowserId('new-recording'); + setRecordingName(''); + setRecordingId(''); + + if (storedRecordingUrl) { + setRecordingUrl(storedRecordingUrl); + } + + navigate('/recording'); + } + else if (location.pathname === '/recording' || + (getTabState('nextTabIsRecording') === 'true' && sessionParam === storedSessionId)) { + setIsRecordingMode(true); + + if (location.pathname !== '/recording') { navigate('/recording'); } + + window.sessionStorage.removeItem('nextTabIsRecording'); + } else if (tabMode === 'main') { + console.log('Tab is in main application mode'); + } else { + const id = getTabState('browserId'); + if (id === 'new-recording' || location.pathname === '/recording') { + setIsRecordingMode(true); + } } - isRecordingInProgress(); - }, []); - + }, [location.pathname, navigate, setBrowserId, setRecordingId, setRecordingName, setRecordingUrl]); + return (
- {!browserId && } + {/* {!browserId && location.pathname !== '/recording' && } */} + {location.pathname !== '/recording' && } }> } /> diff --git a/src/pages/RecordingPage.tsx b/src/pages/RecordingPage.tsx index 259e6a0d..034b2ea8 100644 --- a/src/pages/RecordingPage.tsx +++ b/src/pages/RecordingPage.tsx @@ -44,7 +44,7 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => { const { setId, socket } = useSocketStore(); const { setWidth } = useBrowserDimensionsStore(); - const { browserId, setBrowserId, recordingId, recordingUrl } = useGlobalInfoStore(); + const { browserId, setBrowserId, recordingId, recordingUrl, setRecordingUrl } = useGlobalInfoStore(); const handleShowOutputData = useCallback(() => { setShowOutputData(true); @@ -78,6 +78,11 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => { useEffect(() => { let isCancelled = false; const handleRecording = async () => { + const storedUrl = window.sessionStorage.getItem('recordingUrl'); + if (storedUrl && !recordingUrl) { + setRecordingUrl(storedUrl); + } + const id = await getActiveBrowserId(); if (!isCancelled) { if (id) { @@ -91,13 +96,13 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => { } } }; - + handleRecording(); - + return () => { isCancelled = true; } - }, [setId]); + }, [setId, recordingUrl, setRecordingUrl]); const changeBrowserDimensions = useCallback(() => { if (browserContentRef.current) { diff --git a/src/routes/userRoute.tsx b/src/routes/userRoute.tsx index 4e9bb7c2..fe0a31ff 100644 --- a/src/routes/userRoute.tsx +++ b/src/routes/userRoute.tsx @@ -1,12 +1,56 @@ -import React from 'react'; -import { Navigate, Outlet } from 'react-router-dom'; +import React, { useEffect, useState } from 'react'; +import { Navigate, Outlet, useLocation } from 'react-router-dom'; import { useContext } from 'react'; import { AuthContext } from '../context/auth'; +import { useGlobalInfoStore } from '../context/globalInfo'; const UserRoute = () => { const { state } = useContext(AuthContext); + const location = useLocation(); + const [isCheckingAuth, setIsCheckingAuth] = useState(true); + const { setRecordingUrl } = useGlobalInfoStore(); + + useEffect(() => { + if (location.pathname === '/recording') { + const hasRecordingSession = + window.sessionStorage.getItem('browserId') || + window.sessionStorage.getItem('recordingSessionId'); + + const recordingUrl = window.sessionStorage.getItem('recordingUrl'); + if (recordingUrl) { + setRecordingUrl(recordingUrl); + } + + if (hasRecordingSession) { + console.log('UserRoute: Valid recording session detected, bypassing auth check'); + setIsCheckingAuth(false); + return; + } + } + + const timer = setTimeout(() => { + setIsCheckingAuth(false); + }, 100); + + return () => clearTimeout(timer); + }, [location.pathname]); + + if (isCheckingAuth) { + return null; + } + + if (location.pathname === '/recording') { + const hasRecordingSession = + window.sessionStorage.getItem('browserId') || + window.sessionStorage.getItem('recordingSessionId'); + + if (hasRecordingSession) { + return ; + } + } + return state.user ? : ; }; -export default UserRoute; +export default UserRoute; \ No newline at end of file