Merge pull request #460 from getmaxun/perf-v11

wip: performance
This commit is contained in:
Karishma Shukla
2025-03-12 22:16:31 +05:30
committed by GitHub
29 changed files with 1928 additions and 270 deletions

View File

@@ -56,6 +56,7 @@
"moment-timezone": "^0.5.45",
"node-cron": "^3.0.3",
"pg": "^8.13.0",
"pg-boss": "^10.1.6",
"pkce-challenge": "^4.1.0",
"playwright": "^1.48.2",
"playwright-extra": "^4.3.6",

View File

@@ -18,7 +18,7 @@ export class FrontendPerformanceMonitor {
};
this.lastFrameTime = performance.now();
this.frameCount = 0;
// Start monitoring
this.startMonitoring();
}
@@ -93,6 +93,262 @@ export class FrontendPerformanceMonitor {
}
}
// Enhanced Performance Monitor with Memory Management
export class EnhancedPerformanceMonitor extends FrontendPerformanceMonitor {
private memoryWarningThreshold: number = 100000000; // 100MB
private memoryAlertCallback?: (usage: MemoryInfo) => void;
private frameTimeHistory: number[] = [];
private maxMetricsHistory: number = 100; // Keep only the last 100 readings
private isThrottled: boolean = false;
private rafHandle: number | null = null;
private memoryCheckInterval: NodeJS.Timeout | null = null;
constructor(options?: {
memoryWarningThreshold?: number,
maxMetricsHistory?: number,
memoryAlertCallback?: (usage: MemoryInfo) => void
}) {
super();
if (options) {
if (options.memoryWarningThreshold) {
this.memoryWarningThreshold = options.memoryWarningThreshold;
}
if (options.maxMetricsHistory) {
this.maxMetricsHistory = options.maxMetricsHistory;
}
if (options.memoryAlertCallback) {
this.memoryAlertCallback = options.memoryAlertCallback;
}
}
// Override the parent's monitoring with our enhanced version
this.startEnhancedMonitoring();
}
private startEnhancedMonitoring(): void {
// Stop any existing monitoring from parent class
if (this.rafHandle) {
cancelAnimationFrame(this.rafHandle);
}
if (this.memoryCheckInterval) {
clearInterval(this.memoryCheckInterval);
}
// Enhanced FPS monitoring with frame time tracking
let lastFrameTime = performance.now();
let frameCount = 0;
let frameTimes: number[] = [];
const measureFPS = () => {
const now = performance.now();
const frameTime = now - lastFrameTime;
lastFrameTime = now;
// Track individual frame times for jank detection
frameTimes.push(frameTime);
if (frameTimes.length > 60) { // Keep only last 60 frame times
frameTimes.shift();
}
frameCount++;
this.frameTimeHistory.push(frameTime);
// Calculate FPS every second
if (this.frameTimeHistory.length >= 60) {
const totalTime = this.frameTimeHistory.reduce((sum, time) => sum + time, 0);
const fps = Math.round((this.frameTimeHistory.length * 1000) / totalTime);
// Get metrics from parent class
const metrics = this.getMetrics();
metrics.fps.push(fps);
// Limit metrics history
if (metrics.fps.length > this.maxMetricsHistory) {
metrics.fps.shift();
}
// Detect jank (long frames)
const jankThreshold = 16.7 * 2; // 2x normal frame time at 60fps
const jankFrames = frameTimes.filter(time => time > jankThreshold);
if (jankFrames.length > 10) { // If more than 10 out of 60 frames are janky
this.detectPerformanceIssue('jank', {
jankFrames: jankFrames.length,
averageJankTime: jankFrames.reduce((sum, time) => sum + time, 0) / jankFrames.length
});
}
// Reset for next measurement
this.frameTimeHistory = [];
frameTimes = [];
}
this.rafHandle = requestAnimationFrame(measureFPS);
};
this.rafHandle = requestAnimationFrame(measureFPS);
// Enhanced memory monitoring
if (window.performance && (performance as any).memory) {
this.memoryCheckInterval = setInterval(() => {
const memory = (performance as any).memory;
const memoryInfo = {
usedJSHeapSize: memory.usedJSHeapSize,
totalJSHeapSize: memory.totalJSHeapSize,
timestamp: Date.now()
};
// Get metrics from parent class
const metrics = this.getMetrics();
metrics.memoryUsage.push(memoryInfo);
// Limit metrics history
if (metrics.memoryUsage.length > this.maxMetricsHistory) {
metrics.memoryUsage.shift();
}
// Check for memory warnings
if (memoryInfo.usedJSHeapSize > this.memoryWarningThreshold) {
this.detectPerformanceIssue('memory', memoryInfo);
if (this.memoryAlertCallback) {
this.memoryAlertCallback(memoryInfo);
}
}
// Check for memory leaks (steady increase)
if (metrics.memoryUsage.length >= 10) {
const recentMemory = metrics.memoryUsage.slice(-10);
let increasingCount = 0;
for (let i = 1; i < recentMemory.length; i++) {
if (recentMemory[i].usedJSHeapSize > recentMemory[i - 1].usedJSHeapSize) {
increasingCount++;
}
}
// If memory increased in 8 out of 9 consecutive readings
if (increasingCount >= 8) {
this.detectPerformanceIssue('memoryLeak', {
startMemory: recentMemory[0].usedJSHeapSize,
currentMemory: recentMemory[recentMemory.length - 1].usedJSHeapSize,
increaseRate: (recentMemory[recentMemory.length - 1].usedJSHeapSize - recentMemory[0].usedJSHeapSize) /
(recentMemory[recentMemory.length - 1].timestamp - recentMemory[0].timestamp) * 1000 // bytes per second
});
}
}
}, 1000);
}
}
// Method to detect various performance issues
private detectPerformanceIssue(type: 'jank' | 'memory' | 'memoryLeak', data: any): void {
console.warn(`Performance issue detected: ${type}`, data);
if (type === 'memory' || type === 'memoryLeak') {
// Auto-throttle rendering if memory issues detected
if (!this.isThrottled) {
this.throttleRendering();
}
// Suggest garbage collection
this.suggestGarbageCollection();
}
}
// Get access to the metrics from parent class
private getMetrics(): any {
return (this as any).metrics;
}
// Throttle rendering to reduce memory pressure
private throttleRendering(): void {
this.isThrottled = true;
console.info('Throttling rendering due to memory pressure');
// Application code would implement throttling behavior
}
// Un-throttle rendering when memory pressure is reduced
public unthrottleRendering(): void {
if (this.isThrottled) {
this.isThrottled = false;
console.info('Resuming normal rendering');
}
}
// Suggest garbage collection to the browser
private suggestGarbageCollection(): void {
if (window.gc) {
try {
window.gc();
} catch (e) {
// gc() might not be available without special flags
}
}
// Alternative approach to encourage garbage collection
const largeArray = new Array(1000000).fill(0);
largeArray.length = 0;
}
// Enhanced performance report with more detailed metrics
public getEnhancedPerformanceReport(): EnhancedPerformanceReport {
const baseReport = super.getPerformanceReport();
const metrics = this.getMetrics();
// Calculate 95th percentile render time
const sortedRenderTimes = [...metrics.renderTime].sort((a, b) => a - b);
const idx95 = Math.floor(sortedRenderTimes.length * 0.95);
const renderTime95Percentile = sortedRenderTimes[idx95] || 0;
// Calculate memory growth rate
let memoryGrowthRate = 0;
if (metrics.memoryUsage.length >= 2) {
const first = metrics.memoryUsage[0];
const last = metrics.memoryUsage[metrics.memoryUsage.length - 1];
const timeDiffInSeconds = (last.timestamp - first.timestamp) / 1000;
memoryGrowthRate = timeDiffInSeconds > 0
? (last.usedJSHeapSize - first.usedJSHeapSize) / timeDiffInSeconds
: 0;
}
return {
...baseReport,
renderTime95Percentile,
memoryGrowthRate,
isThrottled: this.isThrottled,
heapUsagePercentage: baseReport.lastMemoryUsage
? (baseReport.lastMemoryUsage.usedJSHeapSize / baseReport.lastMemoryUsage.totalJSHeapSize) * 100
: 0
};
}
// Clean up resources when no longer needed
public dispose(): void {
if (this.rafHandle) {
cancelAnimationFrame(this.rafHandle);
this.rafHandle = null;
}
if (this.memoryCheckInterval) {
clearInterval(this.memoryCheckInterval);
this.memoryCheckInterval = null;
}
}
}
// Extended types
interface EnhancedPerformanceReport extends PerformanceReport {
renderTime95Percentile: number;
memoryGrowthRate: number; // bytes per second
isThrottled: boolean;
heapUsagePercentage: number;
}
// Backend Performance Monitoring
export class BackendPerformanceMonitor {
private metrics: {

View File

@@ -512,6 +512,7 @@
"success": "Erfolg",
"running": "Läuft",
"scheduled": "Geplant",
"queued": "In Warteschlange",
"failed": "Fehlgeschlagen"
},
"run_settings_modal": {

View File

@@ -54,6 +54,12 @@
"label":"URL",
"button":"Start Recording"
},
"warning_modal":{
"title":"Active Browser Detected",
"message": "There is already a browser recording session running. Would you like to discard it and create a new recording?",
"discard_and_create":"Discard & Create New",
"cancel":"Cancel"
},
"edit":"Edit",
"delete":"Delete",
"duplicate":"Duplicate",
@@ -514,6 +520,7 @@
"success": "Success",
"running": "Running",
"scheduled": "Scheduled",
"queued": "Queued",
"failed": "Failed"
},
"run_settings_modal": {

View File

@@ -513,6 +513,7 @@
"success": "Éxito",
"running": "Ejecutando",
"scheduled": "Programado",
"queued": "En cola",
"failed": "Fallido"
},
"run_settings_modal": {

View File

@@ -513,6 +513,7 @@
"success": "成功",
"running": "実行中",
"scheduled": "スケジュール済み",
"queued": "キューに入れました",
"failed": "失敗"
},
"run_settings_modal": {

View File

@@ -513,6 +513,7 @@
"success": "成功",
"running": "运行中",
"scheduled": "已计划",
"queued": "排队",
"failed": "失败"
},
"run_settings_modal": {

View File

@@ -4,6 +4,12 @@ import logger from "../../logger";
/**
* @category Types
*/
/**
* Represents the possible states of a remote browser.
* @category Types
*/
type BrowserState = "recording" | "run";
interface BrowserPoolInfo {
/**
* The instance of remote browser.
@@ -19,6 +25,12 @@ interface BrowserPoolInfo {
* The user ID that owns this browser instance.
*/
userId: string,
/**
* The current state of the browser.
* Can be "recording" or "run".
* @default "recording"
*/
state: BrowserState,
}
/**
@@ -33,7 +45,7 @@ interface PoolDictionary {
/**
* A browser pool is a collection of remote browsers that are initialized and ready to be used.
* Enforces a "1 User - 1 Browser" policy, while allowing multiple users to have their own browser instances.
* Enforces a "1 User - 2 Browser" policy, while allowing multiple users to have their own browser instances.
* Adds the possibility to add, remove and retrieve remote browsers from the pool.
* @category BrowserManagement
*/
@@ -45,12 +57,13 @@ export class BrowserPool {
/**
* Maps user IDs to their browser IDs.
* A user can have up to 2 browsers.
*/
private userToBrowserMap: Map<string, string> = new Map();
private userToBrowserMap: Map<string, string[]> = new Map();
/**
* Adds a remote browser instance to the pool for a specific user.
* If the user already has a browser, the existing browser will be closed and replaced.
* If the user already has two browsers, the oldest browser will be closed and replaced.
*
* @param id remote browser instance's id
* @param browser remote browser instance
@@ -62,38 +75,58 @@ export class BrowserPool {
id: string,
browser: RemoteBrowser,
userId: string,
active: boolean = false
active: boolean = false,
state: BrowserState = "recording"
): boolean => {
// Check if user already has a browser
const existingBrowserId = this.userToBrowserMap.get(userId);
// Check if browser with this ID already exists and belongs to this user
if (this.pool[id] && this.pool[id].userId === userId) {
// Just update the existing browser
this.pool[id] = {
browser,
active,
userId,
state: this.pool[id].state || state,
};
logger.log('debug', `Updated existing browser with id: ${id} for user: ${userId}`);
return false;
}
// Get existing browsers for this user
let userBrowserIds = this.userToBrowserMap.get(userId) || [];
let replaced = false;
if (existingBrowserId) {
// Close and remove the existing browser
if (existingBrowserId !== id) {
this.closeAndDeleteBrowser(existingBrowserId);
replaced = true;
} else {
// If it's the same browser ID, just update the info
this.pool[id] = {
browser,
active,
userId,
};
logger.log('debug', `Updated existing browser with id: ${id} for user: ${userId}`);
// If trying to add a "recording" browser, check if one already exists
if (state === "recording") {
// Check if user already has a recording browser
const hasRecordingBrowser = userBrowserIds.some(browserId =>
this.pool[browserId] && this.pool[browserId].state === "recording"
);
if (hasRecordingBrowser) {
logger.log('debug', `User ${userId} already has a browser in "recording" state`);
return false;
}
}
// For "run" state, check if the user already has the maximum number of browsers (2)
if (userBrowserIds.length >= 2 && !userBrowserIds.includes(id)) {
logger.log('debug', "User already has the maximum number of browsers (2)");
return false;
}
// Add the new browser to the pool
this.pool[id] = {
browser,
active,
userId,
state,
};
// Update the user-to-browser mapping
this.userToBrowserMap.set(userId, id);
if (!userBrowserIds.includes(id)) {
userBrowserIds.push(id);
}
this.userToBrowserMap.set(userId, userBrowserIds);
logger.log('debug', `Remote browser with id: ${id} added to the pool for user: ${userId}`);
return !replaced;
@@ -115,8 +148,16 @@ export class BrowserPool {
// Remove the user-to-browser mapping
const userId = this.pool[id].userId;
if (this.userToBrowserMap.get(userId) === id) {
this.userToBrowserMap.delete(userId);
const userBrowserIds = this.userToBrowserMap.get(userId) || [];
if (userBrowserIds.includes(id)) {
const updatedBrowserIds = userBrowserIds.filter(bid => bid !== id);
if (updatedBrowserIds.length === 0) {
this.userToBrowserMap.delete(userId);
} else {
this.userToBrowserMap.set(userId, updatedBrowserIds);
}
}
// Remove from pool
@@ -139,8 +180,16 @@ export class BrowserPool {
// Remove the user-to-browser mapping
const userId = this.pool[id].userId;
if (this.userToBrowserMap.get(userId) === id) {
this.userToBrowserMap.delete(userId);
const userBrowserIds = this.userToBrowserMap.get(userId) || [];
if (userBrowserIds.includes(id)) {
const updatedBrowserIds = userBrowserIds.filter(bid => bid !== id);
if (updatedBrowserIds.length === 0) {
this.userToBrowserMap.delete(userId);
} else {
this.userToBrowserMap.set(userId, updatedBrowserIds);
}
}
// Remove from pool
@@ -162,25 +211,73 @@ export class BrowserPool {
/**
* Returns the active browser's instance id for a specific user.
* If state is specified, only returns a browser with that exact state.
*
* @param userId the user ID to find the browser for
* @returns the browser ID for the user, or null if no browser exists
* @param state optional browser state filter ("recording" or "run")
* @returns the browser ID for the user, or null if no browser exists with the required state
*/
public getActiveBrowserId = (userId: string): string | null => {
const browserId = this.userToBrowserMap.get(userId);
if (!browserId) {
public getActiveBrowserId = (userId: string, state?: BrowserState): string | null => {
const browserIds = this.userToBrowserMap.get(userId);
if (!browserIds || browserIds.length === 0) {
logger.log('debug', `No browser found for user: ${userId}`);
return null;
}
// Verify the browser still exists in the pool
if (!this.pool[browserId]) {
this.userToBrowserMap.delete(userId);
logger.log('warn', `Browser mapping found for user: ${userId}, but browser doesn't exist in pool`);
// If state is specified, only return browsers with that exact state
if (state) {
// Check browsers in reverse order (newest first) to find one with the specified state
for (let i = browserIds.length - 1; i >= 0; i--) {
const browserId = browserIds[i];
// Verify the browser still exists in the pool
if (!this.pool[browserId]) {
browserIds.splice(i, 1);
continue;
}
// Check if browser matches state filter
if (this.pool[browserId].state === state) {
return browserId;
}
}
// If no browser with matching state, return null
logger.log('debug', `No browser with state ${state} found for user: ${userId}`);
return null;
}
console.log(`Browser Id ${browserId} found for user: ${userId}`);
return browserId;
// If no state specified, return any browser
for (let i = browserIds.length - 1; i >= 0; i--) {
const browserId = browserIds[i];
// Verify the browser still exists in the pool
if (!this.pool[browserId]) {
browserIds.splice(i, 1);
continue;
}
// Return the first browser found
if (this.pool[browserId]) {
console.log(`Active browser Id ${browserId} found for user: ${userId}`);
return browserId;
}
}
// If no active browser, return the most recent one
if (browserIds.length > 0) {
const mostRecentId = browserIds[browserIds.length - 1];
console.log(`No active browser found, returning most recent browser Id ${mostRecentId} for user: ${userId}`);
return mostRecentId;
}
// Clean up the mapping if all browsers were invalid
if (browserIds.length === 0) {
this.userToBrowserMap.delete(userId);
}
logger.log('warn', `Browser mapping found for user: ${userId}, but no valid browsers exist in pool`);
return null;
};
/**
@@ -213,10 +310,62 @@ export class BrowserPool {
logger.log('debug', `Remote browser with id: ${id} set to ${active ? 'active' : 'inactive'}`);
return true;
};
/**
* Sets the state of a browser.
* Only allows one browser in "recording" state per user.
*
* @param id the browser ID
* @param state the new state ("recording" or "run")
* @returns true if successful, false if the browser wasn't found or state change not allowed
*/
public setBrowserState = (id: string, state: BrowserState): boolean => {
if (!this.pool[id]) {
logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`);
return false;
}
// If trying to set to "recording" state, check if another browser is already recording
if (state === "recording") {
const userId = this.pool[id].userId;
const userBrowserIds = this.userToBrowserMap.get(userId) || [];
// Check if any other browser for this user is already in recording state
const hasAnotherRecordingBrowser = userBrowserIds.some(browserId =>
browserId !== id &&
this.pool[browserId] &&
this.pool[browserId].state === "recording"
);
if (hasAnotherRecordingBrowser) {
logger.log('warn', `Cannot set browser ${id} to "recording" state: User ${userId} already has a browser in recording state`);
return false;
}
}
this.pool[id].state = state;
logger.log('debug', `Remote browser with id: ${id} state set to ${state}`);
return true;
};
/**
* Gets the current state of a browser.
*
* @param id the browser ID
* @returns the current state or null if the browser wasn't found
*/
public getBrowserState = (id: string): BrowserState | null => {
if (!this.pool[id]) {
logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`);
return null;
}
return this.pool[id].state;
};
/**
* Returns all browser instances for a specific user.
* Should only be one per the "1 User - 1 Browser" policy, but included for flexibility.
* With the "1 User - 2 Browser" policy, this can return up to 2 browsers.
*
* @param userId the user ID to find browsers for
* @returns an array of browser IDs belonging to the user
@@ -224,24 +373,31 @@ export class BrowserPool {
public getAllBrowserIdsForUser = (userId: string): string[] => {
const browserIds: string[] = [];
// Normally this would just return the one browser from the map
const mappedBrowserId = this.userToBrowserMap.get(userId);
if (mappedBrowserId && this.pool[mappedBrowserId]) {
browserIds.push(mappedBrowserId);
// Get browser IDs from the map
const mappedBrowserIds = this.userToBrowserMap.get(userId) || [];
// Filter to only include IDs that exist in the pool
for (const id of mappedBrowserIds) {
if (this.pool[id]) {
browserIds.push(id);
}
}
// But as a safeguard, also check the entire pool for any browsers assigned to this user
// As a safeguard, also check the entire pool for any browsers assigned to this user
// This helps detect and fix any inconsistencies in the maps
for (const [id, info] of Object.entries(this.pool)) {
if (info.userId === userId && !browserIds.includes(id)) {
browserIds.push(id);
// Fix the map if it's inconsistent
if (!mappedBrowserId) {
this.userToBrowserMap.set(userId, id);
}
}
}
// Update the map if inconsistencies were found
if (browserIds.length > 0 && JSON.stringify(browserIds) !== JSON.stringify(mappedBrowserIds)) {
// Limit to 2 browsers if more were found
const limitedBrowserIds = browserIds.slice(-2);
this.userToBrowserMap.set(userId, limitedBrowserIds);
}
return browserIds;
};
@@ -264,24 +420,81 @@ export class BrowserPool {
* This is a migration helper to support code that hasn't been updated to the user-browser model yet.
*
* @param currentUserId The ID of the current user, which will be prioritized if multiple browsers exist
* @param state Optional state filter to find browsers in a specific state
* @returns A browser ID if one can be determined, or null
*/
public getActiveBrowserForMigration = (currentUserId?: string): string | null => {
public getActiveBrowserForMigration = (currentUserId?: string, state?: BrowserState): string | null => {
// If a current user ID is provided and they have a browser, return that
if (currentUserId) {
const browserForUser = this.getActiveBrowserId(currentUserId);
const browserForUser = this.getActiveBrowserId(currentUserId, state);
if (browserForUser) {
return browserForUser;
}
// If state is specified and no matching browser was found, return null
if (state) {
return null;
}
}
// If only one user has a browser, return that
// If only one user has a browser, try to find a matching browser
if (this.userToBrowserMap.size === 1) {
const userId = Array.from(this.userToBrowserMap.keys())[0];
return this.userToBrowserMap.get(userId) || null;
const browserIds = this.userToBrowserMap.get(userId) || [];
// If state is specified, only look for that state
if (state) {
// Return the active browser that matches the state
for (let i = browserIds.length - 1; i >= 0; i--) {
const bid = browserIds[i];
if (this.pool[bid]?.active && this.pool[bid].state === state) {
return bid;
}
}
// If no active browser with matching state, try to find any browser with matching state
for (let i = browserIds.length - 1; i >= 0; i--) {
const bid = browserIds[i];
if (this.pool[bid] && this.pool[bid].state === state) {
return bid;
}
}
// If still no matching browser, return null
return null;
}
// If no state filter, find any active browser
for (let i = browserIds.length - 1; i >= 0; i--) {
if (this.pool[browserIds[i]]?.active) {
return browserIds[i];
}
}
return browserIds.length > 0 ? browserIds[browserIds.length - 1] : null;
}
// Fall back to the first active browser if any
// Fall back to checking all browsers if no user was specified
if (state) {
// Look for active browsers with the specific state
for (const id of Object.keys(this.pool)) {
if (this.pool[id].active && this.pool[id].state === state) {
return id;
}
}
// Then look for any browser with the specific state
for (const id of Object.keys(this.pool)) {
if (this.pool[id].state === state) {
return id;
}
}
// If no browser with the requested state is found, return null
return null;
}
// If no state filter, find any active browser
for (const id of Object.keys(this.pool)) {
if (this.pool[id].active) {
return id;
@@ -299,7 +512,7 @@ export class BrowserPool {
* If there are multiple active browsers, it returns the first one.
*
* @returns the first remote active browser instance's id from the pool
* @deprecated Use getBrowserIdForUser instead to enforce the 1 User - 1 Browser policy
* @deprecated Use getBrowserIdForUser instead to enforce the 1 User - 2 Browser policy
*/
public getActiveBrowserIdLegacy = (): string | null => {
for (const id of Object.keys(this.pool)) {

View File

@@ -21,12 +21,12 @@ import logger from "../logger";
* @category BrowserManagement-Controller
*/
export const initializeRemoteBrowserForRecording = (userId: string): string => {
const id = getActiveBrowserId(userId) || uuid();
const id = getActiveBrowserIdByState(userId, "recording") || uuid();
createSocketConnection(
io.of(id),
async (socket: Socket) => {
// browser is already active
const activeId = getActiveBrowserId(userId);
const activeId = getActiveBrowserIdByState(userId, "recording");
if (activeId) {
const remoteBrowser = browserPool.getRemoteBrowser(activeId);
remoteBrowser?.updateSocket(socket);
@@ -37,7 +37,7 @@ export const initializeRemoteBrowserForRecording = (userId: string): string => {
await browserSession.initialize(userId);
await browserSession.registerEditorEvents();
await browserSession.subscribeToScreencast();
browserPool.addRemoteBrowser(id, browserSession, userId);
browserPool.addRemoteBrowser(id, browserSession, userId, false, "recording");
}
socket.emit('loaded');
});
@@ -59,7 +59,7 @@ export const createRemoteBrowserForRun = (userId: string): string => {
async (socket: Socket) => {
const browserSession = new RemoteBrowser(socket, userId);
await browserSession.initialize(userId);
browserPool.addRemoteBrowser(id, browserSession, userId);
browserPool.addRemoteBrowser(id, browserSession, userId, false, "run");
socket.emit('ready-for-run');
});
return id;
@@ -92,6 +92,17 @@ export const getActiveBrowserId = (userId: string): string | null => {
return browserPool.getActiveBrowserId(userId);
};
/**
* Returns the id of an active browser with the specified state or null.
* @param userId the user ID to find the browser for
* @param state the browser state to filter by ("recording" or "run")
* @returns {string | null}
* @category BrowserManagement-Controller
*/
export const getActiveBrowserIdByState = (userId: string, state: "recording" | "run"): string | null => {
return browserPool.getActiveBrowserId(userId, state);
};
/**
* Returns the url string from a remote browser if exists in the browser pool.
* @param id instance id of the remote browser
@@ -127,7 +138,7 @@ export const getRemoteBrowserCurrentTabs = (id: string, userId: string): string[
* @category BrowserManagement-Controller
*/
export const interpretWholeWorkflow = async (userId: string) => {
const id = getActiveBrowserId(userId);
const id = getActiveBrowserIdByState(userId, "recording");
if (id) {
const browser = browserPool.getRemoteBrowser(id);
if (browser) {
@@ -147,7 +158,7 @@ export const interpretWholeWorkflow = async (userId: string) => {
* @category BrowserManagement-Controller
*/
export const stopRunningInterpretation = async (userId: string) => {
const id = getActiveBrowserId(userId);
const id = getActiveBrowserIdByState(userId, "recording");
if (id) {
const browser = browserPool.getRemoteBrowser(id);
await browser?.stopCurrentInterpretation();

View File

@@ -56,7 +56,7 @@ const handleWrapper = async (
return;
}
const id = browserPool.getActiveBrowserId(userId);
const id = browserPool.getActiveBrowserId(userId, "recording");
if (id) {
const activeBrowser = browserPool.getRemoteBrowser(id);
if (activeBrowser?.interpreter.interpretationInProgress() && !activeBrowser.interpreter.interpretationIsPaused) {

488
server/src/pgboss-worker.ts Normal file
View File

@@ -0,0 +1,488 @@
/**
* Recording worker using PgBoss for asynchronous browser recording operations
*/
import PgBoss, { Job } from 'pg-boss';
import logger from './logger';
import {
initializeRemoteBrowserForRecording,
destroyRemoteBrowser,
interpretWholeWorkflow,
stopRunningInterpretation,
createRemoteBrowserForRun
} from './browser-management/controller';
import { WorkflowFile } from 'maxun-core';
import Run from './models/Run';
import Robot from './models/Robot';
import { browserPool } from './server';
import { Page } from 'playwright';
import { BinaryOutputService } from './storage/mino';
import { capture } from './utils/analytics';
import { googleSheetUpdateTasks, processGoogleSheetUpdates } from './workflow-management/integrations/gsheet';
import { airtableUpdateTasks, processAirtableUpdates } from './workflow-management/integrations/airtable';
import { RemoteBrowser } from './browser-management/classes/RemoteBrowser';
import { io as serverIo } from "./server";
const pgBossConnectionString = `postgres://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`;
interface InitializeBrowserData {
userId: string;
}
interface InterpretWorkflow {
userId: string;
}
interface StopInterpretWorkflow {
userId: string;
}
interface DestroyBrowserData {
browserId: string;
userId: string;
}
interface ExecuteRunData {
userId: string;
runId: string;
browserId: string;
}
const pgBoss = new PgBoss({connectionString: pgBossConnectionString, schema: 'public'});
/**
* Extract data safely from a job (single job or job array)
*/
function extractJobData<T>(job: Job<T> | Job<T>[]): T {
if (Array.isArray(job)) {
if (job.length === 0) {
throw new Error('Empty job array received');
}
return job[0].data;
}
return job.data;
}
function AddGeneratedFlags(workflow: WorkflowFile) {
const copy = JSON.parse(JSON.stringify(workflow));
for (let i = 0; i < workflow.workflow.length; i++) {
copy.workflow[i].what.unshift({
action: 'flag',
args: ['generated'],
});
}
return copy;
};
/**
* Function to reset browser state without creating a new browser
*/
async function resetBrowserState(browser: RemoteBrowser): Promise<boolean> {
try {
const currentPage = browser.getCurrentPage();
if (!currentPage) {
logger.log('error', 'No current page available to reset browser state');
return false;
}
// Navigate to blank page to reset state
await currentPage.goto('about:blank');
// Clear browser storage
await currentPage.evaluate(() => {
try {
localStorage.clear();
sessionStorage.clear();
} catch (e) {
// Ignore errors in cleanup
}
});
// Clear cookies
const context = currentPage.context();
await context.clearCookies();
return true;
} catch (error) {
logger.log('error', `Failed to reset browser state`);
return false;
}
}
/**
* Modified checkAndProcessQueuedRun function - only changes browser reset logic
*/
async function checkAndProcessQueuedRun(userId: string, browserId: string): Promise<boolean> {
try {
// Find the oldest queued run for this specific browser
const queuedRun = await Run.findOne({
where: {
browserId: browserId,
status: 'queued'
},
order: [['startedAt', 'ASC']]
});
if (!queuedRun) {
logger.log('info', `No queued runs found for browser ${browserId}`);
return false;
}
// Reset the browser state before next run
const browser = browserPool.getRemoteBrowser(browserId);
if (browser) {
logger.log('info', `Resetting browser state for browser ${browserId} before next run`);
await resetBrowserState(browser);
}
// Update the queued run to running status
await queuedRun.update({
status: 'running',
log: 'Run started - using browser from previous run'
});
// Schedule the run execution
await pgBoss.createQueue('execute-run');
const executeJobId = await pgBoss.send('execute-run', {
userId: userId,
runId: queuedRun.runId,
browserId: browserId
});
logger.log('info', `Scheduled queued run ${queuedRun.runId} to use browser ${browserId}, job ID: ${executeJobId}`);
return true;
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error);
logger.log('error', `Error checking for queued runs: ${errorMessage}`);
return false;
}
}
/**
* Modified processRunExecution function - only add browser reset
*/
async function processRunExecution(job: Job<ExecuteRunData>) {
try {
const data = job.data;
logger.log('info', `Processing run execution job for runId: ${data.runId}, browserId: ${data.browserId}`);
// Find the run
const run = await Run.findOne({ where: { runId: data.runId } });
if (!run) {
logger.log('error', `Run ${data.runId} not found in database`);
return { success: false };
}
const plainRun = run.toJSON();
// Find the recording
const recording = await Robot.findOne({ where: { 'recording_meta.id': plainRun.robotMetaId }, raw: true });
if (!recording) {
logger.log('error', `Recording for run ${data.runId} not found`);
// Update run status to failed
await run.update({
status: 'failed',
finishedAt: new Date().toLocaleString(),
log: 'Failed: Recording not found',
});
// Check for queued runs even if this one failed
await checkAndProcessQueuedRun(data.userId, data.browserId);
return { success: false };
}
// Get the browser and execute the run
const browser = browserPool.getRemoteBrowser(plainRun.browserId);
let currentPage = browser?.getCurrentPage();
if (!browser || !currentPage) {
logger.log('error', `Browser or page not available for run ${data.runId}`);
await pgBoss.fail(job.id, "Failed to get browser or page for run");
// Even if this run failed, check for queued runs
await checkAndProcessQueuedRun(data.userId, data.browserId);
return { success: false };
}
try {
// Reset the browser state before executing this run
await resetBrowserState(browser);
// Execute the workflow
const workflow = AddGeneratedFlags(recording.recording);
const interpretationInfo = await browser.interpreter.InterpretRecording(
workflow,
currentPage,
(newPage: Page) => currentPage = newPage,
plainRun.interpreterSettings
);
// Process the results
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput);
// Update the run record with results
await run.update({
...run,
status: 'success',
finishedAt: new Date().toLocaleString(),
browserId: plainRun.browserId,
log: interpretationInfo.log.join('\n'),
serializableOutput: interpretationInfo.serializableOutput,
binaryOutput: uploadedBinaryOutput,
});
// Track extraction metrics
let totalRowsExtracted = 0;
let extractedScreenshotsCount = 0;
let extractedItemsCount = 0;
if (run.dataValues.binaryOutput && run.dataValues.binaryOutput["item-0"]) {
extractedScreenshotsCount = 1;
}
if (run.dataValues.serializableOutput && run.dataValues.serializableOutput["item-0"]) {
const itemsArray = run.dataValues.serializableOutput["item-0"];
extractedItemsCount = itemsArray.length;
totalRowsExtracted = itemsArray.reduce((total, item) => {
return total + Object.keys(item).length;
}, 0);
}
console.log(`Extracted Items Count: ${extractedItemsCount}`);
console.log(`Extracted Screenshots Count: ${extractedScreenshotsCount}`);
console.log(`Total Rows Extracted: ${totalRowsExtracted}`);
// Capture metrics
capture(
'maxun-oss-run-created-manual',
{
runId: data.runId,
user_id: data.userId,
created_at: new Date().toISOString(),
status: 'success',
totalRowsExtracted,
extractedItemsCount,
extractedScreenshotsCount,
}
);
// Schedule updates for Google Sheets and Airtable
try {
googleSheetUpdateTasks[plainRun.runId] = {
robotId: plainRun.robotMetaId,
runId: plainRun.runId,
status: 'pending',
retries: 5,
};
airtableUpdateTasks[plainRun.runId] = {
robotId: plainRun.robotMetaId,
runId: plainRun.runId,
status: 'pending',
retries: 5,
};
processAirtableUpdates();
processGoogleSheetUpdates();
} catch (err: any) {
logger.log('error', `Failed to update Google Sheet for run: ${plainRun.runId}: ${err.message}`);
}
serverIo.of(plainRun.browserId).emit('run-completed', {
runId: data.runId,
robotMetaId: plainRun.robotMetaId,
robotName: recording.recording_meta.name,
status: 'success',
finishedAt: new Date().toLocaleString()
});;
// Check for and process queued runs before destroying the browser
const queuedRunProcessed = await checkAndProcessQueuedRun(data.userId, plainRun.browserId);
// Only destroy the browser if no queued run was found
if (!queuedRunProcessed) {
await destroyRemoteBrowser(plainRun.browserId, data.userId);
logger.log('info', `No queued runs found for browser ${plainRun.browserId}, browser destroyed`);
}
return { success: true };
} catch (executionError: any) {
logger.log('error', `Run execution failed for run ${data.runId}: ${executionError.message}`);
await run.update({
status: 'failed',
finishedAt: new Date().toLocaleString(),
log: `Failed: ${executionError.message}`,
});
// Check for queued runs before destroying the browser
const queuedRunProcessed = await checkAndProcessQueuedRun(data.userId, plainRun.browserId);
// Only destroy the browser if no queued run was found
if (!queuedRunProcessed) {
try {
await destroyRemoteBrowser(plainRun.browserId, data.userId);
logger.log('info', `No queued runs found for browser ${plainRun.browserId}, browser destroyed`);
} catch (cleanupError: any) {
logger.log('warn', `Failed to clean up browser for failed run ${data.runId}: ${cleanupError.message}`);
}
}
// Capture failure metrics
capture(
'maxun-oss-run-created-manual',
{
runId: data.runId,
user_id: data.userId,
created_at: new Date().toISOString(),
status: 'failed',
error_message: executionError.message,
}
);
return { success: false };
}
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error);
logger.log('error', `Failed to process run execution job: ${errorMessage}`);
return { success: false };
}
}
async function registerRunExecutionWorker() {
try {
// Worker for executing runs
await pgBoss.work('execute-run', async (job: Job<ExecuteRunData> | Job<ExecuteRunData>[]) => {
try {
const singleJob = Array.isArray(job) ? job[0] : job;
return await processRunExecution(singleJob);
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error);
logger.log('error', `Run execution job failed: ${errorMessage}`);
throw error;
}
});
// setInterval(checkForStuckQueuedRuns, 30000);
logger.log('info', 'Run execution worker registered successfully');
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error);
logger.log('error', `Failed to register run execution worker: ${errorMessage}`);
}
}
/**
* Initialize PgBoss and register all workers
*/
async function startWorkers() {
try {
logger.log('info', 'Starting PgBoss worker...');
await pgBoss.start();
logger.log('info', 'PgBoss worker started successfully');
// Worker for initializing browser recording
await pgBoss.work('initialize-browser-recording', async (job: Job<InitializeBrowserData> | Job<InitializeBrowserData>[]) => {
try {
const data = extractJobData(job);
const userId = data.userId;
logger.log('info', `Starting browser initialization job for user: ${userId}`);
const browserId = initializeRemoteBrowserForRecording(userId);
logger.log('info', `Browser recording job completed with browserId: ${browserId}`);
return { browserId };
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error);
logger.log('error', `Browser recording job failed: ${errorMessage}`);
throw error;
}
});
// Worker for stopping a browser
await pgBoss.work('destroy-browser', async (job: Job<DestroyBrowserData> | Job<DestroyBrowserData>[]) => {
try {
const data = extractJobData(job);
const { browserId, userId } = data;
logger.log('info', `Starting browser destruction job for browser: ${browserId}`);
const success = await destroyRemoteBrowser(browserId, userId);
logger.log('info', `Browser destruction job completed with result: ${success}`);
return { success };
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error);
logger.log('error', `Destroy browser job failed: ${errorMessage}`);
throw error;
}
});
// Worker for interpreting workflow
await pgBoss.work('interpret-workflow', async (job: Job<InterpretWorkflow> | Job<InterpretWorkflow>[]) => {
try {
const data = extractJobData(job);
const userId = data.userId;
logger.log('info', 'Starting workflow interpretation job');
await interpretWholeWorkflow(userId);
logger.log('info', 'Workflow interpretation job completed');
return { success: true };
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error);
logger.log('error', `Interpret workflow job failed: ${errorMessage}`);
throw error;
}
});
// Worker for stopping workflow interpretation
await pgBoss.work('stop-interpretation', async (job: Job<StopInterpretWorkflow> | Job<StopInterpretWorkflow>[]) => {
try {
const data = extractJobData(job);
const userId = data.userId;
logger.log('info', 'Starting stop interpretation job');
await stopRunningInterpretation(userId);
logger.log('info', 'Stop interpretation job completed');
return { success: true };
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error);
logger.log('error', `Stop interpretation job failed: ${errorMessage}`);
throw error;
}
});
// Register the run execution worker
await registerRunExecutionWorker();
logger.log('info', 'All recording workers registered successfully');
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error);
logger.log('error', `Failed to start PgBoss workers: ${errorMessage}`);
process.exit(1);
}
}
// Start all workers
startWorkers();
// Handle graceful shutdown
process.on('SIGTERM', async () => {
logger.log('info', 'SIGTERM received, shutting down PgBoss...');
await pgBoss.stop();
process.exit(0);
});
process.on('SIGINT', async () => {
logger.log('info', 'SIGINT received, shutting down PgBoss...');
await pgBoss.stop();
process.exit(0);
});
// For use in other files
export { pgBoss };

View File

@@ -9,22 +9,58 @@ import {
getActiveBrowserId,
interpretWholeWorkflow,
stopRunningInterpretation,
getRemoteBrowserCurrentUrl, getRemoteBrowserCurrentTabs,
} from '../browser-management/controller'
getRemoteBrowserCurrentUrl,
getRemoteBrowserCurrentTabs,
getActiveBrowserIdByState,
} from '../browser-management/controller';
import { chromium } from 'playwright-extra';
import stealthPlugin from 'puppeteer-extra-plugin-stealth';
import logger from "../logger";
import { getDecryptedProxyConfig } from './proxy';
import { requireSignIn } from '../middlewares/auth';
import { pgBoss } from '../pgboss-worker';
export const router = Router();
chromium.use(stealthPlugin());
export interface AuthenticatedRequest extends Request {
user?: any;
}
async function waitForJobCompletion(jobId: string, queueName: string, timeout = 15000): Promise<any> {
return new Promise((resolve, reject) => {
const startTime = Date.now();
const checkJobStatus = async () => {
if (Date.now() - startTime > timeout) {
return reject(new Error(`Timeout waiting for job ${jobId} to complete`));
}
try {
const job = await pgBoss.getJobById(queueName, jobId);
if (!job) {
return reject(new Error(`Job ${jobId} not found`));
}
if (job.state === 'completed') {
return resolve(job.output);
}
if (job.state === 'failed') {
return reject(new Error(`Job ${jobId} failed.`));
}
setTimeout(checkJobStatus, 200);
} catch (error) {
reject(error);
}
};
checkJobStatus();
});
}
/**
* Logs information about remote browser recording session.
*/
@@ -33,31 +69,53 @@ router.all('/', requireSignIn, (req, res, next) => {
next() // pass control to the next handler
})
/**
* GET endpoint for starting the remote browser recording session.
* returns session's id
* GET endpoint for starting the remote browser recording session
* Waits for job completion
*/
router.get('/start', requireSignIn, async (req: AuthenticatedRequest, res: Response) => {
if (!req.user) {
return res.status(401).send('User not authenticated');
}
const proxyConfig = await getDecryptedProxyConfig(req.user.id);
// Prepare the proxy options dynamically based on the user's proxy configuration
let proxyOptions: any = {}; // Default to no proxy
if (proxyConfig.proxy_url) {
// Set the server, and if username & password exist, set those as well
proxyOptions = {
server: proxyConfig.proxy_url,
...(proxyConfig.proxy_username && proxyConfig.proxy_password && {
username: proxyConfig.proxy_username,
password: proxyConfig.proxy_password,
}),
};
try {
await pgBoss.createQueue('initialize-browser-recording');
const jobId = await pgBoss.send('initialize-browser-recording', {
userId: req.user.id,
timestamp: new Date().toISOString()
});
if (!jobId) {
const browserId = initializeRemoteBrowserForRecording(req.user.id);
return res.send(browserId);
}
logger.log('info', `Queued browser initialization job: ${jobId}, waiting for completion...`);
try {
const result = await waitForJobCompletion(jobId, 'initialize-browser-recording', 15000);
if (result && result.browserId) {
return res.send(result.browserId);
} else {
return res.send(jobId);
}
} catch (waitError: any) {
return res.send(jobId);
}
} catch (error: any) {
logger.log('error', `Failed to queue browser initialization job: ${error.message}`);
try {
const browserId = initializeRemoteBrowserForRecording(req.user.id);
return res.send( browserId );
} catch (directError: any) {
logger.log('error', `Direct initialization also failed: ${directError.message}`);
return res.status(500).send('Failed to start recording');
}
}
const id = initializeRemoteBrowserForRecording(req.user.id);
return res.send(id);
});
/**
@@ -80,8 +138,38 @@ router.get('/stop/:browserId', requireSignIn, async (req: AuthenticatedRequest,
if (!req.user) {
return res.status(401).send('User not authenticated');
}
const success = await destroyRemoteBrowser(req.params.browserId, req.user?.id);
return res.send(success);
try {
await pgBoss.createQueue('destroy-browser');
const jobId = await pgBoss.send('destroy-browser', {
browserId: req.params.browserId,
userId: req.user.id,
timestamp: new Date().toISOString()
});
if (!jobId) {
const browserId = initializeRemoteBrowserForRecording(req.user.id);
return res.send( browserId );
}
logger.log('info', `Queued browser destruction job: ${jobId}, waiting for completion...`);
try {
const result = await waitForJobCompletion(jobId, 'destroy-browser', 15000);
if (result) {
return res.send(result.success);
} else {
return res.send(false);
}
} catch (waitError: any) {
return res.send(false);
}
} catch (error: any) {
logger.log('error', `Failed to stop browser: ${error.message}`);
return res.status(500).send(false);
}
});
/**
@@ -91,7 +179,7 @@ router.get('/active', requireSignIn, (req: AuthenticatedRequest, res) => {
if (!req.user) {
return res.status(401).send('User not authenticated');
}
const id = getActiveBrowserId(req.user?.id);
const id = getActiveBrowserIdByState(req.user?.id, "recording");
return res.send(id);
});
@@ -102,7 +190,7 @@ router.get('/active/url', requireSignIn, (req: AuthenticatedRequest, res) => {
if (!req.user) {
return res.status(401).send('User not authenticated');
}
const id = getActiveBrowserId(req.user?.id);
const id = getActiveBrowserIdByState(req.user?.id, "recording");
if (id) {
const url = getRemoteBrowserCurrentUrl(id, req.user?.id);
return res.send(url);
@@ -117,7 +205,7 @@ router.get('/active/tabs', requireSignIn, (req: AuthenticatedRequest, res) => {
if (!req.user) {
return res.status(401).send('User not authenticated');
}
const id = getActiveBrowserId(req.user?.id);
const id = getActiveBrowserIdByState(req.user?.id, "recording");
if (id) {
const hosts = getRemoteBrowserCurrentTabs(id, req.user?.id);
return res.send(hosts);
@@ -129,24 +217,77 @@ router.get('/active/tabs', requireSignIn, (req: AuthenticatedRequest, res) => {
* GET endpoint for starting an interpretation of the currently generated workflow.
*/
router.get('/interpret', requireSignIn, async (req: AuthenticatedRequest, res) => {
if (!req.user) {
return res.status(401).send('User not authenticated');
}
try {
if (!req.user) {
return res.status(401).send('User not authenticated');
await pgBoss.createQueue('interpret-workflow');
const jobId = await pgBoss.send('interpret-workflow', {
userId: req.user.id,
timestamp: new Date().toISOString()
});
if (!jobId) {
await interpretWholeWorkflow(req.user?.id);
return res.send('interpretation done');
}
await interpretWholeWorkflow(req.user?.id);
return res.send('interpretation done');
} catch (e) {
return res.send('interpretation failed');
logger.log('info', `Queued interpret workflow job: ${jobId}, waiting for completion...`);
try {
const result = await waitForJobCompletion(jobId, 'interpret-workflow', 15000);
if (result) {
return res.send('interpretation done');
} else {
return res.send('interpretation failed');
}
} catch (waitError: any) {
return res.send('interpretation failed');
}
} catch (error: any) {
logger.log('error', `Failed to stop interpret workflow: ${error.message}`);
return res.status(500).send('interpretation failed');
}
});
/**
* GET endpoint for stopping an ongoing interpretation of the currently generated workflow.
*/
router.get('/interpret/stop', requireSignIn, async (req: AuthenticatedRequest, res) => {
if (!req.user) {
return res.status(401).send('User not authenticated');
}
await stopRunningInterpretation(req.user?.id);
return res.send('interpretation stopped');
try {
await pgBoss.createQueue('stop-interpretation');
const jobId = await pgBoss.send('stop-interpretation', {
userId: req.user.id,
timestamp: new Date().toISOString()
});
if (!jobId) {
await stopRunningInterpretation(req.user?.id);
return res.send('interpretation stopped');
}
logger.log('info', `Queued stop interpret workflow job: ${jobId}, waiting for completion...`);
try {
const result = await waitForJobCompletion(jobId, 'stop-interpretation', 15000);
if (result) {
return res.send('interpretation stopped');
} else {
return res.send('interpretation failed to stop');
}
} catch (waitError: any) {
return res.send('interpretation failed to stop');
}
} catch (error: any) {
logger.log('error', `Failed to stop interpretation: ${error.message}`);
return res.status(500).send('interpretation failed to stop');
}
});
export default router;

View File

@@ -1,6 +1,6 @@
import { Router } from 'express';
import logger from "../logger";
import { createRemoteBrowserForRun, destroyRemoteBrowser } from "../browser-management/controller";
import { createRemoteBrowserForRun, destroyRemoteBrowser, getActiveBrowserIdByState } from "../browser-management/controller";
import { chromium } from 'playwright-extra';
import stealthPlugin from 'puppeteer-extra-plugin-stealth';
import { browserPool } from "../server";
@@ -22,6 +22,7 @@ import { encrypt, decrypt } from '../utils/auth';
import { WorkflowFile } from 'maxun-core';
import { Page } from 'playwright';
import { airtableUpdateTasks, processAirtableUpdates } from '../workflow-management/integrations/airtable';
import { pgBoss } from '../pgboss-worker';
chromium.use(stealthPlugin());
export const router = Router();
@@ -494,6 +495,8 @@ router.delete('/runs/:id', requireSignIn, async (req: AuthenticatedRequest, res)
/**
* PUT endpoint for starting a remote browser instance and saving run metadata to the storage.
* Making it ready for interpretation and returning a runId.
*
* If the user has reached their browser limit, the run will be queued using PgBoss.
*/
router.put('/runs/:id', requireSignIn, async (req: AuthenticatedRequest, res) => {
try {
@@ -525,35 +528,81 @@ router.put('/runs/:id', requireSignIn, async (req: AuthenticatedRequest, res) =>
};
}
console.log(`Proxy config for run: ${JSON.stringify(proxyOptions)}`)
const id = createRemoteBrowserForRun(req.user.id);
console.log(`Proxy config for run: ${JSON.stringify(proxyOptions)}`);
// Generate runId first
const runId = uuid();
// Check if user has reached browser limit
const userBrowserIds = browserPool.getAllBrowserIdsForUser(req.user.id);
const canCreateBrowser = userBrowserIds.length < 2;
if (canCreateBrowser) {
// User has available browser slots, create it directly
const id = createRemoteBrowserForRun(req.user.id);
const run = await Run.create({
status: 'running',
name: recording.recording_meta.name,
robotId: recording.id,
robotMetaId: recording.recording_meta.id,
startedAt: new Date().toLocaleString(),
finishedAt: '',
browserId: id,
interpreterSettings: req.body,
log: '',
runId,
runByUserId: req.user.id,
serializableOutput: {},
binaryOutput: {},
});
const run = await Run.create({
status: 'running',
name: recording.recording_meta.name,
robotId: recording.id,
robotMetaId: recording.recording_meta.id,
startedAt: new Date().toLocaleString(),
finishedAt: '',
browserId: id,
interpreterSettings: req.body,
log: '',
runId,
runByUserId: req.user.id,
serializableOutput: {},
binaryOutput: {},
});
const plainRun = run.toJSON();
const plainRun = run.toJSON();
return res.send({
browserId: id,
runId: plainRun.runId,
robotMetaId: recording.recording_meta.id,
});
return res.send({
browserId: id,
runId: plainRun.runId,
robotMetaId: recording.recording_meta.id,
queued: false
});
} else {
const browserId = getActiveBrowserIdByState(req.user.id, "run")
if (browserId) {
// User has reached the browser limit, queue the run
try {
// Create the run record with 'queued' status
await Run.create({
status: 'queued',
name: recording.recording_meta.name,
robotId: recording.id,
robotMetaId: recording.recording_meta.id,
startedAt: new Date().toLocaleString(),
finishedAt: '',
browserId: browserId, // Random will be updated later
interpreterSettings: req.body,
log: 'Run queued - waiting for available browser slot',
runId,
runByUserId: req.user.id,
serializableOutput: {},
binaryOutput: {},
});
return res.send({
browserId: browserId,
runId: runId,
robotMetaId: recording.recording_meta.id,
queued: true,
});
} catch (queueError: any) {
logger.log('error', `Failed to queue run job: ${queueError.message}`);
return res.status(503).send({ error: 'Unable to queue run, please try again later' });
}
} else {
logger.log('info', "Browser id does not exist");
return res.send('');
}
}
} catch (e) {
const { message } = e as Error;
logger.log('info', `Error while creating a run with robot id: ${req.params.id} - ${message}`);
@@ -608,82 +657,20 @@ router.post('/runs/run/:id', requireSignIn, async (req: AuthenticatedRequest, re
return res.status(404).send(false);
}
// interpret the run in active browser
const browser = browserPool.getRemoteBrowser(plainRun.browserId);
let currentPage = browser?.getCurrentPage();
if (browser && currentPage) {
const workflow = AddGeneratedFlags(recording.recording);
const interpretationInfo = await browser.interpreter.InterpretRecording(
workflow, currentPage, (newPage: Page) => currentPage = newPage, plainRun.interpreterSettings);
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput);
await destroyRemoteBrowser(plainRun.browserId, req.user?.id);
await run.update({
...run,
status: 'success',
finishedAt: new Date().toLocaleString(),
browserId: plainRun.browserId,
log: interpretationInfo.log.join('\n'),
serializableOutput: interpretationInfo.serializableOutput,
binaryOutput: uploadedBinaryOutput,
try {
// Queue the execution job
await pgBoss.createQueue('execute-run');
const jobId = await pgBoss.send('execute-run', {
userId: req.user.id,
runId: req.params.id,
browserId: plainRun.browserId
});
let totalRowsExtracted = 0;
let extractedScreenshotsCount = 0;
let extractedItemsCount = 0;
if (run.dataValues.binaryOutput && run.dataValues.binaryOutput["item-0"]) {
extractedScreenshotsCount = 1;
}
if (run.dataValues.serializableOutput && run.dataValues.serializableOutput["item-0"]) {
const itemsArray = run.dataValues.serializableOutput["item-0"];
extractedItemsCount = itemsArray.length;
totalRowsExtracted = itemsArray.reduce((total, item) => {
return total + Object.keys(item).length;
}, 0);
}
console.log(`Extracted Items Count: ${extractedItemsCount}`);
console.log(`Extracted Screenshots Count: ${extractedScreenshotsCount}`);
console.log(`Total Rows Extracted: ${totalRowsExtracted}`);
capture(
'maxun-oss-run-created-manual',
{
runId: req.params.id,
user_id: req.user?.id,
created_at: new Date().toISOString(),
status: 'success',
totalRowsExtracted,
extractedItemsCount,
extractedScreenshotsCount,
}
)
try {
googleSheetUpdateTasks[plainRun.runId] = {
robotId: plainRun.robotMetaId,
runId: plainRun.runId,
status: 'pending',
retries: 5,
};
airtableUpdateTasks[plainRun.runId] = {
robotId: plainRun.robotMetaId,
runId: plainRun.runId,
status: 'pending',
retries: 5,
};
processAirtableUpdates();
processGoogleSheetUpdates();
} catch (err: any) {
logger.log('error', `Failed to update Google Sheet for run: ${plainRun.runId}: ${err.message}`);
}
return res.send(true);
} else {
throw new Error('Could not destroy browser');
logger.log('info', `Queued run execution job with ID: ${jobId} for run: ${req.params.id}`);
} catch (queueError: any) {
logger.log('error', `Failed to queue run execution`);
}
} catch (e) {
const { message } = e as Error;

View File

@@ -49,7 +49,7 @@ router.get('/params/:browserId', requireSignIn, (req, res) => {
*/
router.delete('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => {
if (!req.user) { return res.status(401).send('User not authenticated'); }
const id = browserPool.getActiveBrowserId(req.user?.id);
const id = browserPool.getActiveBrowserId(req.user?.id, "recording");
if (id) {
const browser = browserPool.getRemoteBrowser(id);
if (browser) {
@@ -66,7 +66,7 @@ router.delete('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) =>
*/
router.post('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => {
if (!req.user) { return res.status(401).send('User not authenticated'); }
const id = browserPool.getActiveBrowserId(req.user?.id);
const id = browserPool.getActiveBrowserId(req.user?.id, "recording");
if (id) {
const browser = browserPool.getRemoteBrowser(id);
logger.log('debug', `Adding pair to workflow`);
@@ -87,7 +87,7 @@ router.post('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => {
*/
router.put('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => {
if (!req.user) { return res.status(401).send('User not authenticated'); }
const id = browserPool.getActiveBrowserId(req.user?.id);
const id = browserPool.getActiveBrowserId(req.user?.id, "recording");
if (id) {
const browser = browserPool.getRemoteBrowser(id);
logger.log('debug', `Updating pair in workflow`);

View File

@@ -23,7 +23,6 @@ import session from 'express-session';
import Run from './models/Run';
const app = express();
app.use(cors({
origin: process.env.PUBLIC_URL ? process.env.PUBLIC_URL : 'http://localhost:5173',
@@ -79,8 +78,11 @@ readdirSync(path.join(__dirname, 'api')).forEach((r) => {
const isProduction = process.env.NODE_ENV === 'production';
const workerPath = path.resolve(__dirname, isProduction ? './worker.js' : './worker.ts');
const recordingWorkerPath = path.resolve(__dirname, isProduction ? './pgboss-worker.js' : './pgboss-worker.ts');
let workerProcess: any;
let recordingWorkerProcess: any;
if (!isProduction) {
workerProcess = fork(workerPath, [], {
execArgv: ['--inspect=5859'],
@@ -94,6 +96,19 @@ if (!isProduction) {
workerProcess.on('exit', (code: any) => {
console.log(`Worker exited with code: ${code}`);
});
recordingWorkerProcess = fork(recordingWorkerPath, [], {
execArgv: ['--inspect=5860'],
});
recordingWorkerProcess.on('message', (message: any) => {
console.log(`Message from recording worker: ${message}`);
});
recordingWorkerProcess.on('error', (error: any) => {
console.error(`Error in recording worker: ${error}`);
});
recordingWorkerProcess.on('exit', (code: any) => {
console.log(`Recording worker exited with code: ${code}`);
});
}
app.get('/', function (req, res) {
@@ -146,7 +161,8 @@ process.on('SIGINT', async () => {
}
if (!isProduction) {
workerProcess.kill();
if (workerProcess) workerProcess.kill();
if (recordingWorkerProcess) recordingWorkerProcess.kill();
}
process.exit();
});

View File

@@ -152,7 +152,7 @@ export class WorkflowGenerator {
});
socket.on('activeIndex', (data) => this.generatedData.lastIndex = parseInt(data));
socket.on('decision', async ({ pair, actionType, decision, userId }) => {
const id = browserPool.getActiveBrowserId(userId);
const id = browserPool.getActiveBrowserId(userId, "recording");
if (id) {
// const activeBrowser = browserPool.getRemoteBrowser(id);
// const currentPage = activeBrowser?.getCurrentPage();
@@ -708,6 +708,7 @@ export class WorkflowGenerator {
public updateSocket = (socket: Socket): void => {
this.socket = socket;
this.registerEventHandlers(socket);
this.initializeSocketListeners();
};
/**

View File

@@ -54,6 +54,7 @@ const BrowserNavBar: FC<NavBarProps> = ({
const handleCurrentUrlChange = useCallback((url: string) => {
handleUrlChanged(url);
setRecordingUrl(url);
window.sessionStorage.setItem('recordingUrl', url);
}, [handleUrlChanged, recordingUrl]);
useEffect(() => {

View File

@@ -14,9 +14,9 @@ import { MoreHoriz } from '@mui/icons-material';
const BrowserRecordingSave = () => {
const { t } = useTranslation();
const [openDiscardModal, setOpenDiscardModal] = useState<boolean>(false);
const [openResetModal, setOpenResetModal] = useState<boolean>(false);
const [anchorEl, setAnchorEl] = React.useState<null | HTMLElement>(null);
const [openDiscardModal, setOpenDiscardModal] = useState(false);
const [openResetModal, setOpenResetModal] = useState(false);
const [anchorEl, setAnchorEl] = React.useState(null);
const { recordingName, browserId, initialUrl, setRecordingUrl, setBrowserId, notify, setCurrentWorkflowActionsState, resetInterpretationLog } = useGlobalInfoStore();
const navigate = useNavigate();
@@ -42,10 +42,25 @@ const BrowserRecordingSave = () => {
const goToMainMenu = async () => {
if (browserId) {
await stopRecording(browserId);
notify('warning', t('browser_recording.notifications.terminated'));
const notificationData = {
type: 'warning',
message: t('browser_recording.notifications.terminated'),
timestamp: Date.now()
};
window.sessionStorage.setItem('pendingNotification', JSON.stringify(notificationData));
if (window.opener) {
window.opener.postMessage({
type: 'recording-notification',
notification: notificationData
}, '*');
}
setBrowserId(null);
window.close();
}
navigate('/');
};
const performReset = () => {
@@ -82,7 +97,13 @@ const BrowserRecordingSave = () => {
socket?.emit('new-recording');
socket.emit('input:url', initialUrl);
// Update the URL in the navbar to match
setRecordingUrl(initialUrl);
let sessionInitialUrl = window.sessionStorage.getItem('initialUrl');
if (sessionInitialUrl) {
setRecordingUrl(sessionInitialUrl);
window.sessionStorage.setItem('recordingUrl', sessionInitialUrl);
} else {
setRecordingUrl(initialUrl);
}
}
// Close the reset confirmation modal
@@ -92,7 +113,7 @@ const BrowserRecordingSave = () => {
notify('info', t('browser_recording.notifications.environment_reset'));
};
const handleClick = (event: React.MouseEvent<HTMLElement>) => {
const handleClick = (event: any) => {
setAnchorEl(event.currentTarget);
};

View File

@@ -78,9 +78,24 @@ export const BrowserWindow = () => {
const { notify } = useGlobalInfoStore();
const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext();
const { addTextStep, addListStep } = useBrowserSteps();
const { state } = useContext(AuthContext);
const { user } = state;
const { user } = state;
useEffect(() => {
if (listSelector) {
window.sessionStorage.setItem('recordingListSelector', listSelector);
}
}, [listSelector]);
useEffect(() => {
const storedListSelector = window.sessionStorage.getItem('recordingListSelector');
// Only restore state if it exists in sessionStorage
if (storedListSelector && !listSelector) {
setListSelector(storedListSelector);
}
}, []);
const onMouseMove = (e: MouseEvent) => {
if (canvasRef && canvasRef.current && highlighterData) {
@@ -211,13 +226,25 @@ export const BrowserWindow = () => {
useEffect(() => {
document.addEventListener('mousemove', onMouseMove, false);
if (socket) {
socket.on("highlighter", highlighterHandler);
socket.off("highlighter", highlighterHandler);
socket.on("highlighter", highlighterHandler);
}
return () => {
document.removeEventListener('mousemove', onMouseMove);
socket?.off("highlighter", highlighterHandler);
document.removeEventListener('mousemove', onMouseMove);
if (socket) {
socket.off("highlighter", highlighterHandler);
}
};
}, [socket, onMouseMove]);
}, [socket, highlighterHandler, onMouseMove, getList, listSelector]);
useEffect(() => {
if (socket && listSelector) {
console.log('Syncing list selector with server:', listSelector);
socket.emit('setGetList', { getList: true });
socket.emit('listSelector', { selector: listSelector });
}
}, [socket, listSelector]);
useEffect(() => {
if (captureStage === 'initial' && listSelector) {

View File

@@ -0,0 +1,241 @@
export class CanvasRenderer {
private canvas: HTMLCanvasElement;
private ctx: CanvasRenderingContext2D;
private offscreenCanvas: OffscreenCanvas | null = null;
private offscreenCtx: CanvasRenderingContext2D | null = null;
private lastFrameRequest: number | null = null;
private imageCache: Map<string, HTMLImageElement> = new Map();
private consecutiveFrameCount: number = 0;
private lastDrawTime: number = 0;
private memoryCheckCounter: number = 0;
private lastMemoryCheck: number = 0;
private memoryThreshold: number = 100000000; // 100MB
constructor(canvas: HTMLCanvasElement) {
this.canvas = canvas;
// Get 2D context with optimized settings
const ctx = canvas.getContext('2d', {
alpha: false, // Disable alpha for better performance
desynchronized: true, // Reduce latency when possible
});
if (!ctx) {
throw new Error('Could not get 2D context from canvas');
}
this.ctx = ctx;
// Apply performance optimizations
this.ctx.imageSmoothingEnabled = false;
// Set up offscreen canvas if supported
if (typeof OffscreenCanvas !== 'undefined') {
this.offscreenCanvas = new OffscreenCanvas(canvas.width, canvas.height);
const offCtx = this.offscreenCanvas.getContext('2d', {
alpha: false
});
if (offCtx) {
this.offscreenCtx = offCtx as unknown as CanvasRenderingContext2D;
this.offscreenCtx.imageSmoothingEnabled = false;
}
}
// Initial timestamp
this.lastDrawTime = performance.now();
this.lastMemoryCheck = performance.now();
}
/**
* Renders a screenshot to the canvas, optimized for performance
*/
public drawScreenshot(
screenshot: string | ImageBitmap | HTMLImageElement,
x: number = 0,
y: number = 0,
width?: number,
height?: number
): void {
// Cancel any pending frame request
if (this.lastFrameRequest !== null) {
cancelAnimationFrame(this.lastFrameRequest);
}
// Check memory usage periodically
this.memoryCheckCounter++;
const now = performance.now();
if (this.memoryCheckCounter >= 30 || now - this.lastMemoryCheck > 5000) {
this.checkMemoryUsage();
this.memoryCheckCounter = 0;
this.lastMemoryCheck = now;
}
// Request a new frame
this.lastFrameRequest = requestAnimationFrame(() => {
this.renderFrame(screenshot, x, y, width, height);
});
}
private renderFrame(
screenshot: string | ImageBitmap | HTMLImageElement,
x: number,
y: number,
width?: number,
height?: number
): void {
// Target context (offscreen if available, otherwise main)
const targetCtx = this.offscreenCtx || this.ctx;
// Start timing the render
const startTime = performance.now();
const timeSinceLastDraw = startTime - this.lastDrawTime;
// Adaptive frame skipping for high-frequency updates
// If we're getting updates faster than 60fps and this isn't the first frame
if (timeSinceLastDraw < 16 && this.consecutiveFrameCount > 5) {
this.consecutiveFrameCount++;
// Skip some frames when we're getting excessive updates
if (this.consecutiveFrameCount % 2 !== 0) {
return;
}
} else {
this.consecutiveFrameCount = 0;
}
try {
if (typeof screenshot === 'string') {
// Check if we have this image in cache
let img = this.imageCache.get(screenshot);
if (!img) {
img = new Image();
img.src = screenshot;
this.imageCache.set(screenshot, img);
// If image isn't loaded yet, draw when it loads
if (!img.complete) {
img.onload = () => {
if (img) {
this.drawScreenshot(img, x, y, width, height);
}
};
return;
}
}
targetCtx.drawImage(
img,
x, y,
width || img.width,
height || img.height
);
} else {
// Draw ImageBitmap or HTMLImageElement directly
targetCtx.drawImage(
screenshot,
x, y,
width || screenshot.width,
height || screenshot.height
);
}
// If using offscreen canvas, copy to main canvas
if (this.offscreenCanvas && this.offscreenCtx) {
if ('transferToImageBitmap' in this.offscreenCanvas) {
// Use more efficient transfer when available
const bitmap = this.offscreenCanvas.transferToImageBitmap();
this.ctx.drawImage(bitmap, 0, 0);
} else {
// Fallback to drawImage
this.ctx.drawImage(this.offscreenCanvas, 0, 0);
}
}
// Update timestamp
this.lastDrawTime = performance.now();
} catch (error) {
console.error('Error rendering frame:', error);
}
}
/**
* Checks current memory usage and cleans up if necessary
*/
private checkMemoryUsage(): void {
if (window.performance && (performance as any).memory) {
const memory = (performance as any).memory;
if (memory.usedJSHeapSize > this.memoryThreshold) {
this.cleanupMemory();
}
}
}
/**
* Cleans up resources to reduce memory usage
*/
private cleanupMemory(): void {
// Limit image cache size
if (this.imageCache.size > 20) {
// Keep only the most recent 10 images
const keysToDelete = Array.from(this.imageCache.keys()).slice(0, this.imageCache.size - 10);
keysToDelete.forEach(key => {
this.imageCache.delete(key);
});
}
// Suggest garbage collection
if (window.gc) {
try {
window.gc();
} catch (e) {
// GC not available, ignore
}
}
}
/**
* Update canvas dimensions
*/
public updateCanvasSize(width: number, height: number): void {
this.canvas.width = width;
this.canvas.height = height;
// Re-apply context settings
this.ctx.imageSmoothingEnabled = false;
// Update offscreen canvas if available
if (this.offscreenCanvas) {
this.offscreenCanvas.width = width;
this.offscreenCanvas.height = height;
if (this.offscreenCtx) {
this.offscreenCtx.imageSmoothingEnabled = false;
}
}
}
/**
* Clean up resources
*/
public dispose(): void {
// Cancel any pending frame requests
if (this.lastFrameRequest !== null) {
cancelAnimationFrame(this.lastFrameRequest);
this.lastFrameRequest = null;
}
// Clear the image cache
this.imageCache.clear();
// Clear canvases
this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
if (this.offscreenCtx && this.offscreenCanvas) {
this.offscreenCtx.clearRect(0, 0, this.offscreenCanvas.width, this.offscreenCanvas.height);
}
}
}

View File

@@ -47,13 +47,27 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => {
};
const exitRecording = useCallback(async () => {
notify('success', t('save_recording.notifications.save_success'));
const notificationData = {
type: 'success',
message: t('save_recording.notifications.save_success'),
timestamp: Date.now()
};
window.sessionStorage.setItem('pendingNotification', JSON.stringify(notificationData));
if (window.opener) {
window.opener.postMessage({
type: 'recording-notification',
notification: notificationData
}, '*');
}
if (browserId) {
await stopRecording(browserId);
}
setBrowserId(null);
navigate('/');
}, [setBrowserId, browserId, notify]);
window.close();
}, [setBrowserId, browserId]);
// notifies backed to save the recording in progress,
// releases resources and changes the view for main page by clearing the global browserId
@@ -142,4 +156,4 @@ const modalStyle = {
height: 'fit-content',
display: 'block',
padding: '20px',
};
};

View File

@@ -1,4 +1,4 @@
import React, { useCallback, useEffect, useRef } from 'react';
import React, { memo, useCallback, useEffect, useRef } from 'react';
import { useSocketStore } from '../../context/socket';
import { useGlobalInfoStore } from "../../context/globalInfo";
import { useActionContext } from '../../context/browserActions';
@@ -6,6 +6,7 @@ import DatePicker from '../pickers/DatePicker';
import Dropdown from '../pickers/Dropdown';
import TimePicker from '../pickers/TimePicker';
import DateTimeLocalPicker from '../pickers/DateTimeLocalPicker';
import { EnhancedPerformanceMonitor } from '../../../perf/performance';
interface CreateRefCallback {
(ref: React.RefObject<HTMLCanvasElement>): void;
@@ -27,6 +28,9 @@ export interface Coordinates {
const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
const performanceMonitor = useRef(new EnhancedPerformanceMonitor());
console.log('Frontend Performance Report:', performanceMonitor.current.getPerformanceReport());
const canvasRef = useRef<HTMLCanvasElement>(null);
const { socket } = useSocketStore();
const { setLastAction, lastAction } = useGlobalInfoStore();
@@ -34,6 +38,9 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
const getTextRef = useRef(getText);
const getListRef = useRef(getList);
const MOUSE_MOVE_THROTTLE = 16; // ~60fps
const lastMouseMoveTime = useRef(0);
const [datePickerInfo, setDatePickerInfo] = React.useState<{
coordinates: Coordinates;
selector: string;
@@ -110,6 +117,7 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
}, [socket]);
const onMouseEvent = useCallback((event: MouseEvent) => {
performanceMonitor.current.measureEventLatency(event);
if (socket && canvasRef.current) {
// Get the canvas bounding rectangle
const rect = canvasRef.current.getBoundingClientRect();
@@ -129,29 +137,38 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
}
notifyLastAction('click');
break;
case 'mousemove':
if (lastMousePosition.current.x !== clickCoordinates.x ||
lastMousePosition.current.y !== clickCoordinates.y) {
case 'mousemove': {
const now = performance.now();
if (now - lastMouseMoveTime.current < MOUSE_MOVE_THROTTLE) {
return;
}
lastMouseMoveTime.current = now;
const dx = Math.abs(lastMousePosition.current.x - clickCoordinates.x);
const dy = Math.abs(lastMousePosition.current.y - clickCoordinates.y);
if (dx > 1 || dy > 1) {
lastMousePosition.current = {
x: clickCoordinates.x,
y: clickCoordinates.y,
};
socket.emit('input:mousemove', {
x: clickCoordinates.x,
y: clickCoordinates.y,
});
socket.emit('input:mousemove', clickCoordinates);
notifyLastAction('move');
}
break;
case 'wheel':
}
// Optimize wheel events
case 'wheel': {
const wheelEvent = event as WheelEvent;
const deltas = {
deltaX: Math.round(wheelEvent.deltaX),
deltaY: Math.round(wheelEvent.deltaY),
};
socket.emit('input:wheel', deltas);
notifyLastAction('scroll');
const deltaX = Math.round(wheelEvent.deltaX / 10) * 10;
const deltaY = Math.round(wheelEvent.deltaY / 10) * 10;
if (Math.abs(deltaX) > 5 || Math.abs(deltaY) > 5) {
socket.emit('input:wheel', { deltaX, deltaY });
notifyLastAction('scroll');
}
break;
}
default:
console.log('Default mouseEvent registered');
return;
@@ -159,6 +176,16 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
}
}, [socket]);
// performance logging
useEffect(() => {
const intervalId = setInterval(() => {
const report = performanceMonitor.current.getPerformanceReport();
console.log('Frontend Performance Report:', report);
}, 5000);
return () => clearInterval(intervalId);
}, []);
const onKeyboardEvent = useCallback((event: KeyboardEvent) => {
if (socket) {
switch (event.type) {
@@ -207,9 +234,14 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
<canvas
tabIndex={0}
ref={canvasRef}
height={400}
width={900}
style={{ display: 'block' }}
height={height}
width={width}
style={{
display: 'block',
imageRendering: 'crisp-edges',
willChange: 'transform',
transform: 'translateZ(0)'
}}
/>
{datePickerInfo && (
<DatePicker
@@ -246,4 +278,4 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
};
export default Canvas;
export default memo(Canvas);

View File

@@ -41,9 +41,14 @@ import { useGlobalInfoStore } from "../../context/globalInfo";
import { checkRunsForRecording, deleteRecordingFromStorage, getStoredRecordings } from "../../api/storage";
import { Add } from "@mui/icons-material";
import { useNavigate } from 'react-router-dom';
import { stopRecording } from "../../api/recording";
import { getActiveBrowserId, stopRecording } from "../../api/recording";
import { GenericModal } from '../ui/GenericModal';
declare global {
interface Window {
openedRecordingWindow?: Window | null;
}
}
/** TODO:
* 1. allow editing existing robot after persisting browser steps
@@ -148,6 +153,8 @@ export const RecordingsTable = ({
const [rows, setRows] = React.useState<Data[]>([]);
const [isModalOpen, setModalOpen] = React.useState(false);
const [searchTerm, setSearchTerm] = React.useState('');
const [isWarningModalOpen, setWarningModalOpen] = React.useState(false);
const [activeBrowserId, setActiveBrowserId] = React.useState('');
const columns = useMemo(() => [
{ id: 'interpret', label: t('recordingtable.run'), minWidth: 80 },
@@ -176,6 +183,30 @@ export const RecordingsTable = ({
setRecordingId } = useGlobalInfoStore();
const navigate = useNavigate();
useEffect(() => {
const handleMessage = (event: any) => {
if (event.data && event.data.type === 'recording-notification') {
const notificationData = event.data.notification;
if (notificationData) {
notify(notificationData.type, notificationData.message);
if ((notificationData.type === 'success' &&
notificationData.message.includes('saved')) ||
(notificationData.type === 'warning' &&
notificationData.message.includes('terminated'))) {
setRerenderRobots(true);
}
}
}
};
window.addEventListener('message', handleMessage);
return () => {
window.removeEventListener('message', handleMessage);
};
}, [notify, setRerenderRobots]);
const handleChangePage = useCallback((event: unknown, newPage: number) => {
setPage(newPage);
}, []);
@@ -233,28 +264,69 @@ export const RecordingsTable = ({
}, [setRecordings, notify, t]);
const handleNewRecording = useCallback(async () => {
if (browserId) {
setBrowserId(null);
await stopRecording(browserId);
const activeBrowserId = await getActiveBrowserId();
if (activeBrowserId) {
setActiveBrowserId(activeBrowserId);
setWarningModalOpen(true);
} else {
setModalOpen(true);
}
setModalOpen(true);
}, [browserId]);
}, []);
const handleStartRecording = useCallback(() => {
setBrowserId('new-recording');
setRecordingName('');
setRecordingId('');
navigate('/recording');
}, [navigate]);
const notifyRecordingTabsToClose = (browserId: string) => {
const closeMessage = {
action: 'close-recording-tab',
browserId: browserId,
timestamp: Date.now()
};
window.sessionStorage.setItem('recordingTabCloseMessage', JSON.stringify(closeMessage));
if (window.openedRecordingWindow && !window.openedRecordingWindow.closed) {
try {
window.openedRecordingWindow.close();
} catch (e) {
console.log('Could not directly close recording window:', e);
}
}
};
const handleDiscardAndCreate = async () => {
if (activeBrowserId) {
await stopRecording(activeBrowserId);
notify('warning', t('browser_recording.notifications.terminated'));
notifyRecordingTabsToClose(activeBrowserId);
}
setWarningModalOpen(false);
setModalOpen(true);
};
const startRecording = () => {
setModalOpen(false);
handleStartRecording();
// Set local state
setBrowserId('new-recording');
setRecordingName('');
setRecordingId('');
window.sessionStorage.setItem('browserId', 'new-recording');
const sessionId = Date.now().toString();
window.sessionStorage.setItem('recordingSessionId', sessionId);
window.sessionStorage.setItem('recordingUrl', recordingUrl);
window.openedRecordingWindow = window.open(`/recording-setup?session=${sessionId}`, '_blank');
window.sessionStorage.setItem('nextTabIsRecording', 'true');
};
const setBrowserRecordingUrl = (event: React.ChangeEvent<HTMLInputElement>) => {
setInitialUrl(event.target.value);
setRecordingUrl(event.target.value);
window.sessionStorage.setItem('initialUrl', event.target.value);
}
useEffect(() => {
@@ -401,6 +473,30 @@ export const RecordingsTable = ({
onPageChange={handleChangePage}
onRowsPerPageChange={handleChangeRowsPerPage}
/>
<GenericModal isOpen={isWarningModalOpen} onClose={() => setWarningModalOpen(false)} modalStyle={modalStyle}>
<div style={{ padding: '10px' }}>
<Typography variant="h6" gutterBottom>{t('recordingtable.warning_modal.title')}</Typography>
<Typography variant="body1" style={{ marginBottom: '20px' }}>
{t('recordingtable.warning_modal.message')}
</Typography>
<Box display="flex" justifyContent="space-between" mt={2}>
<Button
onClick={handleDiscardAndCreate}
variant="contained"
color="error"
>
{t('recordingtable.warning_modal.discard_and_create')}
</Button>
<Button
onClick={() => setWarningModalOpen(false)}
variant="outlined"
>
{t('recordingtable.warning_modal.cancel')}
</Button>
</Box>
</div>
</GenericModal>
<GenericModal isOpen={isModalOpen} onClose={() => setModalOpen(false)} modalStyle={modalStyle}>
<div style={{ padding: '10px' }}>
<Typography variant="h6" gutterBottom>{t('recordingtable.modal.title')}</Typography>

View File

@@ -123,6 +123,7 @@ export const CollapsibleRow = ({ row, handleDelete, isOpen, currentLog, abortRun
{row.status === 'success' && <Chip label={t('runs_table.run_status_chips.success')} color="success" variant="outlined" />}
{row.status === 'running' && <Chip label={t('runs_table.run_status_chips.running')} color="warning" variant="outlined" />}
{row.status === 'scheduled' && <Chip label={t('runs_table.run_status_chips.scheduled')} variant="outlined" />}
{row.status === 'queued' && <Chip label={t('runs_table.run_status_chips.queued')} variant="outlined" />}
{row.status === 'failed' && <Chip label={t('runs_table.run_status_chips.failed')} color="error" variant="outlined" />}
</TableCell>
)

View File

@@ -134,13 +134,13 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
}}>
<div>
<Highlight className="javascript">
{interpretationInProgress ? currentLog : row.log}
{row.status === 'running' ? currentLog : row.log}
</Highlight>
<div style={{ float: "left", clear: "both" }}
ref={logEndRef} />
</div>
</Box>
{interpretationInProgress ? <Button
{row.status === 'running' ? <Button
color="error"
onClick={abortRunHandler}
>
@@ -148,7 +148,7 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
</Button> : null}
</TabPanel>
<TabPanel value='output' sx={{ width: '700px' }}>
{interpretationInProgress ? (
{row.status === 'running' || row.status === 'queued' ? (
<Box sx={{ display: 'flex', alignItems: 'center' }}>
<CircularProgress size={22} sx={{ marginRight: '10px' }} />
{t('run_content.loading')}
@@ -248,4 +248,4 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
</TabContext>
</Box>
);
};
};

View File

@@ -103,6 +103,17 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
setSockets(sockets => [...sockets, socket]);
socket.on('ready-for-run', () => readyForRunHandler(browserId, runId));
socket.on('debugMessage', debugMessageHandler);
socket.on('run-completed', (data) => {
setRerenderRuns(true);
const robotName = data.robotName;
if (data.status === 'success') {
notify('success', t('main_page.notifications.interpretation_success', { name: robotName }));
} else {
notify('error', t('main_page.notifications.interpretation_failed', { name: robotName }));
}
});
setContent('runs');
if (browserId) {
notify('info', t('main_page.notifications.run_started', { name: runningRecordingName }));

View File

@@ -16,10 +16,11 @@ import { NotFoundPage } from '../components/dashboard/NotFound';
export const PageWrapper = () => {
const [open, setOpen] = useState(false);
const [isRecordingMode, setIsRecordingMode] = useState(false);
const navigate = useNavigate();
const { browserId, setBrowserId, notification, recordingName, setRecordingName, recordingId, setRecordingId } = useGlobalInfoStore();
const { browserId, setBrowserId, notification, recordingName, setRecordingName, recordingId, setRecordingId, setRecordingUrl } = useGlobalInfoStore();
const handleEditRecording = (recordingId: string, fileName: string) => {
setRecordingName(fileName);
@@ -35,23 +36,62 @@ export const PageWrapper = () => {
return notification.isOpen;
}
/**
* Get the current tab's state from session storage
*/
const getTabState = (key: string): string | null => {
try {
const value = window.sessionStorage.getItem(key);
return value;
} catch (error) {
return null;
}
};
useEffect(() => {
const isRecordingInProgress = async () => {
const id = await getActiveBrowserId();
if (id) {
setBrowserId(id);
const tabMode = getTabState('tabMode');
const urlParams = new URLSearchParams(window.location.search);
const sessionParam = urlParams.get('session');
const storedSessionId = getTabState('recordingSessionId');
const storedRecordingUrl = getTabState('recordingUrl');
if (location.pathname === '/recording-setup' && sessionParam && sessionParam === storedSessionId) {
setBrowserId('new-recording');
setRecordingName('');
setRecordingId('');
if (storedRecordingUrl) {
setRecordingUrl(storedRecordingUrl);
}
navigate('/recording');
}
else if (location.pathname === '/recording' ||
(getTabState('nextTabIsRecording') === 'true' && sessionParam === storedSessionId)) {
setIsRecordingMode(true);
if (location.pathname !== '/recording') {
navigate('/recording');
}
window.sessionStorage.removeItem('nextTabIsRecording');
} else if (tabMode === 'main') {
console.log('Tab is in main application mode');
} else {
const id = getTabState('browserId');
if (id === 'new-recording' || location.pathname === '/recording') {
setIsRecordingMode(true);
}
}
isRecordingInProgress();
}, []);
}, [location.pathname, navigate, setBrowserId, setRecordingId, setRecordingName, setRecordingUrl]);
return (
<div>
<AuthProvider>
<SocketProvider>
<React.Fragment>
{!browserId && <NavBar recordingName={recordingName} isRecording={!!browserId} />}
{/* {!browserId && location.pathname !== '/recording' && <NavBar recordingName={recordingName} isRecording={!!browserId} />} */}
{location.pathname !== '/recording' && <NavBar recordingName={recordingName} isRecording={false} />}
<Routes>
<Route element={<UserRoute />}>
<Route path="/" element={<Navigate to="/robots" replace />} />

View File

@@ -44,7 +44,7 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
const { setId, socket } = useSocketStore();
const { setWidth } = useBrowserDimensionsStore();
const { browserId, setBrowserId, recordingId, recordingUrl } = useGlobalInfoStore();
const { browserId, setBrowserId, recordingId, recordingUrl, setRecordingUrl } = useGlobalInfoStore();
const handleShowOutputData = useCallback(() => {
setShowOutputData(true);
@@ -78,6 +78,11 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
useEffect(() => {
let isCancelled = false;
const handleRecording = async () => {
const storedUrl = window.sessionStorage.getItem('recordingUrl');
if (storedUrl && !recordingUrl) {
setRecordingUrl(storedUrl);
}
const id = await getActiveBrowserId();
if (!isCancelled) {
if (id) {
@@ -91,13 +96,13 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
}
}
};
handleRecording();
return () => {
isCancelled = true;
}
}, [setId]);
}, [setId, recordingUrl, setRecordingUrl]);
const changeBrowserDimensions = useCallback(() => {
if (browserContentRef.current) {

View File

@@ -1,12 +1,56 @@
import React from 'react';
import { Navigate, Outlet } from 'react-router-dom';
import React, { useEffect, useState } from 'react';
import { Navigate, Outlet, useLocation } from 'react-router-dom';
import { useContext } from 'react';
import { AuthContext } from '../context/auth';
import { useGlobalInfoStore } from '../context/globalInfo';
const UserRoute = () => {
const { state } = useContext(AuthContext);
const location = useLocation();
const [isCheckingAuth, setIsCheckingAuth] = useState(true);
const { setRecordingUrl } = useGlobalInfoStore();
useEffect(() => {
if (location.pathname === '/recording') {
const hasRecordingSession =
window.sessionStorage.getItem('browserId') ||
window.sessionStorage.getItem('recordingSessionId');
const recordingUrl = window.sessionStorage.getItem('recordingUrl');
if (recordingUrl) {
setRecordingUrl(recordingUrl);
}
if (hasRecordingSession) {
console.log('UserRoute: Valid recording session detected, bypassing auth check');
setIsCheckingAuth(false);
return;
}
}
const timer = setTimeout(() => {
setIsCheckingAuth(false);
}, 100);
return () => clearTimeout(timer);
}, [location.pathname]);
if (isCheckingAuth) {
return null;
}
if (location.pathname === '/recording') {
const hasRecordingSession =
window.sessionStorage.getItem('browserId') ||
window.sessionStorage.getItem('recordingSessionId');
if (hasRecordingSession) {
return <Outlet />;
}
}
return state.user ? <Outlet /> : <Navigate to="/login" />;
};
export default UserRoute;
export default UserRoute;