vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #467 from getmaxun/browser-instances

Browse Source 
feat: support multiple browser instances
This commit is contained in:
Karishma Shukla
2025-03-08 22:53:14 +05:30
committed by GitHub
parent ba519a6a1b 0220632aef
commit 56453f2788
12 changed files with 629 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
server/src/api/record.ts
View File

@@ -507,9 +507,9 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) {
}
}
async function readyForRunHandler(browserId: string, id: string) {
async function readyForRunHandler(browserId: string, id: string, userId: string){
try {
const result = await executeRun(id);
const result = await executeRun(id, userId);
if (result && result.success) {
logger.log('info', `Interpretation of ${id} succeeded`);
@@ -517,14 +517,14 @@ async function readyForRunHandler(browserId: string, id: string) {
return result.interpretationInfo;
} else {
logger.log('error', `Interpretation of ${id} failed`);
await destroyRemoteBrowser(browserId);
await destroyRemoteBrowser(browserId, userId);
resetRecordingState(browserId, id);
return null;
}
} catch (error: any) {
logger.error(`Error during readyForRunHandler: ${error.message}`);
await destroyRemoteBrowser(browserId);
await destroyRemoteBrowser(browserId, userId);
return null;
}
}
@@ -546,7 +546,7 @@ function AddGeneratedFlags(workflow: WorkflowFile) {
return copy;
};
async function executeRun(id: string) {
async function executeRun(id: string, userId: string) {
try {
const run = await Run.findOne({ where: { runId: id } });
if (!run) {
@@ -568,7 +568,7 @@ async function executeRun(id: string) {
plainRun.status = 'running';
const browser = browserPool.getRemoteBrowser(plainRun.browserId);
const browser = browserPool.getRemoteBrowser(userId);
if (!browser) {
throw new Error('Could not access browser');
}
@@ -586,7 +586,7 @@ async function executeRun(id: string) {
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput);
await destroyRemoteBrowser(plainRun.browserId);
await destroyRemoteBrowser(plainRun.browserId, userId);
const updatedRun = await run.update({
...run,
@@ -672,12 +672,12 @@ export async function handleRunRecording(id: string, userId: string) {
rejectUnauthorized: false
});
socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId));
socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId, userId));
logger.log('info', `Running Robot: ${id}`);
socket.on('disconnect', () => {
cleanupSocketListeners(socket, browserId, newRunId);
cleanupSocketListeners(socket, browserId, newRunId, userId);
});
// Return the runId immediately, so the client knows the run is started
@@ -688,8 +688,8 @@ export async function handleRunRecording(id: string, userId: string) {
}
}
function cleanupSocketListeners(socket: Socket, browserId: string, id: string) {
socket.off('ready-for-run', () => readyForRunHandler(browserId, id));
function cleanupSocketListeners(socket: Socket, browserId: string, id: string, userId: string) {
socket.off('ready-for-run', () => readyForRunHandler(browserId, id, userId));
logger.log('info', `Cleaned up listeners for browserId: ${browserId}, runId: ${id}`);
}

252
server/src/browser-management/classes/BrowserPool.ts
View File

@@ -15,6 +15,10 @@ interface BrowserPoolInfo {
* @default false
*/
active: boolean,
/**
* The user ID that owns this browser instance.
*/
userId: string,
}
/**
@@ -29,36 +33,101 @@ interface PoolDictionary {
/**
* A browser pool is a collection of remote browsers that are initialized and ready to be used.
* Enforces a "1 User - 1 Browser" policy, while allowing multiple users to have their own browser instances.
* Adds the possibility to add, remove and retrieve remote browsers from the pool.
* It is possible to manage multiple browsers for creating or running a recording.
* @category BrowserManagement
*/
export class BrowserPool {
/**
* Holds all the instances of remote browsers.
*/
private pool: PoolDictionary = {};
/**
* Adds a remote browser instance to the pool indexed by the id.
* Maps user IDs to their browser IDs.
*/
private userToBrowserMap: Map<string, string> = new Map();
/**
* Adds a remote browser instance to the pool for a specific user.
* If the user already has a browser, the existing browser will be closed and replaced.
*
* @param id remote browser instance's id
* @param browser remote browser instance
* @param userId the user ID that owns this browser instance
* @param active states if the browser's instance is being actively used
* @returns true if a new browser was added, false if an existing browser was replaced
*/
public addRemoteBrowser = (id: string, browser: RemoteBrowser, active: boolean = false): void => {
this.pool = {
...this.pool,
[id]: {
browser,
active,
},
public addRemoteBrowser = (
id: string,
browser: RemoteBrowser,
userId: string,
active: boolean = false
): boolean => {
// Check if user already has a browser
const existingBrowserId = this.userToBrowserMap.get(userId);
let replaced = false;
if (existingBrowserId) {
// Close and remove the existing browser
if (existingBrowserId !== id) {
this.closeAndDeleteBrowser(existingBrowserId);
replaced = true;
} else {
// If it's the same browser ID, just update the info
this.pool[id] = {
browser,
active,
userId,
};
logger.log('debug', `Updated existing browser with id: ${id} for user: ${userId}`);
return false;
}
}
logger.log('debug', `Remote browser with id: ${id} added to the pool`);
// Add the new browser to the pool
this.pool[id] = {
browser,
active,
userId,
};
// Update the user-to-browser mapping
this.userToBrowserMap.set(userId, id);
logger.log('debug', `Remote browser with id: ${id} added to the pool for user: ${userId}`);
return !replaced;
};
/**
* Removes the remote browser instance from the pool.
* Note: This doesn't handle browser closing as RemoteBrowser doesn't expose a close method.
* The caller should ensure the browser is properly closed before calling this method.
*
* @param id remote browser instance's id
* @returns true if the browser was removed successfully, false otherwise
*/
public closeAndDeleteBrowser = (id: string): boolean => {
if (!this.pool[id]) {
logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`);
return false;
}
// Remove the user-to-browser mapping
const userId = this.pool[id].userId;
if (this.userToBrowserMap.get(userId) === id) {
this.userToBrowserMap.delete(userId);
}
// Remove from pool
delete this.pool[id];
logger.log('debug', `Remote browser with id: ${id} removed from the pool`);
return true;
};
/**
* Removes the remote browser instance from the pool without attempting to close it.
*
* @param id remote browser instance's id
* @returns true if the browser was removed successfully, false otherwise
*/
@@ -67,13 +136,22 @@ export class BrowserPool {
logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`);
return false;
}
delete (this.pool[id]);
// Remove the user-to-browser mapping
const userId = this.pool[id].userId;
if (this.userToBrowserMap.get(userId) === id) {
this.userToBrowserMap.delete(userId);
}
// Remove from pool
delete this.pool[id];
logger.log('debug', `Remote browser with id: ${id} deleted from the pool`);
return true;
};
/**
* Returns the remote browser instance from the pool.
*
* @param id remote browser instance's id
* @returns remote browser instance or undefined if it does not exist in the pool
*/
@@ -83,18 +161,154 @@ export class BrowserPool {
};
/**
* Returns the active browser's instance id from the pool.
* If there is no active browser, it returns undefined.
* If there are multiple active browsers, it returns the first one.
* @returns the first remote active browser instance's id from the pool
* Returns the active browser's instance id for a specific user.
*
* @param userId the user ID to find the browser for
* @returns the browser ID for the user, or null if no browser exists
*/
public getActiveBrowserId = (): string | null => {
public getActiveBrowserId = (userId: string): string | null => {
const browserId = this.userToBrowserMap.get(userId);
if (!browserId) {
logger.log('debug', `No browser found for user: ${userId}`);
return null;
}
// Verify the browser still exists in the pool
if (!this.pool[browserId]) {
this.userToBrowserMap.delete(userId);
logger.log('warn', `Browser mapping found for user: ${userId}, but browser doesn't exist in pool`);
return null;
}
console.log(`Browser Id ${browserId} found for user: ${userId}`);
return browserId;
};
/**
* Returns the user ID associated with a browser ID.
*
* @param browserId the browser ID to find the user for
* @returns the user ID for the browser, or null if the browser doesn't exist
*/
public getUserForBrowser = (browserId: string): string | null => {
if (!this.pool[browserId]) {
return null;
}
return this.pool[browserId].userId;
};
/**
* Sets the active state of a browser.
*
* @param id the browser ID
* @param active the new active state
* @returns true if successful, false if the browser wasn't found
*/
public setActiveBrowser = (id: string, active: boolean): boolean => {
if (!this.pool[id]) {
logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`);
return false;
}
this.pool[id].active = active;
logger.log('debug', `Remote browser with id: ${id} set to ${active ? 'active' : 'inactive'}`);
return true;
};
/**
* Returns all browser instances for a specific user.
* Should only be one per the "1 User - 1 Browser" policy, but included for flexibility.
*
* @param userId the user ID to find browsers for
* @returns an array of browser IDs belonging to the user
*/
public getAllBrowserIdsForUser = (userId: string): string[] => {
const browserIds: string[] = [];
// Normally this would just return the one browser from the map
const mappedBrowserId = this.userToBrowserMap.get(userId);
if (mappedBrowserId && this.pool[mappedBrowserId]) {
browserIds.push(mappedBrowserId);
}
// But as a safeguard, also check the entire pool for any browsers assigned to this user
// This helps detect and fix any inconsistencies in the maps
for (const [id, info] of Object.entries(this.pool)) {
if (info.userId === userId && !browserIds.includes(id)) {
browserIds.push(id);
// Fix the map if it's inconsistent
if (!mappedBrowserId) {
this.userToBrowserMap.set(userId, id);
}
}
}
return browserIds;
};
/**
* Returns the total number of browsers in the pool.
*/
public getPoolSize = (): number => {
return Object.keys(this.pool).length;
};
/**
* Returns the total number of active users (users with browsers).
*/
public getActiveUserCount = (): number => {
return this.userToBrowserMap.size;
};
/**
* Gets the current active browser for the system if there's only one active user.
* This is a migration helper to support code that hasn't been updated to the user-browser model yet.
*
* @param currentUserId The ID of the current user, which will be prioritized if multiple browsers exist
* @returns A browser ID if one can be determined, or null
*/
public getActiveBrowserForMigration = (currentUserId?: string): string | null => {
// If a current user ID is provided and they have a browser, return that
if (currentUserId) {
const browserForUser = this.getActiveBrowserId(currentUserId);
if (browserForUser) {
return browserForUser;
}
}
// If only one user has a browser, return that
if (this.userToBrowserMap.size === 1) {
const userId = Array.from(this.userToBrowserMap.keys())[0];
return this.userToBrowserMap.get(userId) || null;
}
// Fall back to the first active browser if any
for (const id of Object.keys(this.pool)) {
if (this.pool[id].active) {
return id;
}
}
logger.log('warn', `No active browser in the pool`);
// If all else fails, return the first browser in the pool
const browserIds = Object.keys(this.pool);
return browserIds.length > 0 ? browserIds[0] : null;
};
/**
* Returns the first active browser's instance id from the pool.
* If there is no active browser, it returns null.
* If there are multiple active browsers, it returns the first one.
*
* @returns the first remote active browser instance's id from the pool
* @deprecated Use getBrowserIdForUser instead to enforce the 1 User - 1 Browser policy
*/
public getActiveBrowserIdLegacy = (): string | null => {
for (const id of Object.keys(this.pool)) {
if (this.pool[id].active) {
return id;
}
}
// Don't log a warning since this behavior is expected in the user-browser model
// logger.log('warn', `No active browser in the pool`);
return null;
};
}
}

120
server/src/browser-management/classes/RemoteBrowser.ts
View File

@@ -90,6 +90,12 @@ export class RemoteBrowser {
maxRepeats: 1,
};
/**
* The user ID that owns this browser instance
* @private
*/
private userId: string;
private lastEmittedUrl: string | null = null;
/**
@@ -106,6 +112,7 @@ export class RemoteBrowser {
private screenshotQueue: Buffer[] = [];
private isProcessingScreenshot = false;
private screencastInterval: NodeJS.Timeout | null = null
private isScreencastActive: boolean = false;
/**
* Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and
@@ -113,8 +120,9 @@ export class RemoteBrowser {
* @param socket socket.io socket instance used to communicate with the client side
* @constructor
*/
public constructor(socket: Socket) {
public constructor(socket: Socket, userId: string) {
this.socket = socket;
this.userId = userId;
this.interpreter = new WorkflowInterpreter(socket);
this.generator = new WorkflowGenerator(socket);
}
@@ -193,7 +201,7 @@ export class RemoteBrowser {
const currentUrl = page.url();
if (this.shouldEmitUrlChange(currentUrl)) {
this.lastEmittedUrl = currentUrl;
this.socket.emit('urlChanged', currentUrl);
this.socket.emit('urlChanged', {url: currentUrl, userId: this.userId});
}
}
});
@@ -333,15 +341,40 @@ export class RemoteBrowser {
* @returns void
*/
public registerEditorEvents = (): void => {
this.socket.on('rerender', async () => await this.makeAndEmitScreenshot());
this.socket.on('settings', (settings) => this.interpreterSettings = settings);
this.socket.on('changeTab', async (tabIndex) => await this.changeTab(tabIndex));
this.socket.on('addTab', async () => {
// For each event, include userId to make sure events are handled for the correct browser
logger.log('debug', `Registering editor events for user: ${this.userId}`);
// Listen for specific events for this user
this.socket.on(`rerender:${this.userId}`, async () => {
logger.debug(`Rerender event received for user ${this.userId}`);
await this.makeAndEmitScreenshot();
});
// For backward compatibility, also listen to the general event
this.socket.on('rerender', async () => {
logger.debug(`General rerender event received, checking if for user ${this.userId}`);
await this.makeAndEmitScreenshot();
});
this.socket.on(`settings:${this.userId}`, (settings) => {
this.interpreterSettings = settings;
logger.debug(`Settings updated for user ${this.userId}`);
});
this.socket.on(`changeTab:${this.userId}`, async (tabIndex) => {
logger.debug(`Tab change to ${tabIndex} requested for user ${this.userId}`);
await this.changeTab(tabIndex);
});
this.socket.on(`addTab:${this.userId}`, async () => {
logger.debug(`New tab requested for user ${this.userId}`);
await this.currentPage?.context().newPage();
const lastTabIndex = this.currentPage ? this.currentPage.context().pages().length - 1 : 0;
await this.changeTab(lastTabIndex);
});
this.socket.on('closeTab', async (tabInfo) => {
this.socket.on(`closeTab:${this.userId}`, async (tabInfo) => {
logger.debug(`Close tab ${tabInfo.index} requested for user ${this.userId}`);
const page = this.currentPage?.context().pages()[tabInfo.index];
if (page) {
if (tabInfo.isCurrent) {
@@ -356,24 +389,52 @@ export class RemoteBrowser {
await page.close();
logger.log(
'debug',
`${tabInfo.index} page was closed, new length of pages: ${this.currentPage?.context().pages().length}`
)
`Tab ${tabInfo.index} was closed for user ${this.userId}, new tab count: ${this.currentPage?.context().pages().length}`
);
} else {
logger.log('error', `${tabInfo.index} index out of range of pages`)
logger.log('error', `Tab index ${tabInfo.index} out of range for user ${this.userId}`);
}
});
this.socket.on('setViewportSize', async (data: { width: number, height: number }) => {
this.socket.on(`setViewportSize:${this.userId}`, async (data: { width: number, height: number }) => {
const { width, height } = data;
logger.log('debug', `Received viewport size: width=${width}, height=${height}`);
logger.log('debug', `Viewport size change to width=${width}, height=${height} requested for user ${this.userId}`);
// Update the browser context's viewport dynamically
if (this.context && this.browser) {
this.context = await this.browser.newContext({ viewport: { width, height } });
logger.log('debug', `Viewport size updated to width=${width}, height=${height} for the entire browser context`);
logger.log('debug', `Viewport size updated to width=${width}, height=${height} for user ${this.userId}`);
}
});
}
// For backward compatibility, also register the standard events
this.socket.on('settings', (settings) => this.interpreterSettings = settings);
this.socket.on('changeTab', async (tabIndex) => await this.changeTab(tabIndex));
this.socket.on('addTab', async () => {
await this.currentPage?.context().newPage();
const lastTabIndex = this.currentPage ? this.currentPage.context().pages().length - 1 : 0;
await this.changeTab(lastTabIndex);
});
this.socket.on('closeTab', async (tabInfo) => {
const page = this.currentPage?.context().pages()[tabInfo.index];
if (page) {
if (tabInfo.isCurrent) {
if (this.currentPage?.context().pages()[tabInfo.index + 1]) {
await this.changeTab(tabInfo.index + 1);
} else {
await this.changeTab(tabInfo.index - 1);
}
}
await page.close();
}
});
this.socket.on('setViewportSize', async (data: { width: number, height: number }) => {
const { width, height } = data;
if (this.context && this.browser) {
this.context = await this.browser.newContext({ viewport: { width, height } });
}
});
};
/**
* Subscribes the remote browser for a screencast session
* on [CDP](https://chromedevtools.github.io/devtools-protocol/) level,
@@ -382,16 +443,24 @@ export class RemoteBrowser {
* @returns {Promise<void>}
*/
public subscribeToScreencast = async (): Promise<void> => {
logger.log('debug', `Starting screencast for user: ${this.userId}`);
await this.startScreencast();
if (!this.client) {
logger.log('warn', 'client is not initialized');
return;
}
// Set flag to indicate screencast is active
this.isScreencastActive = true;
this.client.on('Page.screencastFrame', ({ data: base64, sessionId }) => {
// Only process if screencast is still active for this user
if (!this.isScreencastActive) {
return;
}
this.emitScreenshot(Buffer.from(base64, 'base64'))
setTimeout(async () => {
try {
if (!this.client) {
if (!this.client || !this.isScreencastActive) {
logger.log('warn', 'client is not initialized');
return;
}
@@ -410,6 +479,8 @@ export class RemoteBrowser {
*/
public async switchOff(): Promise<void> {
try {
this.isScreencastActive = false;
await this.interpreter.stopInterpretation();
if (this.screencastInterval) {
@@ -553,7 +624,11 @@ export class RemoteBrowser {
//await this.currentPage.setViewportSize({ height: 400, width: 900 })
this.client = await this.currentPage.context().newCDPSession(this.currentPage);
this.socket.emit('urlChanged', this.currentPage.url());
// Include userId in the URL change event
this.socket.emit('urlChanged', {
url: this.currentPage.url(),
userId: this.userId
});
await this.makeAndEmitScreenshot();
await this.subscribeToScreencast();
} else {
@@ -602,6 +677,8 @@ export class RemoteBrowser {
await this.client.send('Page.startScreencast', {
format: SCREENCAST_CONFIG.format,
});
// Set flag to indicate screencast is active
this.isScreencastActive = true;
// Set up screencast frame handler
this.client.on('Page.screencastFrame', async ({ data, sessionId }) => {
@@ -627,6 +704,8 @@ export class RemoteBrowser {
}
try {
// Set flag to indicate screencast is active
this.isScreencastActive = false;
await this.client.send('Page.stopScreencast');
this.screenshotQueue = [];
this.isProcessingScreenshot = false;
@@ -657,8 +736,11 @@ export class RemoteBrowser {
const base64Data = optimizedScreenshot.toString('base64');
const dataWithMimeType = `data:image/jpeg;base64,${base64Data}`;
this.socket.emit('screencast', dataWithMimeType);
logger.debug('Screenshot emitted');
// Emit with user context to ensure the frontend can identify which browser's screenshot this is
this.socket.emit('screencast', {
image: dataWithMimeType,
userId: this.userId
}); logger.debug('Screenshot emitted');
} catch (error) {
logger.error('Screenshot emission failed:', error);
} finally {

30
server/src/browser-management/controller.ts
View File

@@ -21,23 +21,23 @@ import logger from "../logger";
* @category BrowserManagement-Controller
*/
export const initializeRemoteBrowserForRecording = (userId: string): string => {
const id = getActiveBrowserId() || uuid();
const id = getActiveBrowserId(userId) || uuid();
createSocketConnection(
io.of(id),
async (socket: Socket) => {
// browser is already active
const activeId = getActiveBrowserId();
const activeId = getActiveBrowserId(userId);
if (activeId) {
const remoteBrowser = browserPool.getRemoteBrowser(activeId);
remoteBrowser?.updateSocket(socket);
await remoteBrowser?.makeAndEmitScreenshot();
} else {
const browserSession = new RemoteBrowser(socket);
const browserSession = new RemoteBrowser(socket, userId);
browserSession.interpreter.subscribeToPausing();
await browserSession.initialize(userId);
await browserSession.registerEditorEvents();
await browserSession.subscribeToScreencast();
browserPool.addRemoteBrowser(id, browserSession, true);
browserPool.addRemoteBrowser(id, browserSession, userId);
}
socket.emit('loaded');
});
@@ -57,9 +57,9 @@ export const createRemoteBrowserForRun = (userId: string): string => {
createSocketConnectionForRun(
io.of(id),
async (socket: Socket) => {
const browserSession = new RemoteBrowser(socket);
const browserSession = new RemoteBrowser(socket, userId);
await browserSession.initialize(userId);
browserPool.addRemoteBrowser(id, browserSession, true);
browserPool.addRemoteBrowser(id, browserSession, userId);
socket.emit('ready-for-run');
});
return id;
@@ -72,7 +72,7 @@ export const createRemoteBrowserForRun = (userId: string): string => {
* @returns {Promise<boolean>}
* @category BrowserManagement-Controller
*/
export const destroyRemoteBrowser = async (id: string): Promise<boolean> => {
export const destroyRemoteBrowser = async (id: string, userId: string): Promise<boolean> => {
const browserSession = browserPool.getRemoteBrowser(id);
if (browserSession) {
logger.log('debug', `Switching off the browser with id: ${id}`);
@@ -88,8 +88,8 @@ export const destroyRemoteBrowser = async (id: string): Promise<boolean> => {
* @returns {string | null}
* @category BrowserManagement-Controller
*/
export const getActiveBrowserId = (): string | null => {
return browserPool.getActiveBrowserId();
export const getActiveBrowserId = (userId: string): string | null => {
return browserPool.getActiveBrowserId(userId);
};
/**
@@ -98,7 +98,7 @@ export const getActiveBrowserId = (): string | null => {
* @returns {string | undefined}
* @category BrowserManagement-Controller
*/
export const getRemoteBrowserCurrentUrl = (id: string): string | undefined => {
export const getRemoteBrowserCurrentUrl = (id: string, userId: string): string | undefined => {
return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.url();
};
@@ -108,7 +108,7 @@ export const getRemoteBrowserCurrentUrl = (id: string): string | undefined => {
* @return {string[] | undefined}
* @category BrowserManagement-Controller
*/
export const getRemoteBrowserCurrentTabs = (id: string): string[] | undefined => {
export const getRemoteBrowserCurrentTabs = (id: string, userId: string): string[] | undefined => {
return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.context().pages()
.map((page) => {
const parsedUrl = new URL(page.url());
@@ -126,8 +126,8 @@ export const getRemoteBrowserCurrentTabs = (id: string): string[] | undefined =>
* @returns {Promise<void>}
* @category BrowserManagement-Controller
*/
export const interpretWholeWorkflow = async () => {
const id = getActiveBrowserId();
export const interpretWholeWorkflow = async (userId: string) => {
const id = getActiveBrowserId(userId);
if (id) {
const browser = browserPool.getRemoteBrowser(id);
if (browser) {
@@ -146,8 +146,8 @@ export const interpretWholeWorkflow = async () => {
* @returns {Promise<void>}
* @category BrowserManagement-Controller
*/
export const stopRunningInterpretation = async () => {
const id = getActiveBrowserId();
export const stopRunningInterpretation = async (userId: string) => {
const id = getActiveBrowserId(userId);
if (id) {
const browser = browserPool.getRemoteBrowser(id);
await browser?.stopCurrentInterpretation();

191
server/src/browser-management/inputHandlers.ts
View File

@@ -4,6 +4,8 @@
* These functions are called by the client through socket communication.
*/
import { Socket } from 'socket.io';
import { IncomingMessage } from 'http';
import { JwtPayload } from 'jsonwebtoken';
import logger from "../logger";
import { Coordinates, ScrollDeltas, KeyboardInput, DatePickerEventData } from '../types';
@@ -13,6 +15,14 @@ import { Page } from "playwright";
import { throttle } from "../../../src/helpers/inputHelpers";
import { CustomActions } from "../../../src/shared/types";
interface AuthenticatedIncomingMessage extends IncomingMessage {
user?: JwtPayload | string;
}
interface AuthenticatedSocket extends Socket {
request: AuthenticatedIncomingMessage;
}
/**
* A wrapper function for handling user input.
* This function gets the active browser instance from the browser pool
@@ -23,6 +33,7 @@ import { CustomActions } from "../../../src/shared/types";
*
* @param handleCallback The callback handler to be called
* @param args - arguments to be passed to the handler
* @param socket - socket with authenticated request
* @category HelperFunctions
*/
const handleWrapper = async (
@@ -31,9 +42,21 @@ const handleWrapper = async (
page: Page,
args?: any
) => Promise<void>,
args?: any
args?: any,
socket?: AuthenticatedSocket,
) => {
const id = browserPool.getActiveBrowserId();
if (!socket || !socket.request || !socket.request.user || typeof socket.request.user === 'string') {
logger.log('warn', `User not authenticated or invalid JWT payload`);
return;
}
const userId = socket.request.user.id;
if (!userId) {
logger.log('warn', `User ID is missing in JWT payload`);
return;
}
const id = browserPool.getActiveBrowserId(userId);
if (id) {
const activeBrowser = browserPool.getRemoteBrowser(id);
if (activeBrowser?.interpreter.interpretationInProgress() && !activeBrowser.interpreter.interpretationIsPaused) {
@@ -66,12 +89,13 @@ interface CustomActionEventData {
/**
* A wrapper function for handling custom actions.
* @param socket The socket connection
* @param customActionEventData The custom action event data
* @category HelperFunctions
*/
const onGenerateAction = async (customActionEventData: CustomActionEventData) => {
const onGenerateAction = async (socket: AuthenticatedSocket, customActionEventData: CustomActionEventData) => {
logger.log('debug', `Generating ${customActionEventData.action} action emitted from client`);
await handleWrapper(handleGenerateAction, customActionEventData);
await handleWrapper(handleGenerateAction, customActionEventData, socket);
}
/**
@@ -89,12 +113,13 @@ const handleGenerateAction =
/**
* A wrapper function for handling mousedown event.
* @param socket The socket connection
* @param coordinates - coordinates of the mouse click
* @category HelperFunctions
*/
const onMousedown = async (coordinates: Coordinates) => {
const onMousedown = async (socket: AuthenticatedSocket, coordinates: Coordinates) => {
logger.log('debug', 'Handling mousedown event emitted from client');
await handleWrapper(handleMousedown, coordinates);
await handleWrapper(handleMousedown, coordinates, socket);
}
/**
@@ -139,12 +164,13 @@ const handleMousedown = async (generator: WorkflowGenerator, page: Page, { x, y
/**
* A wrapper function for handling the wheel event.
* @param socket The socket connection
* @param scrollDeltas - the scroll deltas of the wheel event
* @category HelperFunctions
*/
const onWheel = async (scrollDeltas: ScrollDeltas) => {
const onWheel = async (socket: AuthenticatedSocket, scrollDeltas: ScrollDeltas) => {
logger.log('debug', 'Handling scroll event emitted from client');
await handleWrapper(handleWheel, scrollDeltas);
await handleWrapper(handleWheel, scrollDeltas, socket);
};
/**
@@ -165,12 +191,13 @@ const handleWheel = async (generator: WorkflowGenerator, page: Page, { deltaX, d
/**
* A wrapper function for handling the mousemove event.
* @param socket The socket connection
* @param coordinates - the coordinates of the mousemove event
* @category HelperFunctions
*/
const onMousemove = async (coordinates: Coordinates) => {
const onMousemove = async (socket: AuthenticatedSocket, coordinates: Coordinates) => {
logger.log('debug', 'Handling mousemove event emitted from client');
await handleWrapper(handleMousemove, coordinates);
await handleWrapper(handleMousemove, coordinates, socket);
}
/**
@@ -199,12 +226,13 @@ const handleMousemove = async (generator: WorkflowGenerator, page: Page, { x, y
/**
* A wrapper function for handling the keydown event.
* @param socket The socket connection
* @param keyboardInput - the keyboard input of the keydown event
* @category HelperFunctions
*/
const onKeydown = async (keyboardInput: KeyboardInput) => {
const onKeydown = async (socket: AuthenticatedSocket, keyboardInput: KeyboardInput) => {
logger.log('debug', 'Handling keydown event emitted from client');
await handleWrapper(handleKeydown, keyboardInput);
await handleWrapper(handleKeydown, keyboardInput, socket);
}
/**
@@ -235,49 +263,95 @@ const handleDateSelection = async (generator: WorkflowGenerator, page: Page, dat
logger.log('debug', `Date ${data.value} selected`);
}
const onDateSelection = async (data: DatePickerEventData) => {
/**
* A wrapper function for handling the date selection event.
* @param socket The socket connection
* @param data - the data of the date selection event
* @category HelperFunctions
*/
const onDateSelection = async (socket: AuthenticatedSocket, data: DatePickerEventData) => {
logger.log('debug', 'Handling date selection event emitted from client');
await handleWrapper(handleDateSelection, data);
await handleWrapper(handleDateSelection, data, socket);
}
/**
* Handles the dropdown selection event.
* @param generator - the workflow generator {@link Generator}
* @param page - the active page of the remote browser
* @param data - the data of the dropdown selection event
* @category BrowserManagement
*/
const handleDropdownSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => {
await generator.onDropdownSelection(page, data);
logger.log('debug', `Dropdown value ${data.value} selected`);
}
const onDropdownSelection = async (data: { selector: string, value: string }) => {
/**
* A wrapper function for handling the dropdown selection event.
* @param socket The socket connection
* @param data - the data of the dropdown selection event
* @category HelperFunctions
*/
const onDropdownSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => {
logger.log('debug', 'Handling dropdown selection event emitted from client');
await handleWrapper(handleDropdownSelection, data);
await handleWrapper(handleDropdownSelection, data, socket);
}
/**
* Handles the time selection event.
* @param generator - the workflow generator {@link Generator}
* @param page - the active page of the remote browser
* @param data - the data of the time selection event
* @category BrowserManagement
*/
const handleTimeSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => {
await generator.onTimeSelection(page, data);
logger.log('debug', `Time value ${data.value} selected`);
}
const onTimeSelection = async (data: { selector: string, value: string }) => {
/**
* A wrapper function for handling the time selection event.
* @param socket The socket connection
* @param data - the data of the time selection event
* @category HelperFunctions
*/
const onTimeSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => {
logger.log('debug', 'Handling time selection event emitted from client');
await handleWrapper(handleTimeSelection, data);
await handleWrapper(handleTimeSelection, data, socket);
}
/**
* Handles the datetime-local selection event.
* @param generator - the workflow generator {@link Generator}
* @param page - the active page of the remote browser
* @param data - the data of the datetime-local selection event
* @category BrowserManagement
*/
const handleDateTimeLocalSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => {
await generator.onDateTimeLocalSelection(page, data);
logger.log('debug', `DateTime Local value ${data.value} selected`);
}
const onDateTimeLocalSelection = async (data: { selector: string, value: string }) => {
/**
* A wrapper function for handling the datetime-local selection event.
* @param socket The socket connection
* @param data - the data of the datetime-local selection event
* @category HelperFunctions
*/
const onDateTimeLocalSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => {
logger.log('debug', 'Handling datetime-local selection event emitted from client');
await handleWrapper(handleDateTimeLocalSelection, data);
await handleWrapper(handleDateTimeLocalSelection, data, socket);
}
/**
* A wrapper function for handling the keyup event.
* @param socket The socket connection
* @param keyboardInput - the keyboard input of the keyup event
* @category HelperFunctions
*/
const onKeyup = async (keyboardInput: KeyboardInput) => {
const onKeyup = async (socket: AuthenticatedSocket, keyboardInput: KeyboardInput) => {
logger.log('debug', 'Handling keyup event emitted from client');
await handleWrapper(handleKeyup, keyboardInput);
await handleWrapper(handleKeyup, keyboardInput, socket);
}
/**
@@ -296,12 +370,13 @@ const handleKeyup = async (generator: WorkflowGenerator, page: Page, key: string
/**
* A wrapper function for handling the url change event.
* @param socket The socket connection
* @param url - the new url of the page
* @category HelperFunctions
*/
const onChangeUrl = async (url: string) => {
const onChangeUrl = async (socket: AuthenticatedSocket, url: string) => {
logger.log('debug', 'Handling change url event emitted from client');
await handleWrapper(handleChangeUrl, url);
await handleWrapper(handleChangeUrl, url, socket);
}
/**
@@ -329,11 +404,12 @@ const handleChangeUrl = async (generator: WorkflowGenerator, page: Page, url: st
/**
* A wrapper function for handling the refresh event.
* @param socket The socket connection
* @category HelperFunctions
*/
const onRefresh = async () => {
const onRefresh = async (socket: AuthenticatedSocket) => {
logger.log('debug', 'Handling refresh event emitted from client');
await handleWrapper(handleRefresh);
await handleWrapper(handleRefresh, undefined, socket);
}
/**
@@ -350,11 +426,12 @@ const handleRefresh = async (generator: WorkflowGenerator, page: Page) => {
/**
* A wrapper function for handling the go back event.
* @param socket The socket connection
* @category HelperFunctions
*/
const onGoBack = async () => {
logger.log('debug', 'Handling refresh event emitted from client');
await handleWrapper(handleGoBack);
const onGoBack = async (socket: AuthenticatedSocket) => {
logger.log('debug', 'Handling go back event emitted from client');
await handleWrapper(handleGoBack, undefined, socket);
}
/**
@@ -372,11 +449,12 @@ const handleGoBack = async (generator: WorkflowGenerator, page: Page) => {
/**
* A wrapper function for handling the go forward event.
* @param socket The socket connection
* @category HelperFunctions
*/
const onGoForward = async () => {
logger.log('debug', 'Handling refresh event emitted from client');
await handleWrapper(handleGoForward);
const onGoForward = async (socket: AuthenticatedSocket) => {
logger.log('debug', 'Handling go forward event emitted from client');
await handleWrapper(handleGoForward, undefined, socket);
}
/**
@@ -394,18 +472,7 @@ const handleGoForward = async (generator: WorkflowGenerator, page: Page) => {
/**
* Helper function for registering the handlers onto established websocket connection.
* Registers:
* - mousedownHandler
* - wheelHandler
* - mousemoveHandler
* - keydownHandler
* - keyupHandler
* - changeUrlHandler
* - refreshHandler
* - goBackHandler
* - goForwardHandler
* - onGenerateAction
* input handlers.
* Registers various input handlers.
*
* All these handlers first generates the workflow pair data
* and then calls the corresponding playwright's function to emulate the input.
@@ -415,21 +482,25 @@ const handleGoForward = async (generator: WorkflowGenerator, page: Page) => {
* @returns void
* @category BrowserManagement
*/
const registerInputHandlers = (socket: Socket) => {
socket.on("input:mousedown", onMousedown);
socket.on("input:wheel", onWheel);
socket.on("input:mousemove", onMousemove);
socket.on("input:keydown", onKeydown);
socket.on("input:keyup", onKeyup);
socket.on("input:url", onChangeUrl);
socket.on("input:refresh", onRefresh);
socket.on("input:back", onGoBack);
socket.on("input:forward", onGoForward);
socket.on("input:date", onDateSelection);
socket.on("input:dropdown", onDropdownSelection);
socket.on("input:time", onTimeSelection);
socket.on("input:datetime-local", onDateTimeLocalSelection);
socket.on("action", onGenerateAction);
const registerInputHandlers = (socket: Socket) => {
// Cast to our authenticated socket type
const authSocket = socket as AuthenticatedSocket;
// Register handlers with the socket
socket.on("input:mousedown", (data) => onMousedown(authSocket, data));
socket.on("input:wheel", (data) => onWheel(authSocket, data));
socket.on("input:mousemove", (data) => onMousemove(authSocket, data));
socket.on("input:keydown", (data) => onKeydown(authSocket, data));
socket.on("input:keyup", (data) => onKeyup(authSocket, data));
socket.on("input:url", (data) => onChangeUrl(authSocket, data));
socket.on("input:refresh", () => onRefresh(authSocket));
socket.on("input:back", () => onGoBack(authSocket));
socket.on("input:forward", () => onGoForward(authSocket));
socket.on("input:date", (data) => onDateSelection(authSocket, data));
socket.on("input:dropdown", (data) => onDropdownSelection(authSocket, data));
socket.on("input:time", (data) => onTimeSelection(authSocket, data));
socket.on("input:datetime-local", (data) => onDateTimeLocalSelection(authSocket, data));
socket.on("action", (data) => onGenerateAction(authSocket, data));
};
export default registerInputHandlers;
export default registerInputHandlers;

46
server/src/routes/record.ts
View File

@@ -76,26 +76,35 @@ router.post('/start', requireSignIn, (req: AuthenticatedRequest, res:Response) =
* GET endpoint for terminating the remote browser recording session.
* returns whether the termination was successful
*/
router.get('/stop/:browserId', requireSignIn, async (req, res) => {
const success = await destroyRemoteBrowser(req.params.browserId);
router.get('/stop/:browserId', requireSignIn, async (req: AuthenticatedRequest, res) => {
if (!req.user) {
return res.status(401).send('User not authenticated');
}
const success = await destroyRemoteBrowser(req.params.browserId, req.user?.id);
return res.send(success);
});
/**
* GET endpoint for getting the id of the active remote browser.
*/
router.get('/active', requireSignIn, (req, res) => {
const id = getActiveBrowserId();
router.get('/active', requireSignIn, (req: AuthenticatedRequest, res) => {
if (!req.user) {
return res.status(401).send('User not authenticated');
}
const id = getActiveBrowserId(req.user?.id);
return res.send(id);
});
/**
* GET endpoint for getting the current url of the active remote browser.
*/
router.get('/active/url', requireSignIn, (req, res) => {
const id = getActiveBrowserId();
router.get('/active/url', requireSignIn, (req: AuthenticatedRequest, res) => {
if (!req.user) {
return res.status(401).send('User not authenticated');
}
const id = getActiveBrowserId(req.user?.id);
if (id) {
const url = getRemoteBrowserCurrentUrl(id);
const url = getRemoteBrowserCurrentUrl(id, req.user?.id);
return res.send(url);
}
return res.send(null);
@@ -104,10 +113,13 @@ router.get('/active/url', requireSignIn, (req, res) => {
/**
* GET endpoint for getting the current tabs of the active remote browser.
*/
router.get('/active/tabs', requireSignIn, (req, res) => {
const id = getActiveBrowserId();
router.get('/active/tabs', requireSignIn, (req: AuthenticatedRequest, res) => {
if (!req.user) {
return res.status(401).send('User not authenticated');
}
const id = getActiveBrowserId(req.user?.id);
if (id) {
const hosts = getRemoteBrowserCurrentTabs(id);
const hosts = getRemoteBrowserCurrentTabs(id, req.user?.id);
return res.send(hosts);
}
return res.send([]);
@@ -116,9 +128,12 @@ router.get('/active/tabs', requireSignIn, (req, res) => {
/**
* GET endpoint for starting an interpretation of the currently generated workflow.
*/
router.get('/interpret', requireSignIn, async (req, res) => {
router.get('/interpret', requireSignIn, async (req: AuthenticatedRequest, res) => {
try {
await interpretWholeWorkflow();
if (!req.user) {
return res.status(401).send('User not authenticated');
}
await interpretWholeWorkflow(req.user?.id);
return res.send('interpretation done');
} catch (e) {
return res.send('interpretation failed');
@@ -128,7 +143,10 @@ router.get('/interpret', requireSignIn, async (req, res) => {
/**
* GET endpoint for stopping an ongoing interpretation of the currently generated workflow.
*/
router.get('/interpret/stop', requireSignIn, async (req, res) => {
await stopRunningInterpretation();
router.get('/interpret/stop', requireSignIn, async (req: AuthenticatedRequest, res) => {
if (!req.user) {
return res.status(401).send('User not authenticated');
}
await stopRunningInterpretation(req.user?.id);
return res.send('interpretation stopped');
});

5
server/src/routes/storage.ts
View File

@@ -617,7 +617,7 @@ router.post('/runs/run/:id', requireSignIn, async (req: AuthenticatedRequest, re
workflow, currentPage, (newPage: Page) => currentPage = newPage, plainRun.interpreterSettings);
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput);
await destroyRemoteBrowser(plainRun.browserId);
await destroyRemoteBrowser(plainRun.browserId, req.user?.id);
await run.update({
...run,
status: 'success',
@@ -900,8 +900,9 @@ router.delete('/schedule/:id', requireSignIn, async (req: AuthenticatedRequest,
/**
* POST endpoint for aborting a current interpretation of the run.
*/
router.post('/runs/abort/:id', requireSignIn, async (req, res) => {
router.post('/runs/abort/:id', requireSignIn, async (req: AuthenticatedRequest, res) => {
try {
if (!req.user) { return res.status(401).send({ error: 'Unauthorized' }); }
const run = await Run.findOne({ where: { runId: req.params.id } });
if (!run) {
return res.status(404).send(false);

16
server/src/routes/workflow.ts
View File

@@ -7,6 +7,7 @@ import logger from "../logger";
import { browserPool } from "../server";
import { requireSignIn } from '../middlewares/auth';
import Robot from '../models/Robot';
import { AuthenticatedRequest } from './record';
export const router = Router();
@@ -46,8 +47,9 @@ router.get('/params/:browserId', requireSignIn, (req, res) => {
/**
* DELETE endpoint for deleting a pair from the generated workflow.
*/
router.delete('/pair/:index', requireSignIn, (req, res) => {
const id = browserPool.getActiveBrowserId();
router.delete('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => {
if (!req.user) { return res.status(401).send('User not authenticated'); }
const id = browserPool.getActiveBrowserId(req.user?.id);
if (id) {
const browser = browserPool.getRemoteBrowser(id);
if (browser) {
@@ -62,8 +64,9 @@ router.delete('/pair/:index', requireSignIn, (req, res) => {
/**
* POST endpoint for adding a pair to the generated workflow.
*/
router.post('/pair/:index', requireSignIn, (req, res) => {
const id = browserPool.getActiveBrowserId();
router.post('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => {
if (!req.user) { return res.status(401).send('User not authenticated'); }
const id = browserPool.getActiveBrowserId(req.user?.id);
if (id) {
const browser = browserPool.getRemoteBrowser(id);
logger.log('debug', `Adding pair to workflow`);
@@ -82,8 +85,9 @@ router.post('/pair/:index', requireSignIn, (req, res) => {
/**
* PUT endpoint for updating a pair in the generated workflow.
*/
router.put('/pair/:index', requireSignIn, (req, res) => {
const id = browserPool.getActiveBrowserId();
router.put('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => {
if (!req.user) { return res.status(401).send('User not authenticated'); }
const id = browserPool.getActiveBrowserId(req.user?.id);
if (id) {
const browser = browserPool.getRemoteBrowser(id);
logger.log('debug', `Updating pair in workflow`);

62
server/src/socket-connection/connection.ts
View File

@@ -1,6 +1,60 @@
import { Namespace, Socket } from 'socket.io';
import { IncomingMessage } from 'http';
import { verify, JwtPayload } from 'jsonwebtoken';
import logger from "../logger";
import registerInputHandlers from '../browser-management/inputHandlers'
import registerInputHandlers from '../browser-management/inputHandlers';
interface AuthenticatedIncomingMessage extends IncomingMessage {
user?: JwtPayload | string;
}
interface AuthenticatedSocket extends Socket {
request: AuthenticatedIncomingMessage;
}
/**
* Socket.io middleware for authentication
* This is a socket.io specific auth handler that doesn't rely on Express middleware
*/
const socketAuthMiddleware = (socket: Socket, next: (err?: Error) => void) => {
const cookies = socket.handshake.headers.cookie;
if (!cookies) {
return next(new Error('Authentication required'));
}
const tokenMatch = cookies.split(';').find(c => c.trim().startsWith('token='));
if (!tokenMatch) {
return next(new Error('Authentication required'));
}
const token = tokenMatch.split('=')[1];
if (!token) {
return next(new Error('Authentication required'));
}
const secret = process.env.JWT_SECRET;
if (!secret) {
return next(new Error('Server configuration error'));
}
verify(token, secret, (err: any, user: any) => {
if (err) {
logger.log('warn', 'JWT verification error:', err);
return next(new Error('Authentication failed'));
}
// Normalize payload key
if (user.userId && !user.id) {
user.id = user.userId;
delete user.userId; // temporary: del the old key for clarity
}
// Attach user to socket request
const authSocket = socket as AuthenticatedSocket;
authSocket.request.user = user;
next();
});
};
/**
* Opens a websocket canal for duplex data transfer and registers all handlers for this data for the recording session.
@@ -13,6 +67,8 @@ export const createSocketConnection = (
io: Namespace,
callback: (socket: Socket) => void,
) => {
io.use(socketAuthMiddleware);
const onConnection = async (socket: Socket) => {
logger.log('info', "Client connected " + socket.id);
registerInputHandlers(socket);
@@ -34,6 +90,8 @@ export const createSocketConnectionForRun = (
io: Namespace,
callback: (socket: Socket) => void,
) => {
io.use(socketAuthMiddleware);
const onConnection = async (socket: Socket) => {
logger.log('info', "Client connected " + socket.id);
socket.on('disconnect', () => logger.log('info', "Client disconnected " + socket.id));
@@ -41,4 +99,4 @@ export const createSocketConnectionForRun = (
}
io.on('connection', onConnection);
};
};

4
server/src/workflow-management/classes/Generator.ts
View File

@@ -151,8 +151,8 @@ export class WorkflowGenerator {
workflow: [],
});
socket.on('activeIndex', (data) => this.generatedData.lastIndex = parseInt(data));
socket.on('decision', async ({ pair, actionType, decision }) => {
const id = browserPool.getActiveBrowserId();
socket.on('decision', async ({ pair, actionType, decision, userId }) => {
const id = browserPool.getActiveBrowserId(userId);
if (id) {
// const activeBrowser = browserPool.getRemoteBrowser(id);
// const currentPage = activeBrowser?.getCurrentPage();

22
server/src/workflow-management/scheduler/index.ts
View File

@@ -92,7 +92,7 @@ function AddGeneratedFlags(workflow: WorkflowFile) {
return copy;
};
async function executeRun(id: string) {
async function executeRun(id: string, userId: string) {
try {
const run = await Run.findOne({ where: { runId: id } });
if (!run) {
@@ -114,7 +114,7 @@ async function executeRun(id: string) {
plainRun.status = 'running';
const browser = browserPool.getRemoteBrowser(plainRun.browserId);
const browser = browserPool.getRemoteBrowser(userId);
if (!browser) {
throw new Error('Could not access browser');
}
@@ -132,7 +132,7 @@ async function executeRun(id: string) {
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput);
await destroyRemoteBrowser(plainRun.browserId);
await destroyRemoteBrowser(plainRun.browserId, userId);
await run.update({
...run,
@@ -207,22 +207,22 @@ async function executeRun(id: string) {
}
}
async function readyForRunHandler(browserId: string, id: string) {
async function readyForRunHandler(browserId: string, id: string, userId: string) {
try {
const interpretation = await executeRun(id);
const interpretation = await executeRun(id, userId);
if (interpretation) {
logger.log('info', `Interpretation of ${id} succeeded`);
} else {
logger.log('error', `Interpretation of ${id} failed`);
await destroyRemoteBrowser(browserId);
await destroyRemoteBrowser(browserId, userId);
}
resetRecordingState(browserId, id);
} catch (error: any) {
logger.error(`Error during readyForRunHandler: ${error.message}`);
await destroyRemoteBrowser(browserId);
await destroyRemoteBrowser(browserId, userId);
}
}
@@ -245,12 +245,12 @@ export async function handleRunRecording(id: string, userId: string) {
rejectUnauthorized: false
});
socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId));
socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId, userId));
logger.log('info', `Running robot: ${id}`);
socket.on('disconnect', () => {
cleanupSocketListeners(socket, browserId, newRunId);
cleanupSocketListeners(socket, browserId, newRunId, userId);
});
} catch (error: any) {
@@ -258,8 +258,8 @@ export async function handleRunRecording(id: string, userId: string) {
}
}
function cleanupSocketListeners(socket: Socket, browserId: string, id: string) {
socket.off('ready-for-run', () => readyForRunHandler(browserId, id));
function cleanupSocketListeners(socket: Socket, browserId: string, id: string, userId: string) {
socket.off('ready-for-run', () => readyForRunHandler(browserId, id, userId));
logger.log('info', `Cleaned up listeners for browserId: ${browserId}, runId: ${id}`);
}

24
src/components/browser/BrowserWindow.tsx
View File

@@ -1,4 +1,4 @@
import React, { useCallback, useEffect, useState } from 'react';
import React, { useCallback, useContext, useEffect, useState } from 'react';
import { useSocketStore } from '../../context/socket';
import { Button } from '@mui/material';
import Canvas from "../recorder/canvas";
@@ -8,6 +8,7 @@ import { useActionContext } from '../../context/browserActions';
import { useBrowserSteps, TextStep } from '../../context/browserSteps';
import { useGlobalInfoStore } from '../../context/globalInfo';
import { useTranslation } from 'react-i18next';
import { AuthContext } from '../../context/auth';
interface ElementInfo {
tagName: string;
@@ -27,6 +28,12 @@ interface AttributeOption {
value: string;
}
interface ScreencastData {
image: string;
userId: string;
}
const getAttributeOptions = (tagName: string, elementInfo: ElementInfo | null): AttributeOption[] => {
if (!elementInfo) return [];
switch (tagName.toLowerCase()) {
@@ -72,6 +79,9 @@ export const BrowserWindow = () => {
const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext();
const { addTextStep, addListStep } = useBrowserSteps();
const { state } = useContext(AuthContext);
const { user } = state;
const onMouseMove = (e: MouseEvent) => {
if (canvasRef && canvasRef.current && highlighterData) {
const canvasRect = canvasRef.current.getBoundingClientRect();
@@ -99,9 +109,15 @@ export const BrowserWindow = () => {
}
}, [getList, resetListState]);
const screencastHandler = useCallback((data: string) => {
setScreenShot(data);
}, [screenShot]);
const screencastHandler = useCallback((data: string | ScreencastData) => {
if (typeof data === 'string') {
setScreenShot(data);
} else if (data && typeof data === 'object' && 'image' in data) {
if (!data.userId || data.userId === user?.id) {
setScreenShot(data.image);
}
}
}, [screenShot, user?.id]);
useEffect(() => {
if (socket) {