From cce663c6fd853f424fd367bb291f6c68b0ad4068 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 5 Mar 2025 21:27:08 +0530 Subject: [PATCH 01/35] feat: pass userId --- server/src/api/record.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/src/api/record.ts b/server/src/api/record.ts index 05560487..b826806d 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -507,9 +507,9 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) { } } -async function readyForRunHandler(browserId: string, id: string) { +async function readyForRunHandler(browserId: string, id: string, userId: string){ try { - const result = await executeRun(id); + const result = await executeRun(id, userId); if (result && result.success) { logger.log('info', `Interpretation of ${id} succeeded`); @@ -546,7 +546,7 @@ function AddGeneratedFlags(workflow: WorkflowFile) { return copy; }; -async function executeRun(id: string) { +async function executeRun(id: string, userId: string) { try { const run = await Run.findOne({ where: { runId: id } }); if (!run) { @@ -568,7 +568,7 @@ async function executeRun(id: string) { plainRun.status = 'running'; - const browser = browserPool.getRemoteBrowser(plainRun.browserId); + const browser = browserPool.getRemoteBrowser(plainRun.browserId, userId); if (!browser) { throw new Error('Could not access browser'); } @@ -672,12 +672,12 @@ export async function handleRunRecording(id: string, userId: string) { rejectUnauthorized: false }); - socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId)); + socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId, userId)); logger.log('info', `Running Robot: ${id}`); socket.on('disconnect', () => { - cleanupSocketListeners(socket, browserId, newRunId); + cleanupSocketListeners(socket, browserId, newRunId, userId); }); // Return the runId immediately, so the client knows the run is started @@ -688,8 +688,8 @@ export async function handleRunRecording(id: string, userId: string) { } } -function cleanupSocketListeners(socket: Socket, browserId: string, id: string) { - socket.off('ready-for-run', () => readyForRunHandler(browserId, id)); +function cleanupSocketListeners(socket: Socket, browserId: string, id: string, userId: string) { + socket.off('ready-for-run', () => readyForRunHandler(browserId, id, userId)); logger.log('info', `Cleaned up listeners for browserId: ${browserId}, runId: ${id}`); } From 1d5ccea0fe7eac713b071defdd998b1436ea1234 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 02:17:50 +0530 Subject: [PATCH 02/35] feat: use Authenticated request to pass userId --- server/src/routes/record.ts | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index 51d3ff92..006fff6b 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -76,8 +76,11 @@ router.post('/start', requireSignIn, (req: AuthenticatedRequest, res:Response) = * GET endpoint for terminating the remote browser recording session. * returns whether the termination was successful */ -router.get('/stop/:browserId', requireSignIn, async (req, res) => { - const success = await destroyRemoteBrowser(req.params.browserId); +router.get('/stop/:browserId', requireSignIn, async (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + const success = await destroyRemoteBrowser(req.user?.id, req.params.browserId); return res.send(success); }); @@ -92,10 +95,13 @@ router.get('/active', requireSignIn, (req, res) => { /** * GET endpoint for getting the current url of the active remote browser. */ -router.get('/active/url', requireSignIn, (req, res) => { +router.get('/active/url', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } const id = getActiveBrowserId(); if (id) { - const url = getRemoteBrowserCurrentUrl(id); + const url = getRemoteBrowserCurrentUrl(id, req.user?.id); return res.send(url); } return res.send(null); @@ -104,10 +110,13 @@ router.get('/active/url', requireSignIn, (req, res) => { /** * GET endpoint for getting the current tabs of the active remote browser. */ -router.get('/active/tabs', requireSignIn, (req, res) => { +router.get('/active/tabs', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } const id = getActiveBrowserId(); if (id) { - const hosts = getRemoteBrowserCurrentTabs(id); + const hosts = getRemoteBrowserCurrentTabs(id, req.user?.id); return res.send(hosts); } return res.send([]); @@ -116,9 +125,12 @@ router.get('/active/tabs', requireSignIn, (req, res) => { /** * GET endpoint for starting an interpretation of the currently generated workflow. */ -router.get('/interpret', requireSignIn, async (req, res) => { +router.get('/interpret', requireSignIn, async (req: AuthenticatedRequest, res) => { try { - await interpretWholeWorkflow(); + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + await interpretWholeWorkflow(req.user?.id); return res.send('interpretation done'); } catch (e) { return res.send('interpretation failed'); @@ -128,7 +140,10 @@ router.get('/interpret', requireSignIn, async (req, res) => { /** * GET endpoint for stopping an ongoing interpretation of the currently generated workflow. */ -router.get('/interpret/stop', requireSignIn, async (req, res) => { - await stopRunningInterpretation(); +router.get('/interpret/stop', requireSignIn, async (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + await stopRunningInterpretation(req.user?.id); return res.send('interpretation stopped'); }); From e79532c9f7e446586a1e6bc16256f782e1a1e667 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 02:35:38 +0530 Subject: [PATCH 03/35] feat: use Authenticated request to pass userId --- server/src/routes/storage.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 5ab9ef7a..e14d5e97 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -617,7 +617,7 @@ router.post('/runs/run/:id', requireSignIn, async (req: AuthenticatedRequest, re workflow, currentPage, (newPage: Page) => currentPage = newPage, plainRun.interpreterSettings); const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(plainRun.browserId); + await destroyRemoteBrowser(req.user?.id, plainRun.browserId); await run.update({ ...run, status: 'success', @@ -900,8 +900,9 @@ router.delete('/schedule/:id', requireSignIn, async (req: AuthenticatedRequest, /** * POST endpoint for aborting a current interpretation of the run. */ -router.post('/runs/abort/:id', requireSignIn, async (req, res) => { +router.post('/runs/abort/:id', requireSignIn, async (req: AuthenticatedRequest, res) => { try { + if (!req.user) { return res.status(401).send({ error: 'Unauthorized' }); } const run = await Run.findOne({ where: { runId: req.params.id } }); if (!run) { return res.status(404).send(false); From 591318728a65fd9edd03417d7af4bd242a8d0c3d Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 02:39:21 +0530 Subject: [PATCH 04/35] feat: use Authenticated request to pass userId --- server/src/api/record.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/api/record.ts b/server/src/api/record.ts index b826806d..b4014c3a 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -517,14 +517,14 @@ async function readyForRunHandler(browserId: string, id: string, userId: string) return result.interpretationInfo; } else { logger.log('error', `Interpretation of ${id} failed`); - await destroyRemoteBrowser(browserId); + await destroyRemoteBrowser(browserId, userId); resetRecordingState(browserId, id); return null; } } catch (error: any) { logger.error(`Error during readyForRunHandler: ${error.message}`); - await destroyRemoteBrowser(browserId); + await destroyRemoteBrowser(browserId, userId); return null; } } @@ -568,7 +568,7 @@ async function executeRun(id: string, userId: string) { plainRun.status = 'running'; - const browser = browserPool.getRemoteBrowser(plainRun.browserId, userId); + const browser = browserPool.getRemoteBrowser(userId); if (!browser) { throw new Error('Could not access browser'); } @@ -586,7 +586,7 @@ async function executeRun(id: string, userId: string) { const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(plainRun.browserId); + await destroyRemoteBrowser(plainRun.browserId, userId); const updatedRun = await run.update({ ...run, From 36e77d1666dd2b7956bfba2664383b850ac005de Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 02:41:11 +0530 Subject: [PATCH 05/35] feat: pass userId to browser pool methods --- .../workflow-management/scheduler/index.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index ade7d969..c5a9d19c 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -92,7 +92,7 @@ function AddGeneratedFlags(workflow: WorkflowFile) { return copy; }; -async function executeRun(id: string) { +async function executeRun(id: string, userId: string) { try { const run = await Run.findOne({ where: { runId: id } }); if (!run) { @@ -114,7 +114,7 @@ async function executeRun(id: string) { plainRun.status = 'running'; - const browser = browserPool.getRemoteBrowser(plainRun.browserId); + const browser = browserPool.getRemoteBrowser(userId); if (!browser) { throw new Error('Could not access browser'); } @@ -132,7 +132,7 @@ async function executeRun(id: string) { const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(plainRun.browserId); + await destroyRemoteBrowser(userId, plainRun.browserId); await run.update({ ...run, @@ -207,22 +207,22 @@ async function executeRun(id: string) { } } -async function readyForRunHandler(browserId: string, id: string) { +async function readyForRunHandler(browserId: string, id: string, userId: string) { try { - const interpretation = await executeRun(id); + const interpretation = await executeRun(id, userId); if (interpretation) { logger.log('info', `Interpretation of ${id} succeeded`); } else { logger.log('error', `Interpretation of ${id} failed`); - await destroyRemoteBrowser(browserId); + await destroyRemoteBrowser(userId, browserId); } resetRecordingState(browserId, id); } catch (error: any) { logger.error(`Error during readyForRunHandler: ${error.message}`); - await destroyRemoteBrowser(browserId); + await destroyRemoteBrowser(userId, browserId); } } @@ -245,12 +245,12 @@ export async function handleRunRecording(id: string, userId: string) { rejectUnauthorized: false }); - socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId)); + socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId, userId)); logger.log('info', `Running robot: ${id}`); socket.on('disconnect', () => { - cleanupSocketListeners(socket, browserId, newRunId); + cleanupSocketListeners(socket, browserId, newRunId, userId); }); } catch (error: any) { @@ -258,8 +258,8 @@ export async function handleRunRecording(id: string, userId: string) { } } -function cleanupSocketListeners(socket: Socket, browserId: string, id: string) { - socket.off('ready-for-run', () => readyForRunHandler(browserId, id)); +function cleanupSocketListeners(socket: Socket, browserId: string, id: string, userId: string) { + socket.off('ready-for-run', () => readyForRunHandler(browserId, id, userId)); logger.log('info', `Cleaned up listeners for browserId: ${browserId}, runId: ${id}`); } From 4059e8bd2671bb843126d3a02648c831ad8b96f7 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 02:50:14 +0530 Subject: [PATCH 06/35] feat: pass userId to browser pool methods --- server/src/browser-management/controller.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index 24a677ce..329ffa1a 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -37,7 +37,7 @@ export const initializeRemoteBrowserForRecording = (userId: string): string => { await browserSession.initialize(userId); await browserSession.registerEditorEvents(); await browserSession.subscribeToScreencast(); - browserPool.addRemoteBrowser(id, browserSession, true); + browserPool.addRemoteBrowser(id, browserSession, userId); } socket.emit('loaded'); }); @@ -59,7 +59,7 @@ export const createRemoteBrowserForRun = (userId: string): string => { async (socket: Socket) => { const browserSession = new RemoteBrowser(socket); await browserSession.initialize(userId); - browserPool.addRemoteBrowser(id, browserSession, true); + browserPool.addRemoteBrowser(id, browserSession, userId); socket.emit('ready-for-run'); }); return id; @@ -72,14 +72,14 @@ export const createRemoteBrowserForRun = (userId: string): string => { * @returns {Promise} * @category BrowserManagement-Controller */ -export const destroyRemoteBrowser = async (id: string): Promise => { +export const destroyRemoteBrowser = async (id: string, userId: string): Promise => { const browserSession = browserPool.getRemoteBrowser(id); if (browserSession) { logger.log('debug', `Switching off the browser with id: ${id}`); await browserSession.stopCurrentInterpretation(); await browserSession.switchOff(); } - return browserPool.deleteRemoteBrowser(id); + return browserPool.deleteRemoteBrowser(userId); }; /** @@ -98,7 +98,7 @@ export const getActiveBrowserId = (): string | null => { * @returns {string | undefined} * @category BrowserManagement-Controller */ -export const getRemoteBrowserCurrentUrl = (id: string): string | undefined => { +export const getRemoteBrowserCurrentUrl = (id: string, userId: string): string | undefined => { return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.url(); }; @@ -108,7 +108,7 @@ export const getRemoteBrowserCurrentUrl = (id: string): string | undefined => { * @return {string[] | undefined} * @category BrowserManagement-Controller */ -export const getRemoteBrowserCurrentTabs = (id: string): string[] | undefined => { +export const getRemoteBrowserCurrentTabs = (id: string, userId: string): string[] | undefined => { return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.context().pages() .map((page) => { const parsedUrl = new URL(page.url()); @@ -126,7 +126,7 @@ export const getRemoteBrowserCurrentTabs = (id: string): string[] | undefined => * @returns {Promise} * @category BrowserManagement-Controller */ -export const interpretWholeWorkflow = async () => { +export const interpretWholeWorkflow = async (userId: string) => { const id = getActiveBrowserId(); if (id) { const browser = browserPool.getRemoteBrowser(id); @@ -146,7 +146,7 @@ export const interpretWholeWorkflow = async () => { * @returns {Promise} * @category BrowserManagement-Controller */ -export const stopRunningInterpretation = async () => { +export const stopRunningInterpretation = async (userId: string) => { const id = getActiveBrowserId(); if (id) { const browser = browserPool.getRemoteBrowser(id); From 57e75a5ce7224f81cdd225aceb327da7e65577e9 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 03:54:44 +0530 Subject: [PATCH 07/35] feat: pass userId to getActiveBrowserId --- server/src/browser-management/controller.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index 329ffa1a..16bbb66d 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -21,12 +21,12 @@ import logger from "../logger"; * @category BrowserManagement-Controller */ export const initializeRemoteBrowserForRecording = (userId: string): string => { - const id = getActiveBrowserId() || uuid(); + const id = getActiveBrowserId(userId) || uuid(); createSocketConnection( io.of(id), async (socket: Socket) => { // browser is already active - const activeId = getActiveBrowserId(); + const activeId = getActiveBrowserId(userId); if (activeId) { const remoteBrowser = browserPool.getRemoteBrowser(activeId); remoteBrowser?.updateSocket(socket); @@ -88,8 +88,8 @@ export const destroyRemoteBrowser = async (id: string, userId: string): Promise< * @returns {string | null} * @category BrowserManagement-Controller */ -export const getActiveBrowserId = (): string | null => { - return browserPool.getActiveBrowserId(); +export const getActiveBrowserId = (userId: string): string | null => { + return browserPool.getActiveBrowserId(userId); }; /** @@ -127,7 +127,7 @@ export const getRemoteBrowserCurrentTabs = (id: string, userId: string): string[ * @category BrowserManagement-Controller */ export const interpretWholeWorkflow = async (userId: string) => { - const id = getActiveBrowserId(); + const id = getActiveBrowserId(userId); if (id) { const browser = browserPool.getRemoteBrowser(id); if (browser) { @@ -147,7 +147,7 @@ export const interpretWholeWorkflow = async (userId: string) => { * @category BrowserManagement-Controller */ export const stopRunningInterpretation = async (userId: string) => { - const id = getActiveBrowserId(); + const id = getActiveBrowserId(userId); if (id) { const browser = browserPool.getRemoteBrowser(id); await browser?.stopCurrentInterpretation(); From da8adcd7ad89e23cbdaae34edbcf5bf61255ac94 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 03:55:48 +0530 Subject: [PATCH 08/35] feat: pass userId to getActiveBrowserId --- server/src/routes/record.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index 006fff6b..5e386468 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -87,8 +87,11 @@ router.get('/stop/:browserId', requireSignIn, async (req: AuthenticatedRequest, /** * GET endpoint for getting the id of the active remote browser. */ -router.get('/active', requireSignIn, (req, res) => { - const id = getActiveBrowserId(); +router.get('/active', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { + return res.status(401).send('User not authenticated'); + } + const id = getActiveBrowserId(req.user?.id); return res.send(id); }); @@ -99,7 +102,7 @@ router.get('/active/url', requireSignIn, (req: AuthenticatedRequest, res) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = getActiveBrowserId(); + const id = getActiveBrowserId(req.user?.id); if (id) { const url = getRemoteBrowserCurrentUrl(id, req.user?.id); return res.send(url); @@ -114,7 +117,7 @@ router.get('/active/tabs', requireSignIn, (req: AuthenticatedRequest, res) => { if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = getActiveBrowserId(); + const id = getActiveBrowserId(req.user?.id); if (id) { const hosts = getRemoteBrowserCurrentTabs(id, req.user?.id); return res.send(hosts); From 0385d29deddae87527b3951fce3b7ebac517a02a Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 03:56:18 +0530 Subject: [PATCH 09/35] feat: pass userId to getActiveBrowserId --- server/src/routes/workflow.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/server/src/routes/workflow.ts b/server/src/routes/workflow.ts index 3c0e356b..3a117ce1 100644 --- a/server/src/routes/workflow.ts +++ b/server/src/routes/workflow.ts @@ -7,6 +7,7 @@ import logger from "../logger"; import { browserPool } from "../server"; import { requireSignIn } from '../middlewares/auth'; import Robot from '../models/Robot'; +import { AuthenticatedRequest } from './record'; export const router = Router(); @@ -46,8 +47,9 @@ router.get('/params/:browserId', requireSignIn, (req, res) => { /** * DELETE endpoint for deleting a pair from the generated workflow. */ -router.delete('/pair/:index', requireSignIn, (req, res) => { - const id = browserPool.getActiveBrowserId(); +router.delete('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { return res.status(401).send('User not authenticated'); } + const id = browserPool.getActiveBrowserId(req.user?.id); if (id) { const browser = browserPool.getRemoteBrowser(id); if (browser) { @@ -62,8 +64,9 @@ router.delete('/pair/:index', requireSignIn, (req, res) => { /** * POST endpoint for adding a pair to the generated workflow. */ -router.post('/pair/:index', requireSignIn, (req, res) => { - const id = browserPool.getActiveBrowserId(); +router.post('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { return res.status(401).send('User not authenticated'); } + const id = browserPool.getActiveBrowserId(req.user?.id); if (id) { const browser = browserPool.getRemoteBrowser(id); logger.log('debug', `Adding pair to workflow`); @@ -82,8 +85,9 @@ router.post('/pair/:index', requireSignIn, (req, res) => { /** * PUT endpoint for updating a pair in the generated workflow. */ -router.put('/pair/:index', requireSignIn, (req, res) => { - const id = browserPool.getActiveBrowserId(); +router.put('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => { + if (!req.user) { return res.status(401).send('User not authenticated'); } + const id = browserPool.getActiveBrowserId(req.user?.id); if (id) { const browser = browserPool.getRemoteBrowser(id); logger.log('debug', `Updating pair in workflow`); From 86be39a609dc19f877da4858d8e671dfec134116 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 04:07:33 +0530 Subject: [PATCH 10/35] feat: pass userId to getActiveBrowserId --- server/src/workflow-management/classes/Generator.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index b4c72e96..8cdc233c 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -151,8 +151,8 @@ export class WorkflowGenerator { workflow: [], }); socket.on('activeIndex', (data) => this.generatedData.lastIndex = parseInt(data)); - socket.on('decision', async ({ pair, actionType, decision }) => { - const id = browserPool.getActiveBrowserId(); + socket.on('decision', async ({ pair, actionType, decision, userId }) => { + const id = browserPool.getActiveBrowserId(userId); if (id) { // const activeBrowser = browserPool.getRemoteBrowser(id); // const currentPage = activeBrowser?.getCurrentPage(); From 0dfbcd3c170c3e5c4acbfd4487a6d37afcca399b Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 04:08:11 +0530 Subject: [PATCH 11/35] feat: pass req to handleWrapper --- server/src/browser-management/inputHandlers.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index bf365053..298a95ce 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -12,6 +12,7 @@ import { WorkflowGenerator } from "../workflow-management/classes/Generator"; import { Page } from "playwright"; import { throttle } from "../../../src/helpers/inputHelpers"; import { CustomActions } from "../../../src/shared/types"; +import { AuthenticatedRequest } from '../routes/record'; /** * A wrapper function for handling user input. @@ -31,9 +32,21 @@ const handleWrapper = async ( page: Page, args?: any ) => Promise, - args?: any + args?: any, + req?: AuthenticatedRequest, ) => { - const id = browserPool.getActiveBrowserId(); + if (!req || !req.user || typeof req.user === 'string') { + logger.log('warn', `User not authenticated or invalid JWT payload`); + return; + } + + const userId = req.user.id; + if (!userId) { + logger.log('warn', `User ID is missing in JWT payload`); + return; + } + + const id = browserPool.getActiveBrowserId(userId); if (id) { const activeBrowser = browserPool.getRemoteBrowser(id); if (activeBrowser?.interpreter.interpretationInProgress() && !activeBrowser.interpreter.interpretationIsPaused) { From 9798631c1696a244a2164af58f0ced266e3a059b Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 04:41:35 +0530 Subject: [PATCH 12/35] feat: accept user id that owns the browser instance --- server/src/browser-management/classes/RemoteBrowser.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 8ff4f601..8bf8380c 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -90,6 +90,12 @@ export class RemoteBrowser { maxRepeats: 1, }; + /** + * The user ID that owns this browser instance + * @private + */ + private userId: string; + private lastEmittedUrl: string | null = null; /** From c8a10c5230bb96d295bb7e46ef4db28fcc40b2f6 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 05:06:58 +0530 Subject: [PATCH 13/35] feat: is screencast active --- server/src/browser-management/classes/RemoteBrowser.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 8bf8380c..2d161d14 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -112,6 +112,7 @@ export class RemoteBrowser { private screenshotQueue: Buffer[] = []; private isProcessingScreenshot = false; private screencastInterval: NodeJS.Timeout | null = null + private isScreencastActive: boolean = false; /** * Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and From 61914a562a5c1c1682ac16d46dd8d660bb9a6e25 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 05:11:06 +0530 Subject: [PATCH 14/35] feat: instantiate userId --- server/src/browser-management/classes/RemoteBrowser.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 2d161d14..8ae755bd 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -120,8 +120,9 @@ export class RemoteBrowser { * @param socket socket.io socket instance used to communicate with the client side * @constructor */ - public constructor(socket: Socket) { + public constructor(socket: Socket, userId: string) { this.socket = socket; + this.userId = userId; this.interpreter = new WorkflowInterpreter(socket); this.generator = new WorkflowGenerator(socket); } From 242529c3fdedf15ab54b67e7c59d26721f7962fb Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 05:16:44 +0530 Subject: [PATCH 15/35] feat: include userId in event data to ensure client can match it to the correct browser --- server/src/browser-management/classes/RemoteBrowser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 8ae755bd..855c5596 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -201,7 +201,7 @@ export class RemoteBrowser { const currentUrl = page.url(); if (this.shouldEmitUrlChange(currentUrl)) { this.lastEmittedUrl = currentUrl; - this.socket.emit('urlChanged', currentUrl); + this.socket.emit('urlChanged', {url: currentUrl, userId: this.userId}); } } }); From aa51b23f999c21c0425e5fcfa0b56ce0108cbc37 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 6 Mar 2025 16:44:19 +0530 Subject: [PATCH 16/35] feat: pass userId --- server/src/browser-management/controller.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index 16bbb66d..92fb6499 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -32,7 +32,7 @@ export const initializeRemoteBrowserForRecording = (userId: string): string => { remoteBrowser?.updateSocket(socket); await remoteBrowser?.makeAndEmitScreenshot(); } else { - const browserSession = new RemoteBrowser(socket); + const browserSession = new RemoteBrowser(socket, userId); browserSession.interpreter.subscribeToPausing(); await browserSession.initialize(userId); await browserSession.registerEditorEvents(); @@ -57,7 +57,7 @@ export const createRemoteBrowserForRun = (userId: string): string => { createSocketConnectionForRun( io.of(id), async (socket: Socket) => { - const browserSession = new RemoteBrowser(socket); + const browserSession = new RemoteBrowser(socket, userId); await browserSession.initialize(userId); browserPool.addRemoteBrowser(id, browserSession, userId); socket.emit('ready-for-run'); From ea43d9d604fdc62cb15dfd12a926d98c230eeb46 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:40:07 +0530 Subject: [PATCH 17/35] feat: browser pool info interface --- .../browser-management/classes/BrowserPool.ts | 85 +------------------ 1 file changed, 4 insertions(+), 81 deletions(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index cd4962a1..21fc4b00 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -15,86 +15,9 @@ interface BrowserPoolInfo { * @default false */ active: boolean, + /** + * The user ID that owns this browser instance. + */ + userId: string, } -/** - * Dictionary of all the active remote browser's instances indexed by their id. - * The value in this dictionary is of type BrowserPoolInfo, - * which provides additional information about the browser's usage. - * @category Types - */ -interface PoolDictionary { - [key: string]: BrowserPoolInfo, -} - -/** - * A browser pool is a collection of remote browsers that are initialized and ready to be used. - * Adds the possibility to add, remove and retrieve remote browsers from the pool. - * It is possible to manage multiple browsers for creating or running a recording. - * @category BrowserManagement - */ -export class BrowserPool { - - /** - * Holds all the instances of remote browsers. - */ - private pool: PoolDictionary = {}; - - /** - * Adds a remote browser instance to the pool indexed by the id. - * @param id remote browser instance's id - * @param browser remote browser instance - * @param active states if the browser's instance is being actively used - */ - public addRemoteBrowser = (id: string, browser: RemoteBrowser, active: boolean = false): void => { - this.pool = { - ...this.pool, - [id]: { - browser, - active, - }, - } - logger.log('debug', `Remote browser with id: ${id} added to the pool`); - }; - - /** - * Removes the remote browser instance from the pool. - * @param id remote browser instance's id - * @returns true if the browser was removed successfully, false otherwise - */ - public deleteRemoteBrowser = (id: string): boolean => { - if (!this.pool[id]) { - logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); - return false; - } - delete (this.pool[id]); - logger.log('debug', `Remote browser with id: ${id} deleted from the pool`); - return true; - }; - - /** - * Returns the remote browser instance from the pool. - * @param id remote browser instance's id - * @returns remote browser instance or undefined if it does not exist in the pool - */ - public getRemoteBrowser = (id: string): RemoteBrowser | undefined => { - logger.log('debug', `Remote browser with id: ${id} retrieved from the pool`); - return this.pool[id]?.browser; - }; - - /** - * Returns the active browser's instance id from the pool. - * If there is no active browser, it returns undefined. - * If there are multiple active browsers, it returns the first one. - * @returns the first remote active browser instance's id from the pool - */ - public getActiveBrowserId = (): string | null => { - for (const id of Object.keys(this.pool)) { - if (this.pool[id].active) { - return id; - } - } - logger.log('warn', `No active browser in the pool`); - return null; - }; -} From ffc030d58b89b281959e11eae7d78b4c98fbf48f Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:40:34 +0530 Subject: [PATCH 18/35] feat: pool dictionary --- server/src/browser-management/classes/BrowserPool.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 21fc4b00..5244fcea 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -21,3 +21,13 @@ interface BrowserPoolInfo { userId: string, } +/** + * Dictionary of all the active remote browser's instances indexed by their id. + * The value in this dictionary is of type BrowserPoolInfo, + * which provides additional information about the browser's usage. + * @category Types + */ +interface PoolDictionary { + [key: string]: BrowserPoolInfo, +} + From 4fad9634e0a939f8af1ba0331641498d426a685b Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:41:03 +0530 Subject: [PATCH 19/35] feat: init new browser pool --- .../browser-management/classes/BrowserPool.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 5244fcea..bac06d2f 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -31,3 +31,32 @@ interface PoolDictionary { [key: string]: BrowserPoolInfo, } +/** + * A browser pool is a collection of remote browsers that are initialized and ready to be used. + * Enforces a "1 User - 1 Browser" policy, while allowing multiple users to have their own browser instances. + * Adds the possibility to add, remove and retrieve remote browsers from the pool. + * @category BrowserManagement + */ +export class BrowserPool { + /** + * Holds all the instances of remote browsers. + */ + private pool: PoolDictionary = {}; + + /** + * Maps user IDs to their browser IDs. + */ + private userToBrowserMap: Map = new Map(); + + /** + * Adds a remote browser instance to the pool for a specific user. + * If the user already has a browser, the existing browser will be closed and replaced. + * + * @param id remote browser instance's id + * @param browser remote browser instance + * @param userId the user ID that owns this browser instance + * @param active states if the browser's instance is being actively used + * @returns true if a new browser was added, false if an existing browser was replaced + */ + +} \ No newline at end of file From 677f05d0da295e77d07863a949a9fea6e3ad5fdc Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:41:25 +0530 Subject: [PATCH 20/35] feat: add new browser --- .../browser-management/classes/BrowserPool.ts | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index bac06d2f..16fa1908 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -58,5 +58,46 @@ export class BrowserPool { * @param active states if the browser's instance is being actively used * @returns true if a new browser was added, false if an existing browser was replaced */ - + public addRemoteBrowser = ( + id: string, + browser: RemoteBrowser, + userId: string, + active: boolean = false + ): boolean => { + // Check if user already has a browser + const existingBrowserId = this.userToBrowserMap.get(userId); + let replaced = false; + + if (existingBrowserId) { + // Close and remove the existing browser + if (existingBrowserId !== id) { + this.closeAndDeleteBrowser(existingBrowserId); + replaced = true; + } else { + // If it's the same browser ID, just update the info + this.pool[id] = { + browser, + active, + userId, + }; + logger.log('debug', `Updated existing browser with id: ${id} for user: ${userId}`); + return false; + } + } + + // Add the new browser to the pool + this.pool[id] = { + browser, + active, + userId, + }; + + // Update the user-to-browser mapping + this.userToBrowserMap.set(userId, id); + + logger.log('debug', `Remote browser with id: ${id} added to the pool for user: ${userId}`); + return !replaced; + }; + + } \ No newline at end of file From cce901962dd807f59ceb98fe0d4437c7ea49470b Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:41:47 +0530 Subject: [PATCH 21/35] feat: close & del browser --- .../browser-management/classes/BrowserPool.ts | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 16fa1908..19719f3e 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -99,5 +99,31 @@ export class BrowserPool { return !replaced; }; - + /** + * Removes the remote browser instance from the pool. + * Note: This doesn't handle browser closing as RemoteBrowser doesn't expose a close method. + * The caller should ensure the browser is properly closed before calling this method. + * + * @param id remote browser instance's id + * @returns true if the browser was removed successfully, false otherwise + */ + public closeAndDeleteBrowser = (id: string): boolean => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return false; + } + + // Remove the user-to-browser mapping + const userId = this.pool[id].userId; + if (this.userToBrowserMap.get(userId) === id) { + this.userToBrowserMap.delete(userId); + } + + // Remove from pool + delete this.pool[id]; + logger.log('debug', `Remote browser with id: ${id} removed from the pool`); + return true; + }; + + } \ No newline at end of file From 2fa75ab8f39efea31fdf7e2ce3e6ed4de27f1563 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:42:07 +0530 Subject: [PATCH 22/35] feat: del browser --- .../browser-management/classes/BrowserPool.ts | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 19719f3e..6f2e2adf 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -125,5 +125,29 @@ export class BrowserPool { return true; }; + /** + * Removes the remote browser instance from the pool without attempting to close it. + * + * @param id remote browser instance's id + * @returns true if the browser was removed successfully, false otherwise + */ + public deleteRemoteBrowser = (id: string): boolean => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return false; + } + + // Remove the user-to-browser mapping + const userId = this.pool[id].userId; + if (this.userToBrowserMap.get(userId) === id) { + this.userToBrowserMap.delete(userId); + } + + // Remove from pool + delete this.pool[id]; + logger.log('debug', `Remote browser with id: ${id} deleted from the pool`); + return true; + }; + } \ No newline at end of file From 63e48cead7f42e899e80c43b10c05d9cdc6e5d40 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:42:23 +0530 Subject: [PATCH 23/35] feat: get browser --- server/src/browser-management/classes/BrowserPool.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 6f2e2adf..d260f16d 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -149,5 +149,16 @@ export class BrowserPool { return true; }; + /** + * Returns the remote browser instance from the pool. + * + * @param id remote browser instance's id + * @returns remote browser instance or undefined if it does not exist in the pool + */ + public getRemoteBrowser = (id: string): RemoteBrowser | undefined => { + logger.log('debug', `Remote browser with id: ${id} retrieved from the pool`); + return this.pool[id]?.browser; + }; + } \ No newline at end of file From e9d3fcebad909aa3aa94ff479537f992f467e853 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:42:49 +0530 Subject: [PATCH 24/35] feat: get active browser id --- .../browser-management/classes/BrowserPool.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index d260f16d..e09f96ef 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -160,5 +160,28 @@ export class BrowserPool { return this.pool[id]?.browser; }; + /** + * Returns the active browser's instance id for a specific user. + * + * @param userId the user ID to find the browser for + * @returns the browser ID for the user, or null if no browser exists + */ + public getActiveBrowserId = (userId: string): string | null => { + const browserId = this.userToBrowserMap.get(userId); + if (!browserId) { + logger.log('debug', `No browser found for user: ${userId}`); + return null; + } + + // Verify the browser still exists in the pool + if (!this.pool[browserId]) { + this.userToBrowserMap.delete(userId); + logger.log('warn', `Browser mapping found for user: ${userId}, but browser doesn't exist in pool`); + return null; + } + console.log(`Browser Id ${browserId} found for user: ${userId}`); + return browserId; + }; + } \ No newline at end of file From 59e494c9be103fef30203258931d41d578493cb5 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:43:06 +0530 Subject: [PATCH 25/35] feat: get user for browser --- .../src/browser-management/classes/BrowserPool.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index e09f96ef..a52c85d6 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -183,5 +183,18 @@ export class BrowserPool { return browserId; }; + /** + * Returns the user ID associated with a browser ID. + * + * @param browserId the browser ID to find the user for + * @returns the user ID for the browser, or null if the browser doesn't exist + */ + public getUserForBrowser = (browserId: string): string | null => { + if (!this.pool[browserId]) { + return null; + } + return this.pool[browserId].userId; + }; + } \ No newline at end of file From 9a1fa3fbce94ff18812b2eb0fccf044bb9d62b69 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:43:45 +0530 Subject: [PATCH 26/35] feat: set active browser --- .../browser-management/classes/BrowserPool.ts | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index a52c85d6..b3002100 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -196,5 +196,23 @@ export class BrowserPool { return this.pool[browserId].userId; }; - + /** + * Sets the active state of a browser. + * + * @param id the browser ID + * @param active the new active state + * @returns true if successful, false if the browser wasn't found + */ + public setActiveBrowser = (id: string, active: boolean): boolean => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return false; + } + + this.pool[id].active = active; + logger.log('debug', `Remote browser with id: ${id} set to ${active ? 'active' : 'inactive'}`); + return true; + }; + + } \ No newline at end of file From c7fd8cb4a9e926545b870e6b0ddc316c2fe23001 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:44:05 +0530 Subject: [PATCH 27/35] feat: get all active browser ids for user --- .../browser-management/classes/BrowserPool.ts | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index b3002100..4d3e54c0 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -214,5 +214,36 @@ export class BrowserPool { return true; }; + /** + * Returns all browser instances for a specific user. + * Should only be one per the "1 User - 1 Browser" policy, but included for flexibility. + * + * @param userId the user ID to find browsers for + * @returns an array of browser IDs belonging to the user + */ + public getAllBrowserIdsForUser = (userId: string): string[] => { + const browserIds: string[] = []; + + // Normally this would just return the one browser from the map + const mappedBrowserId = this.userToBrowserMap.get(userId); + if (mappedBrowserId && this.pool[mappedBrowserId]) { + browserIds.push(mappedBrowserId); + } + + // But as a safeguard, also check the entire pool for any browsers assigned to this user + // This helps detect and fix any inconsistencies in the maps + for (const [id, info] of Object.entries(this.pool)) { + if (info.userId === userId && !browserIds.includes(id)) { + browserIds.push(id); + // Fix the map if it's inconsistent + if (!mappedBrowserId) { + this.userToBrowserMap.set(userId, id); + } + } + } + + return browserIds; + }; + } \ No newline at end of file From b16e7b3834a5dfff2d69e21d86fc362402d49cad Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:44:34 +0530 Subject: [PATCH 28/35] feat: get pool size --- server/src/browser-management/classes/BrowserPool.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 4d3e54c0..68992002 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -245,5 +245,12 @@ export class BrowserPool { return browserIds; }; + /** + * Returns the total number of browsers in the pool. + */ + public getPoolSize = (): number => { + return Object.keys(this.pool).length; + }; + } \ No newline at end of file From 04629c05c447f9b7cdf9bd5bdb5e1eb23a304cfc Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:44:50 +0530 Subject: [PATCH 29/35] feat: get active user count --- server/src/browser-management/classes/BrowserPool.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 68992002..b4e35582 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -252,5 +252,12 @@ export class BrowserPool { return Object.keys(this.pool).length; }; - + /** + * Returns the total number of active users (users with browsers). + */ + public getActiveUserCount = (): number => { + return this.userToBrowserMap.size; + }; + + } \ No newline at end of file From 8e12516985417cd357913ccc7b79f02e8e9ea4c6 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:45:20 +0530 Subject: [PATCH 30/35] feat: getActiveBrowserForMigration <-> backward compatibility --- .../browser-management/classes/BrowserPool.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index b4e35582..3c86a220 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -259,5 +259,39 @@ export class BrowserPool { return this.userToBrowserMap.size; }; + /** + * Gets the current active browser for the system if there's only one active user. + * This is a migration helper to support code that hasn't been updated to the user-browser model yet. + * + * @param currentUserId The ID of the current user, which will be prioritized if multiple browsers exist + * @returns A browser ID if one can be determined, or null + */ + public getActiveBrowserForMigration = (currentUserId?: string): string | null => { + // If a current user ID is provided and they have a browser, return that + if (currentUserId) { + const browserForUser = this.getActiveBrowserId(currentUserId); + if (browserForUser) { + return browserForUser; + } + } + + // If only one user has a browser, return that + if (this.userToBrowserMap.size === 1) { + const userId = Array.from(this.userToBrowserMap.keys())[0]; + return this.userToBrowserMap.get(userId) || null; + } + + // Fall back to the first active browser if any + for (const id of Object.keys(this.pool)) { + if (this.pool[id].active) { + return id; + } + } + + // If all else fails, return the first browser in the pool + const browserIds = Object.keys(this.pool); + return browserIds.length > 0 ? browserIds[0] : null; + }; + } \ No newline at end of file From 6761968f2d4f8a330d6c183aa8f717ce1798007e Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:46:00 +0530 Subject: [PATCH 31/35] feat: get active browser id <-> backward compatibility --- .../browser-management/classes/BrowserPool.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 3c86a220..77c3a637 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -293,5 +293,22 @@ export class BrowserPool { return browserIds.length > 0 ? browserIds[0] : null; }; - + /** + * Returns the first active browser's instance id from the pool. + * If there is no active browser, it returns null. + * If there are multiple active browsers, it returns the first one. + * + * @returns the first remote active browser instance's id from the pool + * @deprecated Use getBrowserIdForUser instead to enforce the 1 User - 1 Browser policy + */ + public getActiveBrowserIdDep = (): string | null => { + for (const id of Object.keys(this.pool)) { + if (this.pool[id].active) { + return id; + } + } + // Don't log a warning since this behavior is expected in the user-browser model + // logger.log('warn', `No active browser in the pool`); + return null; + }; } \ No newline at end of file From 8f95ad82b9b651dbb24c9cc574b7d14d9baed25a Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:46:28 +0530 Subject: [PATCH 32/35] wip: modify socket events based on user --- .../classes/RemoteBrowser.ts | 108 +++++++++++++++--- 1 file changed, 91 insertions(+), 17 deletions(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 855c5596..85d1928c 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -341,15 +341,40 @@ export class RemoteBrowser { * @returns void */ public registerEditorEvents = (): void => { - this.socket.on('rerender', async () => await this.makeAndEmitScreenshot()); - this.socket.on('settings', (settings) => this.interpreterSettings = settings); - this.socket.on('changeTab', async (tabIndex) => await this.changeTab(tabIndex)); - this.socket.on('addTab', async () => { + // For each event, include userId to make sure events are handled for the correct browser + logger.log('debug', `Registering editor events for user: ${this.userId}`); + + // Listen for specific events for this user + this.socket.on(`rerender:${this.userId}`, async () => { + logger.debug(`Rerender event received for user ${this.userId}`); + await this.makeAndEmitScreenshot(); + }); + + // For backward compatibility, also listen to the general event + this.socket.on('rerender', async () => { + logger.debug(`General rerender event received, checking if for user ${this.userId}`); + await this.makeAndEmitScreenshot(); + }); + + this.socket.on(`settings:${this.userId}`, (settings) => { + this.interpreterSettings = settings; + logger.debug(`Settings updated for user ${this.userId}`); + }); + + this.socket.on(`changeTab:${this.userId}`, async (tabIndex) => { + logger.debug(`Tab change to ${tabIndex} requested for user ${this.userId}`); + await this.changeTab(tabIndex); + }); + + this.socket.on(`addTab:${this.userId}`, async () => { + logger.debug(`New tab requested for user ${this.userId}`); await this.currentPage?.context().newPage(); const lastTabIndex = this.currentPage ? this.currentPage.context().pages().length - 1 : 0; await this.changeTab(lastTabIndex); }); - this.socket.on('closeTab', async (tabInfo) => { + + this.socket.on(`closeTab:${this.userId}`, async (tabInfo) => { + logger.debug(`Close tab ${tabInfo.index} requested for user ${this.userId}`); const page = this.currentPage?.context().pages()[tabInfo.index]; if (page) { if (tabInfo.isCurrent) { @@ -364,24 +389,52 @@ export class RemoteBrowser { await page.close(); logger.log( 'debug', - `${tabInfo.index} page was closed, new length of pages: ${this.currentPage?.context().pages().length}` - ) + `Tab ${tabInfo.index} was closed for user ${this.userId}, new tab count: ${this.currentPage?.context().pages().length}` + ); } else { - logger.log('error', `${tabInfo.index} index out of range of pages`) + logger.log('error', `Tab index ${tabInfo.index} out of range for user ${this.userId}`); } }); - this.socket.on('setViewportSize', async (data: { width: number, height: number }) => { + + this.socket.on(`setViewportSize:${this.userId}`, async (data: { width: number, height: number }) => { const { width, height } = data; - logger.log('debug', `Received viewport size: width=${width}, height=${height}`); + logger.log('debug', `Viewport size change to width=${width}, height=${height} requested for user ${this.userId}`); // Update the browser context's viewport dynamically if (this.context && this.browser) { this.context = await this.browser.newContext({ viewport: { width, height } }); - logger.log('debug', `Viewport size updated to width=${width}, height=${height} for the entire browser context`); + logger.log('debug', `Viewport size updated to width=${width}, height=${height} for user ${this.userId}`); } }); - } - + + // For backward compatibility, also register the standard events + this.socket.on('settings', (settings) => this.interpreterSettings = settings); + this.socket.on('changeTab', async (tabIndex) => await this.changeTab(tabIndex)); + this.socket.on('addTab', async () => { + await this.currentPage?.context().newPage(); + const lastTabIndex = this.currentPage ? this.currentPage.context().pages().length - 1 : 0; + await this.changeTab(lastTabIndex); + }); + this.socket.on('closeTab', async (tabInfo) => { + const page = this.currentPage?.context().pages()[tabInfo.index]; + if (page) { + if (tabInfo.isCurrent) { + if (this.currentPage?.context().pages()[tabInfo.index + 1]) { + await this.changeTab(tabInfo.index + 1); + } else { + await this.changeTab(tabInfo.index - 1); + } + } + await page.close(); + } + }); + this.socket.on('setViewportSize', async (data: { width: number, height: number }) => { + const { width, height } = data; + if (this.context && this.browser) { + this.context = await this.browser.newContext({ viewport: { width, height } }); + } + }); + }; /** * Subscribes the remote browser for a screencast session * on [CDP](https://chromedevtools.github.io/devtools-protocol/) level, @@ -390,16 +443,24 @@ export class RemoteBrowser { * @returns {Promise} */ public subscribeToScreencast = async (): Promise => { + logger.log('debug', `Starting screencast for user: ${this.userId}`); await this.startScreencast(); if (!this.client) { logger.log('warn', 'client is not initialized'); return; } + // Set flag to indicate screencast is active + this.isScreencastActive = true; + this.client.on('Page.screencastFrame', ({ data: base64, sessionId }) => { + // Only process if screencast is still active for this user + if (!this.isScreencastActive) { + return; + } this.emitScreenshot(Buffer.from(base64, 'base64')) setTimeout(async () => { try { - if (!this.client) { + if (!this.client || !this.isScreencastActive) { logger.log('warn', 'client is not initialized'); return; } @@ -418,6 +479,8 @@ export class RemoteBrowser { */ public async switchOff(): Promise { try { + this.isScreencastActive = false; + await this.interpreter.stopInterpretation(); if (this.screencastInterval) { @@ -561,7 +624,11 @@ export class RemoteBrowser { //await this.currentPage.setViewportSize({ height: 400, width: 900 }) this.client = await this.currentPage.context().newCDPSession(this.currentPage); - this.socket.emit('urlChanged', this.currentPage.url()); + // Include userId in the URL change event + this.socket.emit('urlChanged', { + url: this.currentPage.url(), + userId: this.userId + }); await this.makeAndEmitScreenshot(); await this.subscribeToScreencast(); } else { @@ -610,6 +677,8 @@ export class RemoteBrowser { await this.client.send('Page.startScreencast', { format: SCREENCAST_CONFIG.format, }); + // Set flag to indicate screencast is active + this.isScreencastActive = true; // Set up screencast frame handler this.client.on('Page.screencastFrame', async ({ data, sessionId }) => { @@ -635,6 +704,8 @@ export class RemoteBrowser { } try { + // Set flag to indicate screencast is active + this.isScreencastActive = false; await this.client.send('Page.stopScreencast'); this.screenshotQueue = []; this.isProcessingScreenshot = false; @@ -665,8 +736,11 @@ export class RemoteBrowser { const base64Data = optimizedScreenshot.toString('base64'); const dataWithMimeType = `data:image/jpeg;base64,${base64Data}`; - this.socket.emit('screencast', dataWithMimeType); - logger.debug('Screenshot emitted'); +// Emit with user context to ensure the frontend can identify which browser's screenshot this is +this.socket.emit('screencast', { + image: dataWithMimeType, + userId: this.userId +}); logger.debug('Screenshot emitted'); } catch (error) { logger.error('Screenshot emission failed:', error); } finally { From a0e7404d25c8675ba29a8b1a438856baa6ba749f Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 7 Mar 2025 22:46:47 +0530 Subject: [PATCH 33/35] feat: get active browser id <-> backward compatibility --- server/src/browser-management/classes/BrowserPool.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 77c3a637..4bbc97af 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -301,7 +301,7 @@ export class BrowserPool { * @returns the first remote active browser instance's id from the pool * @deprecated Use getBrowserIdForUser instead to enforce the 1 User - 1 Browser policy */ - public getActiveBrowserIdDep = (): string | null => { + public getActiveBrowserIdLegacy = (): string | null => { for (const id of Object.keys(this.pool)) { if (this.pool[id].active) { return id; From 6de4cfcafea7651a2cb17bcc43b51a4478d70f53 Mon Sep 17 00:00:00 2001 From: Rohit Date: Sat, 8 Mar 2025 17:09:33 +0530 Subject: [PATCH 34/35] feat: server socket management for multiple browsers --- server/src/browser-management/controller.ts | 2 +- .../src/browser-management/inputHandlers.ts | 182 ++++++++++++------ server/src/routes/record.ts | 2 +- server/src/routes/storage.ts | 2 +- server/src/socket-connection/connection.ts | 62 +++++- .../workflow-management/scheduler/index.ts | 6 +- 6 files changed, 186 insertions(+), 70 deletions(-) diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index 92fb6499..e6619d77 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -79,7 +79,7 @@ export const destroyRemoteBrowser = async (id: string, userId: string): Promise< await browserSession.stopCurrentInterpretation(); await browserSession.switchOff(); } - return browserPool.deleteRemoteBrowser(userId); + return browserPool.deleteRemoteBrowser(id); }; /** diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 298a95ce..92c75e1d 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -4,6 +4,8 @@ * These functions are called by the client through socket communication. */ import { Socket } from 'socket.io'; +import { IncomingMessage } from 'http'; +import { JwtPayload } from 'jsonwebtoken'; import logger from "../logger"; import { Coordinates, ScrollDeltas, KeyboardInput, DatePickerEventData } from '../types'; @@ -12,7 +14,14 @@ import { WorkflowGenerator } from "../workflow-management/classes/Generator"; import { Page } from "playwright"; import { throttle } from "../../../src/helpers/inputHelpers"; import { CustomActions } from "../../../src/shared/types"; -import { AuthenticatedRequest } from '../routes/record'; + +interface AuthenticatedIncomingMessage extends IncomingMessage { + user?: JwtPayload | string; +} + +interface AuthenticatedSocket extends Socket { + request: AuthenticatedIncomingMessage; +} /** * A wrapper function for handling user input. @@ -24,6 +33,7 @@ import { AuthenticatedRequest } from '../routes/record'; * * @param handleCallback The callback handler to be called * @param args - arguments to be passed to the handler + * @param socket - socket with authenticated request * @category HelperFunctions */ const handleWrapper = async ( @@ -33,14 +43,14 @@ const handleWrapper = async ( args?: any ) => Promise, args?: any, - req?: AuthenticatedRequest, + socket?: AuthenticatedSocket, ) => { - if (!req || !req.user || typeof req.user === 'string') { + if (!socket || !socket.request || !socket.request.user || typeof socket.request.user === 'string') { logger.log('warn', `User not authenticated or invalid JWT payload`); return; } - const userId = req.user.id; + const userId = socket.request.user.id; if (!userId) { logger.log('warn', `User ID is missing in JWT payload`); return; @@ -79,12 +89,13 @@ interface CustomActionEventData { /** * A wrapper function for handling custom actions. + * @param socket The socket connection * @param customActionEventData The custom action event data * @category HelperFunctions */ -const onGenerateAction = async (customActionEventData: CustomActionEventData) => { +const onGenerateAction = async (socket: AuthenticatedSocket, customActionEventData: CustomActionEventData) => { logger.log('debug', `Generating ${customActionEventData.action} action emitted from client`); - await handleWrapper(handleGenerateAction, customActionEventData); + await handleWrapper(handleGenerateAction, customActionEventData, socket); } /** @@ -102,12 +113,13 @@ const handleGenerateAction = /** * A wrapper function for handling mousedown event. + * @param socket The socket connection * @param coordinates - coordinates of the mouse click * @category HelperFunctions */ -const onMousedown = async (coordinates: Coordinates) => { +const onMousedown = async (socket: AuthenticatedSocket, coordinates: Coordinates) => { logger.log('debug', 'Handling mousedown event emitted from client'); - await handleWrapper(handleMousedown, coordinates); + await handleWrapper(handleMousedown, coordinates, socket); } /** @@ -152,12 +164,13 @@ const handleMousedown = async (generator: WorkflowGenerator, page: Page, { x, y /** * A wrapper function for handling the wheel event. + * @param socket The socket connection * @param scrollDeltas - the scroll deltas of the wheel event * @category HelperFunctions */ -const onWheel = async (scrollDeltas: ScrollDeltas) => { +const onWheel = async (socket: AuthenticatedSocket, scrollDeltas: ScrollDeltas) => { logger.log('debug', 'Handling scroll event emitted from client'); - await handleWrapper(handleWheel, scrollDeltas); + await handleWrapper(handleWheel, scrollDeltas, socket); }; /** @@ -178,12 +191,13 @@ const handleWheel = async (generator: WorkflowGenerator, page: Page, { deltaX, d /** * A wrapper function for handling the mousemove event. + * @param socket The socket connection * @param coordinates - the coordinates of the mousemove event * @category HelperFunctions */ -const onMousemove = async (coordinates: Coordinates) => { +const onMousemove = async (socket: AuthenticatedSocket, coordinates: Coordinates) => { logger.log('debug', 'Handling mousemove event emitted from client'); - await handleWrapper(handleMousemove, coordinates); + await handleWrapper(handleMousemove, coordinates, socket); } /** @@ -212,12 +226,13 @@ const handleMousemove = async (generator: WorkflowGenerator, page: Page, { x, y /** * A wrapper function for handling the keydown event. + * @param socket The socket connection * @param keyboardInput - the keyboard input of the keydown event * @category HelperFunctions */ -const onKeydown = async (keyboardInput: KeyboardInput) => { +const onKeydown = async (socket: AuthenticatedSocket, keyboardInput: KeyboardInput) => { logger.log('debug', 'Handling keydown event emitted from client'); - await handleWrapper(handleKeydown, keyboardInput); + await handleWrapper(handleKeydown, keyboardInput, socket); } /** @@ -248,49 +263,95 @@ const handleDateSelection = async (generator: WorkflowGenerator, page: Page, dat logger.log('debug', `Date ${data.value} selected`); } -const onDateSelection = async (data: DatePickerEventData) => { +/** + * A wrapper function for handling the date selection event. + * @param socket The socket connection + * @param data - the data of the date selection event + * @category HelperFunctions + */ +const onDateSelection = async (socket: AuthenticatedSocket, data: DatePickerEventData) => { logger.log('debug', 'Handling date selection event emitted from client'); - await handleWrapper(handleDateSelection, data); + await handleWrapper(handleDateSelection, data, socket); } +/** + * Handles the dropdown selection event. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param data - the data of the dropdown selection event + * @category BrowserManagement + */ const handleDropdownSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { await generator.onDropdownSelection(page, data); logger.log('debug', `Dropdown value ${data.value} selected`); } -const onDropdownSelection = async (data: { selector: string, value: string }) => { +/** + * A wrapper function for handling the dropdown selection event. + * @param socket The socket connection + * @param data - the data of the dropdown selection event + * @category HelperFunctions + */ +const onDropdownSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => { logger.log('debug', 'Handling dropdown selection event emitted from client'); - await handleWrapper(handleDropdownSelection, data); + await handleWrapper(handleDropdownSelection, data, socket); } +/** + * Handles the time selection event. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param data - the data of the time selection event + * @category BrowserManagement + */ const handleTimeSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { await generator.onTimeSelection(page, data); logger.log('debug', `Time value ${data.value} selected`); } -const onTimeSelection = async (data: { selector: string, value: string }) => { +/** + * A wrapper function for handling the time selection event. + * @param socket The socket connection + * @param data - the data of the time selection event + * @category HelperFunctions + */ +const onTimeSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => { logger.log('debug', 'Handling time selection event emitted from client'); - await handleWrapper(handleTimeSelection, data); + await handleWrapper(handleTimeSelection, data, socket); } +/** + * Handles the datetime-local selection event. + * @param generator - the workflow generator {@link Generator} + * @param page - the active page of the remote browser + * @param data - the data of the datetime-local selection event + * @category BrowserManagement + */ const handleDateTimeLocalSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => { await generator.onDateTimeLocalSelection(page, data); logger.log('debug', `DateTime Local value ${data.value} selected`); } -const onDateTimeLocalSelection = async (data: { selector: string, value: string }) => { +/** + * A wrapper function for handling the datetime-local selection event. + * @param socket The socket connection + * @param data - the data of the datetime-local selection event + * @category HelperFunctions + */ +const onDateTimeLocalSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => { logger.log('debug', 'Handling datetime-local selection event emitted from client'); - await handleWrapper(handleDateTimeLocalSelection, data); + await handleWrapper(handleDateTimeLocalSelection, data, socket); } /** * A wrapper function for handling the keyup event. + * @param socket The socket connection * @param keyboardInput - the keyboard input of the keyup event * @category HelperFunctions */ -const onKeyup = async (keyboardInput: KeyboardInput) => { +const onKeyup = async (socket: AuthenticatedSocket, keyboardInput: KeyboardInput) => { logger.log('debug', 'Handling keyup event emitted from client'); - await handleWrapper(handleKeyup, keyboardInput); + await handleWrapper(handleKeyup, keyboardInput, socket); } /** @@ -309,12 +370,13 @@ const handleKeyup = async (generator: WorkflowGenerator, page: Page, key: string /** * A wrapper function for handling the url change event. + * @param socket The socket connection * @param url - the new url of the page * @category HelperFunctions */ -const onChangeUrl = async (url: string) => { +const onChangeUrl = async (socket: AuthenticatedSocket, url: string) => { logger.log('debug', 'Handling change url event emitted from client'); - await handleWrapper(handleChangeUrl, url); + await handleWrapper(handleChangeUrl, url, socket); } /** @@ -342,11 +404,12 @@ const handleChangeUrl = async (generator: WorkflowGenerator, page: Page, url: st /** * A wrapper function for handling the refresh event. + * @param socket The socket connection * @category HelperFunctions */ -const onRefresh = async () => { +const onRefresh = async (socket: AuthenticatedSocket) => { logger.log('debug', 'Handling refresh event emitted from client'); - await handleWrapper(handleRefresh); + await handleWrapper(handleRefresh, undefined, socket); } /** @@ -363,11 +426,12 @@ const handleRefresh = async (generator: WorkflowGenerator, page: Page) => { /** * A wrapper function for handling the go back event. + * @param socket The socket connection * @category HelperFunctions */ -const onGoBack = async () => { - logger.log('debug', 'Handling refresh event emitted from client'); - await handleWrapper(handleGoBack); +const onGoBack = async (socket: AuthenticatedSocket) => { + logger.log('debug', 'Handling go back event emitted from client'); + await handleWrapper(handleGoBack, undefined, socket); } /** @@ -385,11 +449,12 @@ const handleGoBack = async (generator: WorkflowGenerator, page: Page) => { /** * A wrapper function for handling the go forward event. + * @param socket The socket connection * @category HelperFunctions */ -const onGoForward = async () => { - logger.log('debug', 'Handling refresh event emitted from client'); - await handleWrapper(handleGoForward); +const onGoForward = async (socket: AuthenticatedSocket) => { + logger.log('debug', 'Handling go forward event emitted from client'); + await handleWrapper(handleGoForward, undefined, socket); } /** @@ -407,18 +472,7 @@ const handleGoForward = async (generator: WorkflowGenerator, page: Page) => { /** * Helper function for registering the handlers onto established websocket connection. - * Registers: - * - mousedownHandler - * - wheelHandler - * - mousemoveHandler - * - keydownHandler - * - keyupHandler - * - changeUrlHandler - * - refreshHandler - * - goBackHandler - * - goForwardHandler - * - onGenerateAction - * input handlers. + * Registers various input handlers. * * All these handlers first generates the workflow pair data * and then calls the corresponding playwright's function to emulate the input. @@ -428,21 +482,25 @@ const handleGoForward = async (generator: WorkflowGenerator, page: Page) => { * @returns void * @category BrowserManagement */ -const registerInputHandlers = (socket: Socket) => { - socket.on("input:mousedown", onMousedown); - socket.on("input:wheel", onWheel); - socket.on("input:mousemove", onMousemove); - socket.on("input:keydown", onKeydown); - socket.on("input:keyup", onKeyup); - socket.on("input:url", onChangeUrl); - socket.on("input:refresh", onRefresh); - socket.on("input:back", onGoBack); - socket.on("input:forward", onGoForward); - socket.on("input:date", onDateSelection); - socket.on("input:dropdown", onDropdownSelection); - socket.on("input:time", onTimeSelection); - socket.on("input:datetime-local", onDateTimeLocalSelection); - socket.on("action", onGenerateAction); +const registerInputHandlers = (socket: Socket) => { + // Cast to our authenticated socket type + const authSocket = socket as AuthenticatedSocket; + + // Register handlers with the socket + socket.on("input:mousedown", (data) => onMousedown(authSocket, data)); + socket.on("input:wheel", (data) => onWheel(authSocket, data)); + socket.on("input:mousemove", (data) => onMousemove(authSocket, data)); + socket.on("input:keydown", (data) => onKeydown(authSocket, data)); + socket.on("input:keyup", (data) => onKeyup(authSocket, data)); + socket.on("input:url", (data) => onChangeUrl(authSocket, data)); + socket.on("input:refresh", () => onRefresh(authSocket)); + socket.on("input:back", () => onGoBack(authSocket)); + socket.on("input:forward", () => onGoForward(authSocket)); + socket.on("input:date", (data) => onDateSelection(authSocket, data)); + socket.on("input:dropdown", (data) => onDropdownSelection(authSocket, data)); + socket.on("input:time", (data) => onTimeSelection(authSocket, data)); + socket.on("input:datetime-local", (data) => onDateTimeLocalSelection(authSocket, data)); + socket.on("action", (data) => onGenerateAction(authSocket, data)); }; -export default registerInputHandlers; +export default registerInputHandlers; \ No newline at end of file diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index 5e386468..495dda5d 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -80,7 +80,7 @@ router.get('/stop/:browserId', requireSignIn, async (req: AuthenticatedRequest, if (!req.user) { return res.status(401).send('User not authenticated'); } - const success = await destroyRemoteBrowser(req.user?.id, req.params.browserId); + const success = await destroyRemoteBrowser(req.params.browserId, req.user?.id); return res.send(success); }); diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index e14d5e97..04c0341c 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -617,7 +617,7 @@ router.post('/runs/run/:id', requireSignIn, async (req: AuthenticatedRequest, re workflow, currentPage, (newPage: Page) => currentPage = newPage, plainRun.interpreterSettings); const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(req.user?.id, plainRun.browserId); + await destroyRemoteBrowser(plainRun.browserId, req.user?.id); await run.update({ ...run, status: 'success', diff --git a/server/src/socket-connection/connection.ts b/server/src/socket-connection/connection.ts index dac7a88e..9fad861b 100644 --- a/server/src/socket-connection/connection.ts +++ b/server/src/socket-connection/connection.ts @@ -1,6 +1,60 @@ import { Namespace, Socket } from 'socket.io'; +import { IncomingMessage } from 'http'; +import { verify, JwtPayload } from 'jsonwebtoken'; import logger from "../logger"; -import registerInputHandlers from '../browser-management/inputHandlers' +import registerInputHandlers from '../browser-management/inputHandlers'; + +interface AuthenticatedIncomingMessage extends IncomingMessage { + user?: JwtPayload | string; +} + +interface AuthenticatedSocket extends Socket { + request: AuthenticatedIncomingMessage; +} + +/** + * Socket.io middleware for authentication + * This is a socket.io specific auth handler that doesn't rely on Express middleware + */ +const socketAuthMiddleware = (socket: Socket, next: (err?: Error) => void) => { + const cookies = socket.handshake.headers.cookie; + if (!cookies) { + return next(new Error('Authentication required')); + } + + const tokenMatch = cookies.split(';').find(c => c.trim().startsWith('token=')); + if (!tokenMatch) { + return next(new Error('Authentication required')); + } + + const token = tokenMatch.split('=')[1]; + if (!token) { + return next(new Error('Authentication required')); + } + + const secret = process.env.JWT_SECRET; + if (!secret) { + return next(new Error('Server configuration error')); + } + + verify(token, secret, (err: any, user: any) => { + if (err) { + logger.log('warn', 'JWT verification error:', err); + return next(new Error('Authentication failed')); + } + + // Normalize payload key + if (user.userId && !user.id) { + user.id = user.userId; + delete user.userId; // temporary: del the old key for clarity + } + + // Attach user to socket request + const authSocket = socket as AuthenticatedSocket; + authSocket.request.user = user; + next(); + }); +}; /** * Opens a websocket canal for duplex data transfer and registers all handlers for this data for the recording session. @@ -13,6 +67,8 @@ export const createSocketConnection = ( io: Namespace, callback: (socket: Socket) => void, ) => { + io.use(socketAuthMiddleware); + const onConnection = async (socket: Socket) => { logger.log('info', "Client connected " + socket.id); registerInputHandlers(socket); @@ -34,6 +90,8 @@ export const createSocketConnectionForRun = ( io: Namespace, callback: (socket: Socket) => void, ) => { + io.use(socketAuthMiddleware); + const onConnection = async (socket: Socket) => { logger.log('info', "Client connected " + socket.id); socket.on('disconnect', () => logger.log('info', "Client disconnected " + socket.id)); @@ -41,4 +99,4 @@ export const createSocketConnectionForRun = ( } io.on('connection', onConnection); -}; +}; \ No newline at end of file diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index c5a9d19c..b67e1ca0 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -132,7 +132,7 @@ async function executeRun(id: string, userId: string) { const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); - await destroyRemoteBrowser(userId, plainRun.browserId); + await destroyRemoteBrowser(plainRun.browserId, userId); await run.update({ ...run, @@ -215,14 +215,14 @@ async function readyForRunHandler(browserId: string, id: string, userId: string) logger.log('info', `Interpretation of ${id} succeeded`); } else { logger.log('error', `Interpretation of ${id} failed`); - await destroyRemoteBrowser(userId, browserId); + await destroyRemoteBrowser(browserId, userId); } resetRecordingState(browserId, id); } catch (error: any) { logger.error(`Error during readyForRunHandler: ${error.message}`); - await destroyRemoteBrowser(userId, browserId); + await destroyRemoteBrowser(browserId, userId); } } From 0220632aef8f27ba29095ffb8c0a0d4f26e79814 Mon Sep 17 00:00:00 2001 From: Rohit Date: Sat, 8 Mar 2025 17:10:30 +0530 Subject: [PATCH 35/35] feat: emit screenshot for unique user --- src/components/browser/BrowserWindow.tsx | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/components/browser/BrowserWindow.tsx b/src/components/browser/BrowserWindow.tsx index ffeb0df5..05c93c1e 100644 --- a/src/components/browser/BrowserWindow.tsx +++ b/src/components/browser/BrowserWindow.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useEffect, useState } from 'react'; +import React, { useCallback, useContext, useEffect, useState } from 'react'; import { useSocketStore } from '../../context/socket'; import { Button } from '@mui/material'; import Canvas from "../recorder/canvas"; @@ -8,6 +8,7 @@ import { useActionContext } from '../../context/browserActions'; import { useBrowserSteps, TextStep } from '../../context/browserSteps'; import { useGlobalInfoStore } from '../../context/globalInfo'; import { useTranslation } from 'react-i18next'; +import { AuthContext } from '../../context/auth'; interface ElementInfo { tagName: string; @@ -27,6 +28,12 @@ interface AttributeOption { value: string; } +interface ScreencastData { + image: string; + userId: string; +} + + const getAttributeOptions = (tagName: string, elementInfo: ElementInfo | null): AttributeOption[] => { if (!elementInfo) return []; switch (tagName.toLowerCase()) { @@ -72,6 +79,9 @@ export const BrowserWindow = () => { const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext(); const { addTextStep, addListStep } = useBrowserSteps(); + const { state } = useContext(AuthContext); + const { user } = state; + const onMouseMove = (e: MouseEvent) => { if (canvasRef && canvasRef.current && highlighterData) { const canvasRect = canvasRef.current.getBoundingClientRect(); @@ -99,9 +109,15 @@ export const BrowserWindow = () => { } }, [getList, resetListState]); - const screencastHandler = useCallback((data: string) => { - setScreenShot(data); - }, [screenShot]); + const screencastHandler = useCallback((data: string | ScreencastData) => { + if (typeof data === 'string') { + setScreenShot(data); + } else if (data && typeof data === 'object' && 'image' in data) { + if (!data.userId || data.userId === user?.id) { + setScreenShot(data.image); + } + } + }, [screenShot, user?.id]); useEffect(() => { if (socket) {