From 66428b7204d004b18b384534012e89baf7ee7e7a Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 30 May 2024 04:49:38 +0530 Subject: [PATCH 01/40] feat: setup server --- server/src/server.ts | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 server/src/server.ts diff --git a/server/src/server.ts b/server/src/server.ts new file mode 100644 index 00000000..4875dd21 --- /dev/null +++ b/server/src/server.ts @@ -0,0 +1,33 @@ +import express from 'express'; +import http from 'http'; +import cors from 'cors'; +import 'dotenv/config'; + +import { record, workflow, storage } from './routes'; +import { BrowserPool } from "./browser-management/classes/BrowserPool"; +import logger from './logger' +import { SERVER_PORT } from "./constants/config"; +import {Server} from "socket.io"; + +const app = express(); +app.use(cors()); +app.use(express.json()); + +const server = http.createServer(app); + +/** + * Globally exported singleton instance of socket.io for socket communication with the client. + * @type {Server} + */ +export const io = new Server(server); + +/** + * {@link BrowserPool} globally exported singleton instance for managing browsers. + */ +export const browserPool = new BrowserPool(); + +app.get('/', function (req, res) { + return res.send('Welcome to the BR recorder server :-)'); +}); + +server.listen(SERVER_PORT, () => logger.log('info',`Server listening on port ${SERVER_PORT}`)); From ea7a3ea2c6e200602a00c53a7a7f6e6a836407e7 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 30 May 2024 04:50:30 +0530 Subject: [PATCH 02/40] feat: import routes for record, storage, workflow --- server/src/server.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/server.ts b/server/src/server.ts index 4875dd21..6b34de08 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -26,6 +26,10 @@ export const io = new Server(server); */ export const browserPool = new BrowserPool(); +app.use('/record', record); +app.use('/workflow', workflow); +app.use('/storage', storage); + app.get('/', function (req, res) { return res.send('Welcome to the BR recorder server :-)'); }); From ba62599fb9b06088cdffc2c65e66ab0c1c92983f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 00:23:12 +0530 Subject: [PATCH 03/40] feat: --- server/src/socket-connection/connection.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 server/src/socket-connection/connection.ts diff --git a/server/src/socket-connection/connection.ts b/server/src/socket-connection/connection.ts new file mode 100644 index 00000000..672e2b54 --- /dev/null +++ b/server/src/socket-connection/connection.ts @@ -0,0 +1,16 @@ +import {Namespace, Socket} from 'socket.io'; +import logger from "../logger"; + +export const createSocketConnection = ( + io: Namespace, + callback: (socket: Socket) => void, + ) => { + const onConnection = async (socket: Socket) => { + logger.log('info',"Client connected " + socket.id); + socket.on('disconnect', () => logger.log('info', "Client disconnected " + socket.id)); + callback(socket); + } + + io.on('connection', onConnection); +}; + From 6731ae6c2524916e132803b0d37f9f692ae8ae9c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 00:24:08 +0530 Subject: [PATCH 04/40] feat(browser): socket connection for run --- server/src/socket-connection/connection.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/src/socket-connection/connection.ts b/server/src/socket-connection/connection.ts index 672e2b54..f96d24aa 100644 --- a/server/src/socket-connection/connection.ts +++ b/server/src/socket-connection/connection.ts @@ -14,3 +14,15 @@ export const createSocketConnection = ( io.on('connection', onConnection); }; +export const createSocketConnectionForRun = ( + io: Namespace, + callback: (socket: Socket) => void, +) => { + const onConnection = async (socket: Socket) => { + logger.log('info',"Client connected " + socket.id); + socket.on('disconnect', () => logger.log('info', "Client disconnected " + socket.id)); + callback(socket); + } + + io.on('connection', onConnection); +}; From c7f0bfa7bc43dd121017b582ae801d6a1f828c1d Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 00:25:02 +0530 Subject: [PATCH 05/40] chore(browser): comments --- server/src/socket-connection/connection.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/src/socket-connection/connection.ts b/server/src/socket-connection/connection.ts index f96d24aa..1390bcbf 100644 --- a/server/src/socket-connection/connection.ts +++ b/server/src/socket-connection/connection.ts @@ -1,6 +1,13 @@ import {Namespace, Socket} from 'socket.io'; import logger from "../logger"; +/** + * Opens a websocket canal for duplex data transfer and registers all handlers for this data for the recording session. + * Uses socket.io dynamic namespaces for multiplexing the traffic from different running remote browser instances. + * @param io dynamic namespace on the socket.io server + * @param callback function called after the connection is created providing the socket resource + * @category BrowserManagement + */ export const createSocketConnection = ( io: Namespace, callback: (socket: Socket) => void, @@ -14,6 +21,13 @@ export const createSocketConnection = ( io.on('connection', onConnection); }; +/** + * Opens a websocket canal for duplex data transfer for the recording run. + * Uses socket.io dynamic namespaces for multiplexing the traffic from different running remote browser instances. + * @param io dynamic namespace on the socket.io server + * @param callback function called after the connection is created providing the socket resource + * @category BrowserManagement + */ export const createSocketConnectionForRun = ( io: Namespace, callback: (socket: Socket) => void, From fc675f4ba2942fc8e2a006e49354c7d11ab19fa0 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 08:56:16 +0530 Subject: [PATCH 06/40] feat: create BrowserPoolInfo & PoolDictionary interfaces --- .../browser-management/classes/BrowserPool.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 server/src/browser-management/classes/BrowserPool.ts diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts new file mode 100644 index 00000000..1fe58219 --- /dev/null +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -0,0 +1,16 @@ +import { RemoteBrowser } from "./RemoteBrowser"; +import logger from "../../logger"; + + +interface BrowserPoolInfo { + + browser: RemoteBrowser, + + active: boolean, +} + + +interface PoolDictionary { + [key: string]: BrowserPoolInfo, +} + From 8ae020b93b3e27d9fd67f8b4947374c0e730583c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 08:56:52 +0530 Subject: [PATCH 07/40] chore: BrowserPoolInfo comments --- .../src/browser-management/classes/BrowserPool.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 1fe58219..d1b1e1b4 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -1,11 +1,19 @@ import { RemoteBrowser } from "./RemoteBrowser"; import logger from "../../logger"; - +/** + * @category Types + */ interface BrowserPoolInfo { - + /** + * The instance of remote browser. + */ browser: RemoteBrowser, - + /** + * States if the browser's instance is being actively used. + * Helps to persist the progress on the frontend when the application has been reloaded. + * @default false + */ active: boolean, } From a7aca9c1a43673a418a192e2334c965afe59b4ac Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 08:57:10 +0530 Subject: [PATCH 08/40] chore: PoolDictionary comments --- server/src/browser-management/classes/BrowserPool.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index d1b1e1b4..5d41eeb3 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -17,7 +17,12 @@ interface BrowserPoolInfo { active: boolean, } - +/** + * Dictionary of all the active remote browser's instances indexed by their id. + * The value in this dictionary is of type BrowserPoolInfo, + * which provides additional information about the browser's usage. + * @category Types + */ interface PoolDictionary { [key: string]: BrowserPoolInfo, } From e68f8b84c8041da05b7aa4bb1b97d2fc4ede8ac9 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 10:24:20 +0530 Subject: [PATCH 09/40] feat: create browser pool --- .../src/browser-management/classes/BrowserPool.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 5d41eeb3..7332b1b1 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -27,3 +27,16 @@ interface PoolDictionary { [key: string]: BrowserPoolInfo, } +/** + * A browser pool is a collection of remote browsers that are initialized and ready to be used. + * Adds the possibility to add, remove and retrieve remote browsers from the pool. + * It is possible to manage multiple browsers for creating or running a recording. + * @category BrowserManagement + */ +export class BrowserPool { + + /** + * Holds all the instances of remote browsers. + */ + private pool : PoolDictionary = {}; +} From 026577bf88ea75586fcabdfa388d4749bc083827 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 10:25:04 +0530 Subject: [PATCH 10/40] feat: add browser instace to pool --- .../browser-management/classes/BrowserPool.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 7332b1b1..df855ae0 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -39,4 +39,21 @@ export class BrowserPool { * Holds all the instances of remote browsers. */ private pool : PoolDictionary = {}; + + /** + * Adds a remote browser instance to the pool indexed by the id. + * @param id remote browser instance's id + * @param browser remote browser instance + * @param active states if the browser's instance is being actively used + */ + public addRemoteBrowser = (id: string, browser: RemoteBrowser, active: boolean = false): void => { + this.pool = { + ...this.pool, + [id]: { + browser, + active, + }, + } + logger.log('debug', `Remote browser with id: ${id} added to the pool`); + }; } From aa9b5d1c433fd3d1836fb6987d4895e2c057c599 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 10:27:51 +0530 Subject: [PATCH 11/40] feat: del browser instace from pool --- .../src/browser-management/classes/BrowserPool.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index df855ae0..7e43654b 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -56,4 +56,19 @@ export class BrowserPool { } logger.log('debug', `Remote browser with id: ${id} added to the pool`); }; + + /** + * Removes the remote browser instance from the pool. + * @param id remote browser instance's id + * @returns true if the browser was removed successfully, false otherwise + */ + public deleteRemoteBrowser = (id: string) : boolean => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return false; + } + delete(this.pool[id]); + logger.log('debug', `Remote browser with id: ${id} deleted from the pool`); + return true; + }; } From 99322a2184cc50d589d8d769f9f0e12feb3fa023 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 10:28:17 +0530 Subject: [PATCH 12/40] feat: get browser instance --- server/src/browser-management/classes/BrowserPool.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index 7e43654b..d7abbbf4 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -71,4 +71,14 @@ export class BrowserPool { logger.log('debug', `Remote browser with id: ${id} deleted from the pool`); return true; }; + + /** + * Returns the remote browser instance from the pool. + * @param id remote browser instance's id + * @returns remote browser instance or undefined if it does not exist in the pool + */ + public getRemoteBrowser = (id: string) : RemoteBrowser | undefined => { + logger.log('debug', `Remote browser with id: ${id} retrieved from the pool`); + return this.pool[id]?.browser; + }; } From 5790c8b442644e648a02296147df7f2ed9e46ec4 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 10:28:46 +0530 Subject: [PATCH 13/40] feat: get active browser instance --- .../browser-management/classes/BrowserPool.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts index d7abbbf4..736189db 100644 --- a/server/src/browser-management/classes/BrowserPool.ts +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -81,4 +81,20 @@ export class BrowserPool { logger.log('debug', `Remote browser with id: ${id} retrieved from the pool`); return this.pool[id]?.browser; }; + + /** + * Returns the active browser's instance id from the pool. + * If there is no active browser, it returns undefined. + * If there are multiple active browsers, it returns the first one. + * @returns the first remote active browser instance's id from the pool + */ + public getActiveBrowserId = () : string | null => { + for (const id of Object.keys(this.pool)) { + if (this.pool[id].active) { + return id; + } + } + logger.log('warn', `No active browser in the pool`); + return null; + }; } From a60527b3b3cc12471eeb7f81c1ba6ad9f613a943 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 10:29:57 +0530 Subject: [PATCH 14/40] feat: basic server side local logger --- server/src/logger.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 server/src/logger.ts diff --git a/server/src/logger.ts b/server/src/logger.ts new file mode 100644 index 00000000..d0ecc24e --- /dev/null +++ b/server/src/logger.ts @@ -0,0 +1,19 @@ +import { createLogger, format, transports } from 'winston'; +import { DEBUG, LOGS_PATH } from "./constants/config"; + +const { combine, timestamp, printf } = format; + +const logger = createLogger({ + format: combine( + timestamp(), + printf(info => `${info.timestamp} ${info.level}: ${info.message}`), + ), + defaultMeta: { service: 'user-service' }, + transports: [ + new transports.Console({ level: DEBUG ? 'info' : 'debug' }), + new transports.File({ filename: `${LOGS_PATH}/error.log`, level: 'error' }), + new transports.File({ filename: `${LOGS_PATH}/combined.log`, level: 'debug' }), + ], +}); + +export default logger; From fa63f4d72bf851974e0235481250b1dbe2ef23d6 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 10:55:04 +0530 Subject: [PATCH 15/40] feat: initiallize RemoteBrowser class --- .../classes/RemoteBrowser.ts | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 server/src/browser-management/classes/RemoteBrowser.ts diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts new file mode 100644 index 00000000..707bcebd --- /dev/null +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -0,0 +1,80 @@ +import { + Page, + Browser, + CDPSession, + BrowserContext, +} from 'playwright'; +import { Socket } from "socket.io"; + +import logger from '../../logger'; +import { InterpreterSettings, RemoteBrowserOptions } from "../../types"; +import { WorkflowGenerator } from "../../workflow-management/classes/Generator"; +import { WorkflowInterpreter } from "../../workflow-management/classes/Interpreter"; + +/** + * This class represents a remote browser instance. + * It is used to allow a variety of interaction with the Playwright's browser instance. + * Every remote browser holds an instance of a generator and interpreter classes with + * the purpose of generating and interpreting workflows. + * @category BrowserManagement + */ +export class RemoteBrowser { + + /** + * Playwright's [browser](https://playwright.dev/docs/api/class-browser) instance. + * @private + */ + private browser: Browser | null = null; + + /** + * The Playwright's [CDPSession](https://playwright.dev/docs/api/class-cdpsession) instance, + * used to talk raw Chrome Devtools Protocol. + * @private + */ + private client : CDPSession | null | undefined = null; + + /** + * Socket.io socket instance enabling communication with the client (frontend) side. + * @private + */ + private socket : Socket; + + /** + * The Playwright's [Page](https://playwright.dev/docs/api/class-page) instance + * as current interactive remote browser's page. + * @private + */ + private currentPage : Page | null | undefined = null; + + /** + * Interpreter settings for any started interpretation. + * @private + */ + private interpreterSettings: InterpreterSettings = { + debug: false, + maxConcurrency: 1, + maxRepeats: 1, + }; + + /** + * {@link WorkflowGenerator} instance specific to the remote browser. + */ + public generator: WorkflowGenerator; + + /** + * {@link WorkflowInterpreter} instance specific to the remote browser. + */ + public interpreter: WorkflowInterpreter; + + /** + * Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and + * assigns the socket instance everywhere. + * @param socket socket.io socket instance used to communicate with the client side + * @constructor + */ + public constructor(socket: Socket) { + this.socket = socket; + this.interpreter = new WorkflowInterpreter(socket); + this.generator = new WorkflowGenerator(socket); + } +} From 15f1b9471fc2f069a792861ffcfcf005c11ee444 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 10:56:25 +0530 Subject: [PATCH 16/40] feat: initiallize remote browser instance async --- .../src/browser-management/classes/RemoteBrowser.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 707bcebd..b71718f0 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -77,4 +77,17 @@ export class RemoteBrowser { this.interpreter = new WorkflowInterpreter(socket); this.generator = new WorkflowGenerator(socket); } + + /** + * An asynchronous constructor for asynchronously initialized properties. + * Must be called right after creating an instance of RemoteBrowser class. + * @param options remote browser options to be used when launching the browser + * @returns {Promise} + */ + public initialize = async(options: RemoteBrowserOptions) : Promise => { + this.browser = (await options.browser.launch(options.launchOptions)); + const context = await this.browser.newContext(); + this.currentPage = await context.newPage(); + this.client = await this.currentPage.context().newCDPSession(this.currentPage); + }; } From 15700fab0c7c58e69a6838f4ea49135cd0cbdaf2 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 10:57:32 +0530 Subject: [PATCH 17/40] feat: registor editor events --- .../classes/RemoteBrowser.ts | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index b71718f0..b502b255 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -90,4 +90,42 @@ export class RemoteBrowser { this.currentPage = await context.newPage(); this.client = await this.currentPage.context().newCDPSession(this.currentPage); }; + + /** + * Registers all event listeners needed for the recording editor session. + * Should be called only once after the full initialization of the remote browser. + * @returns void + */ + public registerEditorEvents = () : void => { + this.socket.on('rerender', async() => await this.makeAndEmitScreenshot()); + this.socket.on('settings', (settings) => this.interpreterSettings = settings); + this.socket.on('changeTab', async(tabIndex) => await this.changeTab(tabIndex)); + this.socket.on('addTab', async () => { + await this.currentPage?.context().newPage(); + const lastTabIndex = this.currentPage ? this.currentPage.context().pages().length - 1 : 0; + await this.changeTab(lastTabIndex); + }); + this.socket.on('closeTab', async (tabInfo) => { + const page = this.currentPage?.context().pages()[tabInfo.index]; + if (page) { + if (tabInfo.isCurrent){ + if (this.currentPage?.context().pages()[tabInfo.index + 1]) { + // next tab + await this.changeTab(tabInfo.index + 1); + } else { + //previous tab + await this.changeTab(tabInfo.index - 1); + } + } + // close the page and log it + await page.close(); + logger.log( + 'debug', + `${tabInfo.index} page was closed, new length of pages: ${this.currentPage?.context().pages().length}` + ) + } else { + logger.log('error', `${tabInfo.index} index out of range of pages`) + } + }); + } } From c42919e374fa05106cbc49c521769298354e12f9 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 10:58:23 +0530 Subject: [PATCH 18/40] feat: subscribe to screencast on CDP --- .../classes/RemoteBrowser.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index b502b255..5573a540 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -128,4 +128,33 @@ export class RemoteBrowser { } }); } + + /** + * Subscribes the remote browser for a screencast session + * on [CDP](https://chromedevtools.github.io/devtools-protocol/) level, + * where screenshot is being sent through the socket + * every time the browser's active page updates. + * @returns {Promise} + */ + public subscribeToScreencast = async() : Promise => { + await this.startScreencast(); + if (!this.client) { + logger.log('warn','client is not initialized'); + return; + } + this.client.on('Page.screencastFrame', ({ data: base64, sessionId }) => { + this.emitScreenshot(base64); + setTimeout(async () => { + try { + if (!this.client) { + logger.log('warn','client is not initialized'); + return; + } + await this.client.send('Page.screencastFrameAck', { sessionId: sessionId }); + } catch (e) { + logger.log('error', e); + } + }, 100); + }); + }; } From 8c23ef965d0792ba092a5cb32ac24fb4ca4d32f4 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 10:59:15 +0530 Subject: [PATCH 19/40] feat: terminate screencast --- .../browser-management/classes/RemoteBrowser.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 5573a540..c8ea375d 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -157,4 +157,21 @@ export class RemoteBrowser { }, 100); }); }; + + /** + * Terminates the screencast session and closes the remote browser. + * If an interpretation was running it will be stopped. + * @returns {Promise} + */ + public switchOff = async() : Promise => { + await this.interpreter.stopInterpretation(); + if (this.browser) { + await this.stopScreencast(); + await this.browser.close(); + } else { + logger.log('error', 'Browser wasn\'t initialized'); + logger.log('error','Switching off the browser failed'); + } + }; + } From 83740fa70c9a7780d5c5ce9c57c21cbbd8f4986f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 10:59:49 +0530 Subject: [PATCH 20/40] feat: make and emit ss to client --- .../browser-management/classes/RemoteBrowser.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index c8ea375d..9016b016 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -174,4 +174,21 @@ export class RemoteBrowser { } }; + /** + * Makes and emits a single screenshot to the client side. + * @returns {Promise} + */ + public makeAndEmitScreenshot = async() : Promise => { + try { + const screenshot = await this.currentPage?.screenshot(); + if (screenshot) { + this.emitScreenshot(screenshot.toString('base64')); + } + } catch (e) { + const { message } = e as Error; + logger.log('error', message); + } + }; + + } From 580e0c046a2f4e592363aa214cab9343cb2b5864 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:00:28 +0530 Subject: [PATCH 21/40] feat: update socket connection --- .../browser-management/classes/RemoteBrowser.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 9016b016..d7ead38f 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -190,5 +190,18 @@ export class RemoteBrowser { } }; - + /** + * Updates the active socket instance. + * This will update all registered events for the socket and + * all the properties using the socket. + * @param socket socket.io socket instance used to communicate with the client side + * @returns void + */ + public updateSocket = (socket: Socket) : void => { + this.socket = socket; + this.registerEditorEvents(); + this.generator?.updateSocket(socket); + this.interpreter?.updateSocket(socket); + }; + } From 30cab6c5b2030e20127a0619f4faf6f1e55b7402 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:01:12 +0530 Subject: [PATCH 22/40] feat: helper method to emit screenshot to client via websocket --- server/src/browser-management/classes/RemoteBrowser.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index d7ead38f..adff9150 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -204,4 +204,14 @@ export class RemoteBrowser { this.interpreter?.updateSocket(socket); }; + /** + * Helper for emitting the screenshot of browser's active page through websocket. + * @param payload the screenshot binary data + * @returns void + */ + private emitScreenshot = (payload: any) : void => { + const dataWithMimeType = ('data:image/jpeg;base64,').concat(payload); + this.socket.emit('screencast', dataWithMimeType); + logger.log('debug',`Screenshot emitted`); + }; } From 44b8393ace5eca551dc4c4b157747c51edd40d20 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:02:21 +0530 Subject: [PATCH 23/40] feat: start & stop screencast --- .../classes/RemoteBrowser.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index adff9150..e62651fb 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -204,6 +204,35 @@ export class RemoteBrowser { this.interpreter?.updateSocket(socket); }; + /** + * Initiates screencast of the remote browser through socket, + * registers listener for rerender event and emits the loaded event. + * Should be called only once after the browser is fully initialized. + * @returns {Promise} + */ + private startScreencast = async() : Promise => { + if (!this.client) { + logger.log('warn','client is not initialized'); + return; + } + await this.client.send('Page.startScreencast', { format: 'jpeg', quality: 75 }); + logger.log('info',`Browser started with screencasting a page.`); + }; + + /** + * Unsubscribes the current page from the screencast session. + * @returns {Promise} + */ + private stopScreencast = async() : Promise => { + if (!this.client) { + logger.log('error','client is not initialized'); + logger.log('error','Screencast stop failed'); + } else { + await this.client.send('Page.stopScreencast'); + logger.log('info', `Browser stopped with screencasting.`); + } + }; + /** * Helper for emitting the screenshot of browser's active page through websocket. * @param payload the screenshot binary data From 8b9ce90a6d4dbd44833e5a37e9414af40cd15281 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:03:01 +0530 Subject: [PATCH 24/40] feat: interpret generated workflow --- .../classes/RemoteBrowser.ts | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index e62651fb..6272395a 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -204,6 +204,42 @@ export class RemoteBrowser { this.interpreter?.updateSocket(socket); }; + /** + * Starts the interpretation of the currently generated workflow. + * @returns {Promise} + */ + public interpretCurrentRecording = async () : Promise => { + logger.log('debug', 'Starting interpretation in the editor'); + if (this.generator) { + const workflow = this.generator.AddGeneratedFlags(this.generator.getWorkflowFile()); + await this.initializeNewPage(); + if (this.currentPage) { + const params = this.generator.getParams(); + if (params) { + this.interpreterSettings.params = params.reduce((acc, param) => { + if (this.interpreterSettings.params && Object.keys(this.interpreterSettings.params).includes(param)) { + return { ...acc, [param]: this.interpreterSettings.params[param] }; + } else { + return { ...acc, [param]: '', } + } + }, {}) + } + logger.log('debug', `Starting interpretation with settings: ${JSON.stringify(this.interpreterSettings, null, 2)}`); + await this.interpreter.interpretRecordingInEditor( + workflow, this.currentPage, + (newPage: Page) => this.currentPage = newPage, + this.interpreterSettings + ); + // clear the active index from generator + this.generator.clearLastIndex(); + } else { + logger.log('error', 'Could not get a new page, returned undefined'); + } + } else { + logger.log('error', 'Generator is not initialized'); + } + }; + /** * Initiates screencast of the remote browser through socket, * registers listener for rerender event and emits the loaded event. From 6bbd71baf40099711af5eeaa6cabc72608bfa9ee Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:03:44 +0530 Subject: [PATCH 25/40] feat: stop interpretaion and get current page --- .../browser-management/classes/RemoteBrowser.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 6272395a..0e17248c 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -240,6 +240,23 @@ export class RemoteBrowser { } }; + /** + * Stops the workflow interpretation and initializes a new page. + * @returns {Promise} + */ + public stopCurrentInterpretation = async () : Promise => { + await this.interpreter.stopInterpretation(); + await this.initializeNewPage(); + }; + + /** + * Returns the current page instance. + * @returns {Page | null | undefined} + */ + public getCurrentPage = () : Page | null | undefined => { + return this.currentPage; + }; + /** * Initiates screencast of the remote browser through socket, * registers listener for rerender event and emits the loaded event. From 93585405bfa76267e45b56e66c0375e295593663 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:04:38 +0530 Subject: [PATCH 26/40] feat: tab management for screencast based on page instance --- .../classes/RemoteBrowser.ts | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 0e17248c..03e2c4ff 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -257,6 +257,28 @@ export class RemoteBrowser { return this.currentPage; }; + /** + * Changes the active page to the page instance on the given index + * available in pages array on the {@link BrowserContext}. + * Automatically stops the screencast session on the previous page and starts the new one. + * @param tabIndex index of the page in the pages array on the {@link BrowserContext} + * @returns {Promise} + */ + private changeTab = async (tabIndex: number) : Promise=> { + const page = this.currentPage?.context().pages()[tabIndex]; + if (page) { + await this.stopScreencast(); + this.currentPage = page; + await this.currentPage.setViewportSize({height: 720, width: 1280}) + this.client = await this.currentPage.context().newCDPSession(this.currentPage); + this.socket.emit('urlChanged', this.currentPage.url()); + await this.makeAndEmitScreenshot(); + await this.subscribeToScreencast(); + } else { + logger.log('error', `${tabIndex} index out of range of pages`) + } + } + /** * Initiates screencast of the remote browser through socket, * registers listener for rerender event and emits the loaded event. From 795e653fd0720b26cf8176c32445b31b58158277 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:05:13 +0530 Subject: [PATCH 27/40] feat: new page intialization --- .../classes/RemoteBrowser.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 03e2c4ff..8c5a6e8b 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -279,6 +279,29 @@ export class RemoteBrowser { } } + /** + * Internal method for a new page initialization. Subscribes this page to the screencast. + * @param options optional page options to be used when creating a new page + * @returns {Promise} + */ + private initializeNewPage = async (options?: Object) : Promise => { + await this.stopScreencast(); + const newPage = options ? await this.browser?.newPage(options) + : await this.browser?.newPage(); + + await this.currentPage?.close(); + this.currentPage = newPage; + if (this.currentPage) { + this.currentPage.on('load', (page) => { + this.socket.emit('urlChanged', page.url()); + }) + this.client = await this.currentPage.context().newCDPSession(this.currentPage); + await this.subscribeToScreencast(); + } else { + logger.log('error', 'Could not get a new page, returned undefined'); + } + }; + /** * Initiates screencast of the remote browser through socket, * registers listener for rerender event and emits the loaded event. From 7c69204f9de503b7e4093eea5fbea4a55da20b43 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:05:45 +0530 Subject: [PATCH 28/40] chore: lint --- .../classes/RemoteBrowser.ts | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 8c5a6e8b..f5527581 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -31,20 +31,20 @@ export class RemoteBrowser { * used to talk raw Chrome Devtools Protocol. * @private */ - private client : CDPSession | null | undefined = null; + private client: CDPSession | null | undefined = null; /** * Socket.io socket instance enabling communication with the client (frontend) side. * @private */ - private socket : Socket; + private socket: Socket; /** * The Playwright's [Page](https://playwright.dev/docs/api/class-page) instance * as current interactive remote browser's page. * @private */ - private currentPage : Page | null | undefined = null; + private currentPage: Page | null | undefined = null; /** * Interpreter settings for any started interpretation. @@ -84,7 +84,7 @@ export class RemoteBrowser { * @param options remote browser options to be used when launching the browser * @returns {Promise} */ - public initialize = async(options: RemoteBrowserOptions) : Promise => { + public initialize = async (options: RemoteBrowserOptions): Promise => { this.browser = (await options.browser.launch(options.launchOptions)); const context = await this.browser.newContext(); this.currentPage = await context.newPage(); @@ -96,10 +96,10 @@ export class RemoteBrowser { * Should be called only once after the full initialization of the remote browser. * @returns void */ - public registerEditorEvents = () : void => { - this.socket.on('rerender', async() => await this.makeAndEmitScreenshot()); + public registerEditorEvents = (): void => { + this.socket.on('rerender', async () => await this.makeAndEmitScreenshot()); this.socket.on('settings', (settings) => this.interpreterSettings = settings); - this.socket.on('changeTab', async(tabIndex) => await this.changeTab(tabIndex)); + this.socket.on('changeTab', async (tabIndex) => await this.changeTab(tabIndex)); this.socket.on('addTab', async () => { await this.currentPage?.context().newPage(); const lastTabIndex = this.currentPage ? this.currentPage.context().pages().length - 1 : 0; @@ -108,7 +108,7 @@ export class RemoteBrowser { this.socket.on('closeTab', async (tabInfo) => { const page = this.currentPage?.context().pages()[tabInfo.index]; if (page) { - if (tabInfo.isCurrent){ + if (tabInfo.isCurrent) { if (this.currentPage?.context().pages()[tabInfo.index + 1]) { // next tab await this.changeTab(tabInfo.index + 1); @@ -120,8 +120,8 @@ export class RemoteBrowser { // close the page and log it await page.close(); logger.log( - 'debug', - `${tabInfo.index} page was closed, new length of pages: ${this.currentPage?.context().pages().length}` + 'debug', + `${tabInfo.index} page was closed, new length of pages: ${this.currentPage?.context().pages().length}` ) } else { logger.log('error', `${tabInfo.index} index out of range of pages`) @@ -136,10 +136,10 @@ export class RemoteBrowser { * every time the browser's active page updates. * @returns {Promise} */ - public subscribeToScreencast = async() : Promise => { + public subscribeToScreencast = async (): Promise => { await this.startScreencast(); if (!this.client) { - logger.log('warn','client is not initialized'); + logger.log('warn', 'client is not initialized'); return; } this.client.on('Page.screencastFrame', ({ data: base64, sessionId }) => { @@ -147,7 +147,7 @@ export class RemoteBrowser { setTimeout(async () => { try { if (!this.client) { - logger.log('warn','client is not initialized'); + logger.log('warn', 'client is not initialized'); return; } await this.client.send('Page.screencastFrameAck', { sessionId: sessionId }); @@ -163,14 +163,14 @@ export class RemoteBrowser { * If an interpretation was running it will be stopped. * @returns {Promise} */ - public switchOff = async() : Promise => { + public switchOff = async (): Promise => { await this.interpreter.stopInterpretation(); if (this.browser) { await this.stopScreencast(); await this.browser.close(); } else { logger.log('error', 'Browser wasn\'t initialized'); - logger.log('error','Switching off the browser failed'); + logger.log('error', 'Switching off the browser failed'); } }; @@ -178,7 +178,7 @@ export class RemoteBrowser { * Makes and emits a single screenshot to the client side. * @returns {Promise} */ - public makeAndEmitScreenshot = async() : Promise => { + public makeAndEmitScreenshot = async (): Promise => { try { const screenshot = await this.currentPage?.screenshot(); if (screenshot) { @@ -197,7 +197,7 @@ export class RemoteBrowser { * @param socket socket.io socket instance used to communicate with the client side * @returns void */ - public updateSocket = (socket: Socket) : void => { + public updateSocket = (socket: Socket): void => { this.socket = socket; this.registerEditorEvents(); this.generator?.updateSocket(socket); @@ -208,7 +208,7 @@ export class RemoteBrowser { * Starts the interpretation of the currently generated workflow. * @returns {Promise} */ - public interpretCurrentRecording = async () : Promise => { + public interpretCurrentRecording = async (): Promise => { logger.log('debug', 'Starting interpretation in the editor'); if (this.generator) { const workflow = this.generator.AddGeneratedFlags(this.generator.getWorkflowFile()); @@ -226,9 +226,9 @@ export class RemoteBrowser { } logger.log('debug', `Starting interpretation with settings: ${JSON.stringify(this.interpreterSettings, null, 2)}`); await this.interpreter.interpretRecordingInEditor( - workflow, this.currentPage, - (newPage: Page) => this.currentPage = newPage, - this.interpreterSettings + workflow, this.currentPage, + (newPage: Page) => this.currentPage = newPage, + this.interpreterSettings ); // clear the active index from generator this.generator.clearLastIndex(); @@ -244,7 +244,7 @@ export class RemoteBrowser { * Stops the workflow interpretation and initializes a new page. * @returns {Promise} */ - public stopCurrentInterpretation = async () : Promise => { + public stopCurrentInterpretation = async (): Promise => { await this.interpreter.stopInterpretation(); await this.initializeNewPage(); }; @@ -253,7 +253,7 @@ export class RemoteBrowser { * Returns the current page instance. * @returns {Page | null | undefined} */ - public getCurrentPage = () : Page | null | undefined => { + public getCurrentPage = (): Page | null | undefined => { return this.currentPage; }; @@ -264,12 +264,12 @@ export class RemoteBrowser { * @param tabIndex index of the page in the pages array on the {@link BrowserContext} * @returns {Promise} */ - private changeTab = async (tabIndex: number) : Promise=> { + private changeTab = async (tabIndex: number): Promise => { const page = this.currentPage?.context().pages()[tabIndex]; if (page) { await this.stopScreencast(); this.currentPage = page; - await this.currentPage.setViewportSize({height: 720, width: 1280}) + await this.currentPage.setViewportSize({ height: 720, width: 1280 }) this.client = await this.currentPage.context().newCDPSession(this.currentPage); this.socket.emit('urlChanged', this.currentPage.url()); await this.makeAndEmitScreenshot(); @@ -284,10 +284,10 @@ export class RemoteBrowser { * @param options optional page options to be used when creating a new page * @returns {Promise} */ - private initializeNewPage = async (options?: Object) : Promise => { + private initializeNewPage = async (options?: Object): Promise => { await this.stopScreencast(); const newPage = options ? await this.browser?.newPage(options) - : await this.browser?.newPage(); + : await this.browser?.newPage(); await this.currentPage?.close(); this.currentPage = newPage; @@ -298,7 +298,7 @@ export class RemoteBrowser { this.client = await this.currentPage.context().newCDPSession(this.currentPage); await this.subscribeToScreencast(); } else { - logger.log('error', 'Could not get a new page, returned undefined'); + logger.log('error', 'Could not get a new page, returned undefined'); } }; @@ -308,23 +308,23 @@ export class RemoteBrowser { * Should be called only once after the browser is fully initialized. * @returns {Promise} */ - private startScreencast = async() : Promise => { + private startScreencast = async (): Promise => { if (!this.client) { - logger.log('warn','client is not initialized'); + logger.log('warn', 'client is not initialized'); return; } await this.client.send('Page.startScreencast', { format: 'jpeg', quality: 75 }); - logger.log('info',`Browser started with screencasting a page.`); + logger.log('info', `Browser started with screencasting a page.`); }; /** * Unsubscribes the current page from the screencast session. * @returns {Promise} */ - private stopScreencast = async() : Promise => { + private stopScreencast = async (): Promise => { if (!this.client) { - logger.log('error','client is not initialized'); - logger.log('error','Screencast stop failed'); + logger.log('error', 'client is not initialized'); + logger.log('error', 'Screencast stop failed'); } else { await this.client.send('Page.stopScreencast'); logger.log('info', `Browser stopped with screencasting.`); @@ -336,9 +336,9 @@ export class RemoteBrowser { * @param payload the screenshot binary data * @returns void */ - private emitScreenshot = (payload: any) : void => { + private emitScreenshot = (payload: any): void => { const dataWithMimeType = ('data:image/jpeg;base64,').concat(payload); this.socket.emit('screencast', dataWithMimeType); - logger.log('debug',`Screenshot emitted`); + logger.log('debug', `Screenshot emitted`); }; } From 8bc5296dbd17ae5efcd718178b888b7405e7c0a6 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:18:02 +0530 Subject: [PATCH 29/40] feat: intialize remote browser for recording via controller --- server/src/browser-management/controller.ts | 46 +++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 server/src/browser-management/controller.ts diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts new file mode 100644 index 00000000..db63c060 --- /dev/null +++ b/server/src/browser-management/controller.ts @@ -0,0 +1,46 @@ +/** + * The main function group which determines the flow of remote browser management. + * Holds the singleton instances of browser pool and socket.io server. + */ +import { Socket } from "socket.io"; +import { uuid } from 'uuidv4'; + +import { createSocketConnection, createSocketConnectionForRun } from "../socket-connection/connection"; +import { io, browserPool } from "../server"; +import { RemoteBrowser } from "./classes/RemoteBrowser"; +import { RemoteBrowserOptions } from "../types"; +import logger from "../logger"; + +/** + * Starts and initializes a {@link RemoteBrowser} instance. + * Creates a new socket connection over a dedicated namespace + * and registers all interaction event handlers. + * Returns the id of an active browser or the new remote browser's generated id. + * @param options {@link RemoteBrowserOptions} to be used when launching the browser + * @returns string + * @category BrowserManagement-Controller + */ +export const initializeRemoteBrowserForRecording = (options: RemoteBrowserOptions): string => { + const id = getActiveBrowserId() || uuid(); + createSocketConnection( + io.of(id), + async (socket: Socket) => { + // browser is already active + const activeId = getActiveBrowserId(); + if (activeId) { + const remoteBrowser = browserPool.getRemoteBrowser(activeId); + remoteBrowser?.updateSocket(socket); + await remoteBrowser?.makeAndEmitScreenshot(); + } else { + const browserSession = new RemoteBrowser(socket); + browserSession.interpreter.subscribeToPausing(); + await browserSession.initialize(options); + await browserSession.registerEditorEvents(); + await browserSession.subscribeToScreencast(); + browserPool.addRemoteBrowser(id, browserSession, true); + } + socket.emit('loaded'); + }); + return id; +}; +}; From 35d3843d6882d025b2ee28077410f9e6d7f04b0d Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:19:01 +0530 Subject: [PATCH 30/40] feat: create remote browser to run interpretation --- server/src/browser-management/controller.ts | 22 +++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index db63c060..731ef030 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -43,4 +43,26 @@ export const initializeRemoteBrowserForRecording = (options: RemoteBrowserOption }); return id; }; + +/** + * Starts and initializes a {@link RemoteBrowser} instance for interpretation. + * Creates a new {@link Socket} connection over a dedicated namespace. + * Returns the new remote browser's generated id. + * @param options {@link RemoteBrowserOptions} to be used when launching the browser + * @returns string + * @category BrowserManagement-Controller + */ +export const createRemoteBrowserForRun = (options: RemoteBrowserOptions): string => { + const id = uuid(); + createSocketConnectionForRun( + io.of(id), + async (socket: Socket) => { + const browserSession = new RemoteBrowser(socket); + await browserSession.initialize(options); + browserPool.addRemoteBrowser(id, browserSession, true); + socket.emit('ready-for-run'); + }); + return id; +}; + }; From 2e207f2f5ef89519fe4725b155bf6497ef58feca Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:19:39 +0530 Subject: [PATCH 31/40] feat: terminate / destory remote browser instance --- server/src/browser-management/controller.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index 731ef030..39b8f931 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -65,4 +65,20 @@ export const createRemoteBrowserForRun = (options: RemoteBrowserOptions): string return id; }; +/** + * Terminates a remote browser recording session + * and removes the browser from the browser pool. + * @param id instance id of the remote browser to be terminated + * @returns {Promise} + * @category BrowserManagement-Controller + */ +export const destroyRemoteBrowser = async (id: string) : Promise => { + const browserSession = browserPool.getRemoteBrowser(id); + if (browserSession) { + logger.log('debug', `Switching off the browser with id: ${id}`); + await browserSession.stopCurrentInterpretation(); + await browserSession.switchOff(); + } + return browserPool.deleteRemoteBrowser(id); +}; }; From 000a047723610fb4aa0ad1c8baa408b9d0159bc6 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:20:14 +0530 Subject: [PATCH 32/40] feat: gget active remote browser id --- server/src/browser-management/controller.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index 39b8f931..eb51a104 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -81,4 +81,14 @@ export const destroyRemoteBrowser = async (id: string) : Promise => { } return browserPool.deleteRemoteBrowser(id); }; + +/** + * Returns the id of an active browser or null. + * Wrapper around {@link browserPool.getActiveBrowserId()} function. + * @returns {string | null} + * @category BrowserManagement-Controller + */ +export const getActiveBrowserId = (): string | null=> { + return browserPool.getActiveBrowserId(); +}; }; From 9c32080d4b869ddb2aa4b716ef1845eea0a7ac9f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:20:45 +0530 Subject: [PATCH 33/40] feat: gget active remote brwoser current url --- server/src/browser-management/controller.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index eb51a104..374f58f3 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -91,4 +91,16 @@ export const destroyRemoteBrowser = async (id: string) : Promise => { export const getActiveBrowserId = (): string | null=> { return browserPool.getActiveBrowserId(); }; + +/** + * Returns the url string from a remote browser if exists in the browser pool. + * @param id instance id of the remote browser + * @returns {string | undefined} + * @category BrowserManagement-Controller + */ +export const getRemoteBrowserCurrentUrl = (id: string): string | undefined => { + return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.url(); +}; + + }; From 85518848bf96642673316f707d143b1d52358900 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:21:16 +0530 Subject: [PATCH 34/40] feat: get remote browser current tabs --- server/src/browser-management/controller.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index 374f58f3..eb41106a 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -102,5 +102,23 @@ export const getRemoteBrowserCurrentUrl = (id: string): string | undefined => { return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.url(); }; +/** + * Returns the array of tab strings from a remote browser if exists in the browser pool. + * @param id instance id of the remote browser + * @return {string[] | undefined} + * @category BrowserManagement-Controller + */ +export const getRemoteBrowserCurrentTabs = (id: string): string[] | undefined => { + return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.context().pages() + .map((page) => { + const parsedUrl = new URL(page.url()); + const host = parsedUrl.hostname.match(/\b(?!www\.)[a-zA-Z0-9]+/g)?.join('.'); + if (host) { + return host; + } + return 'new tab'; + }); +}; + }; From c6d740f3ce7f67a5c415b264e1d49dbebc9faf9c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:22:01 +0530 Subject: [PATCH 35/40] feat: interpret current workflow in active browser instance --- server/src/browser-management/controller.ts | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index eb41106a..8646a5cb 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -120,5 +120,23 @@ export const getRemoteBrowserCurrentTabs = (id: string): string[] | undefined => }); }; - +/** + * Interprets the currently generated workflow in the active browser instance. + * If there is no active browser, the function logs an error. + * @returns {Promise} + * @category BrowserManagement-Controller + */ +export const interpretWholeWorkflow = async() => { + const id = getActiveBrowserId(); + if (id) { + const browser = browserPool.getRemoteBrowser(id); + if (browser) { + await browser.interpretCurrentRecording(); + } else { + logger.log('error', `No active browser with id ${id} found in the browser pool`); + } + } else { + logger.log('error', `Cannot interpret the workflow: bad id ${id}.`); + } +}; }; From fc57daef618598d506e7096376159b8a9483754e Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:22:23 +0530 Subject: [PATCH 36/40] feat:stop interpretatopn of current workflow in active browser instance --- server/src/browser-management/controller.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index 8646a5cb..98962c1e 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -139,4 +139,19 @@ export const interpretWholeWorkflow = async() => { logger.log('error', `Cannot interpret the workflow: bad id ${id}.`); } }; + +/** + * Stops the interpretation of the current workflow in the active browser instance. + * If there is no active browser, the function logs an error. + * @returns {Promise} + * @category BrowserManagement-Controller + */ +export const stopRunningInterpretation = async() => { + const id = getActiveBrowserId(); + if (id) { + const browser = browserPool.getRemoteBrowser(id); + await browser?.stopCurrentInterpretation(); + } else { + logger.log('error', 'Cannot stop interpretation: No active browser or generator.'); + } }; From af897a2b49e7cdd34f5d7a413ca31dbe2dfd20d6 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:22:43 +0530 Subject: [PATCH 37/40] chore: lint --- server/src/browser-management/controller.ts | 134 ++++++++++---------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index 98962c1e..62a3de66 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -21,27 +21,27 @@ import logger from "../logger"; * @category BrowserManagement-Controller */ export const initializeRemoteBrowserForRecording = (options: RemoteBrowserOptions): string => { - const id = getActiveBrowserId() || uuid(); - createSocketConnection( - io.of(id), - async (socket: Socket) => { - // browser is already active - const activeId = getActiveBrowserId(); - if (activeId) { - const remoteBrowser = browserPool.getRemoteBrowser(activeId); - remoteBrowser?.updateSocket(socket); - await remoteBrowser?.makeAndEmitScreenshot(); - } else { - const browserSession = new RemoteBrowser(socket); - browserSession.interpreter.subscribeToPausing(); - await browserSession.initialize(options); - await browserSession.registerEditorEvents(); - await browserSession.subscribeToScreencast(); - browserPool.addRemoteBrowser(id, browserSession, true); - } - socket.emit('loaded'); - }); - return id; + const id = getActiveBrowserId() || uuid(); + createSocketConnection( + io.of(id), + async (socket: Socket) => { + // browser is already active + const activeId = getActiveBrowserId(); + if (activeId) { + const remoteBrowser = browserPool.getRemoteBrowser(activeId); + remoteBrowser?.updateSocket(socket); + await remoteBrowser?.makeAndEmitScreenshot(); + } else { + const browserSession = new RemoteBrowser(socket); + browserSession.interpreter.subscribeToPausing(); + await browserSession.initialize(options); + await browserSession.registerEditorEvents(); + await browserSession.subscribeToScreencast(); + browserPool.addRemoteBrowser(id, browserSession, true); + } + socket.emit('loaded'); + }); + return id; }; /** @@ -53,16 +53,16 @@ export const initializeRemoteBrowserForRecording = (options: RemoteBrowserOption * @category BrowserManagement-Controller */ export const createRemoteBrowserForRun = (options: RemoteBrowserOptions): string => { - const id = uuid(); - createSocketConnectionForRun( - io.of(id), + const id = uuid(); + createSocketConnectionForRun( + io.of(id), async (socket: Socket) => { - const browserSession = new RemoteBrowser(socket); - await browserSession.initialize(options); - browserPool.addRemoteBrowser(id, browserSession, true); - socket.emit('ready-for-run'); - }); - return id; + const browserSession = new RemoteBrowser(socket); + await browserSession.initialize(options); + browserPool.addRemoteBrowser(id, browserSession, true); + socket.emit('ready-for-run'); + }); + return id; }; /** @@ -72,14 +72,14 @@ export const createRemoteBrowserForRun = (options: RemoteBrowserOptions): string * @returns {Promise} * @category BrowserManagement-Controller */ -export const destroyRemoteBrowser = async (id: string) : Promise => { - const browserSession = browserPool.getRemoteBrowser(id); - if (browserSession) { - logger.log('debug', `Switching off the browser with id: ${id}`); - await browserSession.stopCurrentInterpretation(); - await browserSession.switchOff(); - } - return browserPool.deleteRemoteBrowser(id); +export const destroyRemoteBrowser = async (id: string): Promise => { + const browserSession = browserPool.getRemoteBrowser(id); + if (browserSession) { + logger.log('debug', `Switching off the browser with id: ${id}`); + await browserSession.stopCurrentInterpretation(); + await browserSession.switchOff(); + } + return browserPool.deleteRemoteBrowser(id); }; /** @@ -88,8 +88,8 @@ export const destroyRemoteBrowser = async (id: string) : Promise => { * @returns {string | null} * @category BrowserManagement-Controller */ -export const getActiveBrowserId = (): string | null=> { - return browserPool.getActiveBrowserId(); +export const getActiveBrowserId = (): string | null => { + return browserPool.getActiveBrowserId(); }; /** @@ -99,7 +99,7 @@ export const getActiveBrowserId = (): string | null=> { * @category BrowserManagement-Controller */ export const getRemoteBrowserCurrentUrl = (id: string): string | undefined => { - return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.url(); + return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.url(); }; /** @@ -109,15 +109,15 @@ export const getRemoteBrowserCurrentUrl = (id: string): string | undefined => { * @category BrowserManagement-Controller */ export const getRemoteBrowserCurrentTabs = (id: string): string[] | undefined => { - return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.context().pages() - .map((page) => { - const parsedUrl = new URL(page.url()); - const host = parsedUrl.hostname.match(/\b(?!www\.)[a-zA-Z0-9]+/g)?.join('.'); - if (host) { - return host; - } - return 'new tab'; - }); + return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.context().pages() + .map((page) => { + const parsedUrl = new URL(page.url()); + const host = parsedUrl.hostname.match(/\b(?!www\.)[a-zA-Z0-9]+/g)?.join('.'); + if (host) { + return host; + } + return 'new tab'; + }); }; /** @@ -126,18 +126,18 @@ export const getRemoteBrowserCurrentTabs = (id: string): string[] | undefined => * @returns {Promise} * @category BrowserManagement-Controller */ -export const interpretWholeWorkflow = async() => { - const id = getActiveBrowserId(); - if (id) { +export const interpretWholeWorkflow = async () => { + const id = getActiveBrowserId(); + if (id) { const browser = browserPool.getRemoteBrowser(id); - if (browser) { - await browser.interpretCurrentRecording(); - } else { - logger.log('error', `No active browser with id ${id} found in the browser pool`); - } + if (browser) { + await browser.interpretCurrentRecording(); } else { - logger.log('error', `Cannot interpret the workflow: bad id ${id}.`); + logger.log('error', `No active browser with id ${id} found in the browser pool`); } + } else { + logger.log('error', `Cannot interpret the workflow: bad id ${id}.`); + } }; /** @@ -146,12 +146,12 @@ export const interpretWholeWorkflow = async() => { * @returns {Promise} * @category BrowserManagement-Controller */ -export const stopRunningInterpretation = async() => { - const id = getActiveBrowserId(); - if (id) { - const browser = browserPool.getRemoteBrowser(id); - await browser?.stopCurrentInterpretation(); - } else { - logger.log('error', 'Cannot stop interpretation: No active browser or generator.'); - } +export const stopRunningInterpretation = async () => { + const id = getActiveBrowserId(); + if (id) { + const browser = browserPool.getRemoteBrowser(id); + await browser?.stopCurrentInterpretation(); + } else { + logger.log('error', 'Cannot stop interpretation: No active browser or generator.'); + } }; From 3898fb49fd250b8aad35f6c06f9a5f948c05a53f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:30:07 +0530 Subject: [PATCH 38/40] feat(temporary): local paths --- server/src/constants/config.ts | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 server/src/constants/config.ts diff --git a/server/src/constants/config.ts b/server/src/constants/config.ts new file mode 100644 index 00000000..199e36fc --- /dev/null +++ b/server/src/constants/config.ts @@ -0,0 +1,3 @@ +export const SERVER_PORT = process.env.SERVER_PORT ? Number(process.env.SERVER_PORT) : 8080 +export const DEBUG = process.env.DEBUG === 'true' +export const LOGS_PATH = process.env.LOGS_PATH ?? 'server/logs' From 659f4c506f039abcea83ae950fb99b9d97773a08 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:34:50 +0530 Subject: [PATCH 39/40] feat(ts): interface InterpreterSettings --- server/src/types/index.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 server/src/types/index.ts diff --git a/server/src/types/index.ts b/server/src/types/index.ts new file mode 100644 index 00000000..dfcaa12f --- /dev/null +++ b/server/src/types/index.ts @@ -0,0 +1,12 @@ +import {BrowserType, LaunchOptions} from "playwright"; + +/** + * Interpreter settings properties including recording parameters. + * @category Types + */ +export interface InterpreterSettings { + maxConcurrency: number; + maxRepeats: number; + debug: boolean; + params?: any; +} From a1a42a7f0aecc5042317334e284a8d06ddca01c2 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sat, 1 Jun 2024 11:35:18 +0530 Subject: [PATCH 40/40] feat(ts): interface RemoteBrowserOptions --- server/src/types/index.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/server/src/types/index.ts b/server/src/types/index.ts index dfcaa12f..44647d9a 100644 --- a/server/src/types/index.ts +++ b/server/src/types/index.ts @@ -10,3 +10,18 @@ export interface InterpreterSettings { debug: boolean; params?: any; } + + +/** + * Options for the {@link BrowserManagement.launch} method. + * Wraps the Playwright's launchOptions and adds an extra browser option. + * The browser option determines which browser to launch as Playwright + * supports multiple browsers. (chromium, firefox, webkit) + * -- Possible expansion for the future of the browser recorder -- + * @category Types + */ +export interface RemoteBrowserOptions { + browser: BrowserType + launchOptions: LaunchOptions +}; +