diff --git a/server/src/browser-management/classes/BrowserPool.ts b/server/src/browser-management/classes/BrowserPool.ts new file mode 100644 index 00000000..736189db --- /dev/null +++ b/server/src/browser-management/classes/BrowserPool.ts @@ -0,0 +1,100 @@ +import { RemoteBrowser } from "./RemoteBrowser"; +import logger from "../../logger"; + +/** + * @category Types + */ +interface BrowserPoolInfo { + /** + * The instance of remote browser. + */ + browser: RemoteBrowser, + /** + * States if the browser's instance is being actively used. + * Helps to persist the progress on the frontend when the application has been reloaded. + * @default false + */ + active: boolean, +} + +/** + * Dictionary of all the active remote browser's instances indexed by their id. + * The value in this dictionary is of type BrowserPoolInfo, + * which provides additional information about the browser's usage. + * @category Types + */ +interface PoolDictionary { + [key: string]: BrowserPoolInfo, +} + +/** + * A browser pool is a collection of remote browsers that are initialized and ready to be used. + * Adds the possibility to add, remove and retrieve remote browsers from the pool. + * It is possible to manage multiple browsers for creating or running a recording. + * @category BrowserManagement + */ +export class BrowserPool { + + /** + * Holds all the instances of remote browsers. + */ + private pool : PoolDictionary = {}; + + /** + * Adds a remote browser instance to the pool indexed by the id. + * @param id remote browser instance's id + * @param browser remote browser instance + * @param active states if the browser's instance is being actively used + */ + public addRemoteBrowser = (id: string, browser: RemoteBrowser, active: boolean = false): void => { + this.pool = { + ...this.pool, + [id]: { + browser, + active, + }, + } + logger.log('debug', `Remote browser with id: ${id} added to the pool`); + }; + + /** + * Removes the remote browser instance from the pool. + * @param id remote browser instance's id + * @returns true if the browser was removed successfully, false otherwise + */ + public deleteRemoteBrowser = (id: string) : boolean => { + if (!this.pool[id]) { + logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`); + return false; + } + delete(this.pool[id]); + logger.log('debug', `Remote browser with id: ${id} deleted from the pool`); + return true; + }; + + /** + * Returns the remote browser instance from the pool. + * @param id remote browser instance's id + * @returns remote browser instance or undefined if it does not exist in the pool + */ + public getRemoteBrowser = (id: string) : RemoteBrowser | undefined => { + logger.log('debug', `Remote browser with id: ${id} retrieved from the pool`); + return this.pool[id]?.browser; + }; + + /** + * Returns the active browser's instance id from the pool. + * If there is no active browser, it returns undefined. + * If there are multiple active browsers, it returns the first one. + * @returns the first remote active browser instance's id from the pool + */ + public getActiveBrowserId = () : string | null => { + for (const id of Object.keys(this.pool)) { + if (this.pool[id].active) { + return id; + } + } + logger.log('warn', `No active browser in the pool`); + return null; + }; +} diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts new file mode 100644 index 00000000..f5527581 --- /dev/null +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -0,0 +1,344 @@ +import { + Page, + Browser, + CDPSession, + BrowserContext, +} from 'playwright'; +import { Socket } from "socket.io"; + +import logger from '../../logger'; +import { InterpreterSettings, RemoteBrowserOptions } from "../../types"; +import { WorkflowGenerator } from "../../workflow-management/classes/Generator"; +import { WorkflowInterpreter } from "../../workflow-management/classes/Interpreter"; + +/** + * This class represents a remote browser instance. + * It is used to allow a variety of interaction with the Playwright's browser instance. + * Every remote browser holds an instance of a generator and interpreter classes with + * the purpose of generating and interpreting workflows. + * @category BrowserManagement + */ +export class RemoteBrowser { + + /** + * Playwright's [browser](https://playwright.dev/docs/api/class-browser) instance. + * @private + */ + private browser: Browser | null = null; + + /** + * The Playwright's [CDPSession](https://playwright.dev/docs/api/class-cdpsession) instance, + * used to talk raw Chrome Devtools Protocol. + * @private + */ + private client: CDPSession | null | undefined = null; + + /** + * Socket.io socket instance enabling communication with the client (frontend) side. + * @private + */ + private socket: Socket; + + /** + * The Playwright's [Page](https://playwright.dev/docs/api/class-page) instance + * as current interactive remote browser's page. + * @private + */ + private currentPage: Page | null | undefined = null; + + /** + * Interpreter settings for any started interpretation. + * @private + */ + private interpreterSettings: InterpreterSettings = { + debug: false, + maxConcurrency: 1, + maxRepeats: 1, + }; + + /** + * {@link WorkflowGenerator} instance specific to the remote browser. + */ + public generator: WorkflowGenerator; + + /** + * {@link WorkflowInterpreter} instance specific to the remote browser. + */ + public interpreter: WorkflowInterpreter; + + /** + * Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and + * assigns the socket instance everywhere. + * @param socket socket.io socket instance used to communicate with the client side + * @constructor + */ + public constructor(socket: Socket) { + this.socket = socket; + this.interpreter = new WorkflowInterpreter(socket); + this.generator = new WorkflowGenerator(socket); + } + + /** + * An asynchronous constructor for asynchronously initialized properties. + * Must be called right after creating an instance of RemoteBrowser class. + * @param options remote browser options to be used when launching the browser + * @returns {Promise} + */ + public initialize = async (options: RemoteBrowserOptions): Promise => { + this.browser = (await options.browser.launch(options.launchOptions)); + const context = await this.browser.newContext(); + this.currentPage = await context.newPage(); + this.client = await this.currentPage.context().newCDPSession(this.currentPage); + }; + + /** + * Registers all event listeners needed for the recording editor session. + * Should be called only once after the full initialization of the remote browser. + * @returns void + */ + public registerEditorEvents = (): void => { + this.socket.on('rerender', async () => await this.makeAndEmitScreenshot()); + this.socket.on('settings', (settings) => this.interpreterSettings = settings); + this.socket.on('changeTab', async (tabIndex) => await this.changeTab(tabIndex)); + this.socket.on('addTab', async () => { + await this.currentPage?.context().newPage(); + const lastTabIndex = this.currentPage ? this.currentPage.context().pages().length - 1 : 0; + await this.changeTab(lastTabIndex); + }); + this.socket.on('closeTab', async (tabInfo) => { + const page = this.currentPage?.context().pages()[tabInfo.index]; + if (page) { + if (tabInfo.isCurrent) { + if (this.currentPage?.context().pages()[tabInfo.index + 1]) { + // next tab + await this.changeTab(tabInfo.index + 1); + } else { + //previous tab + await this.changeTab(tabInfo.index - 1); + } + } + // close the page and log it + await page.close(); + logger.log( + 'debug', + `${tabInfo.index} page was closed, new length of pages: ${this.currentPage?.context().pages().length}` + ) + } else { + logger.log('error', `${tabInfo.index} index out of range of pages`) + } + }); + } + + /** + * Subscribes the remote browser for a screencast session + * on [CDP](https://chromedevtools.github.io/devtools-protocol/) level, + * where screenshot is being sent through the socket + * every time the browser's active page updates. + * @returns {Promise} + */ + public subscribeToScreencast = async (): Promise => { + await this.startScreencast(); + if (!this.client) { + logger.log('warn', 'client is not initialized'); + return; + } + this.client.on('Page.screencastFrame', ({ data: base64, sessionId }) => { + this.emitScreenshot(base64); + setTimeout(async () => { + try { + if (!this.client) { + logger.log('warn', 'client is not initialized'); + return; + } + await this.client.send('Page.screencastFrameAck', { sessionId: sessionId }); + } catch (e) { + logger.log('error', e); + } + }, 100); + }); + }; + + /** + * Terminates the screencast session and closes the remote browser. + * If an interpretation was running it will be stopped. + * @returns {Promise} + */ + public switchOff = async (): Promise => { + await this.interpreter.stopInterpretation(); + if (this.browser) { + await this.stopScreencast(); + await this.browser.close(); + } else { + logger.log('error', 'Browser wasn\'t initialized'); + logger.log('error', 'Switching off the browser failed'); + } + }; + + /** + * Makes and emits a single screenshot to the client side. + * @returns {Promise} + */ + public makeAndEmitScreenshot = async (): Promise => { + try { + const screenshot = await this.currentPage?.screenshot(); + if (screenshot) { + this.emitScreenshot(screenshot.toString('base64')); + } + } catch (e) { + const { message } = e as Error; + logger.log('error', message); + } + }; + + /** + * Updates the active socket instance. + * This will update all registered events for the socket and + * all the properties using the socket. + * @param socket socket.io socket instance used to communicate with the client side + * @returns void + */ + public updateSocket = (socket: Socket): void => { + this.socket = socket; + this.registerEditorEvents(); + this.generator?.updateSocket(socket); + this.interpreter?.updateSocket(socket); + }; + + /** + * Starts the interpretation of the currently generated workflow. + * @returns {Promise} + */ + public interpretCurrentRecording = async (): Promise => { + logger.log('debug', 'Starting interpretation in the editor'); + if (this.generator) { + const workflow = this.generator.AddGeneratedFlags(this.generator.getWorkflowFile()); + await this.initializeNewPage(); + if (this.currentPage) { + const params = this.generator.getParams(); + if (params) { + this.interpreterSettings.params = params.reduce((acc, param) => { + if (this.interpreterSettings.params && Object.keys(this.interpreterSettings.params).includes(param)) { + return { ...acc, [param]: this.interpreterSettings.params[param] }; + } else { + return { ...acc, [param]: '', } + } + }, {}) + } + logger.log('debug', `Starting interpretation with settings: ${JSON.stringify(this.interpreterSettings, null, 2)}`); + await this.interpreter.interpretRecordingInEditor( + workflow, this.currentPage, + (newPage: Page) => this.currentPage = newPage, + this.interpreterSettings + ); + // clear the active index from generator + this.generator.clearLastIndex(); + } else { + logger.log('error', 'Could not get a new page, returned undefined'); + } + } else { + logger.log('error', 'Generator is not initialized'); + } + }; + + /** + * Stops the workflow interpretation and initializes a new page. + * @returns {Promise} + */ + public stopCurrentInterpretation = async (): Promise => { + await this.interpreter.stopInterpretation(); + await this.initializeNewPage(); + }; + + /** + * Returns the current page instance. + * @returns {Page | null | undefined} + */ + public getCurrentPage = (): Page | null | undefined => { + return this.currentPage; + }; + + /** + * Changes the active page to the page instance on the given index + * available in pages array on the {@link BrowserContext}. + * Automatically stops the screencast session on the previous page and starts the new one. + * @param tabIndex index of the page in the pages array on the {@link BrowserContext} + * @returns {Promise} + */ + private changeTab = async (tabIndex: number): Promise => { + const page = this.currentPage?.context().pages()[tabIndex]; + if (page) { + await this.stopScreencast(); + this.currentPage = page; + await this.currentPage.setViewportSize({ height: 720, width: 1280 }) + this.client = await this.currentPage.context().newCDPSession(this.currentPage); + this.socket.emit('urlChanged', this.currentPage.url()); + await this.makeAndEmitScreenshot(); + await this.subscribeToScreencast(); + } else { + logger.log('error', `${tabIndex} index out of range of pages`) + } + } + + /** + * Internal method for a new page initialization. Subscribes this page to the screencast. + * @param options optional page options to be used when creating a new page + * @returns {Promise} + */ + private initializeNewPage = async (options?: Object): Promise => { + await this.stopScreencast(); + const newPage = options ? await this.browser?.newPage(options) + : await this.browser?.newPage(); + + await this.currentPage?.close(); + this.currentPage = newPage; + if (this.currentPage) { + this.currentPage.on('load', (page) => { + this.socket.emit('urlChanged', page.url()); + }) + this.client = await this.currentPage.context().newCDPSession(this.currentPage); + await this.subscribeToScreencast(); + } else { + logger.log('error', 'Could not get a new page, returned undefined'); + } + }; + + /** + * Initiates screencast of the remote browser through socket, + * registers listener for rerender event and emits the loaded event. + * Should be called only once after the browser is fully initialized. + * @returns {Promise} + */ + private startScreencast = async (): Promise => { + if (!this.client) { + logger.log('warn', 'client is not initialized'); + return; + } + await this.client.send('Page.startScreencast', { format: 'jpeg', quality: 75 }); + logger.log('info', `Browser started with screencasting a page.`); + }; + + /** + * Unsubscribes the current page from the screencast session. + * @returns {Promise} + */ + private stopScreencast = async (): Promise => { + if (!this.client) { + logger.log('error', 'client is not initialized'); + logger.log('error', 'Screencast stop failed'); + } else { + await this.client.send('Page.stopScreencast'); + logger.log('info', `Browser stopped with screencasting.`); + } + }; + + /** + * Helper for emitting the screenshot of browser's active page through websocket. + * @param payload the screenshot binary data + * @returns void + */ + private emitScreenshot = (payload: any): void => { + const dataWithMimeType = ('data:image/jpeg;base64,').concat(payload); + this.socket.emit('screencast', dataWithMimeType); + logger.log('debug', `Screenshot emitted`); + }; +} diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts new file mode 100644 index 00000000..62a3de66 --- /dev/null +++ b/server/src/browser-management/controller.ts @@ -0,0 +1,157 @@ +/** + * The main function group which determines the flow of remote browser management. + * Holds the singleton instances of browser pool and socket.io server. + */ +import { Socket } from "socket.io"; +import { uuid } from 'uuidv4'; + +import { createSocketConnection, createSocketConnectionForRun } from "../socket-connection/connection"; +import { io, browserPool } from "../server"; +import { RemoteBrowser } from "./classes/RemoteBrowser"; +import { RemoteBrowserOptions } from "../types"; +import logger from "../logger"; + +/** + * Starts and initializes a {@link RemoteBrowser} instance. + * Creates a new socket connection over a dedicated namespace + * and registers all interaction event handlers. + * Returns the id of an active browser or the new remote browser's generated id. + * @param options {@link RemoteBrowserOptions} to be used when launching the browser + * @returns string + * @category BrowserManagement-Controller + */ +export const initializeRemoteBrowserForRecording = (options: RemoteBrowserOptions): string => { + const id = getActiveBrowserId() || uuid(); + createSocketConnection( + io.of(id), + async (socket: Socket) => { + // browser is already active + const activeId = getActiveBrowserId(); + if (activeId) { + const remoteBrowser = browserPool.getRemoteBrowser(activeId); + remoteBrowser?.updateSocket(socket); + await remoteBrowser?.makeAndEmitScreenshot(); + } else { + const browserSession = new RemoteBrowser(socket); + browserSession.interpreter.subscribeToPausing(); + await browserSession.initialize(options); + await browserSession.registerEditorEvents(); + await browserSession.subscribeToScreencast(); + browserPool.addRemoteBrowser(id, browserSession, true); + } + socket.emit('loaded'); + }); + return id; +}; + +/** + * Starts and initializes a {@link RemoteBrowser} instance for interpretation. + * Creates a new {@link Socket} connection over a dedicated namespace. + * Returns the new remote browser's generated id. + * @param options {@link RemoteBrowserOptions} to be used when launching the browser + * @returns string + * @category BrowserManagement-Controller + */ +export const createRemoteBrowserForRun = (options: RemoteBrowserOptions): string => { + const id = uuid(); + createSocketConnectionForRun( + io.of(id), + async (socket: Socket) => { + const browserSession = new RemoteBrowser(socket); + await browserSession.initialize(options); + browserPool.addRemoteBrowser(id, browserSession, true); + socket.emit('ready-for-run'); + }); + return id; +}; + +/** + * Terminates a remote browser recording session + * and removes the browser from the browser pool. + * @param id instance id of the remote browser to be terminated + * @returns {Promise} + * @category BrowserManagement-Controller + */ +export const destroyRemoteBrowser = async (id: string): Promise => { + const browserSession = browserPool.getRemoteBrowser(id); + if (browserSession) { + logger.log('debug', `Switching off the browser with id: ${id}`); + await browserSession.stopCurrentInterpretation(); + await browserSession.switchOff(); + } + return browserPool.deleteRemoteBrowser(id); +}; + +/** + * Returns the id of an active browser or null. + * Wrapper around {@link browserPool.getActiveBrowserId()} function. + * @returns {string | null} + * @category BrowserManagement-Controller + */ +export const getActiveBrowserId = (): string | null => { + return browserPool.getActiveBrowserId(); +}; + +/** + * Returns the url string from a remote browser if exists in the browser pool. + * @param id instance id of the remote browser + * @returns {string | undefined} + * @category BrowserManagement-Controller + */ +export const getRemoteBrowserCurrentUrl = (id: string): string | undefined => { + return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.url(); +}; + +/** + * Returns the array of tab strings from a remote browser if exists in the browser pool. + * @param id instance id of the remote browser + * @return {string[] | undefined} + * @category BrowserManagement-Controller + */ +export const getRemoteBrowserCurrentTabs = (id: string): string[] | undefined => { + return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.context().pages() + .map((page) => { + const parsedUrl = new URL(page.url()); + const host = parsedUrl.hostname.match(/\b(?!www\.)[a-zA-Z0-9]+/g)?.join('.'); + if (host) { + return host; + } + return 'new tab'; + }); +}; + +/** + * Interprets the currently generated workflow in the active browser instance. + * If there is no active browser, the function logs an error. + * @returns {Promise} + * @category BrowserManagement-Controller + */ +export const interpretWholeWorkflow = async () => { + const id = getActiveBrowserId(); + if (id) { + const browser = browserPool.getRemoteBrowser(id); + if (browser) { + await browser.interpretCurrentRecording(); + } else { + logger.log('error', `No active browser with id ${id} found in the browser pool`); + } + } else { + logger.log('error', `Cannot interpret the workflow: bad id ${id}.`); + } +}; + +/** + * Stops the interpretation of the current workflow in the active browser instance. + * If there is no active browser, the function logs an error. + * @returns {Promise} + * @category BrowserManagement-Controller + */ +export const stopRunningInterpretation = async () => { + const id = getActiveBrowserId(); + if (id) { + const browser = browserPool.getRemoteBrowser(id); + await browser?.stopCurrentInterpretation(); + } else { + logger.log('error', 'Cannot stop interpretation: No active browser or generator.'); + } +}; diff --git a/server/src/constants/config.ts b/server/src/constants/config.ts new file mode 100644 index 00000000..199e36fc --- /dev/null +++ b/server/src/constants/config.ts @@ -0,0 +1,3 @@ +export const SERVER_PORT = process.env.SERVER_PORT ? Number(process.env.SERVER_PORT) : 8080 +export const DEBUG = process.env.DEBUG === 'true' +export const LOGS_PATH = process.env.LOGS_PATH ?? 'server/logs' diff --git a/server/src/logger.ts b/server/src/logger.ts new file mode 100644 index 00000000..d0ecc24e --- /dev/null +++ b/server/src/logger.ts @@ -0,0 +1,19 @@ +import { createLogger, format, transports } from 'winston'; +import { DEBUG, LOGS_PATH } from "./constants/config"; + +const { combine, timestamp, printf } = format; + +const logger = createLogger({ + format: combine( + timestamp(), + printf(info => `${info.timestamp} ${info.level}: ${info.message}`), + ), + defaultMeta: { service: 'user-service' }, + transports: [ + new transports.Console({ level: DEBUG ? 'info' : 'debug' }), + new transports.File({ filename: `${LOGS_PATH}/error.log`, level: 'error' }), + new transports.File({ filename: `${LOGS_PATH}/combined.log`, level: 'debug' }), + ], +}); + +export default logger; diff --git a/server/src/server.ts b/server/src/server.ts new file mode 100644 index 00000000..6b34de08 --- /dev/null +++ b/server/src/server.ts @@ -0,0 +1,37 @@ +import express from 'express'; +import http from 'http'; +import cors from 'cors'; +import 'dotenv/config'; + +import { record, workflow, storage } from './routes'; +import { BrowserPool } from "./browser-management/classes/BrowserPool"; +import logger from './logger' +import { SERVER_PORT } from "./constants/config"; +import {Server} from "socket.io"; + +const app = express(); +app.use(cors()); +app.use(express.json()); + +const server = http.createServer(app); + +/** + * Globally exported singleton instance of socket.io for socket communication with the client. + * @type {Server} + */ +export const io = new Server(server); + +/** + * {@link BrowserPool} globally exported singleton instance for managing browsers. + */ +export const browserPool = new BrowserPool(); + +app.use('/record', record); +app.use('/workflow', workflow); +app.use('/storage', storage); + +app.get('/', function (req, res) { + return res.send('Welcome to the BR recorder server :-)'); +}); + +server.listen(SERVER_PORT, () => logger.log('info',`Server listening on port ${SERVER_PORT}`)); diff --git a/server/src/socket-connection/connection.ts b/server/src/socket-connection/connection.ts new file mode 100644 index 00000000..1390bcbf --- /dev/null +++ b/server/src/socket-connection/connection.ts @@ -0,0 +1,42 @@ +import {Namespace, Socket} from 'socket.io'; +import logger from "../logger"; + +/** + * Opens a websocket canal for duplex data transfer and registers all handlers for this data for the recording session. + * Uses socket.io dynamic namespaces for multiplexing the traffic from different running remote browser instances. + * @param io dynamic namespace on the socket.io server + * @param callback function called after the connection is created providing the socket resource + * @category BrowserManagement + */ +export const createSocketConnection = ( + io: Namespace, + callback: (socket: Socket) => void, + ) => { + const onConnection = async (socket: Socket) => { + logger.log('info',"Client connected " + socket.id); + socket.on('disconnect', () => logger.log('info', "Client disconnected " + socket.id)); + callback(socket); + } + + io.on('connection', onConnection); +}; + +/** + * Opens a websocket canal for duplex data transfer for the recording run. + * Uses socket.io dynamic namespaces for multiplexing the traffic from different running remote browser instances. + * @param io dynamic namespace on the socket.io server + * @param callback function called after the connection is created providing the socket resource + * @category BrowserManagement + */ +export const createSocketConnectionForRun = ( + io: Namespace, + callback: (socket: Socket) => void, +) => { + const onConnection = async (socket: Socket) => { + logger.log('info',"Client connected " + socket.id); + socket.on('disconnect', () => logger.log('info', "Client disconnected " + socket.id)); + callback(socket); + } + + io.on('connection', onConnection); +}; diff --git a/server/src/types/index.ts b/server/src/types/index.ts new file mode 100644 index 00000000..44647d9a --- /dev/null +++ b/server/src/types/index.ts @@ -0,0 +1,27 @@ +import {BrowserType, LaunchOptions} from "playwright"; + +/** + * Interpreter settings properties including recording parameters. + * @category Types + */ +export interface InterpreterSettings { + maxConcurrency: number; + maxRepeats: number; + debug: boolean; + params?: any; +} + + +/** + * Options for the {@link BrowserManagement.launch} method. + * Wraps the Playwright's launchOptions and adds an extra browser option. + * The browser option determines which browser to launch as Playwright + * supports multiple browsers. (chromium, firefox, webkit) + * -- Possible expansion for the future of the browser recorder -- + * @category Types + */ +export interface RemoteBrowserOptions { + browser: BrowserType + launchOptions: LaunchOptions +}; +