From 04d677505767a18799c3909ad394701d168bb6ea Mon Sep 17 00:00:00 2001 From: Rohit Rajan Date: Sun, 24 Aug 2025 14:22:35 +0530 Subject: [PATCH 1/6] feat: specify commonjs for workers, rm export brwoserpool --- server/src/server.ts | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/server/src/server.ts b/server/src/server.ts index 1e5c5a03..9016b73a 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -10,7 +10,6 @@ import logger from './logger'; import { connectDB, syncDB } from './storage/db' import cookieParser from 'cookie-parser'; import { SERVER_PORT } from "./constants/config"; -import { Server } from "socket.io"; import { readdirSync } from "fs" import { fork } from 'child_process'; import { capture } from "./utils/analytics"; @@ -77,12 +76,12 @@ const server = http.createServer(app); * Globally exported singleton instance of socket.io for socket communication with the client. * @type {Server} */ -export const io = new Server(server); +// export const io = new Server(server); /** * {@link BrowserPool} globally exported singleton instance for managing browsers. */ -export const browserPool = new BrowserPool(); +// export const browserPool = new BrowserPool(); app.use(cookieParser()) @@ -114,8 +113,20 @@ let recordingWorkerProcess: any; if (!isProduction) { workerProcess = fork(workerPath, [], { - execArgv: ['--inspect=5859'], + execArgv: [ + '--require', 'ts-node/register', + '--inspect=5859' + ], + env: { + ...process.env, + TS_NODE_COMPILER_OPTIONS: JSON.stringify({ + "module": "commonjs", + "moduleResolution": "node", + "esModuleInterop": true + }) + } }); + workerProcess.on('message', (message: any) => { console.log(`Message from worker: ${message}`); }); @@ -127,8 +138,20 @@ if (!isProduction) { }); recordingWorkerProcess = fork(recordingWorkerPath, [], { - execArgv: ['--inspect=5860'], + execArgv: [ + '--require', 'ts-node/register', + '--inspect=5860' + ], + env: { + ...process.env, + TS_NODE_COMPILER_OPTIONS: JSON.stringify({ + "module": "commonjs", + "moduleResolution": "node", + "esModuleInterop": true + }) + } }); + recordingWorkerProcess.on('message', (message: any) => { console.log(`Message from recording worker: ${message}`); }); From 63ab4360e311ec45d70051db887dddb5eecf296f Mon Sep 17 00:00:00 2001 From: Rohit Rajan Date: Sun, 24 Aug 2025 15:18:46 +0530 Subject: [PATCH 2/6] feat: init browser pool and io shared instance --- src/shared/instances.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/shared/instances.ts diff --git a/src/shared/instances.ts b/src/shared/instances.ts new file mode 100644 index 00000000..b855ad4f --- /dev/null +++ b/src/shared/instances.ts @@ -0,0 +1,21 @@ +import { BrowserPool } from "../../server/src/browser-management/classes/BrowserPool" +import { Server } from "socket.io"; +import http from 'http'; + +/** + * Shared browser pool instance + */ +export const browserPool = new BrowserPool(); + +/** + * Shared socket.io instance - will be initialized by the main server + */ +export let io: Server; + +/** + * Initialize the socket.io instance (called only by main server) + */ +export function initializeSocketIO(server: http.Server): Server { + io = new Server(server); + return io; +} \ No newline at end of file From 37e9f29d8fcef78eb045a13632de4cf22d2eee4a Mon Sep 17 00:00:00 2001 From: Rohit Rajan Date: Sun, 24 Aug 2025 15:49:29 +0530 Subject: [PATCH 3/6] feat: rm shared instances logic --- src/shared/instances.ts | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 src/shared/instances.ts diff --git a/src/shared/instances.ts b/src/shared/instances.ts deleted file mode 100644 index b855ad4f..00000000 --- a/src/shared/instances.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { BrowserPool } from "../../server/src/browser-management/classes/BrowserPool" -import { Server } from "socket.io"; -import http from 'http'; - -/** - * Shared browser pool instance - */ -export const browserPool = new BrowserPool(); - -/** - * Shared socket.io instance - will be initialized by the main server - */ -export let io: Server; - -/** - * Initialize the socket.io instance (called only by main server) - */ -export function initializeSocketIO(server: http.Server): Server { - io = new Server(server); - return io; -} \ No newline at end of file From 527b23bb92d2f3b267f3b59cff5bc235c07e6928 Mon Sep 17 00:00:00 2001 From: Rohit Rajan Date: Sun, 24 Aug 2025 16:00:13 +0530 Subject: [PATCH 4/6] feat: rand port assign worker, check module required --- server/src/server.ts | 105 +++++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 48 deletions(-) diff --git a/server/src/server.ts b/server/src/server.ts index 9016b73a..d133ca6e 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -1,6 +1,7 @@ import express from 'express'; import path from 'path'; import http from 'http'; +import { Server } from "socket.io"; import cors from 'cors'; import dotenv from 'dotenv'; dotenv.config(); @@ -74,14 +75,13 @@ const server = http.createServer(app); /** * Globally exported singleton instance of socket.io for socket communication with the client. - * @type {Server} */ -// export const io = new Server(server); +export let io: Server; /** * {@link BrowserPool} globally exported singleton instance for managing browsers. */ -// export const browserPool = new BrowserPool(); +export const browserPool = new BrowserPool(); app.use(cookieParser()) @@ -111,11 +111,11 @@ const recordingWorkerPath = path.resolve(__dirname, isProduction ? './pgboss-wor let workerProcess: any; let recordingWorkerProcess: any; -if (!isProduction) { +if (!isProduction && require.main === module) { workerProcess = fork(workerPath, [], { execArgv: [ '--require', 'ts-node/register', - '--inspect=5859' + '--inspect=0' ], env: { ...process.env, @@ -140,7 +140,7 @@ if (!isProduction) { recordingWorkerProcess = fork(recordingWorkerPath, [], { execArgv: [ '--require', 'ts-node/register', - '--inspect=5860' + '--inspect=0' ], env: { ...process.env, @@ -183,49 +183,58 @@ app.use((req, res, next) => { next(); }); -setInterval(() => { - processQueuedRuns(); -}, 5000); +if (require.main === module) { + setInterval(() => { + processQueuedRuns(); + }, 5000); +} -server.listen(SERVER_PORT, '0.0.0.0', async () => { - try { - await connectDB(); - await syncDB(); - logger.log('info', `Server listening on port ${SERVER_PORT}`); - } catch (error: any) { - logger.log('error', `Failed to connect to the database: ${error.message}`); - process.exit(1); - } -}); +if (require.main === module) { + server.listen(SERVER_PORT, '0.0.0.0', async () => { + try { + await connectDB(); + await syncDB(); + + io = new Server(server); + + logger.log('info', `Server listening on port ${SERVER_PORT}`); + } catch (error: any) { + logger.log('error', `Failed to connect to the database: ${error.message}`); + process.exit(1); + } + }); +} -process.on('SIGINT', async () => { - console.log('Main app shutting down...'); - try { - await Run.update( - { - status: 'failed', - finishedAt: new Date().toLocaleString(), - log: 'Process interrupted during execution - worker shutdown' - }, - { - where: { status: 'running' } - } - ); - } catch (error: any) { - console.error('Error updating runs:', error); - } +if (require.main === module) { + process.on('SIGINT', async () => { + console.log('Main app shutting down...'); + try { + await Run.update( + { + status: 'failed', + finishedAt: new Date().toLocaleString(), + log: 'Process interrupted during execution - worker shutdown' + }, + { + where: { status: 'running' } + } + ); + } catch (error: any) { + console.error('Error updating runs:', error); + } - try { - console.log('Closing PostgreSQL connection pool...'); - await pool.end(); - console.log('PostgreSQL connection pool closed'); - } catch (error) { - console.error('Error closing PostgreSQL connection pool:', error); - } + try { + console.log('Closing PostgreSQL connection pool...'); + await pool.end(); + console.log('PostgreSQL connection pool closed'); + } catch (error) { + console.error('Error closing PostgreSQL connection pool:', error); + } - if (!isProduction) { - if (workerProcess) workerProcess.kill(); - if (recordingWorkerProcess) recordingWorkerProcess.kill(); - } - process.exit(); -}); \ No newline at end of file + if (!isProduction) { + if (workerProcess) workerProcess.kill(); + if (recordingWorkerProcess) recordingWorkerProcess.kill(); + } + process.exit(); + }); +} \ No newline at end of file From e0ed2a4f01dc7cd830d6ed33972dc869602f3435 Mon Sep 17 00:00:00 2001 From: Rohit Rajan Date: Mon, 25 Aug 2025 16:12:50 +0530 Subject: [PATCH 5/6] feat: run worker same process non windows --- server/src/server.ts | 109 ++++++++++++++++++++++++------------------- 1 file changed, 62 insertions(+), 47 deletions(-) diff --git a/server/src/server.ts b/server/src/server.ts index d133ca6e..6a48d24e 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -112,55 +112,70 @@ let workerProcess: any; let recordingWorkerProcess: any; if (!isProduction && require.main === module) { - workerProcess = fork(workerPath, [], { - execArgv: [ - '--require', 'ts-node/register', - '--inspect=0' - ], - env: { - ...process.env, - TS_NODE_COMPILER_OPTIONS: JSON.stringify({ - "module": "commonjs", - "moduleResolution": "node", - "esModuleInterop": true - }) - } - }); + if (process.platform === 'win32') { + console.log('Using forked processes for Windows'); + + workerProcess = fork(workerPath, [], { + execArgv: [ + '--require', 'ts-node/register', + '--inspect=0' + ], + env: { + ...process.env, + TS_NODE_COMPILER_OPTIONS: JSON.stringify({ + "module": "commonjs", + "moduleResolution": "node", + "esModuleInterop": true + }) + } + }); - workerProcess.on('message', (message: any) => { - console.log(`Message from worker: ${message}`); - }); - workerProcess.on('error', (error: any) => { - console.error(`Error in worker: ${error}`); - }); - workerProcess.on('exit', (code: any) => { - console.log(`Worker exited with code: ${code}`); - }); + workerProcess.on('message', (message: any) => { + console.log(`Message from worker: ${message}`); + }); + workerProcess.on('error', (error: any) => { + console.error(`Error in worker: ${error}`); + }); + workerProcess.on('exit', (code: any) => { + console.log(`Worker exited with code: ${code}`); + }); - recordingWorkerProcess = fork(recordingWorkerPath, [], { - execArgv: [ - '--require', 'ts-node/register', - '--inspect=0' - ], - env: { - ...process.env, - TS_NODE_COMPILER_OPTIONS: JSON.stringify({ - "module": "commonjs", - "moduleResolution": "node", - "esModuleInterop": true - }) - } - }); + recordingWorkerProcess = fork(recordingWorkerPath, [], { + execArgv: [ + '--require', 'ts-node/register', + '--inspect=0' + ], + env: { + ...process.env, + TS_NODE_COMPILER_OPTIONS: JSON.stringify({ + "module": "commonjs", + "moduleResolution": "node", + "esModuleInterop": true + }) + } + }); - recordingWorkerProcess.on('message', (message: any) => { - console.log(`Message from recording worker: ${message}`); - }); - recordingWorkerProcess.on('error', (error: any) => { - console.error(`Error in recording worker: ${error}`); - }); - recordingWorkerProcess.on('exit', (code: any) => { - console.log(`Recording worker exited with code: ${code}`); - }); + recordingWorkerProcess.on('message', (message: any) => { + console.log(`Message from recording worker: ${message}`); + }); + recordingWorkerProcess.on('error', (error: any) => { + console.error(`Error in recording worker: ${error}`); + }); + recordingWorkerProcess.on('exit', (code: any) => { + console.log(`Recording worker exited with code: ${code}`); + }); + } else { + console.log('Running workers in same process for Unix-like systems'); + (async () => { + try { + await import('./schedule-worker'); + await import('./pgboss-worker'); + console.log('Workers started in main process for memory sharing'); + } catch (error) { + console.error('Failed to start workers in main process:', error); + } + })(); + } } app.get('/', function (req, res) { @@ -231,7 +246,7 @@ if (require.main === module) { console.error('Error closing PostgreSQL connection pool:', error); } - if (!isProduction) { + if (!isProduction && process.platform === 'win32') { if (workerProcess) workerProcess.kill(); if (recordingWorkerProcess) recordingWorkerProcess.kill(); } From f3207adca4905866d7b5c8f9f9db2ef6eb20915c Mon Sep 17 00:00:00 2001 From: Rohit Date: Mon, 25 Aug 2025 18:33:59 +0530 Subject: [PATCH 6/6] feat: start worker based on paltform --- server/src/server.ts | 106 ++++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 67 deletions(-) diff --git a/server/src/server.ts b/server/src/server.ts index 6a48d24e..ca2470c5 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -111,73 +111,6 @@ const recordingWorkerPath = path.resolve(__dirname, isProduction ? './pgboss-wor let workerProcess: any; let recordingWorkerProcess: any; -if (!isProduction && require.main === module) { - if (process.platform === 'win32') { - console.log('Using forked processes for Windows'); - - workerProcess = fork(workerPath, [], { - execArgv: [ - '--require', 'ts-node/register', - '--inspect=0' - ], - env: { - ...process.env, - TS_NODE_COMPILER_OPTIONS: JSON.stringify({ - "module": "commonjs", - "moduleResolution": "node", - "esModuleInterop": true - }) - } - }); - - workerProcess.on('message', (message: any) => { - console.log(`Message from worker: ${message}`); - }); - workerProcess.on('error', (error: any) => { - console.error(`Error in worker: ${error}`); - }); - workerProcess.on('exit', (code: any) => { - console.log(`Worker exited with code: ${code}`); - }); - - recordingWorkerProcess = fork(recordingWorkerPath, [], { - execArgv: [ - '--require', 'ts-node/register', - '--inspect=0' - ], - env: { - ...process.env, - TS_NODE_COMPILER_OPTIONS: JSON.stringify({ - "module": "commonjs", - "moduleResolution": "node", - "esModuleInterop": true - }) - } - }); - - recordingWorkerProcess.on('message', (message: any) => { - console.log(`Message from recording worker: ${message}`); - }); - recordingWorkerProcess.on('error', (error: any) => { - console.error(`Error in recording worker: ${error}`); - }); - recordingWorkerProcess.on('exit', (code: any) => { - console.log(`Recording worker exited with code: ${code}`); - }); - } else { - console.log('Running workers in same process for Unix-like systems'); - (async () => { - try { - await import('./schedule-worker'); - await import('./pgboss-worker'); - console.log('Workers started in main process for memory sharing'); - } catch (error) { - console.error('Failed to start workers in main process:', error); - } - })(); - } -} - app.get('/', function (req, res) { capture( 'maxun-oss-server-run', { @@ -212,6 +145,45 @@ if (require.main === module) { io = new Server(server); + if (!isProduction) { + if (process.platform === 'win32') { + workerProcess = fork(workerPath, [], { + execArgv: ['--inspect=5859'], + }); + workerProcess.on('message', (message: any) => { + console.log(`Message from worker: ${message}`); + }); + workerProcess.on('error', (error: any) => { + console.error(`Error in worker: ${error}`); + }); + workerProcess.on('exit', (code: any) => { + console.log(`Worker exited with code: ${code}`); + }); + + recordingWorkerProcess = fork(recordingWorkerPath, [], { + execArgv: ['--inspect=5860'], + }); + recordingWorkerProcess.on('message', (message: any) => { + console.log(`Message from recording worker: ${message}`); + }); + recordingWorkerProcess.on('error', (error: any) => { + console.error(`Error in recording worker: ${error}`); + }); + recordingWorkerProcess.on('exit', (code: any) => { + console.log(`Recording worker exited with code: ${code}`); + }); + } else { + // Run in same process for non-Windows + try { + await import('./schedule-worker'); + await import('./pgboss-worker'); + console.log('Workers started in main process for memory sharing'); + } catch (error) { + console.error('Failed to start workers in main process:', error); + } + } + } + logger.log('info', `Server listening on port ${SERVER_PORT}`); } catch (error: any) { logger.log('error', `Failed to connect to the database: ${error.message}`);