From d9fcb865075e8d1f84d2e0847df17d06d0744c1c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Mon, 14 Oct 2024 23:32:44 +0530 Subject: [PATCH 01/50] chore(deps): install minio --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 0daed06c..1a6d7d8b 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "jsonwebtoken": "^9.0.2", "loglevel": "^1.8.0", "loglevel-plugin-remote": "^0.6.8", + "minio": "^8.0.1", "moment-timezone": "^0.5.45", "node-cron": "^3.0.3", "pg": "^8.13.0", From 4fc8273393895dba160cd884386dd4e21652a155 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Mon, 14 Oct 2024 23:44:22 +0530 Subject: [PATCH 02/50] refactor: rename to storage --- server/src/storage/config.ts | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 server/src/storage/config.ts diff --git a/server/src/storage/config.ts b/server/src/storage/config.ts new file mode 100644 index 00000000..56c68d8b --- /dev/null +++ b/server/src/storage/config.ts @@ -0,0 +1,35 @@ +import { Sequelize } from 'sequelize'; +import dotenv from 'dotenv'; +import setupAssociations from '../models/associations'; + +dotenv.config(); +const sequelize = new Sequelize( + `postgresql://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`, + { + host: 'localhost', + dialect: 'postgres', + logging: false, + } +); + +export const connectDB = async () => { + try { + await sequelize.authenticate(); + console.log('Database connected successfully'); + } catch (error) { + console.error('Unable to connect to the database:', error); + } +}; + +export const syncDB = async () => { + try { + //setupAssociations(); + await sequelize.sync({ force: false }); // force: true will drop and recreate tables on every run + console.log('Database synced successfully!'); + } catch (error) { + console.error('Failed to sync database:', error); + } +}; + + +export default sequelize; \ No newline at end of file From 05b786ef56622d610bc046f85610427dac7116ba Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Mon, 14 Oct 2024 23:44:58 +0530 Subject: [PATCH 03/50] fix: broken storage config import --- server/src/models/Robot.ts | 2 +- server/src/models/Run.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/models/Robot.ts b/server/src/models/Robot.ts index 2400a917..53ccbb39 100644 --- a/server/src/models/Robot.ts +++ b/server/src/models/Robot.ts @@ -1,5 +1,5 @@ import { Model, DataTypes, Optional } from 'sequelize'; -import sequelize from '../db/config'; +import sequelize from '../storage/config'; import { WorkflowFile, Where, What, WhereWhatPair } from 'maxun-core'; interface RobotMeta { diff --git a/server/src/models/Run.ts b/server/src/models/Run.ts index 5c14dfd9..ddf31fd0 100644 --- a/server/src/models/Run.ts +++ b/server/src/models/Run.ts @@ -1,5 +1,5 @@ import { Model, DataTypes, Optional } from 'sequelize'; -import sequelize from '../db/config'; +import sequelize from '../storage/config'; import Robot from './Robot'; // TODO: From 89629ba9cf869862f6f531b995c2f44cf3ff5a58 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Mon, 14 Oct 2024 23:45:56 +0530 Subject: [PATCH 04/50] refactor: rename to db.ts --- server/src/storage/{config.ts => db.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename server/src/storage/{config.ts => db.ts} (100%) diff --git a/server/src/storage/config.ts b/server/src/storage/db.ts similarity index 100% rename from server/src/storage/config.ts rename to server/src/storage/db.ts From 1308c233596be05faebd132fb0137f35ef654598 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Mon, 14 Oct 2024 23:46:24 +0530 Subject: [PATCH 05/50] fix: broken storage db import --- server/src/models/Robot.ts | 2 +- server/src/models/Run.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/models/Robot.ts b/server/src/models/Robot.ts index 53ccbb39..ed9d7780 100644 --- a/server/src/models/Robot.ts +++ b/server/src/models/Robot.ts @@ -1,5 +1,5 @@ import { Model, DataTypes, Optional } from 'sequelize'; -import sequelize from '../storage/config'; +import sequelize from '../storage/db'; import { WorkflowFile, Where, What, WhereWhatPair } from 'maxun-core'; interface RobotMeta { diff --git a/server/src/models/Run.ts b/server/src/models/Run.ts index ddf31fd0..695c10a6 100644 --- a/server/src/models/Run.ts +++ b/server/src/models/Run.ts @@ -1,5 +1,5 @@ import { Model, DataTypes, Optional } from 'sequelize'; -import sequelize from '../storage/config'; +import sequelize from '../storage/db'; import Robot from './Robot'; // TODO: From 06baca27530379c9d5084763e10f8ab960de7391 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Mon, 14 Oct 2024 23:47:17 +0530 Subject: [PATCH 06/50] fix: broken storage db import --- server/src/server.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/server.ts b/server/src/server.ts index da5d3cbc..eb435345 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -7,7 +7,7 @@ dotenv.config(); import { record, workflow, storage, auth, integration, proxy } from './routes'; import { BrowserPool } from "./browser-management/classes/BrowserPool"; import logger from './logger'; -import { connectDB, syncDB } from './db/config'; +import { connectDB, syncDB } from './storage/db' import bodyParser from 'body-parser'; import cookieParser from 'cookie-parser'; import csrf from 'csurf'; @@ -62,7 +62,7 @@ const workerProcess = fork(path.resolve(__dirname, './worker.ts')); workerProcess.on('message', (message) => { console.log(`Message from worker: ${message}`); }); - workerProcess.on('error', (error) => { +workerProcess.on('error', (error) => { console.error(`Error in worker: ${error}`); }); workerProcess.on('exit', (code) => { @@ -81,6 +81,6 @@ server.listen(SERVER_PORT, async () => { process.on('SIGINT', () => { console.log('Main app shutting down...'); - workerProcess.kill(); + //workerProcess.kill(); process.exit(); }); From 8ae2b9f21fa1d2c6a962ae785cca649ae421a69b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Mon, 14 Oct 2024 23:48:31 +0530 Subject: [PATCH 07/50] feat: setup minio client --- server/src/storage/mino.ts | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 server/src/storage/mino.ts diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts new file mode 100644 index 00000000..b795f2aa --- /dev/null +++ b/server/src/storage/mino.ts @@ -0,0 +1,9 @@ +import { Client } from 'minio'; + +const minioClient = new Client({ + endPoint: process.env.MINIO_ENDPOINT || 'localhost', + port: parseInt(process.env.MINIO_PORT || '9000'), + useSSL: false, + accessKey: process.env.MINIO_ACCESS_KEY || 'minio-access-key', + secretKey: process.env.MINIO_SECRET_KEY || 'minio-secret-key', +}); From 8d01c9a67fdd658fb7d5fe541f4acbb50ac32587 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Mon, 14 Oct 2024 23:51:15 +0530 Subject: [PATCH 08/50] feat: exports --- server/src/storage/mino.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index b795f2aa..6cad167e 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -7,3 +7,6 @@ const minioClient = new Client({ accessKey: process.env.MINIO_ACCESS_KEY || 'minio-access-key', secretKey: process.env.MINIO_SECRET_KEY || 'minio-secret-key', }); + + +export default minioClient; \ No newline at end of file From 93bf6bcd3510775cf6bf666b94bb1b201dfae03f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 00:23:07 +0530 Subject: [PATCH 09/50] ts: set binaryOutput type to Record --- server/src/models/Run.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/models/Run.ts b/server/src/models/Run.ts index 695c10a6..17a53829 100644 --- a/server/src/models/Run.ts +++ b/server/src/models/Run.ts @@ -25,7 +25,7 @@ interface RunAttributes { log: string; runId: string; serializableOutput: Record; - binaryOutput: Record; + binaryOutput: Record; } interface RunCreationAttributes extends Optional { } From 06a9d4b394af1226ae4212fee3fa9a27360ccae6 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 00:26:14 +0530 Subject: [PATCH 10/50] feat: set binaryOutput default value as {} --- server/src/models/Run.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/models/Run.ts b/server/src/models/Run.ts index 17a53829..49fe47f7 100644 --- a/server/src/models/Run.ts +++ b/server/src/models/Run.ts @@ -104,6 +104,7 @@ Run.init( binaryOutput: { type: DataTypes.JSONB, allowNull: true, + defaultValue: {}, }, }, { From db448c130a35e905579777f736a786c23ffdb56e Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 01:05:49 +0530 Subject: [PATCH 11/50] feat: upload binary output to minio --- server/src/models/Run.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/models/Run.ts b/server/src/models/Run.ts index 49fe47f7..003e4ce8 100644 --- a/server/src/models/Run.ts +++ b/server/src/models/Run.ts @@ -1,6 +1,7 @@ import { Model, DataTypes, Optional } from 'sequelize'; import sequelize from '../storage/db'; import Robot from './Robot'; +import minioClient from '../storage/mino'; // TODO: // 1. rename variables @@ -44,6 +45,12 @@ class Run extends Model implements RunAttr public runId!: string; public serializableOutput!: Record; public binaryOutput!: Record; + + public async uploadBinaryOutput(key: string, data: Buffer): Promise { + const bucketName = ''; + await minioClient.putObject(bucketName, key, data); + this.binaryOutput[key] = `minio://${bucketName}/${key}`; + } } Run.init( From a99c6fba356053d723a297862692e1715dc0e98b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 01:06:14 +0530 Subject: [PATCH 12/50] refactor: rename to uploadBinaryOutputToMinioBucket --- server/src/models/Run.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/models/Run.ts b/server/src/models/Run.ts index 003e4ce8..d8abe0dd 100644 --- a/server/src/models/Run.ts +++ b/server/src/models/Run.ts @@ -46,7 +46,7 @@ class Run extends Model implements RunAttr public serializableOutput!: Record; public binaryOutput!: Record; - public async uploadBinaryOutput(key: string, data: Buffer): Promise { + public async uploadBinaryOutputToMinioBucket(key: string, data: Buffer): Promise { const bucketName = ''; await minioClient.putObject(bucketName, key, data); this.binaryOutput[key] = `minio://${bucketName}/${key}`; From 91025ad872c1d5d91ace0a08da6b464125668588 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 01:28:26 +0530 Subject: [PATCH 13/50] feat: get binary output from minio --- server/src/models/Run.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/models/Run.ts b/server/src/models/Run.ts index d8abe0dd..1ea1d4a8 100644 --- a/server/src/models/Run.ts +++ b/server/src/models/Run.ts @@ -51,6 +51,17 @@ class Run extends Model implements RunAttr await minioClient.putObject(bucketName, key, data); this.binaryOutput[key] = `minio://${bucketName}/${key}`; } + + public async getBinaryOutputFromMinioBucket(key: string): Promise { + const bucketName = ''; + const stream = await minioClient.getObject(bucketName, key); + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on('data', (chunk) => chunks.push(chunk)); + stream.on('end', () => resolve(Buffer.concat(chunks))); + stream.on('error', reject); + }); + } } Run.init( From 7c6ef5a491645b62728435fb90c7c5395d08aa5e Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 01:28:50 +0530 Subject: [PATCH 14/50] chore: whitespace --- server/src/storage/mino.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index 6cad167e..936ccc0f 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -8,5 +8,4 @@ const minioClient = new Client({ secretKey: process.env.MINIO_SECRET_KEY || 'minio-secret-key', }); - export default minioClient; \ No newline at end of file From 8965bdd6d13146e275114be0bc7cde00942ee7ca Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 01:46:49 +0530 Subject: [PATCH 15/50] chore: remove todo --- server/src/models/Run.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/models/Run.ts b/server/src/models/Run.ts index 1ea1d4a8..f26b7c61 100644 --- a/server/src/models/Run.ts +++ b/server/src/models/Run.ts @@ -4,9 +4,8 @@ import Robot from './Robot'; import minioClient from '../storage/mino'; // TODO: -// 1. rename variables -// 2. we might not need interpreter settings? -// 3. store binaryOutput in MinIO +// 1. we might not need interpreter settings? + interface InterpreterSettings { maxConcurrency: number; maxRepeats: number; From ade38ebf99c169f0e6816e9a5e1461b1aaecc55d Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 19:16:48 +0530 Subject: [PATCH 16/50] feat: binary output service --- server/src/storage/mino.ts | 42 +++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index 936ccc0f..62290ccb 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -1,4 +1,5 @@ import { Client } from 'minio'; +import Run from '../models/Run'; const minioClient = new Client({ endPoint: process.env.MINIO_ENDPOINT || 'localhost', @@ -8,4 +9,43 @@ const minioClient = new Client({ secretKey: process.env.MINIO_SECRET_KEY || 'minio-secret-key', }); -export default minioClient; \ No newline at end of file +class BinaryOutputService { + private bucketName: string; + + constructor(bucketName: string) { + this.bucketName = bucketName; + } + + /** + * Uploads binary data to Minio and stores references in PostgreSQL. + * @param run - The run object representing the current process. + * @param binaryOutput - The binary output object containing data to upload. + * @returns A map of Minio URLs pointing to the uploaded binary data. + */ + async uploadAndStoreBinaryOutput(run: Run, binaryOutput: Record): Promise> { + const uploadedBinaryOutput: Record = {}; + + for (const key of Object.keys(binaryOutput)) { + const binaryData = binaryOutput[key]; + const bufferData = Buffer.from(binaryData, 'binary'); + const minioKey = `${run.runId}/${key}`; + + await run.uploadBinaryOutputToMinioBucket(minioKey, bufferData); + + // Save the Minio URL in the result object + uploadedBinaryOutput[key] = `minio://${this.bucketName}/${minioKey}`; + } + + // Update the run with the Minio URLs for binary output + await run.update({ + binaryOutput: uploadedBinaryOutput + }); + + return uploadedBinaryOutput; + } +} + + + + +export { minioClient, BinaryOutputService }; \ No newline at end of file From 15ca01b1e9f4d11fd8d9cde13352e23c287cf5e5 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 19:17:17 +0530 Subject: [PATCH 17/50] fix: minioClient input --- server/src/models/Run.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/src/models/Run.ts b/server/src/models/Run.ts index f26b7c61..99184776 100644 --- a/server/src/models/Run.ts +++ b/server/src/models/Run.ts @@ -1,10 +1,7 @@ import { Model, DataTypes, Optional } from 'sequelize'; import sequelize from '../storage/db'; import Robot from './Robot'; -import minioClient from '../storage/mino'; - -// TODO: -// 1. we might not need interpreter settings? +import { minioClient } from '../storage/mino'; interface InterpreterSettings { maxConcurrency: number; From 34e8ed2284b0182a00a94c901c947104112c9e78 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 19:17:54 +0530 Subject: [PATCH 18/50] chore: lint --- server/src/storage/mino.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index 62290ccb..8368e5dd 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -27,7 +27,7 @@ class BinaryOutputService { for (const key of Object.keys(binaryOutput)) { const binaryData = binaryOutput[key]; - const bufferData = Buffer.from(binaryData, 'binary'); + const bufferData = Buffer.from(binaryData, 'binary'); const minioKey = `${run.runId}/${key}`; await run.uploadBinaryOutputToMinioBucket(minioKey, bufferData); @@ -48,4 +48,4 @@ class BinaryOutputService { -export { minioClient, BinaryOutputService }; \ No newline at end of file +export { minioClient, BinaryOutputService }; \ No newline at end of file From 14d97401a5315e008185f71c05fa8524af9f065b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 19:19:36 +0530 Subject: [PATCH 19/50] fix: whitespace --- server/src/storage/mino.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index 8368e5dd..1803e027 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -45,7 +45,4 @@ class BinaryOutputService { } } - - - export { minioClient, BinaryOutputService }; \ No newline at end of file From 8a1215ad577532f655aa849a809d2316de0bafb2 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 21:04:43 +0530 Subject: [PATCH 20/50] fix: broken import for sequelize --- server/src/models/User.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/models/User.ts b/server/src/models/User.ts index 83156fd2..b50c1e0f 100644 --- a/server/src/models/User.ts +++ b/server/src/models/User.ts @@ -1,5 +1,5 @@ import { DataTypes, Model, Optional } from 'sequelize'; -import sequelize from '../db/config'; +import sequelize from '../storage/db'; interface UserAttributes { id: number; From 22c0a1529412c5410c8dddb14b2c7b1d396814a0 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 22:17:16 +0530 Subject: [PATCH 21/50] feat: use binaryOutputService --- server/src/api/record.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/api/record.ts b/server/src/api/record.ts index 5b2e8451..d19c7f93 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -11,6 +11,7 @@ import { createRemoteBrowserForRun, destroyRemoteBrowser } from "../browser-mana import logger from "../logger"; import { browserPool } from "../server"; import { io, Socket } from "socket.io-client"; +import { BinaryOutputService } from "../storage/mino"; const formatRecording = (recordingData: any) => { const recordingMeta = recordingData.recording_meta; @@ -307,6 +308,9 @@ async function executeRun(id: string) { recording.recording, currentPage, plainRun.interpreterSettings ); + const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); + const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); + await destroyRemoteBrowser(plainRun.browserId); const updatedRun = await run.update({ From c53b8837632bd0b4fd4a9575cb574e8fa74bc711 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 22:17:47 +0530 Subject: [PATCH 22/50] feat: use uploaded minio urls --- server/src/api/record.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/api/record.ts b/server/src/api/record.ts index d19c7f93..20e38928 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -320,7 +320,7 @@ async function executeRun(id: string) { browserId: plainRun.browserId, log: interpretationInfo.log.join('\n'), serializableOutput: interpretationInfo.serializableOutput, - binaryOutput: interpretationInfo.binaryOutput, + binaryOutput: uploadedBinaryOutput, }); return { From 6d85bac44a01a2dc4100566e9b6b6ffeef906a26 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 22:20:29 +0530 Subject: [PATCH 23/50] feat: use binaryOutputService --- server/src/routes/storage.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index b43713f6..e7e92e64 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -11,6 +11,7 @@ import { getDecryptedProxyConfig } from './proxy'; import { requireSignIn } from '../middlewares/auth'; import Robot from '../models/Robot'; import Run from '../models/Run'; +import { BinaryOutputService } from '../storage/mino'; import { workflowQueue } from '../worker'; export const router = Router(); @@ -189,6 +190,8 @@ router.post('/runs/run/:id', requireSignIn, async (req, res) => { if (browser && currentPage) { const interpretationInfo = await browser.interpreter.InterpretRecording( recording.recording, currentPage, plainRun.interpreterSettings); + const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); + const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); await destroyRemoteBrowser(plainRun.browserId); await run.update({ ...run, @@ -197,7 +200,7 @@ router.post('/runs/run/:id', requireSignIn, async (req, res) => { browserId: plainRun.browserId, log: interpretationInfo.log.join('\n'), serializableOutput: interpretationInfo.serializableOutput, - binaryOutput: interpretationInfo.binaryOutput, + binaryOutput: uploadedBinaryOutput, }); googleSheetUpdateTasks[req.params.id] = { name: plainRun.name, @@ -279,16 +282,16 @@ router.put('/schedule/:id/', requireSignIn, async (req, res) => { const runId = uuid(); const userId = req.user.id; - await workflowQueue.add( - 'run workflow', + await workflowQueue.add( + 'run workflow', { id, runId, userId }, - { + { repeat: { pattern: cronExpression, tz: timezone - } + } } - ); + ); res.status(200).json({ message: 'success', From 17f0e29b046b98012e1d03a3691edd05eccf25e9 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 22:22:03 +0530 Subject: [PATCH 24/50] feat: use binaryOutputService --- server/src/workflow-management/scheduler/index.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 615a43fb..bb182928 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -8,6 +8,7 @@ import { googleSheetUpdateTasks, processGoogleSheetUpdates } from "../integratio import Robot from "../../models/Robot"; import Run from "../../models/Run"; import { getDecryptedProxyConfig } from "../../routes/proxy"; +import { BinaryOutputService } from "../../storage/mino"; async function createWorkflowAndStoreMetadata(id: string, userId: string) { try { @@ -114,6 +115,9 @@ async function executeRun(id: string) { const interpretationInfo = await browser.interpreter.InterpretRecording( recording.recording, currentPage, plainRun.interpreterSettings); + + const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); + const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); await destroyRemoteBrowser(plainRun.browserId); @@ -124,7 +128,7 @@ async function executeRun(id: string) { browserId: plainRun.browserId, log: interpretationInfo.log.join('\n'), serializableOutput: interpretationInfo.serializableOutput, - binaryOutput: interpretationInfo.binaryOutput, + binaryOutput: uploadedBinaryOutput, }); googleSheetUpdateTasks[id] = { From 4e84909de687f688f7f89789d20a597225ddbfc1 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 22:22:14 +0530 Subject: [PATCH 25/50] chore: lint --- server/src/workflow-management/scheduler/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index bb182928..3ea6fe53 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -115,9 +115,9 @@ async function executeRun(id: string) { const interpretationInfo = await browser.interpreter.InterpretRecording( recording.recording, currentPage, plainRun.interpreterSettings); - - const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); - const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); + + const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); + const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput); await destroyRemoteBrowser(plainRun.browserId); From 479b6584164ad7c267337824cbdfa6e9be9396ca Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 15 Oct 2024 22:47:19 +0530 Subject: [PATCH 26/50] feat: check if minio connected --- server/src/storage/mino.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index 1803e027..ce1a63b0 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -9,6 +9,18 @@ const minioClient = new Client({ secretKey: process.env.MINIO_SECRET_KEY || 'minio-secret-key', }); +minioClient.bucketExists('maxun-test') + .then((exists) => { + if (exists) { + console.log('MinIO was connected successfully.'); + } else { + console.log('Bucket does not exist, but MinIO was connected.'); + } + }) + .catch((err) => { + console.error('Error connecting to MinIO:', err); + }) + class BinaryOutputService { private bucketName: string; From 9f11897f2a20bfead68e537a9cedb2b73ebeedfb Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 00:43:24 +0530 Subject: [PATCH 27/50] feat: pass bucket name --- server/src/models/Run.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/models/Run.ts b/server/src/models/Run.ts index 99184776..4d11aa2c 100644 --- a/server/src/models/Run.ts +++ b/server/src/models/Run.ts @@ -43,13 +43,13 @@ class Run extends Model implements RunAttr public binaryOutput!: Record; public async uploadBinaryOutputToMinioBucket(key: string, data: Buffer): Promise { - const bucketName = ''; + const bucketName = 'maxun-run-screenshots'; await minioClient.putObject(bucketName, key, data); this.binaryOutput[key] = `minio://${bucketName}/${key}`; } public async getBinaryOutputFromMinioBucket(key: string): Promise { - const bucketName = ''; + const bucketName = 'maxun-run-screenshots'; const stream = await minioClient.getObject(bucketName, key); return new Promise((resolve, reject) => { const chunks: Buffer[] = []; From 7c615032ab330e3c378a5847c819607396e0dd85 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 00:45:30 +0530 Subject: [PATCH 28/50] feat: error handling while upload --- server/src/storage/mino.ts | 41 +++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index ce1a63b0..ed6a940f 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -39,19 +39,42 @@ class BinaryOutputService { for (const key of Object.keys(binaryOutput)) { const binaryData = binaryOutput[key]; - const bufferData = Buffer.from(binaryData, 'binary'); - const minioKey = `${run.runId}/${key}`; - await run.uploadBinaryOutputToMinioBucket(minioKey, bufferData); + console.log(`Processing binary output key: ${key}`); + console.log(`Binary data: ${binaryData}`); - // Save the Minio URL in the result object - uploadedBinaryOutput[key] = `minio://${this.bucketName}/${minioKey}`; + if (!binaryData) { + console.error(`No data found for key: ${key}`); + continue; + } + + try { + const bufferData = Buffer.from(binaryData, 'binary'); + const minioKey = `${run.runId}/${key}`; + + console.log(`Uploading data to MinIO with key: ${minioKey}`); + + await run.uploadBinaryOutputToMinioBucket(minioKey, bufferData); + + // Save the Minio URL in the result object + uploadedBinaryOutput[key] = `minio://${this.bucketName}/${minioKey}`; + + console.log(`Successfully uploaded ${key} to MinIO`); + } catch (error) { + console.error(`Error uploading key ${key} to MinIO:`, error); + } } - // Update the run with the Minio URLs for binary output - await run.update({ - binaryOutput: uploadedBinaryOutput - }); + console.log('Uploaded Binary Output:', uploadedBinaryOutput); + + try { + await run.update({ + binaryOutput: uploadedBinaryOutput + }); + console.log('Run successfully updated with binary output'); + } catch (updateError) { + console.error('Error updating run with binary output:', updateError); + } return uploadedBinaryOutput; } From 3270bdd392aaa2527ac29f1f3e6a6c69d0551d49 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 00:46:40 +0530 Subject: [PATCH 29/50] feat: error handling --- server/src/models/Run.ts | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/server/src/models/Run.ts b/server/src/models/Run.ts index 4d11aa2c..13162c77 100644 --- a/server/src/models/Run.ts +++ b/server/src/models/Run.ts @@ -44,19 +44,36 @@ class Run extends Model implements RunAttr public async uploadBinaryOutputToMinioBucket(key: string, data: Buffer): Promise { const bucketName = 'maxun-run-screenshots'; - await minioClient.putObject(bucketName, key, data); - this.binaryOutput[key] = `minio://${bucketName}/${key}`; + + try { + console.log(`Uploading to bucket ${bucketName} with key ${key}`); + await minioClient.putObject(bucketName, key, data); + this.binaryOutput[key] = `minio://${bucketName}/${key}`; + console.log(`Successfully uploaded to MinIO: minio://${bucketName}/${key}`); + } catch (error) { + console.error(`Error uploading to MinIO bucket: ${bucketName} with key: ${key}`, error); + } } public async getBinaryOutputFromMinioBucket(key: string): Promise { const bucketName = 'maxun-run-screenshots'; - const stream = await minioClient.getObject(bucketName, key); - return new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - stream.on('data', (chunk) => chunks.push(chunk)); - stream.on('end', () => resolve(Buffer.concat(chunks))); - stream.on('error', reject); - }); + + try { + console.log(`Fetching from bucket ${bucketName} with key ${key}`); + const stream = await minioClient.getObject(bucketName, key); + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on('data', (chunk) => chunks.push(chunk)); + stream.on('end', () => resolve(Buffer.concat(chunks))); + stream.on('error', (error) => { + console.error('Error while reading the stream from MinIO:', error); + reject(error); + }); + }); + } catch (error) { + console.error(`Error fetching from MinIO bucket: ${bucketName} with key: ${key}`, error); + throw error; + } } } From 3d182a8dff12e1ac49dc5d2532d703f38ec67726 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 01:16:43 +0530 Subject: [PATCH 30/50] feat: handle buffer --- server/src/storage/mino.ts | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index ed6a940f..8c6e9b54 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -38,23 +38,43 @@ class BinaryOutputService { const uploadedBinaryOutput: Record = {}; for (const key of Object.keys(binaryOutput)) { - const binaryData = binaryOutput[key]; + let binaryData = binaryOutput[key]; console.log(`Processing binary output key: ${key}`); - console.log(`Binary data: ${binaryData}`); + console.log(`Binary data:`, binaryData); - if (!binaryData) { - console.error(`No data found for key: ${key}`); + // If the binary data is a string, try parsing it as JSON + if (typeof binaryData === 'string') { + try { + const parsedData = JSON.parse(binaryData); + + // Check if the parsed data has the "type" and "data" fields + if (parsedData && parsedData.type === 'Buffer' && Array.isArray(parsedData.data)) { + // Convert the parsed array into a Buffer + binaryData = Buffer.from(parsedData.data); + console.log(`Successfully parsed and converted binary data to Buffer for key: ${key}`); + } else { + console.error(`Invalid Buffer format for key: ${key}`); + continue; // Skip invalid data + } + } catch (jsonError) { + console.error(`Failed to parse JSON for key: ${key}`, jsonError); + continue; // Skip if parsing fails + } + } + + // Handle cases where data might still be invalid + if (!Buffer.isBuffer(binaryData)) { + console.error(`Binary data for key ${key} is not a valid Buffer.`); continue; } try { - const bufferData = Buffer.from(binaryData, 'binary'); const minioKey = `${run.runId}/${key}`; console.log(`Uploading data to MinIO with key: ${minioKey}`); - await run.uploadBinaryOutputToMinioBucket(minioKey, bufferData); + await run.uploadBinaryOutputToMinioBucket(minioKey, binaryData); // Save the Minio URL in the result object uploadedBinaryOutput[key] = `minio://${this.bucketName}/${minioKey}`; @@ -67,6 +87,7 @@ class BinaryOutputService { console.log('Uploaded Binary Output:', uploadedBinaryOutput); + // Update the run with the Minio URLs for binary output try { await run.update({ binaryOutput: uploadedBinaryOutput From 36f9277db7cb3cd8ed61d270c3b36b4b38c5e188 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 01:25:57 +0530 Subject: [PATCH 31/50] feat: hande data as per interpreter --- server/src/storage/mino.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index 8c6e9b54..26709768 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -40,13 +40,14 @@ class BinaryOutputService { for (const key of Object.keys(binaryOutput)) { let binaryData = binaryOutput[key]; + // Log the key and data console.log(`Processing binary output key: ${key}`); console.log(`Binary data:`, binaryData); - // If the binary data is a string, try parsing it as JSON - if (typeof binaryData === 'string') { + // Check if binaryData is an object with a "data" field (containing the Buffer string) + if (binaryData && typeof binaryData.data === 'string') { try { - const parsedData = JSON.parse(binaryData); + const parsedData = JSON.parse(binaryData.data); // Check if the parsed data has the "type" and "data" fields if (parsedData && parsedData.type === 'Buffer' && Array.isArray(parsedData.data)) { @@ -85,6 +86,7 @@ class BinaryOutputService { } } + // Log the binary output after processing console.log('Uploaded Binary Output:', uploadedBinaryOutput); // Update the run with the Minio URLs for binary output From 3f4b67b05a19e29d39314a262989b324064a6eee Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 01:38:02 +0530 Subject: [PATCH 32/50] feat: add runId check --- server/src/storage/mino.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index 26709768..0a413b42 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -71,7 +71,7 @@ class BinaryOutputService { } try { - const minioKey = `${run.runId}/${key}`; + const minioKey = run.runId ? `${run.runId}/${key}`: key; console.log(`Uploading data to MinIO with key: ${minioKey}`); From 6062b10beb173533afbb67eed452ea410c0a0e7a Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 01:40:20 +0530 Subject: [PATCH 33/50] feat: pass content type --- server/src/models/Run.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/models/Run.ts b/server/src/models/Run.ts index 13162c77..7d581d86 100644 --- a/server/src/models/Run.ts +++ b/server/src/models/Run.ts @@ -47,7 +47,7 @@ class Run extends Model implements RunAttr try { console.log(`Uploading to bucket ${bucketName} with key ${key}`); - await minioClient.putObject(bucketName, key, data); + await minioClient.putObject(bucketName, key, data, data.length, { 'Content-Type': 'image/png' }); this.binaryOutput[key] = `minio://${bucketName}/${key}`; console.log(`Successfully uploaded to MinIO: minio://${bucketName}/${key}`); } catch (error) { From c2356f066e020e4de29fd04683d2a86140881ffa Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 01:40:53 +0530 Subject: [PATCH 34/50] feat: improve error handling --- server/src/models/Run.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/models/Run.ts b/server/src/models/Run.ts index 7d581d86..332fc059 100644 --- a/server/src/models/Run.ts +++ b/server/src/models/Run.ts @@ -44,7 +44,6 @@ class Run extends Model implements RunAttr public async uploadBinaryOutputToMinioBucket(key: string, data: Buffer): Promise { const bucketName = 'maxun-run-screenshots'; - try { console.log(`Uploading to bucket ${bucketName} with key ${key}`); await minioClient.putObject(bucketName, key, data, data.length, { 'Content-Type': 'image/png' }); @@ -52,6 +51,7 @@ class Run extends Model implements RunAttr console.log(`Successfully uploaded to MinIO: minio://${bucketName}/${key}`); } catch (error) { console.error(`Error uploading to MinIO bucket: ${bucketName} with key: ${key}`, error); + throw error; } } From b3d76ae66465f4b8388c1030814c204d7de8b0a9 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 03:07:33 +0530 Subject: [PATCH 35/50] feat: handle binary data --- server/src/storage/mino.ts | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index 0a413b42..74d11054 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -34,66 +34,58 @@ class BinaryOutputService { * @param binaryOutput - The binary output object containing data to upload. * @returns A map of Minio URLs pointing to the uploaded binary data. */ - async uploadAndStoreBinaryOutput(run: Run, binaryOutput: Record): Promise> { + async uploadAndStoreBinaryOutput(run: Run, binaryOutput: Record, runId: string): Promise> { const uploadedBinaryOutput: Record = {}; for (const key of Object.keys(binaryOutput)) { let binaryData = binaryOutput[key]; - // Log the key and data + if (!runId) { + console.error('Run ID is undefined. Cannot upload binary data.'); + continue; + } + console.log(`Processing binary output key: ${key}`); console.log(`Binary data:`, binaryData); - // Check if binaryData is an object with a "data" field (containing the Buffer string) + // Check if binaryData has a valid Buffer structure and parse it if (binaryData && typeof binaryData.data === 'string') { try { const parsedData = JSON.parse(binaryData.data); - - // Check if the parsed data has the "type" and "data" fields if (parsedData && parsedData.type === 'Buffer' && Array.isArray(parsedData.data)) { - // Convert the parsed array into a Buffer binaryData = Buffer.from(parsedData.data); - console.log(`Successfully parsed and converted binary data to Buffer for key: ${key}`); } else { console.error(`Invalid Buffer format for key: ${key}`); - continue; // Skip invalid data + continue; } - } catch (jsonError) { - console.error(`Failed to parse JSON for key: ${key}`, jsonError); - continue; // Skip if parsing fails + } catch (error) { + console.error(`Failed to parse JSON for key: ${key}`, error); + continue; } } - // Handle cases where data might still be invalid + // Handle cases where binaryData might not be a Buffer if (!Buffer.isBuffer(binaryData)) { console.error(`Binary data for key ${key} is not a valid Buffer.`); continue; } try { - const minioKey = run.runId ? `${run.runId}/${key}`: key; - - console.log(`Uploading data to MinIO with key: ${minioKey}`); + const minioKey = `${run.id}/${key}`; await run.uploadBinaryOutputToMinioBucket(minioKey, binaryData); // Save the Minio URL in the result object uploadedBinaryOutput[key] = `minio://${this.bucketName}/${minioKey}`; - - console.log(`Successfully uploaded ${key} to MinIO`); } catch (error) { console.error(`Error uploading key ${key} to MinIO:`, error); } } - // Log the binary output after processing console.log('Uploaded Binary Output:', uploadedBinaryOutput); - // Update the run with the Minio URLs for binary output try { - await run.update({ - binaryOutput: uploadedBinaryOutput - }); + await run.update({ binaryOutput: uploadedBinaryOutput }); console.log('Run successfully updated with binary output'); } catch (updateError) { console.error('Error updating run with binary output:', updateError); From 1f11e773e2d1f240494b1ad381e0831c20c46550 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 03:15:28 +0530 Subject: [PATCH 36/50] feat: run to json --- server/src/storage/mino.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index 74d11054..849060b8 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -34,13 +34,14 @@ class BinaryOutputService { * @param binaryOutput - The binary output object containing data to upload. * @returns A map of Minio URLs pointing to the uploaded binary data. */ - async uploadAndStoreBinaryOutput(run: Run, binaryOutput: Record, runId: string): Promise> { + async uploadAndStoreBinaryOutput(run: Run, binaryOutput: Record): Promise> { const uploadedBinaryOutput: Record = {}; + const plainRun = run.toJSON(); for (const key of Object.keys(binaryOutput)) { let binaryData = binaryOutput[key]; - if (!runId) { + if (!plainRun.runId) { console.error('Run ID is undefined. Cannot upload binary data.'); continue; } @@ -71,7 +72,7 @@ class BinaryOutputService { } try { - const minioKey = `${run.id}/${key}`; + const minioKey = `${plainRun.runId}/${key}`; await run.uploadBinaryOutputToMinioBucket(minioKey, binaryData); From a75d40918b7e56c0a27115a8a84272de128a22cb Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 13:34:40 +0530 Subject: [PATCH 37/50] feat: !upload run binary output from Run model --- server/src/models/Run.ts | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/server/src/models/Run.ts b/server/src/models/Run.ts index 332fc059..93e3c924 100644 --- a/server/src/models/Run.ts +++ b/server/src/models/Run.ts @@ -42,19 +42,6 @@ class Run extends Model implements RunAttr public serializableOutput!: Record; public binaryOutput!: Record; - public async uploadBinaryOutputToMinioBucket(key: string, data: Buffer): Promise { - const bucketName = 'maxun-run-screenshots'; - try { - console.log(`Uploading to bucket ${bucketName} with key ${key}`); - await minioClient.putObject(bucketName, key, data, data.length, { 'Content-Type': 'image/png' }); - this.binaryOutput[key] = `minio://${bucketName}/${key}`; - console.log(`Successfully uploaded to MinIO: minio://${bucketName}/${key}`); - } catch (error) { - console.error(`Error uploading to MinIO bucket: ${bucketName} with key: ${key}`, error); - throw error; - } - } - public async getBinaryOutputFromMinioBucket(key: string): Promise { const bucketName = 'maxun-run-screenshots'; From 04fbf17c3a35bc46f8b1b14f921ffc4a81b8e171 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 13:40:26 +0530 Subject: [PATCH 38/50] feat: move all upload logic to BinaryOutputService --- server/src/storage/mino.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index 849060b8..e8ddd6f3 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -74,7 +74,7 @@ class BinaryOutputService { try { const minioKey = `${plainRun.runId}/${key}`; - await run.uploadBinaryOutputToMinioBucket(minioKey, binaryData); + await this.uploadBinaryOutputToMinioBucket(run, minioKey, binaryData); // Save the Minio URL in the result object uploadedBinaryOutput[key] = `minio://${this.bucketName}/${minioKey}`; @@ -94,6 +94,20 @@ class BinaryOutputService { return uploadedBinaryOutput; } + + async uploadBinaryOutputToMinioBucket(run: Run, key: string, data: Buffer): Promise { + const bucketName = 'maxun-run-screenshots'; + try { + console.log(`Uploading to bucket ${bucketName} with key ${key}`); + await minioClient.putObject(bucketName, key, data, data.length, { 'Content-Type': 'image/png' }); + const plainRun = run.toJSON(); + plainRun.binaryOutput[key] = `minio://${bucketName}/${key}`; + console.log(`Successfully uploaded to MinIO: minio://${bucketName}/${key}`); + } catch (error) { + console.error(`Error uploading to MinIO bucket: ${bucketName} with key: ${key}`, error); + throw error; + } + } } export { minioClient, BinaryOutputService }; \ No newline at end of file From ab5431d2cd8a1d49b0b124b6a926d7bde6b7dada Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 13:40:44 +0530 Subject: [PATCH 39/50] chore: lint --- server/src/storage/mino.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index e8ddd6f3..8fdca12a 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -105,7 +105,7 @@ class BinaryOutputService { console.log(`Successfully uploaded to MinIO: minio://${bucketName}/${key}`); } catch (error) { console.error(`Error uploading to MinIO bucket: ${bucketName} with key: ${key}`, error); - throw error; + throw error; } } } From 76d99ef076c14f7c5262afb9f145870893371282 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 13:41:58 +0530 Subject: [PATCH 40/50] chore: !log binary output --- server/src/storage/mino.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index 8fdca12a..6a27f8a0 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -47,7 +47,6 @@ class BinaryOutputService { } console.log(`Processing binary output key: ${key}`); - console.log(`Binary data:`, binaryData); // Check if binaryData has a valid Buffer structure and parse it if (binaryData && typeof binaryData.data === 'string') { From cac8e1c8ba84238cdccc1352a1b1f7c0a01ce438 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 13:42:42 +0530 Subject: [PATCH 41/50] feat: !get run binary output from Run model --- server/src/models/Run.ts | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/server/src/models/Run.ts b/server/src/models/Run.ts index 93e3c924..a48716d5 100644 --- a/server/src/models/Run.ts +++ b/server/src/models/Run.ts @@ -41,27 +41,6 @@ class Run extends Model implements RunAttr public runId!: string; public serializableOutput!: Record; public binaryOutput!: Record; - - public async getBinaryOutputFromMinioBucket(key: string): Promise { - const bucketName = 'maxun-run-screenshots'; - - try { - console.log(`Fetching from bucket ${bucketName} with key ${key}`); - const stream = await minioClient.getObject(bucketName, key); - return new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - stream.on('data', (chunk) => chunks.push(chunk)); - stream.on('end', () => resolve(Buffer.concat(chunks))); - stream.on('error', (error) => { - console.error('Error while reading the stream from MinIO:', error); - reject(error); - }); - }); - } catch (error) { - console.error(`Error fetching from MinIO bucket: ${bucketName} with key: ${key}`, error); - throw error; - } - } } Run.init( From 43f90810c6e6add8ad38f7617133f8dbc767fddc Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 13:43:21 +0530 Subject: [PATCH 42/50] feat: move all get logic to BinaryOutputService --- server/src/storage/mino.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index 6a27f8a0..68edc464 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -107,6 +107,27 @@ class BinaryOutputService { throw error; } } + + public async getBinaryOutputFromMinioBucket(key: string): Promise { + const bucketName = 'maxun-run-screenshots'; + + try { + console.log(`Fetching from bucket ${bucketName} with key ${key}`); + const stream = await minioClient.getObject(bucketName, key); + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on('data', (chunk) => chunks.push(chunk)); + stream.on('end', () => resolve(Buffer.concat(chunks))); + stream.on('error', (error) => { + console.error('Error while reading the stream from MinIO:', error); + reject(error); + }); + }); + } catch (error) { + console.error(`Error fetching from MinIO bucket: ${bucketName} with key: ${key}`, error); + throw error; + } + } } export { minioClient, BinaryOutputService }; \ No newline at end of file From 35790424d007c605e6406071c66ee66772a7fed9 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 13:44:07 +0530 Subject: [PATCH 43/50] chore: remove unused import --- server/src/models/Run.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/models/Run.ts b/server/src/models/Run.ts index a48716d5..1c8e4dc6 100644 --- a/server/src/models/Run.ts +++ b/server/src/models/Run.ts @@ -1,7 +1,6 @@ import { Model, DataTypes, Optional } from 'sequelize'; import sequelize from '../storage/db'; import Robot from './Robot'; -import { minioClient } from '../storage/mino'; interface InterpreterSettings { maxConcurrency: number; From 77fbc99931ca51e6c4c975deffa89f759e26443d Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 14:01:24 +0530 Subject: [PATCH 44/50] feat: set data type as Buffer --- server/src/workflow-management/classes/Interpreter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/workflow-management/classes/Interpreter.ts b/server/src/workflow-management/classes/Interpreter.ts index fa5e9332..35498796 100644 --- a/server/src/workflow-management/classes/Interpreter.ts +++ b/server/src/workflow-management/classes/Interpreter.ts @@ -228,7 +228,7 @@ export class WorkflowInterpreter { this.serializableData.push(data); this.socket.emit('serializableCallback', data); }, - binaryCallback: async (data: string, mimetype: string) => { + binaryCallback: async (data: Buffer, mimetype: string) => { this.binaryData.push({ mimetype, data: JSON.stringify(data) }); this.socket.emit('binaryCallback', { data, mimetype }); } From fd079ecdffef72b6adf268c49503f29cc49bfece Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 14:12:50 +0530 Subject: [PATCH 45/50] feat: convert data to base64 for interpretation while recording --- server/src/workflow-management/classes/Interpreter.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/workflow-management/classes/Interpreter.ts b/server/src/workflow-management/classes/Interpreter.ts index 35498796..f12f429e 100644 --- a/server/src/workflow-management/classes/Interpreter.ts +++ b/server/src/workflow-management/classes/Interpreter.ts @@ -229,8 +229,10 @@ export class WorkflowInterpreter { this.socket.emit('serializableCallback', data); }, binaryCallback: async (data: Buffer, mimetype: string) => { - this.binaryData.push({ mimetype, data: JSON.stringify(data) }); - this.socket.emit('binaryCallback', { data, mimetype }); + const base64Data = data.toString('base64'); + const binaryObject = { mimetype, data: base64Data }; + this.binaryData.push(binaryObject); + this.socket.emit('binaryCallback', binaryObject); } } From f8db9c57cbdcee496b6351b2bbce298244028dcf Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 14:19:01 +0530 Subject: [PATCH 46/50] fix: revert changes --- server/src/workflow-management/classes/Interpreter.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/server/src/workflow-management/classes/Interpreter.ts b/server/src/workflow-management/classes/Interpreter.ts index f12f429e..fa5e9332 100644 --- a/server/src/workflow-management/classes/Interpreter.ts +++ b/server/src/workflow-management/classes/Interpreter.ts @@ -228,11 +228,9 @@ export class WorkflowInterpreter { this.serializableData.push(data); this.socket.emit('serializableCallback', data); }, - binaryCallback: async (data: Buffer, mimetype: string) => { - const base64Data = data.toString('base64'); - const binaryObject = { mimetype, data: base64Data }; - this.binaryData.push(binaryObject); - this.socket.emit('binaryCallback', binaryObject); + binaryCallback: async (data: string, mimetype: string) => { + this.binaryData.push({ mimetype, data: JSON.stringify(data) }); + this.socket.emit('binaryCallback', { data, mimetype }); } } From 3f964239b1ac35f1ace9f1d9fe30042f1377a219 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 14:47:00 +0530 Subject: [PATCH 47/50] feat: public URLs --- server/src/storage/mino.ts | 80 ++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index 68edc464..917a3f2b 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -39,60 +39,64 @@ class BinaryOutputService { const plainRun = run.toJSON(); for (const key of Object.keys(binaryOutput)) { - let binaryData = binaryOutput[key]; + let binaryData = binaryOutput[key]; - if (!plainRun.runId) { - console.error('Run ID is undefined. Cannot upload binary data.'); - continue; - } - - console.log(`Processing binary output key: ${key}`); - - // Check if binaryData has a valid Buffer structure and parse it - if (binaryData && typeof binaryData.data === 'string') { - try { - const parsedData = JSON.parse(binaryData.data); - if (parsedData && parsedData.type === 'Buffer' && Array.isArray(parsedData.data)) { - binaryData = Buffer.from(parsedData.data); - } else { - console.error(`Invalid Buffer format for key: ${key}`); + if (!plainRun.runId) { + console.error('Run ID is undefined. Cannot upload binary data.'); continue; - } - } catch (error) { - console.error(`Failed to parse JSON for key: ${key}`, error); - continue; } - } - // Handle cases where binaryData might not be a Buffer - if (!Buffer.isBuffer(binaryData)) { - console.error(`Binary data for key ${key} is not a valid Buffer.`); - continue; - } + console.log(`Processing binary output key: ${key}`); - try { - const minioKey = `${plainRun.runId}/${key}`; + // Check if binaryData has a valid Buffer structure and parse it + if (binaryData && typeof binaryData.data === 'string') { + try { + const parsedData = JSON.parse(binaryData.data); + if (parsedData && parsedData.type === 'Buffer' && Array.isArray(parsedData.data)) { + binaryData = Buffer.from(parsedData.data); + } else { + console.error(`Invalid Buffer format for key: ${key}`); + continue; + } + } catch (error) { + console.error(`Failed to parse JSON for key: ${key}`, error); + continue; + } + } - await this.uploadBinaryOutputToMinioBucket(run, minioKey, binaryData); + // Handle cases where binaryData might not be a Buffer + if (!Buffer.isBuffer(binaryData)) { + console.error(`Binary data for key ${key} is not a valid Buffer.`); + continue; + } - // Save the Minio URL in the result object - uploadedBinaryOutput[key] = `minio://${this.bucketName}/${minioKey}`; - } catch (error) { - console.error(`Error uploading key ${key} to MinIO:`, error); - } + try { + const minioKey = `${plainRun.runId}/${key}`; + + await this.uploadBinaryOutputToMinioBucket(run, minioKey, binaryData); + + // Construct the public URL for the uploaded object + const publicUrl = `http://${process.env.MINIO_ENDPOINT}:${process.env.MINIO_PORT}/${this.bucketName}/${minioKey}`; + + // Save the public URL in the result object + uploadedBinaryOutput[key] = publicUrl; + } catch (error) { + console.error(`Error uploading key ${key} to MinIO:`, error); + } } console.log('Uploaded Binary Output:', uploadedBinaryOutput); try { - await run.update({ binaryOutput: uploadedBinaryOutput }); - console.log('Run successfully updated with binary output'); + await run.update({ binaryOutput: uploadedBinaryOutput }); + console.log('Run successfully updated with binary output'); } catch (updateError) { - console.error('Error updating run with binary output:', updateError); + console.error('Error updating run with binary output:', updateError); } return uploadedBinaryOutput; - } +} + async uploadBinaryOutputToMinioBucket(run: Run, key: string, data: Buffer): Promise { const bucketName = 'maxun-run-screenshots'; From 8b26ba17a720061fa2a66d93fc889f4283951b41 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 14:47:24 +0530 Subject: [PATCH 48/50] chore: prettier --- server/src/storage/mino.ts | 85 +++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 43 deletions(-) diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index 917a3f2b..96e3d0c2 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -39,64 +39,63 @@ class BinaryOutputService { const plainRun = run.toJSON(); for (const key of Object.keys(binaryOutput)) { - let binaryData = binaryOutput[key]; + let binaryData = binaryOutput[key]; - if (!plainRun.runId) { - console.error('Run ID is undefined. Cannot upload binary data.'); - continue; - } + if (!plainRun.runId) { + console.error('Run ID is undefined. Cannot upload binary data.'); + continue; + } - console.log(`Processing binary output key: ${key}`); - - // Check if binaryData has a valid Buffer structure and parse it - if (binaryData && typeof binaryData.data === 'string') { - try { - const parsedData = JSON.parse(binaryData.data); - if (parsedData && parsedData.type === 'Buffer' && Array.isArray(parsedData.data)) { - binaryData = Buffer.from(parsedData.data); - } else { - console.error(`Invalid Buffer format for key: ${key}`); - continue; - } - } catch (error) { - console.error(`Failed to parse JSON for key: ${key}`, error); - continue; - } - } - - // Handle cases where binaryData might not be a Buffer - if (!Buffer.isBuffer(binaryData)) { - console.error(`Binary data for key ${key} is not a valid Buffer.`); - continue; - } + console.log(`Processing binary output key: ${key}`); + // Check if binaryData has a valid Buffer structure and parse it + if (binaryData && typeof binaryData.data === 'string') { try { - const minioKey = `${plainRun.runId}/${key}`; - - await this.uploadBinaryOutputToMinioBucket(run, minioKey, binaryData); - - // Construct the public URL for the uploaded object - const publicUrl = `http://${process.env.MINIO_ENDPOINT}:${process.env.MINIO_PORT}/${this.bucketName}/${minioKey}`; - - // Save the public URL in the result object - uploadedBinaryOutput[key] = publicUrl; + const parsedData = JSON.parse(binaryData.data); + if (parsedData && parsedData.type === 'Buffer' && Array.isArray(parsedData.data)) { + binaryData = Buffer.from(parsedData.data); + } else { + console.error(`Invalid Buffer format for key: ${key}`); + continue; + } } catch (error) { - console.error(`Error uploading key ${key} to MinIO:`, error); + console.error(`Failed to parse JSON for key: ${key}`, error); + continue; } + } + + // Handle cases where binaryData might not be a Buffer + if (!Buffer.isBuffer(binaryData)) { + console.error(`Binary data for key ${key} is not a valid Buffer.`); + continue; + } + + try { + const minioKey = `${plainRun.runId}/${key}`; + + await this.uploadBinaryOutputToMinioBucket(run, minioKey, binaryData); + + // Construct the public URL for the uploaded object + const publicUrl = `http://${process.env.MINIO_ENDPOINT}:${process.env.MINIO_PORT}/${this.bucketName}/${minioKey}`; + + // Save the public URL in the result object + uploadedBinaryOutput[key] = publicUrl; + } catch (error) { + console.error(`Error uploading key ${key} to MinIO:`, error); + } } console.log('Uploaded Binary Output:', uploadedBinaryOutput); try { - await run.update({ binaryOutput: uploadedBinaryOutput }); - console.log('Run successfully updated with binary output'); + await run.update({ binaryOutput: uploadedBinaryOutput }); + console.log('Run successfully updated with binary output'); } catch (updateError) { - console.error('Error updating run with binary output:', updateError); + console.error('Error updating run with binary output:', updateError); } return uploadedBinaryOutput; -} - + } async uploadBinaryOutputToMinioBucket(run: Run, key: string, data: Buffer): Promise { const bucketName = 'maxun-run-screenshots'; From bc08759cec57a88205a15726801f183235bbaa67 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 14:53:59 +0530 Subject: [PATCH 49/50] feat: display screenshots --- src/components/molecules/RunContent.tsx | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/components/molecules/RunContent.tsx b/src/components/molecules/RunContent.tsx index 2ac4d369..08b832c4 100644 --- a/src/components/molecules/RunContent.tsx +++ b/src/components/molecules/RunContent.tsx @@ -184,19 +184,16 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe Binary output {Object.keys(row.binaryOutput).map((key) => { try { - const binaryBuffer = JSON.parse(row.binaryOutput[key].data); - const b64 = Buffer.from(binaryBuffer.data).toString('base64'); + const imageUrl = row.binaryOutput[key]; return ( {key}: - Download + Download - {key} + {key} ) } catch (e) { From 6778aa827851c58a18c7077620da8ba36670b138 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 16 Oct 2024 14:54:41 +0530 Subject: [PATCH 50/50] chore: remove Buffer import --- src/components/molecules/RunContent.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/components/molecules/RunContent.tsx b/src/components/molecules/RunContent.tsx index 08b832c4..56e6b9d7 100644 --- a/src/components/molecules/RunContent.tsx +++ b/src/components/molecules/RunContent.tsx @@ -7,7 +7,6 @@ import { TabPanel, TabContext } from "@mui/lab"; import SettingsIcon from '@mui/icons-material/Settings'; import ImageIcon from '@mui/icons-material/Image'; import ArticleIcon from '@mui/icons-material/Article'; -import { Buffer } from 'buffer'; import { useEffect, useState } from "react"; import AssignmentIcon from '@mui/icons-material/Assignment'; import Table from '@mui/material/Table';