Merge pull request #61 from amhsirak/develop

feat: storage
This commit is contained in:
Karishma Shukla
2024-10-10 03:31:53 +05:30
committed by GitHub
18 changed files with 539 additions and 242 deletions

4
server/src/api/run.ts Normal file
View File

@@ -0,0 +1,4 @@
import { readFile, readFiles } from "../workflow-management/storage";
import { Router, Request, Response } from 'express';
import { requireAPIKey } from "../middlewares/api";
const router = Router();

View File

@@ -1,5 +1,6 @@
import { Sequelize } from 'sequelize';
import dotenv from 'dotenv';
import setupAssociations from '../models/associations';
dotenv.config();
const sequelize = new Sequelize(
@@ -22,6 +23,7 @@ export const connectDB = async () => {
export const syncDB = async () => {
try {
//setupAssociations();
await sequelize.sync({ force: false }); // force: true will drop and recreate tables on every run
console.log('Database synced successfully!');
} catch (error) {

View File

@@ -0,0 +1,55 @@
import { Model, DataTypes, Optional } from 'sequelize';
import sequelize from '../db/config';
import { WorkflowFile, Where, What, WhereWhatPair } from 'maxun-core';
interface RobotMeta {
name: string;
id: string;
createdAt: string;
pairs: number;
updatedAt: string;
params: any[];
}
interface RobotWorkflow {
workflow: WhereWhatPair[];
}
interface RobotAttributes {
id: string;
recording_meta: RobotMeta;
recording: RobotWorkflow;
}
interface RobotCreationAttributes extends Optional<RobotAttributes, 'id'> { }
class Robot extends Model<RobotAttributes, RobotCreationAttributes> implements RobotAttributes {
public id!: string;
public recording_meta!: RobotMeta;
public recording!: RobotWorkflow;
}
Robot.init(
{
id: {
type: DataTypes.UUID,
defaultValue: DataTypes.UUIDV4,
primaryKey: true,
},
recording_meta: {
type: DataTypes.JSONB,
allowNull: false,
},
recording: {
type: DataTypes.JSONB,
allowNull: false,
},
},
{
sequelize,
tableName: 'robot',
timestamps: false,
}
);
export default Robot;

116
server/src/models/Run.ts Normal file
View File

@@ -0,0 +1,116 @@
import { Model, DataTypes, Optional } from 'sequelize';
import sequelize from '../db/config';
import Robot from './Robot';
// TODO:
// 1. rename variables
// 2. we might not need interpreter settings?
// 3. store binaryOutput in MinIO
interface InterpreterSettings {
maxConcurrency: number;
maxRepeats: number;
debug: boolean;
}
interface RunAttributes {
id: string;
status: string;
name: string;
robotId: string;
robotMetaId: string;
startedAt: string;
finishedAt: string;
browserId: string;
interpreterSettings: InterpreterSettings;
log: string;
runId: string;
serializableOutput: Record<string, any[]>;
binaryOutput: Record<string, any>;
}
interface RunCreationAttributes extends Optional<RunAttributes, 'id'> { }
class Run extends Model<RunAttributes, RunCreationAttributes> implements RunAttributes {
public id!: string;
public status!: string;
public name!: string;
public robotId!: string;
public robotMetaId!: string;
public startedAt!: string;
public finishedAt!: string;
public browserId!: string;
public interpreterSettings!: InterpreterSettings;
public log!: string;
public runId!: string;
public serializableOutput!: Record<string, any[]>;
public binaryOutput!: Record<string, any>;
}
Run.init(
{
id: {
type: DataTypes.UUID,
defaultValue: DataTypes.UUIDV4,
primaryKey: true,
},
status: {
type: DataTypes.STRING(50),
allowNull: false,
},
name: {
type: DataTypes.STRING(255),
allowNull: false,
},
robotId: {
type: DataTypes.UUID,
allowNull: false,
references: {
model: Robot,
key: 'id',
},
},
robotMetaId: {
type: DataTypes.UUID,
allowNull: false,
},
startedAt: {
type: DataTypes.STRING(255),
allowNull: false,
},
finishedAt: {
type: DataTypes.STRING(255),
allowNull: false,
},
browserId: {
type: DataTypes.UUID,
allowNull: false,
},
interpreterSettings: {
type: DataTypes.JSONB,
allowNull: false,
},
log: {
type: DataTypes.TEXT,
allowNull: true,
},
runId: {
type: DataTypes.UUID,
allowNull: false,
},
serializableOutput: {
type: DataTypes.JSONB,
allowNull: true,
},
binaryOutput: {
type: DataTypes.JSONB,
allowNull: true,
},
},
{
sequelize,
tableName: 'run',
timestamps: false,
}
);
export default Run;

View File

@@ -0,0 +1,7 @@
import Robot from './Robot';
import Run from './Run';
export default function setupAssociations() {
Run.belongsTo(Robot, { foreignKey: 'robotId' });
Robot.hasMany(Run, { foreignKey: 'robotId' });
}

View File

@@ -11,12 +11,14 @@ import cron from 'node-cron';
import { googleSheetUpdateTasks, processGoogleSheetUpdates } from '../workflow-management/integrations/gsheet';
import { getDecryptedProxyConfig } from './proxy';
import { requireSignIn } from '../middlewares/auth';
// import { workflowQueue } from '../worker';
import Robot from '../models/Robot';
import Run from '../models/Run';
import { workflowQueue } from '../worker';
// todo: move from here
export const getRecordingByFileName = async (fileName: string): Promise<any | null> => {
try {
const recording = await readFile(`./../storage/recordings/${fileName}.waw.json`)
const recording = await readFile(`./../storage/recordings/${fileName}.json`)
const parsedRecording = JSON.parse(recording);
return parsedRecording;
} catch (error: any) {
@@ -40,7 +42,7 @@ router.all('/', requireSignIn, (req, res, next) => {
*/
router.get('/recordings', requireSignIn, async (req, res) => {
try {
const data = await readFiles('./../storage/recordings/');
const data = await Robot.findAll();
return res.send(data);
} catch (e) {
logger.log('info', 'Error while reading recordings');
@@ -51,13 +53,15 @@ router.get('/recordings', requireSignIn, async (req, res) => {
/**
* DELETE endpoint for deleting a recording from the storage.
*/
router.delete('/recordings/:fileName', requireSignIn, async (req, res) => {
router.delete('/recordings/:id', requireSignIn, async (req, res) => {
try {
await deleteFile(`./../storage/recordings/${req.params.fileName}.waw.json`);
await Robot.destroy({
where: { 'recording_meta.id': req.params.id }
});
return res.send(true);
} catch (e) {
const { message } = e as Error;
logger.log('info', `Error while deleting a recording with name: ${req.params.fileName}.waw.json`);
logger.log('info', `Error while deleting a recording with name: ${req.params.fileName}.json`);
return res.send(false);
}
});
@@ -67,7 +71,7 @@ router.delete('/recordings/:fileName', requireSignIn, async (req, res) => {
*/
router.get('/runs', requireSignIn, async (req, res) => {
try {
const data = await readFiles('./../storage/runs/');
const data = await Run.findAll();
return res.send(data);
} catch (e) {
logger.log('info', 'Error while reading runs');
@@ -78,9 +82,9 @@ router.get('/runs', requireSignIn, async (req, res) => {
/**
* DELETE endpoint for deleting a run from the storage.
*/
router.delete('/runs/:fileName', requireSignIn, async (req, res) => {
router.delete('/runs/:id', requireSignIn, async (req, res) => {
try {
await deleteFile(`./../storage/runs/${req.params.fileName}.json`);
await Run.destroy({ where: { runId: req.params.id } });
return res.send(true);
} catch (e) {
const { message } = e as Error;
@@ -93,9 +97,17 @@ router.delete('/runs/:fileName', requireSignIn, async (req, res) => {
* PUT endpoint for starting a remote browser instance and saving run metadata to the storage.
* Making it ready for interpretation and returning a runId.
*/
router.put('/runs/:fileName', requireSignIn, async (req, res) => {
router.put('/runs/:id', requireSignIn, async (req, res) => {
try {
const recording = await getRecordingByFileName(req.params.fileName);
console.log(`Params recieved:`, req.params)
const recording = await Robot.findOne({
where: {
'recording_meta.id': req.params.id
},
raw: true
});
console.log(`Recording found:`, recording)
if (!recording || !recording.recording_meta || !recording.recording_meta.id) {
return res.status(404).send({ error: 'Recording not found' });
@@ -124,33 +136,41 @@ router.put('/runs/:fileName', requireSignIn, async (req, res) => {
const runId = uuid();
const run_meta = {
const run = await Run.create({
status: 'RUNNING',
name: req.params.fileName,
recordingId: recording.recording_meta.id,
name: recording.recording_meta.name,
robotId: recording.id,
robotMetaId: recording.recording_meta.id,
startedAt: new Date().toLocaleString(),
finishedAt: '',
browserId: id,
interpreterSettings: req.body,
log: '',
runId,
};
fs.mkdirSync('../storage/runs', { recursive: true })
await saveFile(
`../storage/runs/${req.params.fileName}_${runId}.json`,
JSON.stringify({ ...run_meta }, null, 2)
);
logger.log('debug', `Created run with name: ${req.params.fileName}.json`);
serializableOutput: {},
binaryOutput: {},
});
console.log('Run meta:', run_meta);
const plainRun = run.toJSON();
console.log(`Created run (plain object):`, plainRun);
// // we need to handle this via DB
// fs.mkdirSync('../storage/runs', { recursive: true })
// await saveFile(
// `../storage/runs/${req.params.fileName}_${runId}.json`,
// JSON.stringify({ ...run_meta }, null, 2)
// );
// logger.log('debug', `Created run with name: ${req.params.fileName}.json`);
// console.log('Run meta:', run_meta);
return res.send({
browserId: id,
runId: runId,
runId: plainRun.runId,
});
} catch (e) {
const { message } = e as Error;
logger.log('info', `Error while creating a run with name: ${req.params.fileName}.json`);
logger.log('info', `Error while creating a run with recording id: ${req.params.id} - ${message}`);
return res.send('');
}
});
@@ -158,15 +178,19 @@ router.put('/runs/:fileName', requireSignIn, async (req, res) => {
/**
* GET endpoint for getting a run from the storage.
*/
router.get('/runs/run/:fileName/:runId', requireSignIn, async (req, res) => {
router.get('/runs/run/:id', requireSignIn, async (req, res) => {
try {
console.log(`Params for GET /runs/run/:id`, req.params.id)
// read the run from storage
const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`)
const parsedRun = JSON.parse(run);
return res.send(parsedRun);
const run = await Run.findOne({ where: { runId: req.params.runId }, raw: true });
//const parsedRun = JSON.parse(run);
if (!run) {
return res.status(404).send(null);
}
return res.send(run);
} catch (e) {
const { message } = e as Error;
logger.log('error', `Error ${message} while reading a run with name: ${req.params.fileName}_${req.params.runId}.json`);
logger.log('error', `Error ${message} while reading a run with id: ${req.params.id}.json`);
return res.send(null);
}
});
@@ -174,38 +198,48 @@ router.get('/runs/run/:fileName/:runId', requireSignIn, async (req, res) => {
/**
* PUT endpoint for finishing a run and saving it to the storage.
*/
router.post('/runs/run/:fileName/:runId', requireSignIn, async (req, res) => {
router.post('/runs/run/:id', requireSignIn, async (req, res) => {
try {
const recording = await readFile(`./../storage/recordings/${req.params.fileName}.waw.json`)
const parsedRecording = JSON.parse(recording);
// const recording = await readFile(`./../storage/recordings/${req.params.fileName}.json`)
// const parsedRecording = JSON.parse(recording);
const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`)
const parsedRun = JSON.parse(run);
// const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`)
// const parsedRun = JSON.parse(run);
console.log(`Params for POST /runs/run/:id`, req.params.id)
const run = await Run.findOne({ where: { runId: req.params.id } });
if (!run) {
return res.status(404).send(false);
}
console.log(`found run: ${run}`)
const plainRun = run.toJSON();
const recording = await Robot.findOne({ where: { 'recording_meta.id': plainRun.robotMetaId }, raw: true });
if (!recording) {
return res.status(404).send(false);
}
// interpret the run in active browser
const browser = browserPool.getRemoteBrowser(parsedRun.browserId);
const browser = browserPool.getRemoteBrowser(plainRun.browserId);
const currentPage = browser?.getCurrentPage();
if (browser && currentPage) {
const interpretationInfo = await browser.interpreter.InterpretRecording(
parsedRecording.recording, currentPage, parsedRun.interpreterSettings);
await destroyRemoteBrowser(parsedRun.browserId);
const run_meta = {
...parsedRun,
recording.recording, currentPage, plainRun.interpreterSettings);
await destroyRemoteBrowser(plainRun.browserId);
await run.update({
...run,
status: 'success',
finishedAt: new Date().toLocaleString(),
browserId: parsedRun.browserId,
browserId: plainRun.browserId,
log: interpretationInfo.log.join('\n'),
serializableOutput: interpretationInfo.serializableOutput,
binaryOutput: interpretationInfo.binaryOutput,
};
fs.mkdirSync('../storage/runs', { recursive: true })
await saveFile(
`../storage/runs/${parsedRun.name}_${req.params.runId}.json`,
JSON.stringify(run_meta, null, 2)
);
googleSheetUpdateTasks[req.params.runId] = {
name: parsedRun.name,
runId: req.params.runId,
});
googleSheetUpdateTasks[req.params.id] = {
name: plainRun.name,
runId: plainRun.runId,
status: 'pending',
retries: 5,
};
@@ -216,15 +250,15 @@ router.post('/runs/run/:fileName/:runId', requireSignIn, async (req, res) => {
}
} catch (e) {
const { message } = e as Error;
logger.log('info', `Error while running a recording with name: ${req.params.fileName}_${req.params.runId}.json`);
logger.log('info', `Error while running a recording with id: ${req.params.id} - ${message}`);
return res.send(false);
}
});
router.put('/schedule/:fileName/', requireSignIn, async (req, res) => {
router.put('/schedule/:id/', requireSignIn, async (req, res) => {
console.log(req.body);
try {
const { fileName } = req.params;
const { id } = req.params;
const {
runEvery,
runEveryUnit,
@@ -233,7 +267,7 @@ router.put('/schedule/:fileName/', requireSignIn, async (req, res) => {
timezone
} = req.body;
if (!fileName || !runEvery || !runEveryUnit || !startFrom || !atTime || !timezone) {
if (!id || !runEvery || !runEveryUnit || !startFrom || !atTime || !timezone) {
return res.status(400).json({ error: 'Missing required parameters' });
}
@@ -281,17 +315,18 @@ router.put('/schedule/:fileName/', requireSignIn, async (req, res) => {
}
const runId = uuid();
const userId = req.user.id;
// await workflowQueue.add(
// 'run workflow',
// { fileName, runId },
// {
// repeat: {
// pattern: cronExpression,
// tz: timezone
// }
// }
// );
await workflowQueue.add(
'run workflow',
{ id, runId, userId },
{
repeat: {
pattern: cronExpression,
tz: timezone
}
}
);
res.status(200).json({
message: 'success',
@@ -316,12 +351,16 @@ router.put('/schedule/:fileName/', requireSignIn, async (req, res) => {
/**
* POST endpoint for aborting a current interpretation of the run.
*/
router.post('/runs/abort/:fileName/:runId', requireSignIn, async (req, res) => {
router.post('/runs/abort/:id', requireSignIn, async (req, res) => {
try {
const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`)
const parsedRun = JSON.parse(run);
console.log(`Params for POST /runs/abort/:id`, req.params.id)
const run = await Run.findOne({ where: { runId: req.params.id } });
if (!run) {
return res.status(404).send(false);
}
const plainRun = run.toJSON();
const browser = browserPool.getRemoteBrowser(parsedRun.browserId);
const browser = browserPool.getRemoteBrowser(plainRun.browserId);
const currentLog = browser?.interpreter.debugMessages.join('/n');
const serializableOutput = browser?.interpreter.serializableData.reduce((reducedObject, item, index) => {
return {
@@ -335,19 +374,21 @@ router.post('/runs/abort/:fileName/:runId', requireSignIn, async (req, res) => {
...reducedObject,
}
}, {});
const run_meta = {
...parsedRun,
await run.update({
...run,
status: 'aborted',
finishedAt: null,
browserId: null,
finishedAt: new Date().toLocaleString(),
browserId: plainRun.browserId,
log: currentLog,
};
serializableOutput,
binaryOutput,
});
fs.mkdirSync('../storage/runs', { recursive: true })
await saveFile(
`../storage/runs/${parsedRun.name}_${req.params.runId}.json`,
JSON.stringify({ ...run_meta, serializableOutput, binaryOutput }, null, 2)
);
// fs.mkdirSync('../storage/runs', { recursive: true })
// await saveFile(
// `../storage/runs/${run.name}_${req.params.runId}.json`,
// JSON.stringify({ ...run_meta, serializableOutput, binaryOutput }, null, 2)
// );
return res.send(true);
} catch (e) {
const { message } = e as Error;

View File

@@ -7,6 +7,7 @@ import logger from "../logger";
import { browserPool } from "../server";
import { readFile } from "../workflow-management/storage";
import { requireSignIn } from '../middlewares/auth';
import Robot from '../models/Robot';
export const router = Router();
@@ -102,23 +103,60 @@ router.put('/pair/:index', requireSignIn, (req, res) => {
/**
* PUT endpoint for updating the currently generated workflow file from the one in the storage.
*/
router.put('/:browserId/:fileName', requireSignIn, async (req, res) => {
// router.put('/:browserId/:fileName', requireSignIn, async (req, res) => {
// try {
// const browser = browserPool.getRemoteBrowser(req.params.browserId);
// logger.log('debug', `Updating workflow file`);
// if (browser && browser.generator) {
// const recording = await readFile(`./../storage/recordings/${req.params.fileName}.json`)
// const parsedRecording = JSON.parse(recording);
// if (parsedRecording.recording) {
// browser.generator?.updateWorkflowFile(parsedRecording.recording, parsedRecording.recording_meta);
// const workflowFile = browser.generator?.getWorkflowFile();
// return res.send(workflowFile);
// }
// }
// return res.send(null);
// } catch (e) {
// const { message } = e as Error;
// logger.log('info', `Error while reading a recording with name: ${req.params.fileName}.json`);
// return res.send(null);
// }
// });
router.put('/:browserId/:robotId', requireSignIn, async (req, res) => {
try {
const browser = browserPool.getRemoteBrowser(req.params.browserId);
logger.log('debug', `Updating workflow file`);
logger.log('debug', `Updating workflow for Robot ID: ${req.params.robotId}`);
if (browser && browser.generator) {
const recording = await readFile(`./../storage/recordings/${req.params.fileName}.waw.json`)
const parsedRecording = JSON.parse(recording);
if (parsedRecording.recording) {
browser.generator?.updateWorkflowFile(parsedRecording.recording, parsedRecording.recording_meta);
const workflowFile = browser.generator?.getWorkflowFile();
const robot = await Robot.findByPk(req.params.robotId);
if (!robot) {
logger.log('info', `Robot not found with ID: ${req.params.robotId}`);
return res.status(404).send({ error: 'Robot not found' });
}
const { recording, recording_meta } = robot;
if (recording && recording.workflow) {
browser.generator.updateWorkflowFile(recording, recording_meta);
const workflowFile = browser.generator.getWorkflowFile();
return res.send(workflowFile);
} else {
logger.log('info', `Invalid recording data for Robot ID: ${req.params.robotId}`);
return res.status(400).send({ error: 'Invalid recording data' });
}
}
return res.send(null);
logger.log('info', `Browser or generator not available for ID: ${req.params.browserId}`);
return res.status(400).send({ error: 'Browser or generator not available' });
} catch (e) {
const { message } = e as Error;
logger.log('info', `Error while reading a recording with name: ${req.params.fileName}.waw.json`);
return res.send(null);
logger.log('error', `Error while updating workflow for Robot ID: ${req.params.robotId}. Error: ${message}`);
return res.status(500).send({ error: 'Internal server error' });
}
});
export default router;

View File

@@ -20,9 +20,9 @@ connection.on('error', (err) => {
const workflowQueue = new Queue('workflow', { connection });
const worker = new Worker('workflow', async job => {
const { fileName, runId } = job.data;
const { runId, userId } = job.data;
try {
const result = await handleRunRecording(fileName, runId);
const result = await handleRunRecording(runId, userId);
return result;
} catch (error) {
logger.error('Error running workflow:', error);
@@ -31,11 +31,11 @@ const worker = new Worker('workflow', async job => {
}, { connection });
worker.on('completed', async (job: any) => {
logger.log(`info`, `Job ${job.id} completed for ${job.data.fileName}_${job.data.runId}`);
logger.log(`info`, `Job ${job.id} completed for ${job.data.runId}`);
});
worker.on('failed', async (job: any, err) => {
logger.log(`error`, `Job ${job.id} failed for ${job.data.fileName}_${job.data.runId}:`, err);
logger.log(`error`, `Job ${job.id} failed for ${job.data.runId}:`, err);
});
console.log('Worker is running...');

View File

@@ -14,6 +14,8 @@ import {
} from "../selector";
import { CustomActions } from "../../../../src/shared/types";
import { workflow } from "../../routes";
import Robot from "../../models/Robot";
import Run from "../../models/Run";
import { saveFile } from "../storage";
import fs from "fs";
import { getBestSelectorForAction } from "../utils";
@@ -486,11 +488,12 @@ export class WorkflowGenerator {
updatedAt: new Date().toLocaleString(),
params: this.getParams() || [],
}
fs.mkdirSync('../storage/recordings', { recursive: true })
await saveFile(
`../storage/recordings/${fileName}.waw.json`,
JSON.stringify({ recording_meta: this.recordingMeta, recording }, null, 2)
);
const robot = await Robot.create({
recording_meta: this.recordingMeta,
recording: recording,
});
logger.log('info', `Robot saved with id: ${robot.id}`);
}
catch (e) {
const { message } = e as Error;

View File

@@ -8,55 +8,76 @@ import logger from '../../logger';
import { browserPool } from "../../server";
import { googleSheetUpdateTasks, processGoogleSheetUpdates } from "../integrations/gsheet";
import { getRecordingByFileName } from "../../routes/storage";
import Robot from "../../models/Robot";
import Run from "../../models/Run";
import { getDecryptedProxyConfig } from "../../routes/proxy";
async function runWorkflow(fileName: string, runId: string) {
if (!runId) {
runId = uuid();
async function runWorkflow(id: string, userId: string) {
if (!id) {
id = uuid();
}
const recording = await getRecordingByFileName(fileName);
const recording = await Robot.findOne({
where: {
'recording_meta.id': id
},
raw: true
});
if (!recording || !recording.recording_meta || !recording.recording_meta.id) {
logger.log('info', `Recording with name: ${fileName} not found`);
return {
success: false,
error: `Recording with name: ${fileName} not found`,
};
}
if (!recording || !recording.recording_meta || !recording.recording_meta.id) {
return {
success: false,
error: 'Recording not found'
};
}
const proxyConfig = await getDecryptedProxyConfig(userId);
let proxyOptions: any = {};
if (proxyConfig.proxy_url) {
proxyOptions = {
server: proxyConfig.proxy_url,
...(proxyConfig.proxy_username && proxyConfig.proxy_password && {
username: proxyConfig.proxy_username,
password: proxyConfig.proxy_password,
}),
};
}
try {
const browserId = createRemoteBrowserForRun({
browser: chromium,
launchOptions: { headless: true }
launchOptions: {
headless: true,
proxy: proxyOptions.server ? proxyOptions : undefined,
}
});
const run_meta = {
const run = await Run.create({
status: 'Scheduled',
name: fileName,
recordingId: recording.recording_meta.id,
name: recording.recording_meta.name,
robotId: recording.id,
robotMetaId: recording.recording_meta.id,
startedAt: new Date().toLocaleString(),
finishedAt: '',
browserId: browserId,
browserId: id,
interpreterSettings: { maxConcurrency: 1, maxRepeats: 1, debug: true },
log: '',
runId: runId,
};
runId: id,
serializableOutput: {},
binaryOutput: {},
});
fs.mkdirSync('../storage/runs', { recursive: true });
await saveFile(
`../storage/runs/${fileName}_${runId}.json`,
JSON.stringify(run_meta, null, 2)
);
logger.log('debug', `Scheduled run with name: ${fileName}_${runId}.json`);
const plainRun = run.toJSON();
return {
browserId,
runId
runId: plainRun.runId,
}
} catch (e) {
const { message } = e as Error;
logger.log('info', `Error while scheduling a run with name: ${fileName}_${runId}.json`);
logger.log('info', `Error while scheduling a run with id: ${id}`);
console.log(message);
return {
success: false,
@@ -65,21 +86,29 @@ async function runWorkflow(fileName: string, runId: string) {
}
}
async function executeRun(fileName: string, runId: string) {
async function executeRun(id: string) {
try {
const recording = await readFile(`./../storage/recordings/${fileName}.waw.json`);
const parsedRecording = JSON.parse(recording);
const run = await Run.findOne({ where: { runId: id } });
if (!run) {
return {
success: false,
error: 'Run not found'
}
}
const run = await readFile(`./../storage/runs/${fileName}_${runId}.json`);
const parsedRun = JSON.parse(run);
const plainRun = run.toJSON();
parsedRun.status = 'running';
await saveFile(
`../storage/runs/${fileName}_${runId}.json`,
JSON.stringify(parsedRun, null, 2)
);
const recording = await Robot.findOne({ where: { 'recording_meta.id': plainRun.robotMetaId }, raw: true });
if (!recording) {
return {
success: false,
error: 'Recording not found'
}
}
const browser = browserPool.getRemoteBrowser(parsedRun.browserId);
plainRun.status = 'running';
const browser = browserPool.getRemoteBrowser(plainRun.browserId);
if (!browser) {
throw new Error('Could not access browser');
}
@@ -90,61 +119,47 @@ async function executeRun(fileName: string, runId: string) {
}
const interpretationInfo = await browser.interpreter.InterpretRecording(
parsedRecording.recording, currentPage, parsedRun.interpreterSettings);
recording.recording, currentPage, plainRun.interpreterSettings);
await destroyRemoteBrowser(parsedRun.browserId);
await destroyRemoteBrowser(plainRun.browserId);
const updated_run_meta = {
...parsedRun,
await run.update({
...run,
status: 'success',
finishedAt: new Date().toLocaleString(),
browserId: parsedRun.browserId,
browserId: plainRun.browserId,
log: interpretationInfo.log.join('\n'),
serializableOutput: interpretationInfo.serializableOutput,
binaryOutput: interpretationInfo.binaryOutput,
};
});
await saveFile(
`../storage/runs/${fileName}_${runId}.json`,
JSON.stringify(updated_run_meta, null, 2)
);
googleSheetUpdateTasks[runId] = {
name: parsedRun.name,
runId: runId,
googleSheetUpdateTasks[id] = {
name: plainRun.name,
runId: id,
status: 'pending',
retries: 5,
};
processGoogleSheetUpdates();
return true;
} catch (error: any) {
logger.log('info', `Error while running a recording with name: ${fileName}_${runId}.json`);
logger.log('info', `Error while running a recording with id: ${id} - ${error.message}`);
console.log(error.message);
const errorRun = await readFile(`./../storage/runs/${fileName}_${runId}.json`);
const parsedErrorRun = JSON.parse(errorRun);
parsedErrorRun.status = 'ERROR';
parsedErrorRun.log += `\nError: ${error.message}`;
await saveFile(
`../storage/runs/${fileName}_${runId}.json`,
JSON.stringify(parsedErrorRun, null, 2)
);
return false;
}
}
async function readyForRunHandler(browserId: string, fileName: string, runId: string) {
async function readyForRunHandler(browserId: string, id: string) {
try {
const interpretation = await executeRun(fileName, runId);
const interpretation = await executeRun(id);
if (interpretation) {
logger.log('info', `Interpretation of ${fileName} succeeded`);
logger.log('info', `Interpretation of ${id} succeeded`);
} else {
logger.log('error', `Interpretation of ${fileName} failed`);
logger.log('error', `Interpretation of ${id} failed`);
await destroyRemoteBrowser(browserId);
}
resetRecordingState(browserId, fileName, runId);
resetRecordingState(browserId, id);
} catch (error: any) {
logger.error(`Error during readyForRunHandler: ${error.message}`);
@@ -152,20 +167,18 @@ async function readyForRunHandler(browserId: string, fileName: string, runId: st
}
}
function resetRecordingState(browserId: string, fileName: string, runId: string) {
function resetRecordingState(browserId: string, id: string) {
browserId = '';
fileName = '';
runId = '';
logger.log(`info`, `reset values for ${browserId}, ${fileName}, and ${runId}`);
id = '';
}
export async function handleRunRecording(fileName: string, runId: string) {
export async function handleRunRecording(id: string, userId: string) {
try {
const result = await runWorkflow(fileName, runId);
const result = await runWorkflow(id, userId);
const { browserId, runId: newRunId } = result;
if (!browserId || !newRunId) {
throw new Error('browserId or runId is undefined');
if (!browserId || !newRunId || !userId) {
throw new Error('browserId or runId or userId is undefined');
}
const socket = io(`http://localhost:8080/${browserId}`, {
@@ -173,9 +186,9 @@ export async function handleRunRecording(fileName: string, runId: string) {
rejectUnauthorized: false
});
socket.on('ready-for-run', () => readyForRunHandler(browserId, fileName, newRunId));
socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId));
logger.log('info', `Running recording: ${fileName}`);
logger.log('info', `Running recording: ${id}`);
socket.on('disconnect', () => {
cleanupSocketListeners(socket, browserId, newRunId);
@@ -186,9 +199,9 @@ export async function handleRunRecording(fileName: string, runId: string) {
}
}
function cleanupSocketListeners(socket: Socket, browserId: string, runId: string) {
socket.off('ready-for-run', () => readyForRunHandler(browserId, '', runId));
logger.log('info', `Cleaned up listeners for browserId: ${browserId}, runId: ${runId}`);
function cleanupSocketListeners(socket: Socket, browserId: string, id: string) {
socket.off('ready-for-run', () => readyForRunHandler(browserId, id));
logger.log('info', `Cleaned up listeners for browserId: ${browserId}, runId: ${id}`);
}
export { runWorkflow };

View File

@@ -32,13 +32,13 @@ export const getStoredRuns = async (): Promise<string[] | null> => {
}
};
export const deleteRecordingFromStorage = async (fileName: string): Promise<boolean> => {
export const deleteRecordingFromStorage = async (id: string): Promise<boolean> => {
try {
const response = await axios.delete(`http://localhost:8080/storage/recordings/${fileName}`);
const response = await axios.delete(`http://localhost:8080/storage/recordings/${id}`);
if (response.status === 200) {
return response.data;
} else {
throw new Error(`Couldn't delete stored recording ${fileName}`);
throw new Error(`Couldn't delete stored recording ${id}`);
}
} catch(error: any) {
console.log(error);
@@ -46,13 +46,13 @@ export const deleteRecordingFromStorage = async (fileName: string): Promise<bool
}
};
export const deleteRunFromStorage = async (fileName: string): Promise<boolean> => {
export const deleteRunFromStorage = async (id: string): Promise<boolean> => {
try {
const response = await axios.delete(`http://localhost:8080/storage/runs/${fileName}`);
const response = await axios.delete(`http://localhost:8080/storage/runs/${id}`);
if (response.status === 200) {
return response.data;
} else {
throw new Error(`Couldn't delete stored recording ${fileName}`);
throw new Error(`Couldn't delete stored recording ${id}`);
}
} catch(error: any) {
console.log(error);
@@ -60,13 +60,13 @@ export const deleteRunFromStorage = async (fileName: string): Promise<boolean> =
}
};
export const editRecordingFromStorage = async (browserId: string, fileName: string): Promise<WorkflowFile | null> => {
export const editRecordingFromStorage = async (browserId: string, robotId: string): Promise<WorkflowFile | null> => {
try {
const response = await axios.put(`http://localhost:8080/workflow/${browserId}/${fileName}`);
const response = await axios.put(`http://localhost:8080/workflow/${browserId}/${robotId}`);
if (response.status === 200) {
return response.data;
} else {
throw new Error(`Couldn't edit stored recording ${fileName}`);
throw new Error(`Couldn't edit stored recording ${robotId}`);
}
} catch(error: any) {
console.log(error);
@@ -74,15 +74,15 @@ export const editRecordingFromStorage = async (browserId: string, fileName: stri
}
};
export const createRunForStoredRecording = async (fileName: string, settings: RunSettings): Promise<CreateRunResponse> => {
export const createRunForStoredRecording = async (id: string, settings: RunSettings): Promise<CreateRunResponse> => {
try {
const response = await axios.put(
`http://localhost:8080/storage/runs/${fileName}`,
`http://localhost:8080/storage/runs/${id}`,
{...settings});
if (response.status === 200) {
return response.data;
} else {
throw new Error(`Couldn't create a run for a recording ${fileName}`);
throw new Error(`Couldn't create a run for a recording ${id}`);
}
} catch(error: any) {
console.log(error);
@@ -90,13 +90,13 @@ export const createRunForStoredRecording = async (fileName: string, settings: Ru
}
}
export const interpretStoredRecording = async (fileName: string, runId: string): Promise<boolean> => {
export const interpretStoredRecording = async (id: string): Promise<boolean> => {
try {
const response = await axios.post(`http://localhost:8080/storage/runs/run/${fileName}/${runId}`);
const response = await axios.post(`http://localhost:8080/storage/runs/run/${id}`);
if (response.status === 200) {
return response.data;
} else {
throw new Error(`Couldn't run a recording ${fileName}`);
throw new Error(`Couldn't run a recording ${id}`);
}
} catch(error: any) {
console.log(error);
@@ -104,13 +104,13 @@ export const interpretStoredRecording = async (fileName: string, runId: string):
}
}
export const notifyAboutAbort = async (fileName: string, runId:string): Promise<boolean> => {
export const notifyAboutAbort = async (id:string): Promise<boolean> => {
try {
const response = await axios.post(`http://localhost:8080/storage/runs/abort/${fileName}/${runId}`);
const response = await axios.post(`http://localhost:8080/storage/runs/abort/${id}`);
if (response.status === 200) {
return response.data;
} else {
throw new Error(`Couldn't abort a running recording ${fileName} with id ${runId}`);
throw new Error(`Couldn't abort a running recording with id ${id}`);
}
} catch(error: any) {
console.log(error);

View File

@@ -65,7 +65,7 @@ export const CollapsibleRow = ({ row, handleDelete, isOpen, currentLog, abortRun
return (
<TableCell key={column.id} align={column.align}>
<IconButton aria-label="add" size= "small" onClick={() => {
deleteRunFromStorage(`${row.name}_${row.runId}`).then((result: boolean) => {
deleteRunFromStorage(`${row.runId}`).then((result: boolean) => {
if (result) {
handleDelete();
}

View File

@@ -16,7 +16,7 @@ import { useGlobalInfoStore } from "../../context/globalInfo";
import { deleteRecordingFromStorage, getStoredRecordings } from "../../api/storage";
interface Column {
id: 'interpret' | 'name' | 'createdAt' | 'edit' | 'updatedAt' | 'delete' | 'schedule' | 'integrate';
id: 'id' | 'interpret' | 'name' | 'createdAt' | 'edit' | 'updatedAt' | 'delete' | 'schedule' | 'integrate';
label: string;
minWidth?: number;
align?: 'right';
@@ -24,6 +24,7 @@ interface Column {
}
const columns: readonly Column[] = [
{ id: 'id', label: 'ID', minWidth: 80 },
{ id: 'interpret', label: 'Run', minWidth: 80 },
{ id: 'name', label: 'Name', minWidth: 80 },
{
@@ -61,7 +62,7 @@ const columns: readonly Column[] = [
];
interface Data {
id: number;
id: string;
name: string;
createdAt: string;
updatedAt: string;
@@ -70,10 +71,10 @@ interface Data {
}
interface RecordingsTableProps {
handleEditRecording: (fileName: string) => void;
handleRunRecording: (fileName: string, params: string[]) => void;
handleScheduleRecording: (fileName: string, params: string[]) => void;
handleIntegrateRecording: (fileName: string, params: string[]) => void;
handleEditRecording: (id: string, fileName: string) => void;
handleRunRecording: (id: string,fileName: string, params: string[]) => void;
handleScheduleRecording: (id: string,fileName: string, params: string[]) => void;
handleIntegrateRecording: (id: string,fileName: string, params: string[]) => void;
}
export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handleScheduleRecording, handleIntegrateRecording }: RecordingsTableProps) => {
@@ -96,13 +97,12 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl
const recordings = await getStoredRecordings();
if (recordings) {
const parsedRows: Data[] = [];
recordings.map((recording, index) => {
const parsedRecording = JSON.parse(recording);
if (parsedRecording.recording_meta) {
recordings.map((recording: any, index: number) => {
if (recording && recording.recording_meta) {
parsedRows.push({
id: index,
...parsedRecording.recording_meta,
content: parsedRecording.recording
...recording.recording_meta,
content: recording.recording
});
}
});
@@ -156,14 +156,14 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl
case 'interpret':
return (
<TableCell key={column.id} align={column.align}>
<InterpretButton handleInterpret={() => handleRunRecording(row.name, row.params || [])} />
<InterpretButton handleInterpret={() => handleRunRecording(row.id, row.name, row.params || [])} />
</TableCell>
);
case 'edit':
return (
<TableCell key={column.id} align={column.align}>
<IconButton aria-label="add" size="small" onClick={() => {
handleEditRecording(row.name);
handleEditRecording(row.id, row.name);
}} sx={{ '&:hover': { color: '#1976d2', backgroundColor: 'transparent' } }}>
<Edit />
</IconButton>
@@ -172,20 +172,20 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl
case 'schedule':
return (
<TableCell key={column.id} align={column.align}>
<ScheduleButton handleSchedule={() => handleScheduleRecording(row.name, row.params || [])} />
<ScheduleButton handleSchedule={() => handleScheduleRecording(row.id, row.name, row.params || [])} />
</TableCell>
);
case 'integrate':
return (
<TableCell key={column.id} align={column.align}>
<IntegrateButton handleIntegrate={() => handleIntegrateRecording(row.name, row.params || [])} />
<IntegrateButton handleIntegrate={() => handleIntegrateRecording(row.id, row.name, row.params || [])} />
</TableCell>
);
case 'delete':
return (
<TableCell key={column.id} align={column.align}>
<IconButton aria-label="add" size="small" onClick={() => {
deleteRecordingFromStorage(row.name).then((result: boolean) => {
deleteRecordingFromStorage(row.id).then((result: boolean) => {
if (result) {
setRows([]);
notify('success', 'Recording deleted successfully');

View File

@@ -77,11 +77,11 @@ export const RunsTable = (
const runs = await getStoredRuns();
if (runs) {
const parsedRows: Data[] = [];
runs.map((run, index) => {
const parsedRun = JSON.parse(run);
runs.map((run: any, index) => {
// const run = JSON.parse(run);
parsedRows.push({
id: index,
...parsedRun,
...run,
});
});
setRows(parsedRows);

View File

@@ -6,68 +6,78 @@ import { ScheduleSettings, ScheduleSettingsModal } from "../molecules/ScheduleSe
import { IntegrationSettings, IntegrationSettingsModal } from "../molecules/IntegrationSettings";
interface RecordingsProps {
handleEditRecording: (fileName: string) => void;
handleEditRecording: (id: string, fileName: string) => void;
handleRunRecording: (settings: RunSettings) => void;
handleScheduleRecording: (settings: ScheduleSettings) => void;
handleIntegrateRecording: (settings: IntegrationSettings) => void;
setFileName: (fileName: string) => void;
setRecordingInfo: (id: string, name: string) => void;
}
export const Recordings = ({ handleEditRecording, handleRunRecording, setFileName, handleScheduleRecording, handleIntegrateRecording }: RecordingsProps) => {
export const Recordings = ({ handleEditRecording, handleRunRecording, setRecordingInfo, handleScheduleRecording, handleIntegrateRecording }: RecordingsProps) => {
const [runSettingsAreOpen, setRunSettingsAreOpen] = useState(false);
const [scheduleSettingsAreOpen, setScheduleSettingsAreOpen] = useState(false);
const [integrateSettingsAreOpen, setIntegrateSettingsAreOpen] = useState(false);
const [params, setParams] = useState<string[]>([]);
const [selectedRecordingId, setSelectedRecordingId] = useState<string>('');
const handleSettingsAndIntegrate = (fileName: string, params: string[]) => {
const handleSettingsAndIntegrate = (id: string, name: string, params: string[]) => {
if (params.length === 0) {
setIntegrateSettingsAreOpen(true);
setFileName(fileName);
setRecordingInfo(id, name);
setSelectedRecordingId(id);
} else {
setParams(params);
setIntegrateSettingsAreOpen(true);
setFileName(fileName);
setRecordingInfo(id, name);
setSelectedRecordingId(id);
}
}
const handleSettingsAndRun = (fileName: string, params: string[]) => {
const handleSettingsAndRun = (id: string, name: string, params: string[]) => {
if (params.length === 0) {
setRunSettingsAreOpen(true);
setFileName(fileName);
setRecordingInfo(id, name);
setSelectedRecordingId(id);
} else {
setParams(params);
setRunSettingsAreOpen(true);
setFileName(fileName);
setRecordingInfo(id, name);
setSelectedRecordingId(id);
}
}
const handleSettingsAndSchedule = (fileName: string, params: string[]) => {
const handleSettingsAndSchedule = (id: string, name: string, params: string[]) => {
if (params.length === 0) {
setScheduleSettingsAreOpen(true);
setFileName(fileName);
setRecordingInfo(id, name);
setSelectedRecordingId(id);
} else {
setParams(params);
setScheduleSettingsAreOpen(true);
setFileName(fileName);
setRecordingInfo(id, name);
setSelectedRecordingId(id);
}
}
const handleClose = () => {
setParams([]);
setRunSettingsAreOpen(false);
setFileName('');
setRecordingInfo('', '');
setSelectedRecordingId('');
}
const handleIntegrateClose = () => {
setParams([]);
setIntegrateSettingsAreOpen(false);
setFileName('');
setRecordingInfo('', '');
setSelectedRecordingId('');
}
const handleScheduleClose = () => {
setParams([]);
setScheduleSettingsAreOpen(false);
setFileName('');
setRecordingInfo('', '');
setSelectedRecordingId('');
}
return (
@@ -98,4 +108,4 @@ export const Recordings = ({ handleEditRecording, handleRunRecording, setFileNam
</Grid>
</React.Fragment>
);
}
}

View File

@@ -16,6 +16,8 @@ interface GlobalInfo {
setRerenderRuns: (rerenderRuns: boolean) => void;
recordingLength: number;
setRecordingLength: (recordingLength: number) => void;
recordingId: string | null;
setRecordingId: (newId: string | null) => void;
recordingName: string;
setRecordingName: (recordingName: string) => void;
};
@@ -29,6 +31,7 @@ class GlobalInfoStore implements Partial<GlobalInfo> {
message: '',
isOpen: false,
};
recordingId = null;
recordings: string[] = [];
rerenderRuns = false;
recordingName = '';
@@ -46,6 +49,7 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
const [recordings, setRecordings] = useState<string[]>(globalInfoStore.recordings);
const [rerenderRuns, setRerenderRuns] = useState<boolean>(globalInfoStore.rerenderRuns);
const [recordingLength, setRecordingLength] = useState<number>(globalInfoStore.recordingLength);
const [recordingId, setRecordingId] = useState<string | null>(globalInfoStore.recordingId);
const [recordingName, setRecordingName] = useState<string>(globalInfoStore.recordingName);
const notify = (severity: 'error' | 'warning' | 'info' | 'success', message: string) => {
@@ -79,6 +83,8 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
setRerenderRuns,
recordingLength,
setRecordingLength,
recordingId,
setRecordingId,
recordingName,
setRecordingName
}}

View File

@@ -15,7 +15,7 @@ import { ScheduleSettings } from "../components/molecules/ScheduleSettings";
import { IntegrationSettings } from "../components/molecules/IntegrationSettings";
interface MainPageProps {
handleEditRecording: (fileName: string) => void;
handleEditRecording: (id: string, fileName: string) => void;
}
export interface CreateRunResponse {
@@ -32,6 +32,7 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => {
const [content, setContent] = React.useState('recordings');
const [sockets, setSockets] = React.useState<Socket[]>([]);
const [runningRecordingId, setRunningRecordingId] = React.useState('');
const [runningRecordingName, setRunningRecordingName] = React.useState('');
const [currentInterpretationLog, setCurrentInterpretationLog] = React.useState('');
const [ids, setIds] = React.useState<CreateRunResponse>({
@@ -45,7 +46,7 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => {
const abortRunHandler = (runId: string) => {
aborted = true;
notifyAboutAbort(runningRecordingName, runId).then(async (response) => {
notifyAboutAbort(runId).then(async (response) => {
if (response) {
notify('success', `Interpretation of ${runningRecordingName} aborted successfully`);
await stopRecording(ids.browserId);
@@ -55,12 +56,13 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => {
})
}
const setFileName = (fileName: string) => {
setRunningRecordingName(fileName);
const setRecordingInfo = (id: string, name: string) => {
setRunningRecordingId(id);
setRunningRecordingName(name);
}
const readyForRunHandler = useCallback((browserId: string, runId: string) => {
interpretStoredRecording(runningRecordingName, runId).then(async (interpretation: boolean) => {
interpretStoredRecording(runId).then(async (interpretation: boolean) => {
if (!aborted) {
if (interpretation) {
notify('success', `Interpretation of ${runningRecordingName} succeeded`);
@@ -82,7 +84,7 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => {
}, [currentInterpretationLog])
const handleRunRecording = useCallback((settings: RunSettings) => {
createRunForStoredRecording(runningRecordingName, settings).then(({ browserId, runId }: CreateRunResponse) => {
createRunForStoredRecording(runningRecordingId, settings).then(({ browserId, runId }: CreateRunResponse) => {
setIds({ browserId, runId });
const socket =
io(`http://localhost:8080/${browserId}`, {
@@ -98,7 +100,7 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => {
} else {
notify('error', `Failed to run recording: ${runningRecordingName}`);
}
});
})
return (socket: Socket, browserId: string, runId: string) => {
socket.off('ready-for-run', () => readyForRunHandler(browserId, runId));
socket.off('debugMessage', debugMessageHandler);
@@ -133,7 +135,7 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => {
return <Recordings
handleEditRecording={handleEditRecording}
handleRunRecording={handleRunRecording}
setFileName={setFileName}
setRecordingInfo={setRecordingInfo}
handleScheduleRecording={handleScheduleRecording}
handleIntegrateRecording={handleIntegrateRecording}
/>;

View File

@@ -18,20 +18,20 @@ export const PageWrapper = () => {
const navigate = useNavigate();
const { browserId, setBrowserId, notification, recordingName, setRecordingName } = useGlobalInfoStore();
const { browserId, setBrowserId, notification, recordingName, setRecordingName, recordingId, setRecordingId } = useGlobalInfoStore();
const handleNewRecording = () => {
setBrowserId('new-recording');
setRecordingName('');
setRecordingId('');
navigate('/recording');
}
const handleEditRecording = (fileName: string) => {
const handleEditRecording = (recordingId: string, fileName: string) => {
setRecordingName(fileName);
setRecordingId(recordingId);
setBrowserId('new-recording');
navigate('/recording');
}
const isNotification = (): boolean => {