diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 74faa504..6e74a287 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -11,6 +11,8 @@ import cron from 'node-cron'; import { googleSheetUpdateTasks, processGoogleSheetUpdates } from '../workflow-management/integrations/gsheet'; import { getDecryptedProxyConfig } from './proxy'; import { requireSignIn } from '../middlewares/auth'; +import Robot from '../models/Robot'; +import Run from '../models/Run'; // import { workflowQueue } from '../worker'; // todo: move from here @@ -40,7 +42,7 @@ router.all('/', requireSignIn, (req, res, next) => { */ router.get('/recordings', requireSignIn, async (req, res) => { try { - const data = await readFiles('./../storage/recordings/'); + const data = await Robot.findAll(); return res.send(data); } catch (e) { logger.log('info', 'Error while reading recordings'); @@ -51,9 +53,9 @@ router.get('/recordings', requireSignIn, async (req, res) => { /** * DELETE endpoint for deleting a recording from the storage. */ -router.delete('/recordings/:fileName', requireSignIn, async (req, res) => { +router.delete('/recordings/:id', requireSignIn, async (req, res) => { try { - await deleteFile(`./../storage/recordings/${req.params.fileName}.json`); + await Robot.destroy({ where: { id: req.params.id } }); return res.send(true); } catch (e) { const { message } = e as Error; @@ -67,7 +69,7 @@ router.delete('/recordings/:fileName', requireSignIn, async (req, res) => { */ router.get('/runs', requireSignIn, async (req, res) => { try { - const data = await readFiles('./../storage/runs/'); + const data = await Run.findAll(); return res.send(data); } catch (e) { logger.log('info', 'Error while reading runs'); @@ -78,9 +80,9 @@ router.get('/runs', requireSignIn, async (req, res) => { /** * DELETE endpoint for deleting a run from the storage. */ -router.delete('/runs/:fileName', requireSignIn, async (req, res) => { +router.delete('/runs/:id', requireSignIn, async (req, res) => { try { - await deleteFile(`./../storage/runs/${req.params.fileName}.json`); + await Run.destroy({ where: { id: req.params.id } }); return res.send(true); } catch (e) { const { message } = e as Error; @@ -93,9 +95,14 @@ router.delete('/runs/:fileName', requireSignIn, async (req, res) => { * PUT endpoint for starting a remote browser instance and saving run metadata to the storage. * Making it ready for interpretation and returning a runId. */ -router.put('/runs/:fileName', requireSignIn, async (req, res) => { +router.put('/runs/:id', requireSignIn, async (req, res) => { try { - const recording = await getRecordingByFileName(req.params.fileName); + console.log(`Params recieved:`,req.params ) + const recording = await Robot.findOne({ + where: { + 'recording_meta.id': req.params.id + } + }); if (!recording || !recording.recording_meta || !recording.recording_meta.id) { return res.status(404).send({ error: 'Recording not found' }); @@ -124,29 +131,35 @@ router.put('/runs/:fileName', requireSignIn, async (req, res) => { const runId = uuid(); - const run_meta = { + const run = await Run.create({ status: 'RUNNING', name: req.params.fileName, - recordingId: recording.recording_meta.id, + robotId: recording.recording_meta.id, startedAt: new Date().toLocaleString(), finishedAt: '', browserId: id, interpreterSettings: req.body, log: '', runId, - }; - fs.mkdirSync('../storage/runs', { recursive: true }) - await saveFile( - `../storage/runs/${req.params.fileName}_${runId}.json`, - JSON.stringify({ ...run_meta }, null, 2) - ); - logger.log('debug', `Created run with name: ${req.params.fileName}.json`); + serializableOutput: {}, + binaryOutput: {}, + }); - console.log('Run meta:', run_meta); + // // we need to handle this via DB + // fs.mkdirSync('../storage/runs', { recursive: true }) + // await saveFile( + // `../storage/runs/${req.params.fileName}_${runId}.json`, + // JSON.stringify({ ...run_meta }, null, 2) + // ); + // logger.log('debug', `Created run with name: ${req.params.fileName}.json`); + + // console.log('Run meta:', run_meta); + + logger.log('debug', `Created run with id: ${run.id}`); return res.send({ browserId: id, - runId: runId, + runId: run.id, }); } catch (e) { const { message } = e as Error; @@ -158,12 +171,16 @@ router.put('/runs/:fileName', requireSignIn, async (req, res) => { /** * GET endpoint for getting a run from the storage. */ -router.get('/runs/run/:fileName/:runId', requireSignIn, async (req, res) => { +router.get('/runs/run/:id', requireSignIn, async (req, res) => { try { + console.log(`Params for GET /runs/run/:id`, req.params.id) // read the run from storage - const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`) - const parsedRun = JSON.parse(run); - return res.send(parsedRun); + const run = await Run.findByPk(req.params.id); + //const parsedRun = JSON.parse(run); + if (!run) { + return res.status(404).send(null); + } + return res.send(run); } catch (e) { const { message } = e as Error; logger.log('error', `Error ${message} while reading a run with name: ${req.params.fileName}_${req.params.runId}.json`); @@ -174,38 +191,44 @@ router.get('/runs/run/:fileName/:runId', requireSignIn, async (req, res) => { /** * PUT endpoint for finishing a run and saving it to the storage. */ -router.post('/runs/run/:fileName/:runId', requireSignIn, async (req, res) => { +router.post('/runs/run/:id', requireSignIn, async (req, res) => { try { - const recording = await readFile(`./../storage/recordings/${req.params.fileName}.json`) - const parsedRecording = JSON.parse(recording); + // const recording = await readFile(`./../storage/recordings/${req.params.fileName}.json`) + // const parsedRecording = JSON.parse(recording); - const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`) - const parsedRun = JSON.parse(run); + // const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`) + // const parsedRun = JSON.parse(run); + console.log(`Params for POST /runs/run/:id`, req.params.id) + + const run = await Run.findByPk(req.params.id); + if (!run) { + return res.status(404).send(false); + } + + const recording = await Robot.findByPk(run.robotId); + if (!recording) { + return res.status(404).send(false); + } // interpret the run in active browser - const browser = browserPool.getRemoteBrowser(parsedRun.browserId); + const browser = browserPool.getRemoteBrowser(run.browserId); const currentPage = browser?.getCurrentPage(); if (browser && currentPage) { const interpretationInfo = await browser.interpreter.InterpretRecording( - parsedRecording.recording, currentPage, parsedRun.interpreterSettings); - await destroyRemoteBrowser(parsedRun.browserId); - const run_meta = { - ...parsedRun, + recording.recording, currentPage, run.interpreterSettings); + await destroyRemoteBrowser(run.browserId); + await run.update({ + ...run, status: 'success', finishedAt: new Date().toLocaleString(), - browserId: parsedRun.browserId, + browserId: run.browserId, log: interpretationInfo.log.join('\n'), serializableOutput: interpretationInfo.serializableOutput, binaryOutput: interpretationInfo.binaryOutput, - }; - fs.mkdirSync('../storage/runs', { recursive: true }) - await saveFile( - `../storage/runs/${parsedRun.name}_${req.params.runId}.json`, - JSON.stringify(run_meta, null, 2) - ); + }); googleSheetUpdateTasks[req.params.runId] = { - name: parsedRun.name, - runId: req.params.runId, + name: run.name, + runId: run.id, status: 'pending', retries: 5, }; @@ -316,12 +339,16 @@ router.put('/schedule/:fileName/', requireSignIn, async (req, res) => { /** * POST endpoint for aborting a current interpretation of the run. */ -router.post('/runs/abort/:fileName/:runId', requireSignIn, async (req, res) => { +router.post('/runs/abort/:id', requireSignIn, async (req, res) => { try { - const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`) - const parsedRun = JSON.parse(run); + console.log(`Params for POST /runs/abort/:id`, req.params.id) + const run = await Run.findByPk(req.params.id); + if (!run) { + return res.status(404).send(false); + } + //const parsedRun = JSON.parse(run); - const browser = browserPool.getRemoteBrowser(parsedRun.browserId); + const browser = browserPool.getRemoteBrowser(run.browserId); const currentLog = browser?.interpreter.debugMessages.join('/n'); const serializableOutput = browser?.interpreter.serializableData.reduce((reducedObject, item, index) => { return { @@ -335,19 +362,19 @@ router.post('/runs/abort/:fileName/:runId', requireSignIn, async (req, res) => { ...reducedObject, } }, {}); - const run_meta = { - ...parsedRun, + await run.update({ + ...run, status: 'aborted', - finishedAt: null, - browserId: null, + finishedAt: new Date().toLocaleString(), + browserId: run.browserId, log: currentLog, - }; + }); - fs.mkdirSync('../storage/runs', { recursive: true }) - await saveFile( - `../storage/runs/${parsedRun.name}_${req.params.runId}.json`, - JSON.stringify({ ...run_meta, serializableOutput, binaryOutput }, null, 2) - ); + // fs.mkdirSync('../storage/runs', { recursive: true }) + // await saveFile( + // `../storage/runs/${run.name}_${req.params.runId}.json`, + // JSON.stringify({ ...run_meta, serializableOutput, binaryOutput }, null, 2) + // ); return res.send(true); } catch (e) { const { message } = e as Error;