wip: use db model instead of file system

server/src/routes/storage.ts

@@ -11,6 +11,8 @@ import cron from 'node-cron';
 import { googleSheetUpdateTasks, processGoogleSheetUpdates } from '../workflow-management/integrations/gsheet';
 import { getDecryptedProxyConfig } from './proxy';
 import { requireSignIn } from '../middlewares/auth';
+import Robot from '../models/Robot';
+import Run from '../models/Run';
 // import { workflowQueue } from '../worker';
 
 // todo: move from here
@@ -40,7 +42,7 @@ router.all('/', requireSignIn, (req, res, next) => {
  */
 router.get('/recordings', requireSignIn, async (req, res) => {
   try {
-    const data = await readFiles('./../storage/recordings/');
+    const data = await Robot.findAll();
     return res.send(data);
   } catch (e) {
     logger.log('info', 'Error while reading recordings');
@@ -51,9 +53,9 @@ router.get('/recordings', requireSignIn, async (req, res) => {
 /**
  * DELETE endpoint for deleting a recording from the storage.
  */
-router.delete('/recordings/:fileName', requireSignIn, async (req, res) => {
+router.delete('/recordings/:id', requireSignIn, async (req, res) => {
   try {
-    await deleteFile(`./../storage/recordings/${req.params.fileName}.json`);
+    await Robot.destroy({ where: { id: req.params.id } });
     return res.send(true);
   } catch (e) {
     const { message } = e as Error;
@@ -67,7 +69,7 @@ router.delete('/recordings/:fileName', requireSignIn, async (req, res) => {
  */
 router.get('/runs', requireSignIn, async (req, res) => {
   try {
-    const data = await readFiles('./../storage/runs/');
+    const data = await Run.findAll();
     return res.send(data);
   } catch (e) {
     logger.log('info', 'Error while reading runs');
@@ -78,9 +80,9 @@ router.get('/runs', requireSignIn, async (req, res) => {
 /**
  * DELETE endpoint for deleting a run from the storage.
  */
-router.delete('/runs/:fileName', requireSignIn, async (req, res) => {
+router.delete('/runs/:id', requireSignIn, async (req, res) => {
   try {
-    await deleteFile(`./../storage/runs/${req.params.fileName}.json`);
+    await Run.destroy({ where: { id: req.params.id } });
     return res.send(true);
   } catch (e) {
     const { message } = e as Error;
@@ -93,9 +95,14 @@ router.delete('/runs/:fileName', requireSignIn, async (req, res) => {
  * PUT endpoint for starting a remote browser instance and saving run metadata to the storage.
  * Making it ready for interpretation and returning a runId.
  */
-router.put('/runs/:fileName', requireSignIn, async (req, res) => {
+router.put('/runs/:id', requireSignIn, async (req, res) => {
   try {
-    const recording = await getRecordingByFileName(req.params.fileName);
+    console.log(`Params recieved:`,req.params	)
+    const recording = await Robot.findOne({
+      where: { 
+        'recording_meta.id': req.params.id 
+      }
+    });
 
     if (!recording || !recording.recording_meta || !recording.recording_meta.id) {
       return res.status(404).send({ error: 'Recording not found' });
@@ -124,29 +131,35 @@ router.put('/runs/:fileName', requireSignIn, async (req, res) => {
 
     const runId = uuid();
 
-    const run_meta = {
+    const run = await Run.create({
       status: 'RUNNING',
       name: req.params.fileName,
-      recordingId: recording.recording_meta.id,
+      robotId: recording.recording_meta.id,
       startedAt: new Date().toLocaleString(),
       finishedAt: '',
       browserId: id,
       interpreterSettings: req.body,
       log: '',
       runId,
-    };
-    fs.mkdirSync('../storage/runs', { recursive: true })
-    await saveFile(
-      `../storage/runs/${req.params.fileName}_${runId}.json`,
-      JSON.stringify({ ...run_meta }, null, 2)
-    );
-    logger.log('debug', `Created run with name: ${req.params.fileName}.json`);
+      serializableOutput: {},
+      binaryOutput: {},
+    });
 
-    console.log('Run meta:', run_meta);
+    // // we need to handle this via DB
+    // fs.mkdirSync('../storage/runs', { recursive: true })
+    // await saveFile(
+    //   `../storage/runs/${req.params.fileName}_${runId}.json`,
+    //   JSON.stringify({ ...run_meta }, null, 2)
+    // );
+    // logger.log('debug', `Created run with name: ${req.params.fileName}.json`);
+
+    // console.log('Run meta:', run_meta);
+
+    logger.log('debug', `Created run with id: ${run.id}`);
 
     return res.send({
       browserId: id,
-      runId: runId,
+      runId: run.id,
     });
   } catch (e) {
     const { message } = e as Error;
@@ -158,12 +171,16 @@ router.put('/runs/:fileName', requireSignIn, async (req, res) => {
 /**
  * GET endpoint for getting a run from the storage.
  */
-router.get('/runs/run/:fileName/:runId', requireSignIn, async (req, res) => {
+router.get('/runs/run/:id', requireSignIn, async (req, res) => {
   try {
+    console.log(`Params for GET /runs/run/:id`, req.params.id)
     // read the run from storage
-    const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`)
-    const parsedRun = JSON.parse(run);
-    return res.send(parsedRun);
+    const run = await Run.findByPk(req.params.id);
+    //const parsedRun = JSON.parse(run);
+    if (!run) {
+      return res.status(404).send(null);
+    }
+    return res.send(run);
   } catch (e) {
     const { message } = e as Error;
     logger.log('error', `Error ${message} while reading a run with name: ${req.params.fileName}_${req.params.runId}.json`);
@@ -174,38 +191,44 @@ router.get('/runs/run/:fileName/:runId', requireSignIn, async (req, res) => {
 /**
  * PUT endpoint for finishing a run and saving it to the storage.
  */
-router.post('/runs/run/:fileName/:runId', requireSignIn, async (req, res) => {
+router.post('/runs/run/:id', requireSignIn, async (req, res) => {
   try {
-    const recording = await readFile(`./../storage/recordings/${req.params.fileName}.json`)
-    const parsedRecording = JSON.parse(recording);
+    // const recording = await readFile(`./../storage/recordings/${req.params.fileName}.json`)
+    // const parsedRecording = JSON.parse(recording);
 
-    const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`)
-    const parsedRun = JSON.parse(run);
+    // const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`)
+    // const parsedRun = JSON.parse(run);
+    console.log(`Params for POST /runs/run/:id`, req.params.id)
+
+    const run = await Run.findByPk(req.params.id);
+    if (!run) {
+      return res.status(404).send(false);
+    }
+
+    const recording = await Robot.findByPk(run.robotId);
+    if (!recording) {
+      return res.status(404).send(false);
+    }
 
     // interpret the run in active browser
-    const browser = browserPool.getRemoteBrowser(parsedRun.browserId);
+    const browser = browserPool.getRemoteBrowser(run.browserId);
     const currentPage = browser?.getCurrentPage();
     if (browser && currentPage) {
       const interpretationInfo = await browser.interpreter.InterpretRecording(
-        parsedRecording.recording, currentPage, parsedRun.interpreterSettings);
-      await destroyRemoteBrowser(parsedRun.browserId);
-      const run_meta = {
-        ...parsedRun,
+        recording.recording, currentPage, run.interpreterSettings);
+      await destroyRemoteBrowser(run.browserId);
+      await run.update({
+        ...run,
         status: 'success',
         finishedAt: new Date().toLocaleString(),
-        browserId: parsedRun.browserId,
+        browserId: run.browserId,
         log: interpretationInfo.log.join('\n'),
         serializableOutput: interpretationInfo.serializableOutput,
         binaryOutput: interpretationInfo.binaryOutput,
-      };
-      fs.mkdirSync('../storage/runs', { recursive: true })
-      await saveFile(
-        `../storage/runs/${parsedRun.name}_${req.params.runId}.json`,
-        JSON.stringify(run_meta, null, 2)
-      );
+      });
       googleSheetUpdateTasks[req.params.runId] = {
-        name: parsedRun.name,
-        runId: req.params.runId,
+        name: run.name,
+        runId: run.id,
         status: 'pending',
         retries: 5,
       };
@@ -316,12 +339,16 @@ router.put('/schedule/:fileName/', requireSignIn, async (req, res) => {
 /**
  * POST endpoint for aborting a current interpretation of the run.
  */
-router.post('/runs/abort/:fileName/:runId', requireSignIn, async (req, res) => {
+router.post('/runs/abort/:id', requireSignIn, async (req, res) => {
   try {
-    const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`)
-    const parsedRun = JSON.parse(run);
+    console.log(`Params for POST /runs/abort/:id`, req.params.id)
+    const run = await Run.findByPk(req.params.id);
+    if (!run) {
+      return res.status(404).send(false);
+    }
+    //const parsedRun = JSON.parse(run);
 
-    const browser = browserPool.getRemoteBrowser(parsedRun.browserId);
+    const browser = browserPool.getRemoteBrowser(run.browserId);
     const currentLog = browser?.interpreter.debugMessages.join('/n');
     const serializableOutput = browser?.interpreter.serializableData.reduce((reducedObject, item, index) => {
       return {
@@ -335,19 +362,19 @@ router.post('/runs/abort/:fileName/:runId', requireSignIn, async (req, res) => {
         ...reducedObject,
       }
     }, {});
-    const run_meta = {
-      ...parsedRun,
+    await run.update({
+      ...run,
       status: 'aborted',
-      finishedAt: null,
-      browserId: null,
+      finishedAt: new Date().toLocaleString(),
+      browserId: run.browserId,
       log: currentLog,
-    };
+    });
 
-    fs.mkdirSync('../storage/runs', { recursive: true })
-    await saveFile(
-      `../storage/runs/${parsedRun.name}_${req.params.runId}.json`,
-      JSON.stringify({ ...run_meta, serializableOutput, binaryOutput }, null, 2)
-    );
+    // fs.mkdirSync('../storage/runs', { recursive: true })
+    // await saveFile(
+    //   `../storage/runs/${run.name}_${req.params.runId}.json`,
+    //   JSON.stringify({ ...run_meta, serializableOutput, binaryOutput }, null, 2)
+    // );
     return res.send(true);
   } catch (e) {
     const { message } = e as Error;