vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #64 from amhsirak/develop

Browse Source 
chore: local filesystem storage cleanup
This commit is contained in:
Karishma Shukla
2024-10-10 04:38:28 +05:30
committed by GitHub
parent bd4f86a4fe b44ba5778e
commit 22662ec537
2 changed files with 12 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
server/src/routes/storage.ts
View File

@@ -1,10 +1,8 @@
import { Router } from 'express';
import logger from "../logger";
import { deleteFile, readFile, readFiles, saveFile } from "../workflow-management/storage";
import { createRemoteBrowserForRun, destroyRemoteBrowser } from "../browser-management/controller";
import { chromium } from "playwright";
import { browserPool } from "../server";
import fs from "fs";
import { uuid } from "uuidv4";
import moment from 'moment-timezone';
import cron from 'node-cron';
@@ -15,18 +13,6 @@ import Robot from '../models/Robot';
import Run from '../models/Run';
import { workflowQueue } from '../worker';
// todo: move from here
export const getRecordingByFileName = async (fileName: string): Promise<any | null> => {
try {
const recording = await readFile(`./../storage/recordings/${fileName}.json`)
const parsedRecording = JSON.parse(recording);
return parsedRecording;
} catch (error: any) {
console.error(`Error while getting recording for fileName ${fileName}:`, error.message);
return null;
}
};
export const router = Router();
/**
@@ -99,7 +85,6 @@ router.delete('/runs/:id', requireSignIn, async (req, res) => {
*/
router.put('/runs/:id', requireSignIn, async (req, res) => {
try {
console.log(`Params recieved:`, req.params)
const recording = await Robot.findOne({
where: {
'recording_meta.id': req.params.id
@@ -107,8 +92,6 @@ router.put('/runs/:id', requireSignIn, async (req, res) => {
raw: true
});
console.log(`Recording found:`, recording)
if (!recording || !recording.recording_meta || !recording.recording_meta.id) {
return res.status(404).send({ error: 'Recording not found' });
}
@@ -153,17 +136,6 @@ router.put('/runs/:id', requireSignIn, async (req, res) => {
const plainRun = run.toJSON();
console.log(`Created run (plain object):`, plainRun);
// // we need to handle this via DB
// fs.mkdirSync('../storage/runs', { recursive: true })
// await saveFile(
// `../storage/runs/${req.params.fileName}_${runId}.json`,
// JSON.stringify({ ...run_meta }, null, 2)
// );
// logger.log('debug', `Created run with name: ${req.params.fileName}.json`);
// console.log('Run meta:', run_meta);
return res.send({
browserId: id,
runId: plainRun.runId,
@@ -180,10 +152,7 @@ router.put('/runs/:id', requireSignIn, async (req, res) => {
*/
router.get('/runs/run/:id', requireSignIn, async (req, res) => {
try {
console.log(`Params for GET /runs/run/:id`, req.params.id)
// read the run from storage
const run = await Run.findOne({ where: { runId: req.params.runId }, raw: true });
//const parsedRun = JSON.parse(run);
if (!run) {
return res.status(404).send(null);
}
@@ -200,11 +169,6 @@ router.get('/runs/run/:id', requireSignIn, async (req, res) => {
*/
router.post('/runs/run/:id', requireSignIn, async (req, res) => {
try {
// const recording = await readFile(`./../storage/recordings/${req.params.fileName}.json`)
// const parsedRecording = JSON.parse(recording);
// const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`)
// const parsedRun = JSON.parse(run);
console.log(`Params for POST /runs/run/:id`, req.params.id)
const run = await Run.findOne({ where: { runId: req.params.id } });
@@ -212,8 +176,6 @@ router.post('/runs/run/:id', requireSignIn, async (req, res) => {
return res.status(404).send(false);
}
console.log(`found run: ${run}`)
const plainRun = run.toJSON();
const recording = await Robot.findOne({ where: { 'recording_meta.id': plainRun.robotMetaId }, raw: true });
@@ -353,7 +315,6 @@ router.put('/schedule/:id/', requireSignIn, async (req, res) => {
*/
router.post('/runs/abort/:id', requireSignIn, async (req, res) => {
try {
console.log(`Params for POST /runs/abort/:id`, req.params.id)
const run = await Run.findOne({ where: { runId: req.params.id } });
if (!run) {
return res.status(404).send(false);
@@ -383,12 +344,6 @@ router.post('/runs/abort/:id', requireSignIn, async (req, res) => {
serializableOutput,
binaryOutput,
});
// fs.mkdirSync('../storage/runs', { recursive: true })
// await saveFile(
// `../storage/runs/${run.name}_${req.params.runId}.json`,
// JSON.stringify({ ...run_meta, serializableOutput, binaryOutput }, null, 2)
// );
return res.send(true);
} catch (e) {
const { message } = e as Error;

42
server/src/routes/workflow.ts
View File

@@ -5,7 +5,6 @@
import { Router } from 'express';
import logger from "../logger";
import { browserPool } from "../server";
import { readFile } from "../workflow-management/storage";
import { requireSignIn } from '../middlewares/auth';
import Robot from '../models/Robot';
@@ -103,38 +102,21 @@ router.put('/pair/:index', requireSignIn, (req, res) => {
/**
* PUT endpoint for updating the currently generated workflow file from the one in the storage.
*/
// router.put('/:browserId/:fileName', requireSignIn, async (req, res) => {
// try {
// const browser = browserPool.getRemoteBrowser(req.params.browserId);
// logger.log('debug', `Updating workflow file`);
// if (browser && browser.generator) {
// const recording = await readFile(`./../storage/recordings/${req.params.fileName}.json`)
// const parsedRecording = JSON.parse(recording);
// if (parsedRecording.recording) {
// browser.generator?.updateWorkflowFile(parsedRecording.recording, parsedRecording.recording_meta);
// const workflowFile = browser.generator?.getWorkflowFile();
// return res.send(workflowFile);
// }
// }
// return res.send(null);
// } catch (e) {
// const { message } = e as Error;
// logger.log('info', `Error while reading a recording with name: ${req.params.fileName}.json`);
// return res.send(null);
// }
// });
router.put('/:browserId/:robotId', requireSignIn, async (req, res) => {
router.put('/:browserId/:id', requireSignIn, async (req, res) => {
try {
const browser = browserPool.getRemoteBrowser(req.params.browserId);
logger.log('debug', `Updating workflow for Robot ID: ${req.params.robotId}`);
logger.log('debug', `Updating workflow for Robot: ${req.params.id}`);
if (browser && browser.generator) {
const robot = await Robot.findByPk(req.params.robotId);
const robot = await Robot.findOne({
where: {
'recording_meta.id': req.params.id
},
raw: true
});
if (!robot) {
logger.log('info', `Robot not found with ID: ${req.params.robotId}`);
logger.log('info', `Robot not found with ID: ${req.params.id}`);
return res.status(404).send({ error: 'Robot not found' });
}
@@ -145,16 +127,16 @@ router.put('/:browserId/:robotId', requireSignIn, async (req, res) => {
const workflowFile = browser.generator.getWorkflowFile();
return res.send(workflowFile);
} else {
logger.log('info', `Invalid recording data for Robot ID: ${req.params.robotId}`);
logger.log('info', `Invalid recording data for Robot ID: ${req.params.id}`);
return res.status(400).send({ error: 'Invalid recording data' });
}
}
logger.log('info', `Browser or generator not available for ID: ${req.params.browserId}`);
logger.log('info', `Browser or generator not available for ID: ${req.params.id}`);
return res.status(400).send({ error: 'Browser or generator not available' });
} catch (e) {
const { message } = e as Error;
logger.log('error', `Error while updating workflow for Robot ID: ${req.params.robotId}. Error: ${message}`);
logger.log('error', `Error while updating workflow for Robot ID: ${req.params.id}. Error: ${message}`);
return res.status(500).send({ error: 'Internal server error' });
}
});