vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip: use db model instead of file system

Browse Source
This commit is contained in:
karishmas6
2024-10-09 23:08:30 +05:30
parent 565d86ec54
commit 7e22582997
1 changed files with 83 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
server/src/routes/storage.ts
View File

@@ -11,6 +11,8 @@ import cron from 'node-cron';
import { googleSheetUpdateTasks, processGoogleSheetUpdates } from '../workflow-management/integrations/gsheet'; import { googleSheetUpdateTasks, processGoogleSheetUpdates } from '../workflow-management/integrations/gsheet';
import { getDecryptedProxyConfig } from './proxy'; import { getDecryptedProxyConfig } from './proxy';
import { requireSignIn } from '../middlewares/auth'; import { requireSignIn } from '../middlewares/auth';
import Robot from '../models/Robot';
import Run from '../models/Run';
// import { workflowQueue } from '../worker'; // import { workflowQueue } from '../worker';
// todo: move from here // todo: move from here
@@ -40,7 +42,7 @@ router.all('/', requireSignIn, (req, res, next) => {
*/ */
router.get('/recordings', requireSignIn, async (req, res) => { router.get('/recordings', requireSignIn, async (req, res) => {
try { try {
const data = await readFiles('./../storage/recordings/'); const data = await Robot.findAll();
return res.send(data); return res.send(data);
} catch (e) { } catch (e) {
logger.log('info', 'Error while reading recordings'); logger.log('info', 'Error while reading recordings');
@@ -51,9 +53,9 @@ router.get('/recordings', requireSignIn, async (req, res) => {
/** /**
* DELETE endpoint for deleting a recording from the storage. * DELETE endpoint for deleting a recording from the storage.
*/ */
router.delete('/recordings/:fileName', requireSignIn, async (req, res) => { router.delete('/recordings/:id', requireSignIn, async (req, res) => {
try { try {
await deleteFile(`./../storage/recordings/${req.params.fileName}.json`); await Robot.destroy({ where: { id: req.params.id } });
return res.send(true); return res.send(true);
} catch (e) { } catch (e) {
const { message } = e as Error; const { message } = e as Error;
@@ -67,7 +69,7 @@ router.delete('/recordings/:fileName', requireSignIn, async (req, res) => {
*/ */
router.get('/runs', requireSignIn, async (req, res) => { router.get('/runs', requireSignIn, async (req, res) => {
try { try {
const data = await readFiles('./../storage/runs/'); const data = await Run.findAll();
return res.send(data); return res.send(data);
} catch (e) { } catch (e) {
logger.log('info', 'Error while reading runs'); logger.log('info', 'Error while reading runs');
@@ -78,9 +80,9 @@ router.get('/runs', requireSignIn, async (req, res) => {
/** /**
* DELETE endpoint for deleting a run from the storage. * DELETE endpoint for deleting a run from the storage.
*/ */
router.delete('/runs/:fileName', requireSignIn, async (req, res) => { router.delete('/runs/:id', requireSignIn, async (req, res) => {
try { try {
await deleteFile(`./../storage/runs/${req.params.fileName}.json`); await Run.destroy({ where: { id: req.params.id } });
return res.send(true); return res.send(true);
} catch (e) { } catch (e) {
const { message } = e as Error; const { message } = e as Error;
@@ -93,9 +95,14 @@ router.delete('/runs/:fileName', requireSignIn, async (req, res) => {
* PUT endpoint for starting a remote browser instance and saving run metadata to the storage. * PUT endpoint for starting a remote browser instance and saving run metadata to the storage.
* Making it ready for interpretation and returning a runId. * Making it ready for interpretation and returning a runId.
*/ */
router.put('/runs/:fileName', requireSignIn, async (req, res) => { router.put('/runs/:id', requireSignIn, async (req, res) => {
try { try {
const recording = await getRecordingByFileName(req.params.fileName); console.log(`Params recieved:`,req.params )
const recording = await Robot.findOne({
where: {
'recording_meta.id': req.params.id
}
});
if (!recording || !recording.recording_meta || !recording.recording_meta.id) { if (!recording || !recording.recording_meta || !recording.recording_meta.id) {
return res.status(404).send({ error: 'Recording not found' }); return res.status(404).send({ error: 'Recording not found' });
@@ -124,29 +131,35 @@ router.put('/runs/:fileName', requireSignIn, async (req, res) => {
const runId = uuid(); const runId = uuid();
const run_meta = { const run = await Run.create({
status: 'RUNNING', status: 'RUNNING',
name: req.params.fileName, name: req.params.fileName,
recordingId: recording.recording_meta.id, robotId: recording.recording_meta.id,
startedAt: new Date().toLocaleString(), startedAt: new Date().toLocaleString(),
finishedAt: '', finishedAt: '',
browserId: id, browserId: id,
interpreterSettings: req.body, interpreterSettings: req.body,
log: '', log: '',
runId, runId,
}; serializableOutput: {},
fs.mkdirSync('../storage/runs', { recursive: true }) binaryOutput: {},
await saveFile( });
`../storage/runs/${req.params.fileName}_${runId}.json`,
JSON.stringify({ ...run_meta }, null, 2)
);
logger.log('debug', `Created run with name: ${req.params.fileName}.json`);
console.log('Run meta:', run_meta); // // we need to handle this via DB
// fs.mkdirSync('../storage/runs', { recursive: true })
// await saveFile(
// `../storage/runs/${req.params.fileName}_${runId}.json`,
// JSON.stringify({ ...run_meta }, null, 2)
// );
// logger.log('debug', `Created run with name: ${req.params.fileName}.json`);
// console.log('Run meta:', run_meta);
logger.log('debug', `Created run with id: ${run.id}`);
return res.send({ return res.send({
browserId: id, browserId: id,
runId: runId, runId: run.id,
}); });
} catch (e) { } catch (e) {
const { message } = e as Error; const { message } = e as Error;
@@ -158,12 +171,16 @@ router.put('/runs/:fileName', requireSignIn, async (req, res) => {
/** /**
* GET endpoint for getting a run from the storage. * GET endpoint for getting a run from the storage.
*/ */
router.get('/runs/run/:fileName/:runId', requireSignIn, async (req, res) => { router.get('/runs/run/:id', requireSignIn, async (req, res) => {
try { try {
console.log(`Params for GET /runs/run/:id`, req.params.id)
// read the run from storage // read the run from storage
const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`) const run = await Run.findByPk(req.params.id);
const parsedRun = JSON.parse(run); //const parsedRun = JSON.parse(run);
return res.send(parsedRun); if (!run) {
return res.status(404).send(null);
}
return res.send(run);
} catch (e) { } catch (e) {
const { message } = e as Error; const { message } = e as Error;
logger.log('error', `Error ${message} while reading a run with name: ${req.params.fileName}_${req.params.runId}.json`); logger.log('error', `Error ${message} while reading a run with name: ${req.params.fileName}_${req.params.runId}.json`);
@@ -174,38 +191,44 @@ router.get('/runs/run/:fileName/:runId', requireSignIn, async (req, res) => {
/** /**
* PUT endpoint for finishing a run and saving it to the storage. * PUT endpoint for finishing a run and saving it to the storage.
*/ */
router.post('/runs/run/:fileName/:runId', requireSignIn, async (req, res) => { router.post('/runs/run/:id', requireSignIn, async (req, res) => {
try { try {
const recording = await readFile(`./../storage/recordings/${req.params.fileName}.json`) // const recording = await readFile(`./../storage/recordings/${req.params.fileName}.json`)
const parsedRecording = JSON.parse(recording); // const parsedRecording = JSON.parse(recording);
const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`) // const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`)
const parsedRun = JSON.parse(run); // const parsedRun = JSON.parse(run);
console.log(`Params for POST /runs/run/:id`, req.params.id)
const run = await Run.findByPk(req.params.id);
if (!run) {
return res.status(404).send(false);
}
const recording = await Robot.findByPk(run.robotId);
if (!recording) {
return res.status(404).send(false);
}
// interpret the run in active browser // interpret the run in active browser
const browser = browserPool.getRemoteBrowser(parsedRun.browserId); const browser = browserPool.getRemoteBrowser(run.browserId);
const currentPage = browser?.getCurrentPage(); const currentPage = browser?.getCurrentPage();
if (browser && currentPage) { if (browser && currentPage) {
const interpretationInfo = await browser.interpreter.InterpretRecording( const interpretationInfo = await browser.interpreter.InterpretRecording(
parsedRecording.recording, currentPage, parsedRun.interpreterSettings); recording.recording, currentPage, run.interpreterSettings);
await destroyRemoteBrowser(parsedRun.browserId); await destroyRemoteBrowser(run.browserId);
const run_meta = { await run.update({
...parsedRun, ...run,
status: 'success', status: 'success',
finishedAt: new Date().toLocaleString(), finishedAt: new Date().toLocaleString(),
browserId: parsedRun.browserId, browserId: run.browserId,
log: interpretationInfo.log.join('\n'), log: interpretationInfo.log.join('\n'),
serializableOutput: interpretationInfo.serializableOutput, serializableOutput: interpretationInfo.serializableOutput,
binaryOutput: interpretationInfo.binaryOutput, binaryOutput: interpretationInfo.binaryOutput,
}; });
fs.mkdirSync('../storage/runs', { recursive: true })
await saveFile(
`../storage/runs/${parsedRun.name}_${req.params.runId}.json`,
JSON.stringify(run_meta, null, 2)
);
googleSheetUpdateTasks[req.params.runId] = { googleSheetUpdateTasks[req.params.runId] = {
name: parsedRun.name, name: run.name,
runId: req.params.runId, runId: run.id,
status: 'pending', status: 'pending',
retries: 5, retries: 5,
}; };
@@ -316,12 +339,16 @@ router.put('/schedule/:fileName/', requireSignIn, async (req, res) => {
/** /**
* POST endpoint for aborting a current interpretation of the run. * POST endpoint for aborting a current interpretation of the run.
*/ */
router.post('/runs/abort/:fileName/:runId', requireSignIn, async (req, res) => { router.post('/runs/abort/:id', requireSignIn, async (req, res) => {
try { try {
const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`) console.log(`Params for POST /runs/abort/:id`, req.params.id)
const parsedRun = JSON.parse(run); const run = await Run.findByPk(req.params.id);
if (!run) {
return res.status(404).send(false);
}
//const parsedRun = JSON.parse(run);
const browser = browserPool.getRemoteBrowser(parsedRun.browserId); const browser = browserPool.getRemoteBrowser(run.browserId);
const currentLog = browser?.interpreter.debugMessages.join('/n'); const currentLog = browser?.interpreter.debugMessages.join('/n');
const serializableOutput = browser?.interpreter.serializableData.reduce((reducedObject, item, index) => { const serializableOutput = browser?.interpreter.serializableData.reduce((reducedObject, item, index) => {
return { return {
@@ -335,19 +362,19 @@ router.post('/runs/abort/:fileName/:runId', requireSignIn, async (req, res) => {
...reducedObject, ...reducedObject,
} }
}, {}); }, {});
const run_meta = { await run.update({
...parsedRun, ...run,
status: 'aborted', status: 'aborted',
finishedAt: null, finishedAt: new Date().toLocaleString(),
browserId: null, browserId: run.browserId,
log: currentLog, log: currentLog,
}; });
fs.mkdirSync('../storage/runs', { recursive: true }) // fs.mkdirSync('../storage/runs', { recursive: true })
await saveFile( // await saveFile(
`../storage/runs/${parsedRun.name}_${req.params.runId}.json`, // `../storage/runs/${run.name}_${req.params.runId}.json`,
JSON.stringify({ ...run_meta, serializableOutput, binaryOutput }, null, 2) // JSON.stringify({ ...run_meta, serializableOutput, binaryOutput }, null, 2)
); // );
return res.send(true); return res.send(true);
} catch (e) { } catch (e) {
const { message } = e as Error; const { message } = e as Error;