vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #60 from amhsirak/develop

Browse Source 
feat: robot & run metadata updates
This commit is contained in:
Karishma Shukla
2024-10-08 23:39:14 +05:30
committed by GitHub
parent 9c6e6d417a 9ce4740c9f
commit affc36f7ed
5 changed files with 62 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
server/src/api/record.ts
View File

@@ -21,7 +21,7 @@ const formatRecording = (recordingData: any) => {
return {
id: recordingMeta.id,
name: recordingMeta.name,
createdAt: new Date(recordingMeta.create_date).getTime(),
createdAt: new Date(recordingMeta.createdAt).getTime(),
inputParameters,
};
};
@@ -75,7 +75,7 @@ const formatRecordingById = (recordingData: any) => {
return {
id: recordingMeta.id,
name: recordingMeta.name,
createdAt: new Date(recordingMeta.create_date).getTime(),
createdAt: new Date(recordingMeta.createdAt).getTime(),
inputParameters,
};
};

57
server/src/routes/storage.ts
View File

@@ -11,7 +11,19 @@ import cron from 'node-cron';
import { googleSheetUpdateTasks, processGoogleSheetUpdates } from '../workflow-management/integrations/gsheet';
import { getDecryptedProxyConfig } from './proxy';
import { requireSignIn } from '../middlewares/auth';
import { workflowQueue } from '../worker';
// import { workflowQueue } from '../worker';
// todo: move from here
export const getRecordingByFileName = async (fileName: string): Promise<any | null> => {
try {
const recording = await readFile(`./../storage/recordings/${fileName}.waw.json`)
const parsedRecording = JSON.parse(recording);
return parsedRecording;
} catch (error: any) {
console.error(`Error while getting recording for fileName ${fileName}:`, error.message);
return null;
}
};
export const router = Router();
@@ -83,6 +95,12 @@ router.delete('/runs/:fileName', requireSignIn, async (req, res) => {
*/
router.put('/runs/:fileName', requireSignIn, async (req, res) => {
try {
const recording = await getRecordingByFileName(req.params.fileName);
if (!recording || !recording.recording_meta || !recording.recording_meta.id) {
return res.status(404).send({ error: 'Recording not found' });
}
const proxyConfig = await getDecryptedProxyConfig(req.user.id);
let proxyOptions: any = {};
@@ -109,10 +127,9 @@ router.put('/runs/:fileName', requireSignIn, async (req, res) => {
const run_meta = {
status: 'RUNNING',
name: req.params.fileName,
recordingId: recording.recording_meta.id,
startedAt: new Date().toLocaleString(),
finishedAt: '',
duration: '',
task: req.body.params ? 'task' : '',
browserId: id,
interpreterSettings: req.body,
log: '',
@@ -171,22 +188,11 @@ router.post('/runs/run/:fileName/:runId', requireSignIn, async (req, res) => {
if (browser && currentPage) {
const interpretationInfo = await browser.interpreter.InterpretRecording(
parsedRecording.recording, currentPage, parsedRun.interpreterSettings);
const duration = Math.round((new Date().getTime() - new Date(parsedRun.startedAt).getTime()) / 1000);
const durString = (() => {
if (duration < 60) {
return `${duration} s`;
}
else {
const minAndS = (duration / 60).toString().split('.');
return `${minAndS[0]} m ${minAndS[1]} s`;
}
})();
await destroyRemoteBrowser(parsedRun.browserId);
const run_meta = {
...parsedRun,
status: 'success',
finishedAt: new Date().toLocaleString(),
duration: durString,
browserId: parsedRun.browserId,
log: interpretationInfo.log.join('\n'),
serializableOutput: interpretationInfo.serializableOutput,
@@ -276,16 +282,16 @@ router.put('/schedule/:fileName/', requireSignIn, async (req, res) => {
const runId = uuid();
await workflowQueue.add(
'run workflow',
{ fileName, runId },
{
repeat: {
pattern: cronExpression,
tz: timezone
}
}
);
// await workflowQueue.add(
// 'run workflow',
// { fileName, runId },
// {
// repeat: {
// pattern: cronExpression,
// tz: timezone
// }
// }
// );
res.status(200).json({
message: 'success',
@@ -331,9 +337,8 @@ router.post('/runs/abort/:fileName/:runId', requireSignIn, async (req, res) => {
}, {});
const run_meta = {
...parsedRun,
status: 'ABORTED',
status: 'aborted',
finishedAt: null,
duration: '',
browserId: null,
log: currentLog,
};

16
server/src/workflow-management/classes/Generator.ts
View File

@@ -18,6 +18,7 @@ import { saveFile } from "../storage";
import fs from "fs";
import { getBestSelectorForAction } from "../utils";
import { browserPool } from "../../server";
import { uuid } from "uuidv4";
interface PersistedGeneratedData {
lastUsedSelector: string;
@@ -29,9 +30,10 @@ interface PersistedGeneratedData {
interface MetaData {
name: string;
create_date: string;
id: string;
createdAt: string;
pairs: number;
update_date: string;
updatedAt: string;
params: string[],
}
@@ -84,9 +86,10 @@ export class WorkflowGenerator {
*/
private recordingMeta: MetaData = {
name: '',
create_date: '',
id: '',
createdAt: '',
pairs: 0,
update_date: '',
updatedAt: '',
params: [],
}
@@ -477,9 +480,10 @@ export class WorkflowGenerator {
try {
this.recordingMeta = {
name: fileName,
create_date: this.recordingMeta.create_date || new Date().toLocaleString(),
id: uuid(),
createdAt: this.recordingMeta.createdAt || new Date().toLocaleString(),
pairs: recording.workflow.length,
update_date: new Date().toLocaleString(),
updatedAt: new Date().toLocaleString(),
params: this.getParams() || [],
}
fs.mkdirSync('../storage/recordings', { recursive: true })

22
server/src/workflow-management/scheduler/index.ts
View File

@@ -7,24 +7,34 @@ import { createRemoteBrowserForRun, destroyRemoteBrowser } from '../../browser-m
import logger from '../../logger';
import { browserPool } from "../../server";
import { googleSheetUpdateTasks, processGoogleSheetUpdates } from "../integrations/gsheet";
import { getRecordingByFileName } from "../../routes/storage";
async function runWorkflow(fileName: string, runId: string) {
if (!runId) {
runId = uuid();
}
const recording = await getRecordingByFileName(fileName);
if (!recording || !recording.recording_meta || !recording.recording_meta.id) {
logger.log('info', `Recording with name: ${fileName} not found`);
return {
success: false,
error: `Recording with name: ${fileName} not found`,
};
}
try {
const browserId = createRemoteBrowserForRun({
browser: chromium,
launchOptions: { headless: true }
});
const run_meta = {
status: 'SCHEDULED',
status: 'Scheduled',
name: fileName,
recordingId: recording.recording_meta.id,
startedAt: new Date().toLocaleString(),
finishedAt: '',
duration: '',
task: '', // Optionally set based on workflow
browserId: browserId,
interpreterSettings: { maxConcurrency: 1, maxRepeats: 1, debug: true },
log: '',
@@ -63,7 +73,7 @@ async function executeRun(fileName: string, runId: string) {
const run = await readFile(`./../storage/runs/${fileName}_${runId}.json`);
const parsedRun = JSON.parse(run);
parsedRun.status = 'RUNNING';
parsedRun.status = 'running';
await saveFile(
`../storage/runs/${fileName}_${runId}.json`,
JSON.stringify(parsedRun, null, 2)
@@ -82,16 +92,12 @@ async function executeRun(fileName: string, runId: string) {
const interpretationInfo = await browser.interpreter.InterpretRecording(
parsedRecording.recording, currentPage, parsedRun.interpreterSettings);
const duration = Math.round((new Date().getTime() - new Date(parsedRun.startedAt).getTime()) / 1000);
const durString = duration < 60 ? `${duration} s` : `${Math.floor(duration / 60)} m ${duration % 60} s`;
await destroyRemoteBrowser(parsedRun.browserId);
const updated_run_meta = {
...parsedRun,
status: 'success',
finishedAt: new Date().toLocaleString(),
duration: durString,
browserId: parsedRun.browserId,
log: interpretationInfo.log.join('\n'),
serializableOutput: interpretationInfo.serializableOutput,

10
src/components/molecules/RecordingsTable.tsx
View File

@@ -16,7 +16,7 @@ import { useGlobalInfoStore } from "../../context/globalInfo";
import { deleteRecordingFromStorage, getStoredRecordings } from "../../api/storage";
interface Column {
id: 'interpret' | 'name' | 'create_date' | 'edit' | 'update_date' | 'delete' | 'schedule' | 'integrate';
id: 'interpret' | 'name' | 'createdAt' | 'edit' | 'updatedAt' | 'delete' | 'schedule' | 'integrate';
label: string;
minWidth?: number;
align?: 'right';
@@ -27,7 +27,7 @@ const columns: readonly Column[] = [
{ id: 'interpret', label: 'Run', minWidth: 80 },
{ id: 'name', label: 'Name', minWidth: 80 },
{
id: 'create_date',
id: 'createdAt',
label: 'Created at',
minWidth: 80,
//format: (value: string) => value.toLocaleString('en-US'),
@@ -48,7 +48,7 @@ const columns: readonly Column[] = [
minWidth: 80,
},
{
id: 'update_date',
id: 'updatedAt',
label: 'Updated at',
minWidth: 80,
//format: (value: string) => value.toLocaleString('en-US'),
@@ -63,8 +63,8 @@ const columns: readonly Column[] = [
interface Data {
id: number;
name: string;
create_date: string;
update_date: string;
createdAt: string;
updatedAt: string;
content: WorkflowFile;
params: string[];
}