vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: screenshot minio upload

Browse Source
This commit is contained in:
Rohit Rajan
2025-09-10 13:43:56 +05:30
parent b0f2c85863
commit 0b8fe5c831
3 changed files with 37 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
server/src/api/record.ts
View File

@@ -612,11 +612,22 @@ async function executeRun(id: string, userId: string) {
log: interpretationInfo.log.join('\n'),
});
// Upload binary output to MinIO and update run with MinIO URLs
const finalRun = await Run.findOne({ where: { runId: id } });
if (finalRun && finalRun.binaryOutput && Object.keys(finalRun.binaryOutput).length > 0) {
try {
const binaryService = new BinaryOutputService('maxun-run-screenshots');
await binaryService.uploadAndStoreBinaryOutput(finalRun, finalRun.binaryOutput);
logger.log('info', `Uploaded binary output to MinIO for API run ${id}`);
} catch (minioError: any) {
logger.log('error', `Failed to upload binary output to MinIO for API run ${id}: ${minioError.message}`);
}
}
let totalSchemaItemsExtracted = 0;
let totalListItemsExtracted = 0;
let extractedScreenshotsCount = 0;
const finalRun = await Run.findOne({ where: { runId: id } });
if (finalRun) {
if (finalRun.serializableOutput) {
if (finalRun.serializableOutput.scrapeSchema) {

14
server/src/pgboss-worker.ts
View File

@@ -19,6 +19,7 @@ import { googleSheetUpdateTasks, processGoogleSheetUpdates } from './workflow-ma
import { airtableUpdateTasks, processAirtableUpdates } from './workflow-management/integrations/airtable';
import { io as serverIo } from "./server";
import { sendWebhook } from './routes/webhook';
import { BinaryOutputService } from './storage/mino';
if (!process.env.DB_USER || !process.env.DB_PASSWORD || !process.env.DB_HOST || !process.env.DB_PORT || !process.env.DB_NAME) {
throw new Error('Failed to start pgboss worker: one or more required environment variables are missing.');
@@ -242,11 +243,22 @@ async function processRunExecution(job: Job<ExecuteRunData>) {
log: interpretationInfo.log.join('\n')
});
// Upload binary output to MinIO and update run with MinIO URLs
const updatedRun = await Run.findOne({ where: { runId: data.runId } });
if (updatedRun && updatedRun.binaryOutput && Object.keys(updatedRun.binaryOutput).length > 0) {
try {
const binaryService = new BinaryOutputService('maxun-run-screenshots');
await binaryService.uploadAndStoreBinaryOutput(updatedRun, updatedRun.binaryOutput);
logger.log('info', `Uploaded binary output to MinIO for run ${data.runId}`);
} catch (minioError: any) {
logger.log('error', `Failed to upload binary output to MinIO for run ${data.runId}: ${minioError.message}`);
}
}
let totalSchemaItemsExtracted = 0;
let totalListItemsExtracted = 0;
let extractedScreenshotsCount = 0;
const updatedRun = await Run.findOne({ where: { runId: data.runId } });
if (updatedRun) {
if (updatedRun.serializableOutput) {
if (updatedRun.serializableOutput.scrapeSchema) {

13
server/src/workflow-management/scheduler/index.ts
View File

@@ -176,12 +176,23 @@ async function executeRun(id: string, userId: string) {
log: interpretationInfo.log.join('\n'),
});
// Upload binary output to MinIO and update run with MinIO URLs
const updatedRun = await Run.findOne({ where: { runId: id } });
if (updatedRun && updatedRun.binaryOutput && Object.keys(updatedRun.binaryOutput).length > 0) {
try {
const binaryService = new BinaryOutputService('maxun-run-screenshots');
await binaryService.uploadAndStoreBinaryOutput(updatedRun, updatedRun.binaryOutput);
logger.log('info', `Uploaded binary output to MinIO for scheduled run ${id}`);
} catch (minioError: any) {
logger.log('error', `Failed to upload binary output to MinIO for scheduled run ${id}: ${minioError.message}`);
}
}
// Get metrics from persisted data for analytics and webhooks
let totalSchemaItemsExtracted = 0;
let totalListItemsExtracted = 0;
let extractedScreenshotsCount = 0;
const updatedRun = await Run.findOne({ where: { runId: id } });
if (updatedRun) {
if (updatedRun.serializableOutput) {
if (updatedRun.serializableOutput.scrapeSchema) {