vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: partial data recovery

Browse Source
This commit is contained in:
Rohit Rajan
2025-12-04 17:16:47 +05:30
parent 021cdada46
commit 0bc6c12820
1 changed files with 19 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
server/src/server.ts
View File

@@ -280,27 +280,39 @@ if (require.main === module) {
const run = await Run.findOne({ where: { browserId, status: 'running' } }); const run = await Run.findOne({ where: { browserId, status: 'running' } });
if (run) { if (run) {
const limitedData = { const limitedData = {
scrapeSchemaOutput: browser.interpreter.serializableDataByType?.scrapeSchema scrapeSchemaOutput: browser.interpreter.serializableDataByType?.scrapeSchema || {},
? { "schema-tabular": browser.interpreter.serializableDataByType.scrapeSchema }
: {},
scrapeListOutput: browser.interpreter.serializableDataByType?.scrapeList || {}, scrapeListOutput: browser.interpreter.serializableDataByType?.scrapeList || {},
binaryOutput: browser.interpreter.binaryData || [] binaryOutput: browser.interpreter.binaryData || []
}; };
const binaryOutputRecord = limitedData.binaryOutput.reduce((acc: Record<string, any>, item: any, index: number) => { const binaryOutputRecord = limitedData.binaryOutput.reduce((acc: Record<string, any>, item: any, index: number) => {
acc[`item-${index}`] = item; const key = item.name || `Screenshot ${index + 1}`;
acc[key] = { data: item.data, mimeType: item.mimeType };
return acc; return acc;
}, {}); }, {});
let uploadedBinaryOutput = {};
if (Object.keys(binaryOutputRecord).length > 0) {
try {
const { BinaryOutputService } = require('./storage/mino');
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, binaryOutputRecord);
logger.log('info', `Successfully uploaded ${Object.keys(uploadedBinaryOutput).length} screenshots to MinIO for interrupted run`);
} catch (minioError: any) {
logger.log('error', `Failed to upload binary data to MinIO during shutdown: ${minioError.message}`);
uploadedBinaryOutput = binaryOutputRecord;
}
}
await run.update({ await run.update({
status: 'failed', status: 'failed',
finishedAt: new Date().toLocaleString(), finishedAt: new Date().toLocaleString(),
log: 'Process interrupted during execution - partial data preserved', log: 'Process interrupted during execution - partial data preserved',
serializableOutput: { serializableOutput: {
scrapeSchema: Object.values(limitedData.scrapeSchemaOutput), scrapeSchema: limitedData.scrapeSchemaOutput,
scrapeList: Object.values(limitedData.scrapeListOutput), scrapeList: limitedData.scrapeListOutput,
}, },
binaryOutput: binaryOutputRecord binaryOutput: uploadedBinaryOutput
}); });
} }
} }