diff --git a/server/src/api/record.ts b/server/src/api/record.ts index 2d6077a9..9eec3edb 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -17,6 +17,8 @@ import { AuthenticatedRequest } from "../routes/record" import {capture} from "../utils/analytics"; import { Page } from "playwright"; import { WorkflowFile } from "maxun-core"; +import { googleSheetUpdateTasks, processGoogleSheetUpdates } from "../workflow-management/integrations/gsheet"; +import { airtableUpdateTasks, processAirtableUpdates } from "../workflow-management/integrations/airtable"; chromium.use(stealthPlugin()); const formatRecording = (recordingData: any) => { @@ -338,14 +340,29 @@ function formatRunResponse(run: any) { runByUserId: run.runByUserId, runByScheduleId: run.runByScheduleId, runByAPI: run.runByAPI, - data: {}, - screenshot: null, + data: { + textData: [], + listData: [] + }, + screenshots: [] as any[], }; - if (run.serializableOutput && run.serializableOutput['item-0']) { - formattedRun.data = run.serializableOutput['item-0']; - } else if (run.binaryOutput && run.binaryOutput['item-0']) { - formattedRun.screenshot = run.binaryOutput['item-0']; + if (run.serializableOutput) { + if (run.serializableOutput.scrapeSchema && run.serializableOutput.scrapeSchema.length > 0) { + formattedRun.data.textData = run.serializableOutput.scrapeSchema; + } + + if (run.serializableOutput.scrapeList && run.serializableOutput.scrapeList.length > 0) { + formattedRun.data.listData = run.serializableOutput.scrapeList; + } + } + + if (run.binaryOutput) { + Object.keys(run.binaryOutput).forEach(key => { + if (run.binaryOutput[key]) { + formattedRun.screenshots.push(run.binaryOutput[key]); + } + }); } return formattedRun; @@ -568,7 +585,7 @@ async function executeRun(id: string, userId: string) { plainRun.status = 'running'; - const browser = browserPool.getRemoteBrowser(userId); + const browser = browserPool.getRemoteBrowser(plainRun.browserId); if (!browser) { throw new Error('Could not access browser'); } @@ -606,24 +623,36 @@ async function executeRun(id: string, userId: string) { binaryOutput: uploadedBinaryOutput, }); - let totalRowsExtracted = 0; + let totalSchemaItemsExtracted = 0; + let totalListItemsExtracted = 0; let extractedScreenshotsCount = 0; - let extractedItemsCount = 0; - - if (updatedRun.dataValues.binaryOutput && updatedRun.dataValues.binaryOutput["item-0"]) { - extractedScreenshotsCount = 1; + + if (categorizedOutput.scrapeSchema) { + Object.values(categorizedOutput.scrapeSchema).forEach((schemaResult: any) => { + if (Array.isArray(schemaResult)) { + totalSchemaItemsExtracted += schemaResult.length; + } else if (schemaResult && typeof schemaResult === 'object') { + totalSchemaItemsExtracted += 1; + } + }); } - - if (updatedRun.dataValues.serializableOutput && updatedRun.dataValues.serializableOutput["item-0"]) { - const itemsArray = run.dataValues.serializableOutput["item-0"]; - extractedItemsCount = itemsArray.length; - - totalRowsExtracted = itemsArray.reduce((total, item) => { - return total + Object.keys(item).length; - }, 0); + + if (categorizedOutput.scrapeList) { + Object.values(categorizedOutput.scrapeList).forEach((listResult: any) => { + if (Array.isArray(listResult)) { + totalListItemsExtracted += listResult.length; + } + }); } - - console.log(`Extracted Items Count: ${extractedItemsCount}`); + + if (uploadedBinaryOutput) { + extractedScreenshotsCount = Object.keys(uploadedBinaryOutput).length; + } + + const totalRowsExtracted = totalSchemaItemsExtracted + totalListItemsExtracted; + + console.log(`Extracted Schema Items Count: ${totalSchemaItemsExtracted}`); + console.log(`Extracted List Items Count: ${totalListItemsExtracted}`); console.log(`Extracted Screenshots Count: ${extractedScreenshotsCount}`); console.log(`Total Rows Extracted: ${totalRowsExtracted}`); @@ -631,12 +660,34 @@ async function executeRun(id: string, userId: string) { runId: id, created_at: new Date().toISOString(), status: 'success', - extractedItemsCount, totalRowsExtracted, + schemaItemsExtracted: totalSchemaItemsExtracted, + listItemsExtracted: totalListItemsExtracted, extractedScreenshotsCount, } ) + try { + googleSheetUpdateTasks[id] = { + robotId: plainRun.robotMetaId, + runId: id, + status: 'pending', + retries: 5, + }; + + airtableUpdateTasks[id] = { + robotId: plainRun.robotMetaId, + runId: id, + status: 'pending', + retries: 5, + }; + + processAirtableUpdates(); + processGoogleSheetUpdates(); + } catch (err: any) { + logger.log('error', `Failed to update Google Sheet for run: ${plainRun.runId}: ${err.message}`); + } + return { success: true, interpretationInfo: updatedRun.toJSON() @@ -790,7 +841,7 @@ router.post("/robots/:id/runs", requireAPIKey, async (req: AuthenticatedRequest, if (!req.user) { return res.status(401).json({ ok: false, error: 'Unauthorized' }); } - const runId = await handleRunRecording(req.params.id, req.user.dataValues.id); + const runId = await handleRunRecording(req.params.id, req.user.id); if (!runId) { throw new Error('Run ID is undefined');