From 2ffbdc7d0ad5331da3a93b8673278612f3825ceb Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 29 Apr 2025 20:12:11 +0530 Subject: [PATCH] feat: revamp gsheet integration multiple actions --- .../integrations/gsheet.ts | 213 ++++++++++++++---- 1 file changed, 172 insertions(+), 41 deletions(-) diff --git a/server/src/workflow-management/integrations/gsheet.ts b/server/src/workflow-management/integrations/gsheet.ts index b492a7df..3fbe1229 100644 --- a/server/src/workflow-management/integrations/gsheet.ts +++ b/server/src/workflow-management/integrations/gsheet.ts @@ -10,6 +10,12 @@ interface GoogleSheetUpdateTask { retries: number; } +interface SerializableOutput { + scrapeSchema?: any[]; + scrapeList?: any[]; + other?: any[]; +} + const MAX_RETRIES = 5; export let googleSheetUpdateTasks: { [runId: string]: GoogleSheetUpdateTask } = {}; @@ -25,18 +31,6 @@ export async function updateGoogleSheet(robotId: string, runId: string) { const plainRun = run.toJSON(); if (plainRun.status === 'success') { - let data: { [key: string]: any }[] = []; - if (plainRun.serializableOutput && Object.keys(plainRun.serializableOutput).length > 0) { - data = plainRun.serializableOutput['item-0'] as { [key: string]: any }[]; - - } else if (plainRun.binaryOutput && plainRun.binaryOutput['item-0']) { - // Handle binaryOutput by setting the URL as a data entry - const binaryUrl = plainRun.binaryOutput['item-0'] as string; - - // Create a placeholder object with the binary URL - data = [{ "Screenshot URL": binaryUrl }]; - } - const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); if (!robot) { @@ -44,35 +38,169 @@ export async function updateGoogleSheet(robotId: string, runId: string) { } const plainRobot = robot.toJSON(); - const spreadsheetId = plainRobot.google_sheet_id; - if (plainRobot.google_sheet_email && spreadsheetId) { - console.log(`Preparing to write data to Google Sheet for robot: ${robotId}, spreadsheetId: ${spreadsheetId}`); - - await writeDataToSheet(robotId, spreadsheetId, data); - console.log(`Data written to Google Sheet successfully for Robot: ${robotId} and Run: ${runId}`); - } else { + + if (!plainRobot.google_sheet_email || !spreadsheetId) { console.log('Google Sheets integration not configured.'); + return; } + + console.log(`Preparing to write data to Google Sheet for robot: ${robotId}, spreadsheetId: ${spreadsheetId}`); + + const serializableOutput = plainRun.serializableOutput as SerializableOutput; + + if (serializableOutput) { + if (serializableOutput.scrapeSchema && serializableOutput.scrapeSchema.length > 0) { + await processOutputType( + robotId, + spreadsheetId, + 'Text', + serializableOutput.scrapeSchema, + plainRobot + ); + } + + if (serializableOutput.scrapeList && serializableOutput.scrapeList.length > 0) { + await processOutputType( + robotId, + spreadsheetId, + 'List', + serializableOutput.scrapeList, + plainRobot + ); + } + + if (serializableOutput.other && serializableOutput.other.length > 0) { + await processOutputType( + robotId, + spreadsheetId, + 'Other', + serializableOutput.other, + plainRobot + ); + } + } + + if (plainRun.binaryOutput && Object.keys(plainRun.binaryOutput).length > 0) { + const screenshots = Object.entries(plainRun.binaryOutput).map(([key, url]) => ({ + "Screenshot Key": key, + "Screenshot URL": url + })); + + await processOutputType( + robotId, + spreadsheetId, + 'Screenshot', + [screenshots], + plainRobot + ); + } + + console.log(`Data written to Google Sheet successfully for Robot: ${robotId} and Run: ${runId}`); } else { console.log('Run status is not success or serializableOutput is missing.'); } } catch (error: any) { console.error(`Failed to write data to Google Sheet for Robot: ${robotId} and Run: ${runId}: ${error.message}`); + throw error; } -}; +} -export async function writeDataToSheet(robotId: string, spreadsheetId: string, data: any[]) { +async function processOutputType( + robotId: string, + spreadsheetId: string, + outputType: string, + outputData: any[], + robotConfig: any +) { + for (let i = 0; i < outputData.length; i++) { + const data = outputData[i]; + + if (!data || data.length === 0) { + console.log(`No data to write for ${outputType}-${i}. Skipping.`); + continue; + } + + const sheetName = `${outputType}-${i}`; + + await ensureSheetExists(spreadsheetId, sheetName, robotConfig); + + await writeDataToSheet(robotId, spreadsheetId, data, sheetName, robotConfig); + console.log(`Data written to ${sheetName} sheet for ${outputType} data`); + } +} + +async function ensureSheetExists(spreadsheetId: string, sheetName: string, robotConfig: any) { try { - const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + const oauth2Client = getOAuth2Client(robotConfig); + const sheets = google.sheets({ version: 'v4', auth: oauth2Client }); + + const response = await sheets.spreadsheets.get({ + spreadsheetId, + fields: 'sheets.properties.title' + }); + + const existingSheets = response.data.sheets?.map(sheet => sheet.properties?.title) || []; + + if (!existingSheets.includes(sheetName)) { + await sheets.spreadsheets.batchUpdate({ + spreadsheetId, + requestBody: { + requests: [ + { + addSheet: { + properties: { + title: sheetName + } + } + } + ] + } + }); + console.log(`Created new sheet: ${sheetName}`); + } + } catch (error: any) { + logger.log('error', `Error ensuring sheet exists: ${error.message}`); + throw error; + } +} +function getOAuth2Client(robotConfig: any) { + const oauth2Client = new google.auth.OAuth2( + process.env.GOOGLE_CLIENT_ID, + process.env.GOOGLE_CLIENT_SECRET, + process.env.GOOGLE_REDIRECT_URI + ); + + oauth2Client.setCredentials({ + access_token: robotConfig.google_access_token, + refresh_token: robotConfig.google_refresh_token, + }); + + return oauth2Client; +} + +export async function writeDataToSheet( + robotId: string, + spreadsheetId: string, + data: any[], + sheetName: string = 'Sheet1', + robotConfig?: any +) { + try { + let robot = robotConfig; + if (!robot) { - throw new Error(`Robot not found for robotId: ${robotId}`); + robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + + if (!robot) { + throw new Error(`Robot not found for robotId: ${robotId}`); + } + + robot = robot.toJSON(); } - const plainRobot = robot.toJSON(); - - if (!plainRobot.google_access_token || !plainRobot.google_refresh_token) { + if (!robot.google_access_token || !robot.google_refresh_token) { throw new Error('Google Sheets access not configured for user'); } @@ -83,16 +211,19 @@ export async function writeDataToSheet(robotId: string, spreadsheetId: string, d ); oauth2Client.setCredentials({ - access_token: plainRobot.google_access_token, - refresh_token: plainRobot.google_refresh_token, + access_token: robot.google_access_token, + refresh_token: robot.google_refresh_token, }); oauth2Client.on('tokens', async (tokens) => { - if (tokens.refresh_token) { - await robot.update({ google_refresh_token: tokens.refresh_token }); - } - if (tokens.access_token) { - await robot.update({ google_access_token: tokens.access_token }); + if (tokens.refresh_token || tokens.access_token) { + const robotModel = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + if (robotModel) { + const updateData: any = {}; + if (tokens.refresh_token) updateData.google_refresh_token = tokens.refresh_token; + if (tokens.access_token) updateData.google_access_token = tokens.access_token; + await robotModel.update(updateData); + } } }); @@ -100,7 +231,7 @@ export async function writeDataToSheet(robotId: string, spreadsheetId: string, d const checkResponse = await sheets.spreadsheets.values.get({ spreadsheetId, - range: 'Sheet1!1:1', + range: `${sheetName}!1:1`, }); if (!data || data.length === 0) { @@ -109,7 +240,6 @@ export async function writeDataToSheet(robotId: string, spreadsheetId: string, d } const expectedHeaders = Object.keys(data[0]); - const rows = data.map(item => Object.values(item)); const existingHeaders = @@ -129,28 +259,28 @@ export async function writeDataToSheet(robotId: string, spreadsheetId: string, d if (isSheetEmpty || !headersMatch) { resource = { values: [expectedHeaders, ...rows] }; - console.log('Including headers in the append operation.'); + console.log(`Including headers in the append operation for sheet ${sheetName}.`); } else { resource = { values: rows }; - console.log('Headers already exist and match, only appending data rows.'); + console.log(`Headers already exist and match in sheet ${sheetName}, only appending data rows.`); } - console.log('Attempting to write to spreadsheet:', spreadsheetId); + console.log(`Attempting to write to spreadsheet: ${spreadsheetId}, sheet: ${sheetName}`); const response = await sheets.spreadsheets.values.append({ spreadsheetId, - range: 'Sheet1!A1', + range: `${sheetName}!A1`, valueInputOption: 'USER_ENTERED', requestBody: resource, }); if (response.status === 200) { - console.log('Data successfully appended to Google Sheet.'); + console.log(`Data successfully appended to sheet: ${sheetName}`); } else { console.error('Google Sheets append failed:', response); } - logger.log(`info`, `Data written to Google Sheet: ${spreadsheetId}`); + logger.log(`info`, `Data written to Google Sheet: ${spreadsheetId}, sheet: ${sheetName}`); } catch (error: any) { logger.log(`error`, `Error writing data to Google Sheet: ${error.message}`); throw error; @@ -169,6 +299,7 @@ export const processGoogleSheetUpdates = async () => { try { await updateGoogleSheet(task.robotId, task.runId); console.log(`Successfully updated Google Sheet for runId: ${runId}`); + googleSheetUpdateTasks[runId].status = 'completed'; delete googleSheetUpdateTasks[runId]; } catch (error: any) { console.error(`Failed to update Google Sheets for run ${task.runId}:`, error);