feat: revamp gsheet integration multiple actions

server/src/workflow-management/integrations/gsheet.ts

@@ -10,6 +10,12 @@ interface GoogleSheetUpdateTask {
   retries: number;
 }
 
+interface SerializableOutput {
+  scrapeSchema?: any[];
+  scrapeList?: any[];
+  other?: any[];
+}
+
 const MAX_RETRIES = 5;
 
 export let googleSheetUpdateTasks: { [runId: string]: GoogleSheetUpdateTask } = {};
@@ -25,18 +31,6 @@ export async function updateGoogleSheet(robotId: string, runId: string) {
     const plainRun = run.toJSON();
 
     if (plainRun.status === 'success') {
-      let data: { [key: string]: any }[] = [];
-      if (plainRun.serializableOutput && Object.keys(plainRun.serializableOutput).length > 0) {
-        data = plainRun.serializableOutput['item-0'] as { [key: string]: any }[];
-    
-      } else if (plainRun.binaryOutput && plainRun.binaryOutput['item-0']) {
-        // Handle binaryOutput by setting the URL as a data entry
-        const binaryUrl = plainRun.binaryOutput['item-0'] as string;
-    
-        // Create a placeholder object with the binary URL
-        data = [{ "Screenshot URL": binaryUrl }];
-      }
-
       const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } });
 
       if (!robot) {
@@ -44,35 +38,169 @@ export async function updateGoogleSheet(robotId: string, runId: string) {
       }
 
       const plainRobot = robot.toJSON();
-
       const spreadsheetId = plainRobot.google_sheet_id;
-      if (plainRobot.google_sheet_email && spreadsheetId) {
-        console.log(`Preparing to write data to Google Sheet for robot: ${robotId}, spreadsheetId: ${spreadsheetId}`);
       
-        await writeDataToSheet(robotId, spreadsheetId, data);
+      if (!plainRobot.google_sheet_email || !spreadsheetId) {
-        console.log(`Data written to Google Sheet successfully for Robot: ${robotId} and Run: ${runId}`);
-      } else {
         console.log('Google Sheets integration not configured.');
+        return;
       }
+
+      console.log(`Preparing to write data to Google Sheet for robot: ${robotId}, spreadsheetId: ${spreadsheetId}`);
+      
+      const serializableOutput = plainRun.serializableOutput as SerializableOutput;
+      
+      if (serializableOutput) {
+        if (serializableOutput.scrapeSchema && serializableOutput.scrapeSchema.length > 0) {
+          await processOutputType(
+            robotId, 
+            spreadsheetId, 
+            'Text', 
+            serializableOutput.scrapeSchema, 
+            plainRobot
+          );
+        }
+        
+        if (serializableOutput.scrapeList && serializableOutput.scrapeList.length > 0) {
+          await processOutputType(
+            robotId, 
+            spreadsheetId, 
+            'List', 
+            serializableOutput.scrapeList, 
+            plainRobot
+          );
+        }
+        
+        if (serializableOutput.other && serializableOutput.other.length > 0) {
+          await processOutputType(
+            robotId, 
+            spreadsheetId, 
+            'Other', 
+            serializableOutput.other, 
+            plainRobot
+          );
+        }
+      }
+      
+      if (plainRun.binaryOutput && Object.keys(plainRun.binaryOutput).length > 0) {
+        const screenshots = Object.entries(plainRun.binaryOutput).map(([key, url]) => ({
+          "Screenshot Key": key,
+          "Screenshot URL": url
+        }));
+        
+        await processOutputType(
+          robotId,
+          spreadsheetId,
+          'Screenshot',
+          [screenshots],
+          plainRobot
+        );
+      }
+      
+      console.log(`Data written to Google Sheet successfully for Robot: ${robotId} and Run: ${runId}`);
     } else {
       console.log('Run status is not success or serializableOutput is missing.');
     }
   } catch (error: any) {
     console.error(`Failed to write data to Google Sheet for Robot: ${robotId} and Run: ${runId}: ${error.message}`);
+    throw error;
   }
-};
+}
 
-export async function writeDataToSheet(robotId: string, spreadsheetId: string, data: any[]) {
+async function processOutputType(
-  try {
+  robotId: string, 
-    const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } });
+  spreadsheetId: string, 
+  outputType: string, 
+  outputData: any[], 
+  robotConfig: any
+) {
+  for (let i = 0; i < outputData.length; i++) {
+    const data = outputData[i];
     
-    if (!robot) {
+    if (!data || data.length === 0) {
-      throw new Error(`Robot not found for robotId: ${robotId}`);
+      console.log(`No data to write for ${outputType}-${i}. Skipping.`);
+      continue;
     }
     
-    const plainRobot = robot.toJSON();
+    const sheetName = `${outputType}-${i}`;
     
-    if (!plainRobot.google_access_token || !plainRobot.google_refresh_token) {
+    await ensureSheetExists(spreadsheetId, sheetName, robotConfig);
+    
+    await writeDataToSheet(robotId, spreadsheetId, data, sheetName, robotConfig);
+    console.log(`Data written to ${sheetName} sheet for ${outputType} data`);
+  }
+}
+
+async function ensureSheetExists(spreadsheetId: string, sheetName: string, robotConfig: any) {
+  try {
+    const oauth2Client = getOAuth2Client(robotConfig);
+    const sheets = google.sheets({ version: 'v4', auth: oauth2Client });
+    
+    const response = await sheets.spreadsheets.get({
+      spreadsheetId,
+      fields: 'sheets.properties.title'
+    });
+    
+    const existingSheets = response.data.sheets?.map(sheet => sheet.properties?.title) || [];
+    
+    if (!existingSheets.includes(sheetName)) {
+      await sheets.spreadsheets.batchUpdate({
+        spreadsheetId,
+        requestBody: {
+          requests: [
+            {
+              addSheet: {
+                properties: {
+                  title: sheetName
+                }
+              }
+            }
+          ]
+        }
+      });
+      console.log(`Created new sheet: ${sheetName}`);
+    }
+  } catch (error: any) {
+    logger.log('error', `Error ensuring sheet exists: ${error.message}`);
+    throw error;
+  }
+}
+
+function getOAuth2Client(robotConfig: any) {
+  const oauth2Client = new google.auth.OAuth2(
+    process.env.GOOGLE_CLIENT_ID,
+    process.env.GOOGLE_CLIENT_SECRET,
+    process.env.GOOGLE_REDIRECT_URI
+  );
+
+  oauth2Client.setCredentials({
+    access_token: robotConfig.google_access_token,
+    refresh_token: robotConfig.google_refresh_token,
+  });
+
+  return oauth2Client;
+}
+
+export async function writeDataToSheet(
+  robotId: string, 
+  spreadsheetId: string, 
+  data: any[], 
+  sheetName: string = 'Sheet1',
+  robotConfig?: any
+) {
+  try {
+    let robot = robotConfig;
+    
+    if (!robot) {
+      robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } });
+
+      if (!robot) {
+        throw new Error(`Robot not found for robotId: ${robotId}`);
+      }
+      
+      robot = robot.toJSON();
+    }
+
+    if (!robot.google_access_token || !robot.google_refresh_token) {
       throw new Error('Google Sheets access not configured for user');
     }
 
@@ -83,16 +211,19 @@ export async function writeDataToSheet(robotId: string, spreadsheetId: string, d
     );
 
     oauth2Client.setCredentials({
-      access_token: plainRobot.google_access_token,
+      access_token: robot.google_access_token,
-      refresh_token: plainRobot.google_refresh_token,
+      refresh_token: robot.google_refresh_token,
     });
 
     oauth2Client.on('tokens', async (tokens) => {
-      if (tokens.refresh_token) {
+      if (tokens.refresh_token || tokens.access_token) {
-        await robot.update({ google_refresh_token: tokens.refresh_token });
+        const robotModel = await Robot.findOne({ where: { 'recording_meta.id': robotId } });
-      }
+        if (robotModel) {
-      if (tokens.access_token) {
+          const updateData: any = {};
-        await robot.update({ google_access_token: tokens.access_token });
+          if (tokens.refresh_token) updateData.google_refresh_token = tokens.refresh_token;
+          if (tokens.access_token) updateData.google_access_token = tokens.access_token;
+          await robotModel.update(updateData);
+        }
       }
     });
 
@@ -100,7 +231,7 @@ export async function writeDataToSheet(robotId: string, spreadsheetId: string, d
 
     const checkResponse = await sheets.spreadsheets.values.get({
       spreadsheetId,
-      range: 'Sheet1!1:1', 
+      range: `${sheetName}!1:1`, 
     });
 
     if (!data || data.length === 0) {
@@ -109,7 +240,6 @@ export async function writeDataToSheet(robotId: string, spreadsheetId: string, d
     }
 
     const expectedHeaders = Object.keys(data[0]);
-
     const rows = data.map(item => Object.values(item));
 
     const existingHeaders = 
@@ -129,28 +259,28 @@ export async function writeDataToSheet(robotId: string, spreadsheetId: string, d
     
     if (isSheetEmpty || !headersMatch) {
       resource = { values: [expectedHeaders, ...rows] };
-      console.log('Including headers in the append operation.');
+      console.log(`Including headers in the append operation for sheet ${sheetName}.`);
     } else {
       resource = { values: rows };
-      console.log('Headers already exist and match, only appending data rows.');
+      console.log(`Headers already exist and match in sheet ${sheetName}, only appending data rows.`);
     }
 
-    console.log('Attempting to write to spreadsheet:', spreadsheetId);
+    console.log(`Attempting to write to spreadsheet: ${spreadsheetId}, sheet: ${sheetName}`);
 
     const response = await sheets.spreadsheets.values.append({
       spreadsheetId,
-      range: 'Sheet1!A1',
+      range: `${sheetName}!A1`,
       valueInputOption: 'USER_ENTERED',
       requestBody: resource,
     });
 
     if (response.status === 200) {
-      console.log('Data successfully appended to Google Sheet.');
+      console.log(`Data successfully appended to sheet: ${sheetName}`);
     } else {
       console.error('Google Sheets append failed:', response);
     }
 
-    logger.log(`info`, `Data written to Google Sheet: ${spreadsheetId}`);
+    logger.log(`info`, `Data written to Google Sheet: ${spreadsheetId}, sheet: ${sheetName}`);
   } catch (error: any) {
     logger.log(`error`, `Error writing data to Google Sheet: ${error.message}`);
     throw error;
@@ -169,6 +299,7 @@ export const processGoogleSheetUpdates = async () => {
         try {
           await updateGoogleSheet(task.robotId, task.runId);
           console.log(`Successfully updated Google Sheet for runId: ${runId}`);
+          googleSheetUpdateTasks[runId].status = 'completed';
           delete googleSheetUpdateTasks[runId];
         } catch (error: any) {
           console.error(`Failed to update Google Sheets for run ${task.runId}:`, error);