feat: revamp gsheet integration multiple actions
This commit is contained in:
Rohit
@@ -10,6 +10,12 @@ interface GoogleSheetUpdateTask {
|
||||
retries: number;
|
||||
}
|
||||
|
||||
interface SerializableOutput {
|
||||
scrapeSchema?: any[];
|
||||
scrapeList?: any[];
|
||||
other?: any[];
|
||||
}
|
||||
|
||||
const MAX_RETRIES = 5;
|
||||
|
||||
export let googleSheetUpdateTasks: { [runId: string]: GoogleSheetUpdateTask } = {};
|
||||
@@ -25,18 +31,6 @@ export async function updateGoogleSheet(robotId: string, runId: string) {
|
||||
const plainRun = run.toJSON();
|
||||
|
||||
if (plainRun.status === 'success') {
|
||||
let data: { [key: string]: any }[] = [];
|
||||
if (plainRun.serializableOutput && Object.keys(plainRun.serializableOutput).length > 0) {
|
||||
data = plainRun.serializableOutput['item-0'] as { [key: string]: any }[];
|
||||
|
||||
} else if (plainRun.binaryOutput && plainRun.binaryOutput['item-0']) {
|
||||
// Handle binaryOutput by setting the URL as a data entry
|
||||
const binaryUrl = plainRun.binaryOutput['item-0'] as string;
|
||||
|
||||
// Create a placeholder object with the binary URL
|
||||
data = [{ "Screenshot URL": binaryUrl }];
|
||||
}
|
||||
|
||||
const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } });
|
||||
|
||||
if (!robot) {
|
||||
@@ -44,35 +38,169 @@ export async function updateGoogleSheet(robotId: string, runId: string) {
|
||||
}
|
||||
|
||||
const plainRobot = robot.toJSON();
|
||||
|
||||
const spreadsheetId = plainRobot.google_sheet_id;
|
||||
if (plainRobot.google_sheet_email && spreadsheetId) {
|
||||
console.log(`Preparing to write data to Google Sheet for robot: ${robotId}, spreadsheetId: ${spreadsheetId}`);
|
||||
|
||||
await writeDataToSheet(robotId, spreadsheetId, data);
|
||||
console.log(`Data written to Google Sheet successfully for Robot: ${robotId} and Run: ${runId}`);
|
||||
} else {
|
||||
|
||||
if (!plainRobot.google_sheet_email || !spreadsheetId) {
|
||||
console.log('Google Sheets integration not configured.');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Preparing to write data to Google Sheet for robot: ${robotId}, spreadsheetId: ${spreadsheetId}`);
|
||||
|
||||
const serializableOutput = plainRun.serializableOutput as SerializableOutput;
|
||||
|
||||
if (serializableOutput) {
|
||||
if (serializableOutput.scrapeSchema && serializableOutput.scrapeSchema.length > 0) {
|
||||
await processOutputType(
|
||||
robotId,
|
||||
spreadsheetId,
|
||||
'Text',
|
||||
serializableOutput.scrapeSchema,
|
||||
plainRobot
|
||||
);
|
||||
}
|
||||
|
||||
if (serializableOutput.scrapeList && serializableOutput.scrapeList.length > 0) {
|
||||
await processOutputType(
|
||||
robotId,
|
||||
spreadsheetId,
|
||||
'List',
|
||||
serializableOutput.scrapeList,
|
||||
plainRobot
|
||||
);
|
||||
}
|
||||
|
||||
if (serializableOutput.other && serializableOutput.other.length > 0) {
|
||||
await processOutputType(
|
||||
robotId,
|
||||
spreadsheetId,
|
||||
'Other',
|
||||
serializableOutput.other,
|
||||
plainRobot
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (plainRun.binaryOutput && Object.keys(plainRun.binaryOutput).length > 0) {
|
||||
const screenshots = Object.entries(plainRun.binaryOutput).map(([key, url]) => ({
|
||||
"Screenshot Key": key,
|
||||
"Screenshot URL": url
|
||||
}));
|
||||
|
||||
await processOutputType(
|
||||
robotId,
|
||||
spreadsheetId,
|
||||
'Screenshot',
|
||||
[screenshots],
|
||||
plainRobot
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`Data written to Google Sheet successfully for Robot: ${robotId} and Run: ${runId}`);
|
||||
} else {
|
||||
console.log('Run status is not success or serializableOutput is missing.');
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.error(`Failed to write data to Google Sheet for Robot: ${robotId} and Run: ${runId}: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export async function writeDataToSheet(robotId: string, spreadsheetId: string, data: any[]) {
|
||||
async function processOutputType(
|
||||
robotId: string,
|
||||
spreadsheetId: string,
|
||||
outputType: string,
|
||||
outputData: any[],
|
||||
robotConfig: any
|
||||
) {
|
||||
for (let i = 0; i < outputData.length; i++) {
|
||||
const data = outputData[i];
|
||||
|
||||
if (!data || data.length === 0) {
|
||||
console.log(`No data to write for ${outputType}-${i}. Skipping.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const sheetName = `${outputType}-${i}`;
|
||||
|
||||
await ensureSheetExists(spreadsheetId, sheetName, robotConfig);
|
||||
|
||||
await writeDataToSheet(robotId, spreadsheetId, data, sheetName, robotConfig);
|
||||
console.log(`Data written to ${sheetName} sheet for ${outputType} data`);
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureSheetExists(spreadsheetId: string, sheetName: string, robotConfig: any) {
|
||||
try {
|
||||
const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } });
|
||||
const oauth2Client = getOAuth2Client(robotConfig);
|
||||
const sheets = google.sheets({ version: 'v4', auth: oauth2Client });
|
||||
|
||||
const response = await sheets.spreadsheets.get({
|
||||
spreadsheetId,
|
||||
fields: 'sheets.properties.title'
|
||||
});
|
||||
|
||||
const existingSheets = response.data.sheets?.map(sheet => sheet.properties?.title) || [];
|
||||
|
||||
if (!existingSheets.includes(sheetName)) {
|
||||
await sheets.spreadsheets.batchUpdate({
|
||||
spreadsheetId,
|
||||
requestBody: {
|
||||
requests: [
|
||||
{
|
||||
addSheet: {
|
||||
properties: {
|
||||
title: sheetName
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
});
|
||||
console.log(`Created new sheet: ${sheetName}`);
|
||||
}
|
||||
} catch (error: any) {
|
||||
logger.log('error', `Error ensuring sheet exists: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function getOAuth2Client(robotConfig: any) {
|
||||
const oauth2Client = new google.auth.OAuth2(
|
||||
process.env.GOOGLE_CLIENT_ID,
|
||||
process.env.GOOGLE_CLIENT_SECRET,
|
||||
process.env.GOOGLE_REDIRECT_URI
|
||||
);
|
||||
|
||||
oauth2Client.setCredentials({
|
||||
access_token: robotConfig.google_access_token,
|
||||
refresh_token: robotConfig.google_refresh_token,
|
||||
});
|
||||
|
||||
return oauth2Client;
|
||||
}
|
||||
|
||||
export async function writeDataToSheet(
|
||||
robotId: string,
|
||||
spreadsheetId: string,
|
||||
data: any[],
|
||||
sheetName: string = 'Sheet1',
|
||||
robotConfig?: any
|
||||
) {
|
||||
try {
|
||||
let robot = robotConfig;
|
||||
|
||||
if (!robot) {
|
||||
throw new Error(`Robot not found for robotId: ${robotId}`);
|
||||
robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } });
|
||||
|
||||
if (!robot) {
|
||||
throw new Error(`Robot not found for robotId: ${robotId}`);
|
||||
}
|
||||
|
||||
robot = robot.toJSON();
|
||||
}
|
||||
|
||||
const plainRobot = robot.toJSON();
|
||||
|
||||
if (!plainRobot.google_access_token || !plainRobot.google_refresh_token) {
|
||||
if (!robot.google_access_token || !robot.google_refresh_token) {
|
||||
throw new Error('Google Sheets access not configured for user');
|
||||
}
|
||||
|
||||
@@ -83,16 +211,19 @@ export async function writeDataToSheet(robotId: string, spreadsheetId: string, d
|
||||
);
|
||||
|
||||
oauth2Client.setCredentials({
|
||||
access_token: plainRobot.google_access_token,
|
||||
refresh_token: plainRobot.google_refresh_token,
|
||||
access_token: robot.google_access_token,
|
||||
refresh_token: robot.google_refresh_token,
|
||||
});
|
||||
|
||||
oauth2Client.on('tokens', async (tokens) => {
|
||||
if (tokens.refresh_token) {
|
||||
await robot.update({ google_refresh_token: tokens.refresh_token });
|
||||
}
|
||||
if (tokens.access_token) {
|
||||
await robot.update({ google_access_token: tokens.access_token });
|
||||
if (tokens.refresh_token || tokens.access_token) {
|
||||
const robotModel = await Robot.findOne({ where: { 'recording_meta.id': robotId } });
|
||||
if (robotModel) {
|
||||
const updateData: any = {};
|
||||
if (tokens.refresh_token) updateData.google_refresh_token = tokens.refresh_token;
|
||||
if (tokens.access_token) updateData.google_access_token = tokens.access_token;
|
||||
await robotModel.update(updateData);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -100,7 +231,7 @@ export async function writeDataToSheet(robotId: string, spreadsheetId: string, d
|
||||
|
||||
const checkResponse = await sheets.spreadsheets.values.get({
|
||||
spreadsheetId,
|
||||
range: 'Sheet1!1:1',
|
||||
range: `${sheetName}!1:1`,
|
||||
});
|
||||
|
||||
if (!data || data.length === 0) {
|
||||
@@ -109,7 +240,6 @@ export async function writeDataToSheet(robotId: string, spreadsheetId: string, d
|
||||
}
|
||||
|
||||
const expectedHeaders = Object.keys(data[0]);
|
||||
|
||||
const rows = data.map(item => Object.values(item));
|
||||
|
||||
const existingHeaders =
|
||||
@@ -129,28 +259,28 @@ export async function writeDataToSheet(robotId: string, spreadsheetId: string, d
|
||||
|
||||
if (isSheetEmpty || !headersMatch) {
|
||||
resource = { values: [expectedHeaders, ...rows] };
|
||||
console.log('Including headers in the append operation.');
|
||||
console.log(`Including headers in the append operation for sheet ${sheetName}.`);
|
||||
} else {
|
||||
resource = { values: rows };
|
||||
console.log('Headers already exist and match, only appending data rows.');
|
||||
console.log(`Headers already exist and match in sheet ${sheetName}, only appending data rows.`);
|
||||
}
|
||||
|
||||
console.log('Attempting to write to spreadsheet:', spreadsheetId);
|
||||
console.log(`Attempting to write to spreadsheet: ${spreadsheetId}, sheet: ${sheetName}`);
|
||||
|
||||
const response = await sheets.spreadsheets.values.append({
|
||||
spreadsheetId,
|
||||
range: 'Sheet1!A1',
|
||||
range: `${sheetName}!A1`,
|
||||
valueInputOption: 'USER_ENTERED',
|
||||
requestBody: resource,
|
||||
});
|
||||
|
||||
if (response.status === 200) {
|
||||
console.log('Data successfully appended to Google Sheet.');
|
||||
console.log(`Data successfully appended to sheet: ${sheetName}`);
|
||||
} else {
|
||||
console.error('Google Sheets append failed:', response);
|
||||
}
|
||||
|
||||
logger.log(`info`, `Data written to Google Sheet: ${spreadsheetId}`);
|
||||
logger.log(`info`, `Data written to Google Sheet: ${spreadsheetId}, sheet: ${sheetName}`);
|
||||
} catch (error: any) {
|
||||
logger.log(`error`, `Error writing data to Google Sheet: ${error.message}`);
|
||||
throw error;
|
||||
@@ -169,6 +299,7 @@ export const processGoogleSheetUpdates = async () => {
|
||||
try {
|
||||
await updateGoogleSheet(task.robotId, task.runId);
|
||||
console.log(`Successfully updated Google Sheet for runId: ${runId}`);
|
||||
googleSheetUpdateTasks[runId].status = 'completed';
|
||||
delete googleSheetUpdateTasks[runId];
|
||||
} catch (error: any) {
|
||||
console.error(`Failed to update Google Sheets for run ${task.runId}:`, error);
|
||||
|
||||
Reference in New Issue
Block a user