From d0ba425a9209a7f33b6da8d427c455ad95f28d4b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 01:24:18 +0530 Subject: [PATCH 001/109] feat: integrate column --- src/components/molecules/RecordingsTable.tsx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index 429b8234..760d4e92 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -15,7 +15,7 @@ import { useGlobalInfoStore } from "../../context/globalInfo"; import { deleteRecordingFromStorage, getStoredRecordings } from "../../api/storage"; interface Column { - id: 'interpret' | 'name' | 'create_date' | 'edit' | 'pairs' | 'update_date' | 'delete' | 'schedule'; + id: 'interpret' | 'name' | 'create_date' | 'edit' | 'pairs' | 'update_date' | 'delete' | 'schedule' | 'integrate'; label: string; minWidth?: number; align?: 'right'; @@ -46,6 +46,11 @@ const columns: readonly Column[] = [ label: 'Schedule', minWidth: 80, }, + { + id: 'integrate', + label: 'Integrate', + minWidth: 80, + }, { id: 'update_date', label: 'Updated at', From daaae649dd5bd2a9c9002b24332052f1beb7eb69 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 01:30:48 +0530 Subject: [PATCH 002/109] feat: integration ui --- src/components/molecules/RecordingsTable.tsx | 24 +++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index 760d4e92..93917d41 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -78,9 +78,10 @@ interface RecordingsTableProps { handleEditRecording: (fileName: string) => void; handleRunRecording: (fileName: string, params: string[]) => void; handleScheduleRecording: (fileName: string, params: string[]) => void; + handleIntegrateRecording: (fileName: string, params: string[]) => void; } -export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handleScheduleRecording }: RecordingsTableProps) => { +export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handleScheduleRecording, handleIntegrateRecording }: RecordingsTableProps) => { const [page, setPage] = React.useState(0); const [rowsPerPage, setRowsPerPage] = React.useState(10); const [rows, setRows] = React.useState([]); @@ -180,6 +181,12 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl handleScheduleRecording(row.name, row.params || [])} /> ); + case 'integrate': + return ( + + handleIntegrateRecording(row.name, row.params || [])} /> + + ); case 'delete': return ( @@ -250,4 +257,19 @@ const ScheduleButton = ({ handleSchedule }: ScheduleButtonProps) => { ) +} + +interface IntegrateButtonProps { + handleIntegrate: () => void; +} + +const IntegrateButton = ({ handleIntegrate }: IntegrateButtonProps) => { + return ( + { + handleIntegrate(); + }} + sx={{ '&:hover': { color: '#1976d2', backgroundColor: 'transparent' } }}> + + + ) } \ No newline at end of file From 8bc25f3854c676a69b3c82aac59acb5b5084cfaa Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 01:31:13 +0530 Subject: [PATCH 003/109] chore: lint --- src/components/molecules/RecordingsTable.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index 93917d41..6cc75d51 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -181,7 +181,7 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl handleScheduleRecording(row.name, row.params || [])} /> ); - case 'integrate': + case 'integrate': return ( handleIntegrateRecording(row.name, row.params || [])} /> From 61d6e177d60e9711b34209c38f6d44afb26b3470 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 01:41:01 +0530 Subject: [PATCH 004/109] feat: LinkIcon icon --- src/components/molecules/RecordingsTable.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index 6cc75d51..191ee5cb 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -11,6 +11,7 @@ import { useEffect } from "react"; import { WorkflowFile } from "maxun-core"; import { IconButton } from "@mui/material"; import { Schedule, DeleteForever, Edit, PlayCircle } from "@mui/icons-material"; +import LinkIcon from '@mui/icons-material/Link'; import { useGlobalInfoStore } from "../../context/globalInfo"; import { deleteRecordingFromStorage, getStoredRecordings } from "../../api/storage"; @@ -269,7 +270,7 @@ const IntegrateButton = ({ handleIntegrate }: IntegrateButtonProps) => { handleIntegrate(); }} sx={{ '&:hover': { color: '#1976d2', backgroundColor: 'transparent' } }}> - + ) } \ No newline at end of file From da82a4ba57a3c36875ee3c8365817d108817d4e9 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 01:45:14 +0530 Subject: [PATCH 005/109] feat(wip): handle integrate recording --- src/components/organisms/Recordings.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/components/organisms/Recordings.tsx b/src/components/organisms/Recordings.tsx index d5fb6bc5..10ecfa89 100644 --- a/src/components/organisms/Recordings.tsx +++ b/src/components/organisms/Recordings.tsx @@ -8,12 +8,14 @@ interface RecordingsProps { handleEditRecording: (fileName: string) => void; handleRunRecording: (settings: RunSettings) => void; handleScheduleRecording: (settings: ScheduleSettings) => void; + handleIntegrateRecording: (settings: ScheduleSettings) => void; setFileName: (fileName: string) => void; } -export const Recordings = ({ handleEditRecording, handleRunRecording, setFileName, handleScheduleRecording }: RecordingsProps) => { +export const Recordings = ({ handleEditRecording, handleRunRecording, setFileName, handleScheduleRecording, handleIntegrateRecording }: RecordingsProps) => { const [runSettingsAreOpen, setRunSettingsAreOpen] = useState(false); const [scheduleSettingsAreOpen, setScheduleSettingsAreOpen] = useState(false); + const [integrateSettingsAreOpen, setIntegrateSettingsAreOpen] = useState(false); const [params, setParams] = useState([]); const handleSettingsAndRun = (fileName: string, params: string[]) => { @@ -68,6 +70,7 @@ export const Recordings = ({ handleEditRecording, handleRunRecording, setFileNam handleEditRecording={handleEditRecording} handleRunRecording={handleSettingsAndRun} handleScheduleRecording={handleSettingsAndSchedule} + handleIntegrateRecording={handleSettingsAndSchedule} /> From 835227af5ebb9fed7f23f867ab5d9fdae6567141 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 01:47:17 +0530 Subject: [PATCH 006/109] feat(wip): handle integrate recording --- src/pages/MainPage.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pages/MainPage.tsx b/src/pages/MainPage.tsx index 9b67f629..6ffbb63c 100644 --- a/src/pages/MainPage.tsx +++ b/src/pages/MainPage.tsx @@ -120,6 +120,7 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => { handleRunRecording={handleRunRecording} setFileName={setFileName} handleScheduleRecording={handleScheduleRecording} + handleIntegrateRecording={handleScheduleRecording} />; case 'runs': return Date: Sun, 15 Sep 2024 16:52:40 +0530 Subject: [PATCH 007/109] chore(deps): install google-auth-library --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index addce564..cc7f73e8 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "dotenv": "^16.0.0", "express": "^4.17.2", "fortawesome": "^0.0.1-security", + "google-auth-library": "^9.14.1", "ioredis": "^5.4.1", "joi": "^17.6.0", "loglevel": "^1.8.0", From 97809671775dd3d3933625da60834ce37dac5bbe Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 17:54:11 +0530 Subject: [PATCH 008/109] feat(wip): google auth route --- server/src/routes/auth.ts | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 server/src/routes/auth.ts diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts new file mode 100644 index 00000000..b00e5ec9 --- /dev/null +++ b/server/src/routes/auth.ts @@ -0,0 +1,4 @@ +import { Router } from 'express'; +export const router = Router(); + +router.post('/auth/google', googleLogin) \ No newline at end of file From fe80d84e5f005ed171969b37aa72bda7b02e26e4 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 17:54:38 +0530 Subject: [PATCH 009/109] feat(wip): google auth route --- server/src/routes/auth.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index b00e5ec9..74a33d93 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -1,4 +1,6 @@ import { Router } from 'express'; export const router = Router(); -router.post('/auth/google', googleLogin) \ No newline at end of file +router.post('/auth/google', (req, res) => { + res.send('Google auth'); +}); \ No newline at end of file From 454938c768a80fe80aa2eeaad18ee459fe90ee7c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 17:56:27 +0530 Subject: [PATCH 010/109] feat: export auth router --- server/src/routes/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/routes/index.ts b/server/src/routes/index.ts index ff6f8402..19257cf1 100644 --- a/server/src/routes/index.ts +++ b/server/src/routes/index.ts @@ -1,9 +1,11 @@ import { router as record } from './record'; import { router as workflow } from './workflow'; import { router as storage } from './storage'; +import { router as auth } from './auth'; export { record, workflow, storage, + auth, }; From 90df8ffb0a927a7a5a24d9631ce244a4b28112e6 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 17:57:45 +0530 Subject: [PATCH 011/109] feat: use auth router --- server/src/server.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/server.ts b/server/src/server.ts index a81a216c..07f13284 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -3,7 +3,7 @@ import http from 'http'; import cors from 'cors'; import 'dotenv/config'; -import { record, workflow, storage } from './routes'; +import { record, workflow, storage, auth } from './routes'; import { BrowserPool } from "./browser-management/classes/BrowserPool"; import logger from './logger' import { SERVER_PORT } from "./constants/config"; @@ -30,6 +30,7 @@ export const browserPool = new BrowserPool(); app.use('/record', record); app.use('/workflow', workflow); app.use('/storage', storage); +app.use('/auth', auth); app.get('/', function (req, res) { return res.send('Maxun server started 🚀'); From 846c0011b270db1c7f3b47aac722828937ad23ae Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 17:58:12 +0530 Subject: [PATCH 012/109] chore: remove unsued import --- server/src/server.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/server.ts b/server/src/server.ts index 07f13284..29709fa6 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -2,13 +2,11 @@ import express from 'express'; import http from 'http'; import cors from 'cors'; import 'dotenv/config'; - import { record, workflow, storage, auth } from './routes'; import { BrowserPool } from "./browser-management/classes/BrowserPool"; import logger from './logger' import { SERVER_PORT } from "./constants/config"; import { Server } from "socket.io"; -import { worker } from './workflow-management/scheduler'; const app = express(); app.use(cors()); From 19e703c8b7205d97348ed1860b23917db31c4a94 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 18:30:34 +0530 Subject: [PATCH 013/109] chore(deps): install googleapis --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index cc7f73e8..39b00555 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "express": "^4.17.2", "fortawesome": "^0.0.1-security", "google-auth-library": "^9.14.1", + "googleapis": "^144.0.0", "ioredis": "^5.4.1", "joi": "^17.6.0", "loglevel": "^1.8.0", From 7f5a5839ec12376b1cb6312ddab3f975d070d12e Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 18:44:27 +0530 Subject: [PATCH 014/109] feat: intialize google oauth flow --- server/src/routes/auth.ts | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index 74a33d93..b3c2d9a3 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -1,6 +1,24 @@ -import { Router } from 'express'; -export const router = Router(); +import { Router } from 'express';; +import { google } from "googleapis"; +import { OAuth2Client } from 'google-auth-library' -router.post('/auth/google', (req, res) => { - res.send('Google auth'); +export const router = Router() + +const oauth2Client = new OAuth2Client( + '_CLIENT_ID', + '_CLIENT_SECRET', + '_REDIRECT_URI' +); + +// initialize Google OAuth 2.0 flow +router.get('/auth/google', (req, res) => { + const url = oauth2Client.generateAuthUrl({ + access_type: 'offline', + scope: [ + 'https://www.googleapis.com/auth/userinfo.profile', + 'https://www.googleapis.com/auth/drive.file', + 'https://www.googleapis.com/auth/spreadsheets' + ] + }); + res.redirect(url); }); \ No newline at end of file From 13ecfae2d61b3e41fca8c9ebef7888ff66128d93 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 19:15:26 +0530 Subject: [PATCH 015/109] feat: callback google auth --- server/src/routes/auth.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index b3c2d9a3..3b30328a 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -21,4 +21,18 @@ router.get('/auth/google', (req, res) => { ] }); res.redirect(url); -}); \ No newline at end of file +}); + +// Callback route for Google OAuth 2.0 +router.get('/auth/google/callback', async (req, res) => { + const { code } = req.query; + try { + const { tokens } = await oauth2Client.getToken(code); + oauth2Client.setCredentials(tokens); + // Store tokens securely (e.g., in a database) + res.send('Authentication successful'); + } catch (error) { + console.error('Error during authentication:', error); + res.status(500).send('Authentication failed'); + } + }); \ No newline at end of file From a9868f255a47891f5c607900180f2ad4a64d80c7 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 19:16:23 +0530 Subject: [PATCH 016/109] feat: get spreadsheets from drive --- server/src/routes/auth.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index 3b30328a..845975fa 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -35,4 +35,19 @@ router.get('/auth/google/callback', async (req, res) => { console.error('Error during authentication:', error); res.status(500).send('Authentication failed'); } - }); \ No newline at end of file + }); + +router.get('/sheets', async (req, res) => { + try { + const drive = google.drive({ version: 'v3', auth: oauth2Client }); + const response = await drive.files.list({ + q: "mimeType='application/vnd.google-apps.spreadsheet'", + fields: 'files(id, name)' + }); + res.json(response.data.files); + } catch (error) { + console.error('Error listing sheets:', error); + res.status(500).send('Failed to list sheets'); + } + }); + \ No newline at end of file From ed72facddcfd24701b3e422fd36f8fae7094b32f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 19:17:18 +0530 Subject: [PATCH 017/109] feat: read gsheet file --- server/src/routes/auth.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index 845975fa..344f5f61 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -50,4 +50,17 @@ router.get('/sheets', async (req, res) => { res.status(500).send('Failed to list sheets'); } }); - \ No newline at end of file + +router..get('/sheets/:sheetId', async (req, res) => { + try { + const sheets = google.sheets({ version: 'v4', auth: oauth2Client }); + const response = await sheets.spreadsheets.values.get({ + spreadsheetId: req.params.sheetId, + range: 'Sheet1', // Adjust range as needed + }); + res.json(response.data.values); + } catch (error) { + console.error('Error reading sheet:', error); + res.status(500).send('Failed to read sheet'); + } + }); \ No newline at end of file From c0729bc0d7b01002d41b5c110cde22d8df1a9cfa Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 19:17:33 +0530 Subject: [PATCH 018/109] fix: remove extra . --- server/src/routes/auth.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index 344f5f61..d28f63e0 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -51,7 +51,7 @@ router.get('/sheets', async (req, res) => { } }); -router..get('/sheets/:sheetId', async (req, res) => { +router.get('/sheets/:sheetId', async (req, res) => { try { const sheets = google.sheets({ version: 'v4', auth: oauth2Client }); const response = await sheets.spreadsheets.values.get({ From 582f37bf349d8e4027c1f49d900481f8f0756669 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 19:18:26 +0530 Subject: [PATCH 019/109] feat: write to gsheet --- server/src/routes/auth.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index d28f63e0..be0c3992 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -63,4 +63,22 @@ router.get('/sheets/:sheetId', async (req, res) => { console.error('Error reading sheet:', error); res.status(500).send('Failed to read sheet'); } + }); + +router.post('/sheets/:sheetId', async (req, res) => { + try { + const sheets = google.sheets({ version: 'v4', auth: oauth2Client }); + const response = await sheets.spreadsheets.values.append({ + spreadsheetId: req.params.sheetId, + range: 'Sheet1', // Adjust range as needed + valueInputOption: 'USER_ENTERED', + resource: { + values: [req.body.values], // Expect an array of values in the request body + }, + }); + res.json(response.data); + } catch (error) { + console.error('Error writing to sheet:', error); + res.status(500).send('Failed to write to sheet'); + } }); \ No newline at end of file From 70136f9ca98b84d3b680c3f3d0b5fd44472882da Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 19:18:41 +0530 Subject: [PATCH 020/109] chore: lint --- server/src/routes/auth.ts | 94 +++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index be0c3992..f240bdf0 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -12,73 +12,73 @@ const oauth2Client = new OAuth2Client( // initialize Google OAuth 2.0 flow router.get('/auth/google', (req, res) => { - const url = oauth2Client.generateAuthUrl({ - access_type: 'offline', - scope: [ - 'https://www.googleapis.com/auth/userinfo.profile', - 'https://www.googleapis.com/auth/drive.file', - 'https://www.googleapis.com/auth/spreadsheets' - ] - }); - res.redirect(url); + const url = oauth2Client.generateAuthUrl({ + access_type: 'offline', + scope: [ + 'https://www.googleapis.com/auth/userinfo.profile', + 'https://www.googleapis.com/auth/drive.file', + 'https://www.googleapis.com/auth/spreadsheets' + ] + }); + res.redirect(url); }); // Callback route for Google OAuth 2.0 router.get('/auth/google/callback', async (req, res) => { const { code } = req.query; try { - const { tokens } = await oauth2Client.getToken(code); - oauth2Client.setCredentials(tokens); - // Store tokens securely (e.g., in a database) - res.send('Authentication successful'); + const { tokens } = await oauth2Client.getToken(code); + oauth2Client.setCredentials(tokens); + // Store tokens securely (e.g., in a database) + res.send('Authentication successful'); } catch (error) { - console.error('Error during authentication:', error); - res.status(500).send('Authentication failed'); + console.error('Error during authentication:', error); + res.status(500).send('Authentication failed'); } - }); +}); router.get('/sheets', async (req, res) => { try { - const drive = google.drive({ version: 'v3', auth: oauth2Client }); - const response = await drive.files.list({ - q: "mimeType='application/vnd.google-apps.spreadsheet'", - fields: 'files(id, name)' - }); - res.json(response.data.files); + const drive = google.drive({ version: 'v3', auth: oauth2Client }); + const response = await drive.files.list({ + q: "mimeType='application/vnd.google-apps.spreadsheet'", + fields: 'files(id, name)' + }); + res.json(response.data.files); } catch (error) { - console.error('Error listing sheets:', error); - res.status(500).send('Failed to list sheets'); + console.error('Error listing sheets:', error); + res.status(500).send('Failed to list sheets'); } - }); +}); router.get('/sheets/:sheetId', async (req, res) => { try { - const sheets = google.sheets({ version: 'v4', auth: oauth2Client }); - const response = await sheets.spreadsheets.values.get({ - spreadsheetId: req.params.sheetId, - range: 'Sheet1', // Adjust range as needed - }); - res.json(response.data.values); + const sheets = google.sheets({ version: 'v4', auth: oauth2Client }); + const response = await sheets.spreadsheets.values.get({ + spreadsheetId: req.params.sheetId, + range: 'Sheet1', // Adjust range as needed + }); + res.json(response.data.values); } catch (error) { - console.error('Error reading sheet:', error); - res.status(500).send('Failed to read sheet'); + console.error('Error reading sheet:', error); + res.status(500).send('Failed to read sheet'); } - }); +}); router.post('/sheets/:sheetId', async (req, res) => { try { - const sheets = google.sheets({ version: 'v4', auth: oauth2Client }); - const response = await sheets.spreadsheets.values.append({ - spreadsheetId: req.params.sheetId, - range: 'Sheet1', // Adjust range as needed - valueInputOption: 'USER_ENTERED', - resource: { - values: [req.body.values], // Expect an array of values in the request body - }, - }); - res.json(response.data); + const sheets = google.sheets({ version: 'v4', auth: oauth2Client }); + const response = await sheets.spreadsheets.values.append({ + spreadsheetId: req.params.sheetId, + range: 'Sheet1', // Adjust range as needed + valueInputOption: 'USER_ENTERED', + resource: { + values: [req.body.values], // Expect an array of values in the request body + }, + }); + res.json(response.data); } catch (error) { - console.error('Error writing to sheet:', error); - res.status(500).send('Failed to write to sheet'); + console.error('Error writing to sheet:', error); + res.status(500).send('Failed to write to sheet'); } - }); \ No newline at end of file +}); \ No newline at end of file From be006fcc7c6113ad36245933a025ad5a181a0c15 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 19:22:03 +0530 Subject: [PATCH 021/109] fix: ts errors --- server/src/routes/auth.ts | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index f240bdf0..cdb2980c 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -25,17 +25,21 @@ router.get('/auth/google', (req, res) => { // Callback route for Google OAuth 2.0 router.get('/auth/google/callback', async (req, res) => { - const { code } = req.query; - try { - const { tokens } = await oauth2Client.getToken(code); - oauth2Client.setCredentials(tokens); - // Store tokens securely (e.g., in a database) - res.send('Authentication successful'); - } catch (error) { - console.error('Error during authentication:', error); - res.status(500).send('Authentication failed'); + const code = req.query.code; + if (typeof code !== 'string') { + res.status(400).send('Invalid authorization code'); + return; } -}); + try { + const { tokens } = await oauth2Client.getToken(code); + oauth2Client.setCredentials(tokens); + // Store tokens securely (e.g., in a database) + res.send('Authentication successful'); + } catch (error) { + console.error('Error during authentication:', error); + res.status(500).send('Authentication failed'); + } + }); router.get('/sheets', async (req, res) => { try { From 5b6ec16613889093559977807aa0f346af3e1218 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 19:23:04 +0530 Subject: [PATCH 022/109] chore: todo --- server/src/routes/auth.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index cdb2980c..b2b69eb8 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -33,7 +33,7 @@ router.get('/auth/google/callback', async (req, res) => { try { const { tokens } = await oauth2Client.getToken(code); oauth2Client.setCredentials(tokens); - // Store tokens securely (e.g., in a database) + // TODO: Store tokens securely (e.g., in a database) res.send('Authentication successful'); } catch (error) { console.error('Error during authentication:', error); From b906821ca1dcb6215c57fb42010e63aff89cd88f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 19:25:18 +0530 Subject: [PATCH 023/109] fix: ts errors --- server/src/routes/auth.ts | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index b2b69eb8..7349d81e 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -1,5 +1,5 @@ import { Router } from 'express';; -import { google } from "googleapis"; +import { google, } from "googleapis"; import { OAuth2Client } from 'google-auth-library' export const router = Router() @@ -71,18 +71,22 @@ router.get('/sheets/:sheetId', async (req, res) => { router.post('/sheets/:sheetId', async (req, res) => { try { - const sheets = google.sheets({ version: 'v4', auth: oauth2Client }); - const response = await sheets.spreadsheets.values.append({ - spreadsheetId: req.params.sheetId, - range: 'Sheet1', // Adjust range as needed - valueInputOption: 'USER_ENTERED', - resource: { - values: [req.body.values], // Expect an array of values in the request body - }, - }); - res.json(response.data); + const sheets = google.sheets({ version: 'v4', auth: oauth2Client }); + + const request: sheets_v4.Params$Resource$Spreadsheets$Values$Append = { + spreadsheetId: req.params.sheetId, + range: 'Sheet1', // Adjust range as needed + valueInputOption: 'USER_ENTERED', + requestBody: { + values: [req.body.values], // Expect an array of values in the request body + }, + }; + + const response = await sheets.spreadsheets.values.append(request); + + res.json(response.data); } catch (error) { - console.error('Error writing to sheet:', error); - res.status(500).send('Failed to write to sheet'); + console.error('Error writing to sheet:', error); + res.status(500).send('Failed to write to sheet'); } -}); \ No newline at end of file + }); \ No newline at end of file From 6005ffb3f30cfc6a53e463f1e4f4cfbdbd95f2e6 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Sun, 15 Sep 2024 19:26:16 +0530 Subject: [PATCH 024/109] fix: import sheets_v4 --- server/src/routes/auth.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index 7349d81e..00f51aa2 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -1,5 +1,5 @@ import { Router } from 'express';; -import { google, } from "googleapis"; +import { google, sheets_v4 } from "googleapis"; import { OAuth2Client } from 'google-auth-library' export const router = Router() From ff2aed9c0b891fdd2a2890a407fd38d23e4aa277 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 12:57:14 +0530 Subject: [PATCH 025/109] chore(deps): install @react-oauth/google --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 39b00555..bb991aa6 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "@mui/icons-material": "^5.5.1", "@mui/lab": "^5.0.0-alpha.80", "@mui/material": "^5.6.2", + "@react-oauth/google": "^0.12.1", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.1.1", "@testing-library/user-event": "^13.5.0", From 6385a064cd8f87d1e87d4dad0962637173ad7492 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 17:16:15 +0530 Subject: [PATCH 026/109] feat(wip): integration settings --- src/components/organisms/Recordings.tsx | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/components/organisms/Recordings.tsx b/src/components/organisms/Recordings.tsx index 10ecfa89..969ab4ac 100644 --- a/src/components/organisms/Recordings.tsx +++ b/src/components/organisms/Recordings.tsx @@ -3,6 +3,7 @@ import { RecordingsTable } from "../molecules/RecordingsTable"; import { Grid } from "@mui/material"; import { RunSettings, RunSettingsModal } from "../molecules/RunSettings"; import { ScheduleSettings, ScheduleSettingsModal } from "../molecules/ScheduleSettings"; +import { IntegrationSettings, IntegrationSettingsModal } from "../molecules/IntegrationSettings"; interface RecordingsProps { handleEditRecording: (fileName: string) => void; @@ -18,6 +19,17 @@ export const Recordings = ({ handleEditRecording, handleRunRecording, setFileNam const [integrateSettingsAreOpen, setIntegrateSettingsAreOpen] = useState(false); const [params, setParams] = useState([]); + const handleSettingsAndIntegrate = (fileName: string, params: string[]) => { + if (params.length === 0) { + setIntegrateSettingsAreOpen(true); + setFileName(fileName); + } else { + setParams(params); + setIntegrateSettingsAreOpen(true); + setFileName(fileName); + } + } + const handleSettingsAndRun = (fileName: string, params: string[]) => { if (params.length === 0) { setRunSettingsAreOpen(true); From 95c5a3d8e921dc27311d4767a36da4e2f93fb299 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 17:25:15 +0530 Subject: [PATCH 027/109] fix: use handleSetttingAndIntegrate --- src/components/organisms/Recordings.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/organisms/Recordings.tsx b/src/components/organisms/Recordings.tsx index 969ab4ac..45d3fc6f 100644 --- a/src/components/organisms/Recordings.tsx +++ b/src/components/organisms/Recordings.tsx @@ -82,7 +82,7 @@ export const Recordings = ({ handleEditRecording, handleRunRecording, setFileNam handleEditRecording={handleEditRecording} handleRunRecording={handleSettingsAndRun} handleScheduleRecording={handleSettingsAndSchedule} - handleIntegrateRecording={handleSettingsAndSchedule} + handleIntegrateRecording={handleSettingsAndIntegrate} /> From b99e1f84c6a3e8a17ca4dafc3592c4d493cd43cc Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 17:28:23 +0530 Subject: [PATCH 028/109] feat(wip): integration settings modal --- src/components/organisms/Recordings.tsx | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/components/organisms/Recordings.tsx b/src/components/organisms/Recordings.tsx index 45d3fc6f..33474ca5 100644 --- a/src/components/organisms/Recordings.tsx +++ b/src/components/organisms/Recordings.tsx @@ -58,6 +58,12 @@ export const Recordings = ({ handleEditRecording, handleRunRecording, setFileNam setFileName(''); } + const handleIntegrateClose = () => { + setParams([]); + setIntegrateSettingsAreOpen(false); + setFileName(''); + } + const handleScheduleClose = () => { setParams([]); setScheduleSettingsAreOpen(false); @@ -76,6 +82,12 @@ export const Recordings = ({ handleEditRecording, handleRunRecording, setFileNam handleClose={handleScheduleClose} handleStart={(settings) => handleScheduleRecording(settings)} /> + handleIntegrateRecording(settings)} + /> Date: Tue, 17 Sep 2024 17:28:43 +0530 Subject: [PATCH 029/109] feat(wip): integration settings UI --- .../molecules/IntegrationSettings.tsx | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 src/components/molecules/IntegrationSettings.tsx diff --git a/src/components/molecules/IntegrationSettings.tsx b/src/components/molecules/IntegrationSettings.tsx new file mode 100644 index 00000000..0c4ce109 --- /dev/null +++ b/src/components/molecules/IntegrationSettings.tsx @@ -0,0 +1,114 @@ +import React, { useState } from 'react'; +import { GenericModal } from "../atoms/GenericModal"; +import { MenuItem, TextField, Typography } from "@mui/material"; +import { Dropdown } from "../atoms/DropdownMui"; +import Button from "@mui/material/Button"; +import { modalStyle } from "./AddWhereCondModal"; + +interface IntegrationSettingsProps { + isOpen: boolean; + handleStart: (settings: IntegrationSettings) => void; + handleClose: () => void; + isTask: boolean; + params?: string[]; +} + +export interface IntegrationSettings { + maxConcurrency: number; + maxRepeats: number; + debug: boolean; + params?: any; +} + +export const IntegrationSettingsModal = ({ isOpen, handleStart, handleClose, isTask, params }: IntegrationSettingsProps) => { + + const [settings, setSettings] = React.useState({ + maxConcurrency: 1, + maxRepeats: 1, + debug: true, + }); + + return ( + +
+ {isTask + ? + ( + + Recording parameters: + {params?.map((item, index) => { + return setSettings( + { + ...settings, + params: settings.params + ? { + ...settings.params, + [item]: e.target.value, + } + : { + [item]: e.target.value, + }, + })} + /> + })} + ) + : null + } + Interpreter settings: + setSettings( + { + ...settings, + maxConcurrency: parseInt(e.target.value), + })} + defaultValue={settings.maxConcurrency} + /> + setSettings( + { + ...settings, + maxRepeats: parseInt(e.target.value), + })} + defaultValue={settings.maxRepeats} + /> + setSettings( + { + ...settings, + debug: e.target.value === "true", + })} + > + true + false + + +
+ + ); +} From 509a576c7ff3f817526b3e4a2de9a4058e4a9ce2 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 17:30:43 +0530 Subject: [PATCH 030/109] fix: use IntegrationSettings --- src/components/organisms/Recordings.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/organisms/Recordings.tsx b/src/components/organisms/Recordings.tsx index 33474ca5..40d6db87 100644 --- a/src/components/organisms/Recordings.tsx +++ b/src/components/organisms/Recordings.tsx @@ -9,7 +9,7 @@ interface RecordingsProps { handleEditRecording: (fileName: string) => void; handleRunRecording: (settings: RunSettings) => void; handleScheduleRecording: (settings: ScheduleSettings) => void; - handleIntegrateRecording: (settings: ScheduleSettings) => void; + handleIntegrateRecording: (settings: IntegrationSettings) => void; setFileName: (fileName: string) => void; } From 42566867db39d9bd17e88c3aedb9488ad5f57f76 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 18:06:49 +0530 Subject: [PATCH 031/109] feat: pass handle integrate recording --- src/pages/MainPage.tsx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/pages/MainPage.tsx b/src/pages/MainPage.tsx index 6ffbb63c..51ceb02c 100644 --- a/src/pages/MainPage.tsx +++ b/src/pages/MainPage.tsx @@ -9,6 +9,7 @@ import { io, Socket } from "socket.io-client"; import { stopRecording } from "../api/recording"; import { RunSettings } from "../components/molecules/RunSettings"; import { ScheduleSettings } from "../components/molecules/ScheduleSettings"; +import { IntegrationSettings } from "../components/molecules/IntegrationSettings"; interface MainPageProps { handleEditRecording: (fileName: string) => void; @@ -112,6 +113,10 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => { }); } + const handleIntegrateRecording = (settings: IntegrationSettings) => { + console.log(`Integration settings:`, settings); + } + const DisplayContent = () => { switch (content) { case 'recordings': @@ -120,7 +125,7 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => { handleRunRecording={handleRunRecording} setFileName={setFileName} handleScheduleRecording={handleScheduleRecording} - handleIntegrateRecording={handleScheduleRecording} + handleIntegrateRecording={handleIntegrateRecording} />; case 'runs': return Date: Tue, 17 Sep 2024 20:10:51 +0530 Subject: [PATCH 032/109] feat: gsheet ui --- .../molecules/IntegrationSettings.tsx | 130 ++++++++---------- 1 file changed, 54 insertions(+), 76 deletions(-) diff --git a/src/components/molecules/IntegrationSettings.tsx b/src/components/molecules/IntegrationSettings.tsx index 0c4ce109..7e118e68 100644 --- a/src/components/molecules/IntegrationSettings.tsx +++ b/src/components/molecules/IntegrationSettings.tsx @@ -1,33 +1,35 @@ import React, { useState } from 'react'; import { GenericModal } from "../atoms/GenericModal"; import { MenuItem, TextField, Typography } from "@mui/material"; -import { Dropdown } from "../atoms/DropdownMui"; import Button from "@mui/material/Button"; import { modalStyle } from "./AddWhereCondModal"; -interface IntegrationSettingsProps { +interface GoogleSheetsIntegrationProps { isOpen: boolean; - handleStart: (settings: IntegrationSettings) => void; + handleSubmit: (data: GoogleSheetsSettings) => void; handleClose: () => void; - isTask: boolean; - params?: string[]; } -export interface IntegrationSettings { - maxConcurrency: number; - maxRepeats: number; - debug: boolean; - params?: any; +export interface GoogleSheetsSettings { + credentials: string; + spreadsheetId: string; + range: string; + data: string; } -export const IntegrationSettingsModal = ({ isOpen, handleStart, handleClose, isTask, params }: IntegrationSettingsProps) => { +export const GoogleSheetsIntegrationModal = ({ isOpen, handleSubmit, handleClose }: GoogleSheetsIntegrationProps) => { - const [settings, setSettings] = React.useState({ - maxConcurrency: 1, - maxRepeats: 1, - debug: true, + const [settings, setSettings] = useState({ + credentials: '', + spreadsheetId: '', + range: '', + data: '', }); + const handleChange = (field: keyof GoogleSheetsSettings) => (e: React.ChangeEvent) => { + setSettings({ ...settings, [field]: e.target.value }); + }; + return ( - {isTask - ? - ( - - Recording parameters: - {params?.map((item, index) => { - return setSettings( - { - ...settings, - params: settings.params - ? { - ...settings.params, - [item]: e.target.value, - } - : { - [item]: e.target.value, - }, - })} - /> - })} - ) - : null - } - Interpreter settings: + Google Sheets Integration + setSettings( - { - ...settings, - maxConcurrency: parseInt(e.target.value), - })} - defaultValue={settings.maxConcurrency} + value={settings.credentials} + onChange={handleChange('credentials')} + fullWidth /> + setSettings( - { - ...settings, - maxRepeats: parseInt(e.target.value), - })} - defaultValue={settings.maxRepeats} + value={settings.spreadsheetId} + onChange={handleChange('spreadsheetId')} + fullWidth /> - setSettings( - { - ...settings, - debug: e.target.value === "true", - })} - > - true - false - - + + + + + + ); -} +}; From 74b2e5935182763dd1c054a32c0a278349dfb98c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 20:16:50 +0530 Subject: [PATCH 033/109] refactor: variable names --- src/components/molecules/IntegrationSettings.tsx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/components/molecules/IntegrationSettings.tsx b/src/components/molecules/IntegrationSettings.tsx index 7e118e68..f744fff8 100644 --- a/src/components/molecules/IntegrationSettings.tsx +++ b/src/components/molecules/IntegrationSettings.tsx @@ -4,29 +4,29 @@ import { MenuItem, TextField, Typography } from "@mui/material"; import Button from "@mui/material/Button"; import { modalStyle } from "./AddWhereCondModal"; -interface GoogleSheetsIntegrationProps { +interface IntegrationProps { isOpen: boolean; - handleSubmit: (data: GoogleSheetsSettings) => void; + handleSubmit: (data: IntegrationSettings) => void; handleClose: () => void; } -export interface GoogleSheetsSettings { +export interface IntegrationSettings { credentials: string; spreadsheetId: string; range: string; data: string; } -export const GoogleSheetsIntegrationModal = ({ isOpen, handleSubmit, handleClose }: GoogleSheetsIntegrationProps) => { +export const IntegrationModal = ({ isOpen, handleSubmit, handleClose }: IntegrationProps) => { - const [settings, setSettings] = useState({ + const [settings, setSettings] = useState({ credentials: '', spreadsheetId: '', range: '', data: '', }); - const handleChange = (field: keyof GoogleSheetsSettings) => (e: React.ChangeEvent) => { + const handleChange = (field: keyof IntegrationSettings) => (e: React.ChangeEvent) => { setSettings({ ...settings, [field]: e.target.value }); }; From 2a39cb5d5c9d7421c9eeaa7b81768e494289302f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 20:20:07 +0530 Subject: [PATCH 034/109] refactor: rename settings modal --- src/components/molecules/IntegrationSettings.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/molecules/IntegrationSettings.tsx b/src/components/molecules/IntegrationSettings.tsx index f744fff8..7a91c804 100644 --- a/src/components/molecules/IntegrationSettings.tsx +++ b/src/components/molecules/IntegrationSettings.tsx @@ -17,7 +17,7 @@ export interface IntegrationSettings { data: string; } -export const IntegrationModal = ({ isOpen, handleSubmit, handleClose }: IntegrationProps) => { +export const IntegrationSettingsModal = ({ isOpen, handleSubmit, handleClose }: IntegrationProps) => { const [settings, setSettings] = useState({ credentials: '', From 8bb291836382ced3fee28e40b79c883a7598edeb Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 20:20:43 +0530 Subject: [PATCH 035/109] fix: remove unwanted props --- src/components/organisms/Recordings.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/components/organisms/Recordings.tsx b/src/components/organisms/Recordings.tsx index 40d6db87..e7c65ce4 100644 --- a/src/components/organisms/Recordings.tsx +++ b/src/components/organisms/Recordings.tsx @@ -84,8 +84,6 @@ export const Recordings = ({ handleEditRecording, handleRunRecording, setFileNam /> handleIntegrateRecording(settings)} /> From 5c0998ae017e2ef5a4b4030829d560f0d47df59b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 20:24:36 +0530 Subject: [PATCH 036/109] refactor: use handleStart prop --- src/components/molecules/IntegrationSettings.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/components/molecules/IntegrationSettings.tsx b/src/components/molecules/IntegrationSettings.tsx index 7a91c804..d7bdcba9 100644 --- a/src/components/molecules/IntegrationSettings.tsx +++ b/src/components/molecules/IntegrationSettings.tsx @@ -6,7 +6,7 @@ import { modalStyle } from "./AddWhereCondModal"; interface IntegrationProps { isOpen: boolean; - handleSubmit: (data: IntegrationSettings) => void; + handleStart: (data: IntegrationSettings) => void; handleClose: () => void; } @@ -17,7 +17,7 @@ export interface IntegrationSettings { data: string; } -export const IntegrationSettingsModal = ({ isOpen, handleSubmit, handleClose }: IntegrationProps) => { +export const IntegrationSettingsModal = ({ isOpen, handleStart, handleClose }: IntegrationProps) => { const [settings, setSettings] = useState({ credentials: '', @@ -83,7 +83,7 @@ export const IntegrationSettingsModal = ({ isOpen, handleSubmit, handleClose }: fullWidth /> - From a2d04b3d5b03e8417aacc81df3572f0fc80c7f59 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 20:25:10 +0530 Subject: [PATCH 037/109] chore: lint --- .../molecules/IntegrationSettings.tsx | 140 +++++++++--------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/src/components/molecules/IntegrationSettings.tsx b/src/components/molecules/IntegrationSettings.tsx index d7bdcba9..c8c64169 100644 --- a/src/components/molecules/IntegrationSettings.tsx +++ b/src/components/molecules/IntegrationSettings.tsx @@ -5,88 +5,88 @@ import Button from "@mui/material/Button"; import { modalStyle } from "./AddWhereCondModal"; interface IntegrationProps { - isOpen: boolean; - handleStart: (data: IntegrationSettings) => void; - handleClose: () => void; + isOpen: boolean; + handleStart: (data: IntegrationSettings) => void; + handleClose: () => void; } export interface IntegrationSettings { - credentials: string; - spreadsheetId: string; - range: string; - data: string; + credentials: string; + spreadsheetId: string; + range: string; + data: string; } export const IntegrationSettingsModal = ({ isOpen, handleStart, handleClose }: IntegrationProps) => { - const [settings, setSettings] = useState({ - credentials: '', - spreadsheetId: '', - range: '', - data: '', - }); + const [settings, setSettings] = useState({ + credentials: '', + spreadsheetId: '', + range: '', + data: '', + }); - const handleChange = (field: keyof IntegrationSettings) => (e: React.ChangeEvent) => { - setSettings({ ...settings, [field]: e.target.value }); - }; + const handleChange = (field: keyof IntegrationSettings) => (e: React.ChangeEvent) => { + setSettings({ ...settings, [field]: e.target.value }); + }; - return ( - -
- Google Sheets Integration + return ( + +
+ Google Sheets Integration - + - + - + - + - -
- - ); + +
+ + ); }; From fcd11604414ef8d86302d3ba2fcd2e4e339f8a72 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 20:38:32 +0530 Subject: [PATCH 038/109] feat: upload service account credentials --- server/src/routes/integration.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 server/src/routes/integration.ts diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts new file mode 100644 index 00000000..70e3425e --- /dev/null +++ b/server/src/routes/integration.ts @@ -0,0 +1,12 @@ +import { Router } from 'express';; +import { google, sheets_v4 } from "googleapis"; + +export const router = Router() + + +router.post('/upload-credentials', (req, res) => { + const credentials = req.body.credentials; + + if (!credentials) { + return res.status(400).json({ message: 'Credentials are required.' }); + } \ No newline at end of file From 84a4afa7fc7ef6b55a3f3debb88bbd407fe5545e Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 20:39:42 +0530 Subject: [PATCH 039/109] feat: upload service account credentials --- server/src/routes/integration.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index 70e3425e..07d04bcf 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -1,5 +1,7 @@ import { Router } from 'express';; import { google, sheets_v4 } from "googleapis"; +import fs from 'fs'; +import path from 'path'; export const router = Router() @@ -9,4 +11,10 @@ router.post('/upload-credentials', (req, res) => { if (!credentials) { return res.status(400).json({ message: 'Credentials are required.' }); - } \ No newline at end of file + } + // Todo: Store the credentials in a secure place (for test, we store them locally) + const storedCredentialsPath = path.join(__dirname, 'service_account_credentials.json'); + fs.writeFileSync(storedCredentialsPath, JSON.stringify(credentials)); + + res.status(200).json({ message: 'Service Account credentials saved successfully.' }); +}); \ No newline at end of file From fe255fe3183fb511a68e55cda17baa5ac068a984 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 20:47:36 +0530 Subject: [PATCH 040/109] feat(wip): write to sheets --- server/src/routes/integration.ts | 40 +++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index 07d04bcf..9febe002 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -17,4 +17,42 @@ router.post('/upload-credentials', (req, res) => { fs.writeFileSync(storedCredentialsPath, JSON.stringify(credentials)); res.status(200).json({ message: 'Service Account credentials saved successfully.' }); -}); \ No newline at end of file +}); + +router.post('/write-to-sheet', async (req, res) => { + try { + const { spreadsheetId, range, values } = req.body; + + // Load the stored credentials + const credentialsPath = path.join(__dirname, 'service_account_credentials.json'); + if (!fs.existsSync(credentialsPath)) { + return res.status(400).json({ message: 'No credentials found. Please provide credentials first.' }); + } + + const credentials = JSON.parse(fs.readFileSync(credentialsPath, 'utf-8')); + + // Authenticate with Google using the service account credentials + const auth = new google.auth.GoogleAuth({ + credentials, + scopes: ['https://www.googleapis.com/auth/spreadsheets'], + }); + + const sheets = google.sheets({ version: 'v4', auth }); + + // Write data to the provided Google Sheet and range + await sheets.spreadsheets.values.append({ + spreadsheetId, + range, + valueInputOption: 'USER_ENTERED', + resource: { + values: values, // Expecting an array of arrays, like [['data1', 'data2'], ['data3', 'data4']] + }, + }); + + res.status(200).json({ message: 'Data written to Google Sheet successfully.' }); + } catch (error) { + console.error('Error writing to sheet:', error); + res.status(500).json({ message: 'Failed to write to Google Sheet.', error }); + } + }); + \ No newline at end of file From 8417c2f3e8815a5b2e10bedb6512e41117a7b39c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 20:58:09 +0530 Subject: [PATCH 041/109] fix: ts for spreadsheets.values.appen --- server/src/routes/integration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index 9febe002..91150b3d 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -44,7 +44,7 @@ router.post('/write-to-sheet', async (req, res) => { spreadsheetId, range, valueInputOption: 'USER_ENTERED', - resource: { + requestBody: { values: values, // Expecting an array of arrays, like [['data1', 'data2'], ['data3', 'data4']] }, }); From 81456ee8baf158dd19458b0a5a48264fdae7b1d9 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 20:58:30 +0530 Subject: [PATCH 042/109] chore: lint --- server/src/routes/integration.ts | 76 ++++++++++++++++---------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index 91150b3d..ee38c369 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -5,54 +5,52 @@ import path from 'path'; export const router = Router() - router.post('/upload-credentials', (req, res) => { const credentials = req.body.credentials; - + if (!credentials) { - return res.status(400).json({ message: 'Credentials are required.' }); + return res.status(400).json({ message: 'Credentials are required.' }); } // Todo: Store the credentials in a secure place (for test, we store them locally) - const storedCredentialsPath = path.join(__dirname, 'service_account_credentials.json'); - fs.writeFileSync(storedCredentialsPath, JSON.stringify(credentials)); + const storedCredentialsPath = path.join(__dirname, 'service_account_credentials.json'); + fs.writeFileSync(storedCredentialsPath, JSON.stringify(credentials)); - res.status(200).json({ message: 'Service Account credentials saved successfully.' }); + res.status(200).json({ message: 'Service Account credentials saved successfully.' }); }); router.post('/write-to-sheet', async (req, res) => { try { - const { spreadsheetId, range, values } = req.body; - - // Load the stored credentials - const credentialsPath = path.join(__dirname, 'service_account_credentials.json'); - if (!fs.existsSync(credentialsPath)) { - return res.status(400).json({ message: 'No credentials found. Please provide credentials first.' }); - } - - const credentials = JSON.parse(fs.readFileSync(credentialsPath, 'utf-8')); - - // Authenticate with Google using the service account credentials - const auth = new google.auth.GoogleAuth({ - credentials, - scopes: ['https://www.googleapis.com/auth/spreadsheets'], - }); - - const sheets = google.sheets({ version: 'v4', auth }); - - // Write data to the provided Google Sheet and range - await sheets.spreadsheets.values.append({ - spreadsheetId, - range, - valueInputOption: 'USER_ENTERED', - requestBody: { - values: values, // Expecting an array of arrays, like [['data1', 'data2'], ['data3', 'data4']] - }, - }); - - res.status(200).json({ message: 'Data written to Google Sheet successfully.' }); + const { spreadsheetId, range, values } = req.body; + + // Load the stored credentials + const credentialsPath = path.join(__dirname, 'service_account_credentials.json'); + if (!fs.existsSync(credentialsPath)) { + return res.status(400).json({ message: 'No credentials found. Please provide credentials first.' }); + } + + const credentials = JSON.parse(fs.readFileSync(credentialsPath, 'utf-8')); + + // Authenticate with Google using the service account credentials + const auth = new google.auth.GoogleAuth({ + credentials, + scopes: ['https://www.googleapis.com/auth/spreadsheets'], + }); + + const sheets = google.sheets({ version: 'v4', auth }); + + // Write data to the provided Google Sheet and range + await sheets.spreadsheets.values.append({ + spreadsheetId, + range, + valueInputOption: 'USER_ENTERED', + requestBody: { + values: values, // Expecting an array of arrays, like [['data1', 'data2'], ['data3', 'data4']] + }, + }); + + res.status(200).json({ message: 'Data written to Google Sheet successfully.' }); } catch (error) { - console.error('Error writing to sheet:', error); - res.status(500).json({ message: 'Failed to write to Google Sheet.', error }); + console.error('Error writing to sheet:', error); + res.status(500).json({ message: 'Failed to write to Google Sheet.', error }); } - }); - \ No newline at end of file +}); From 60b8dc134f54dbffbc2f3374c7eb1c96d186a0cd Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 21:00:13 +0530 Subject: [PATCH 043/109] feat: global export integration --- server/src/routes/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/routes/index.ts b/server/src/routes/index.ts index 19257cf1..72be1ebd 100644 --- a/server/src/routes/index.ts +++ b/server/src/routes/index.ts @@ -2,10 +2,12 @@ import { router as record } from './record'; import { router as workflow } from './workflow'; import { router as storage } from './storage'; import { router as auth } from './auth'; +import { router as integration } from './integration'; export { record, workflow, storage, auth, + integration, }; From f43524c650e4c8e12caecce9e6e0a2d2e296fb2a Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 21:03:15 +0530 Subject: [PATCH 044/109] feat: use integration routes --- server/src/server.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/server.ts b/server/src/server.ts index 29709fa6..3e553367 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -2,7 +2,7 @@ import express from 'express'; import http from 'http'; import cors from 'cors'; import 'dotenv/config'; -import { record, workflow, storage, auth } from './routes'; +import { record, workflow, storage, auth, integration } from './routes'; import { BrowserPool } from "./browser-management/classes/BrowserPool"; import logger from './logger' import { SERVER_PORT } from "./constants/config"; @@ -29,6 +29,7 @@ app.use('/record', record); app.use('/workflow', workflow); app.use('/storage', storage); app.use('/auth', auth); +app.use('/integration', integration); app.get('/', function (req, res) { return res.send('Maxun server started 🚀'); From 0225529bc4ed3dcf73365aad929cf1578adcbb0d Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 21:05:13 +0530 Subject: [PATCH 045/109] feat: rm TextField for data --- src/components/molecules/IntegrationSettings.tsx | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/components/molecules/IntegrationSettings.tsx b/src/components/molecules/IntegrationSettings.tsx index c8c64169..ed6bea50 100644 --- a/src/components/molecules/IntegrationSettings.tsx +++ b/src/components/molecules/IntegrationSettings.tsx @@ -73,16 +73,6 @@ export const IntegrationSettingsModal = ({ isOpen, handleStart, handleClose }: I fullWidth /> - - From 48a9ced868341fc645e3063dc8bb0cb05ef3b84c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 21:10:01 +0530 Subject: [PATCH 046/109] feat: !accept values from req.body --- server/src/routes/integration.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index ee38c369..f1659e13 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -20,7 +20,13 @@ router.post('/upload-credentials', (req, res) => { router.post('/write-to-sheet', async (req, res) => { try { - const { spreadsheetId, range, values } = req.body; + const { spreadsheetId, range } = req.body; + + // Todo: remove this. This is just for testing purposes. + const values = [ + ['Scraped Data 1', 'More Data'], + ['Scraped Data 2', 'More Data'], + ]; // Load the stored credentials const credentialsPath = path.join(__dirname, 'service_account_credentials.json'); From 6312753d77185b16ccba63910d4fe0e5e9bb536a Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 22:36:40 +0530 Subject: [PATCH 047/109] feat: handle upload credentials --- src/api/interpretation.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/api/interpretation.ts diff --git a/src/api/interpretation.ts b/src/api/interpretation.ts new file mode 100644 index 00000000..7c83b37f --- /dev/null +++ b/src/api/interpretation.ts @@ -0,0 +1,13 @@ +import { default as axios } from "axios"; + +const handleUploadCredentials = async (credentials: any) => { + try { + await axios.post('http://localhost:8080/integration/upload-credentials', { credentials: JSON.parse(credentials) }); + alert('Service Account credentials saved successfully.'); + } catch (error) { + console.error('Error uploading credentials:', error); + alert('Failed to upload credentials.'); + } + }; + + From 00db87580e38ce7b476621725eeb72c92ee5a69f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 22:36:54 +0530 Subject: [PATCH 048/109] chore: lint --- src/api/interpretation.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/api/interpretation.ts b/src/api/interpretation.ts index 7c83b37f..0215292f 100644 --- a/src/api/interpretation.ts +++ b/src/api/interpretation.ts @@ -2,12 +2,12 @@ import { default as axios } from "axios"; const handleUploadCredentials = async (credentials: any) => { try { - await axios.post('http://localhost:8080/integration/upload-credentials', { credentials: JSON.parse(credentials) }); - alert('Service Account credentials saved successfully.'); + await axios.post('http://localhost:8080/integration/upload-credentials', { credentials: JSON.parse(credentials) }); + alert('Service Account credentials saved successfully.'); } catch (error) { - console.error('Error uploading credentials:', error); - alert('Failed to upload credentials.'); + console.error('Error uploading credentials:', error); + alert('Failed to upload credentials.'); } - }; +}; From 365ff8f9e18becb0074d38ef01b0fd04310dc379 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 22:38:32 +0530 Subject: [PATCH 049/109] feat: handle write to sheet --- src/api/interpretation.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/api/interpretation.ts b/src/api/interpretation.ts index 0215292f..ca93c1a3 100644 --- a/src/api/interpretation.ts +++ b/src/api/interpretation.ts @@ -10,4 +10,13 @@ const handleUploadCredentials = async (credentials: any) => { } }; +const handleWriteToSheet = async (spreadsheetId: any, range: any) => { + try { + await axios.post('http://localhost:8080/integration/write-to-sheet', { spreadsheetId, range }); + alert('Data written to Google Sheet successfully.'); + } catch (error) { + console.error('Error writing to Google Sheet:', error); + alert('Failed to write to Google Sheet.'); + } + }; From 719093b57558322648350c69e14e0ba79f7e3e49 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 22:39:05 +0530 Subject: [PATCH 050/109] feat: exports --- src/api/interpretation.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/api/interpretation.ts b/src/api/interpretation.ts index ca93c1a3..c498e652 100644 --- a/src/api/interpretation.ts +++ b/src/api/interpretation.ts @@ -1,6 +1,6 @@ import { default as axios } from "axios"; -const handleUploadCredentials = async (credentials: any) => { +export const handleUploadCredentials = async (credentials: any) => { try { await axios.post('http://localhost:8080/integration/upload-credentials', { credentials: JSON.parse(credentials) }); alert('Service Account credentials saved successfully.'); @@ -10,13 +10,13 @@ const handleUploadCredentials = async (credentials: any) => { } }; -const handleWriteToSheet = async (spreadsheetId: any, range: any) => { +export const handleWriteToSheet = async (spreadsheetId: any, range: any) => { try { - await axios.post('http://localhost:8080/integration/write-to-sheet', { spreadsheetId, range }); - alert('Data written to Google Sheet successfully.'); + await axios.post('http://localhost:8080/integration/write-to-sheet', { spreadsheetId, range }); + alert('Data written to Google Sheet successfully.'); } catch (error) { - console.error('Error writing to Google Sheet:', error); - alert('Failed to write to Google Sheet.'); + console.error('Error writing to Google Sheet:', error); + alert('Failed to write to Google Sheet.'); } - }; +}; From 40b469fb8ff1656a25aa73fed6cc5baba952d911 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 22:39:28 +0530 Subject: [PATCH 051/109] chore: ts types --- src/api/interpretation.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/api/interpretation.ts b/src/api/interpretation.ts index c498e652..7d835cec 100644 --- a/src/api/interpretation.ts +++ b/src/api/interpretation.ts @@ -1,5 +1,6 @@ import { default as axios } from "axios"; +// todo: proper typescript types for params export const handleUploadCredentials = async (credentials: any) => { try { await axios.post('http://localhost:8080/integration/upload-credentials', { credentials: JSON.parse(credentials) }); From 5253608955e68843bc05281e35e7719864398c1d Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 23:07:11 +0530 Subject: [PATCH 052/109] feat(wip): send credentials + handle write to sheet --- src/pages/MainPage.tsx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/pages/MainPage.tsx b/src/pages/MainPage.tsx index 51ceb02c..97eadb4a 100644 --- a/src/pages/MainPage.tsx +++ b/src/pages/MainPage.tsx @@ -5,6 +5,7 @@ import { Recordings } from "../components/organisms/Recordings"; import { Runs } from "../components/organisms/Runs"; import { useGlobalInfoStore } from "../context/globalInfo"; import { createRunForStoredRecording, interpretStoredRecording, notifyAboutAbort, scheduleStoredRecording } from "../api/storage"; +import { handleUploadCredentials, handleWriteToSheet } from "../api/interpretation" import { io, Socket } from "socket.io-client"; import { stopRecording } from "../api/recording"; import { RunSettings } from "../components/molecules/RunSettings"; @@ -114,7 +115,11 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => { } const handleIntegrateRecording = (settings: IntegrationSettings) => { - console.log(`Integration settings:`, settings); + handleUploadCredentials(settings.credentials).then(() => { + handleWriteToSheet(settings.spreadsheetId, settings.range).then(() => { + notify('success', `Data written to Google Sheet successfully`); + }); + }); } const DisplayContent = () => { From 34e370333e2ab81c25c441241b9f8d7fd93d5dbc Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 17 Sep 2024 23:50:57 +0530 Subject: [PATCH 053/109] feat: use resource & proper creds --- server/src/routes/integration.ts | 18 +++++++++++++----- .../routes/service_account_credentials.json | 1 + 2 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 server/src/routes/service_account_credentials.json diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index f1659e13..d5f1cf9b 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -28,6 +28,10 @@ router.post('/write-to-sheet', async (req, res) => { ['Scraped Data 2', 'More Data'], ]; + const resource = { + values, + }; + // Load the stored credentials const credentialsPath = path.join(__dirname, 'service_account_credentials.json'); if (!fs.existsSync(credentialsPath)) { @@ -38,9 +42,15 @@ router.post('/write-to-sheet', async (req, res) => { // Authenticate with Google using the service account credentials const auth = new google.auth.GoogleAuth({ - credentials, + credentials: { + client_email: credentials.client_email, + private_key: credentials.private_key, + }, scopes: ['https://www.googleapis.com/auth/spreadsheets'], }); + const authToken = await auth.getClient(); + console.log('authToken:', authToken); + // return authToken; const sheets = google.sheets({ version: 'v4', auth }); @@ -49,10 +59,8 @@ router.post('/write-to-sheet', async (req, res) => { spreadsheetId, range, valueInputOption: 'USER_ENTERED', - requestBody: { - values: values, // Expecting an array of arrays, like [['data1', 'data2'], ['data3', 'data4']] - }, - }); + requestBody: resource, + }); res.status(200).json({ message: 'Data written to Google Sheet successfully.' }); } catch (error) { diff --git a/server/src/routes/service_account_credentials.json b/server/src/routes/service_account_credentials.json new file mode 100644 index 00000000..602fcbe0 --- /dev/null +++ b/server/src/routes/service_account_credentials.json @@ -0,0 +1 @@ +{"type":"service_account","project_id":"maxun12345","private_key_id":"45859ac1196ba6a371bf5427eb990e44b61d4237","private_key":"-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC0Sx73gWatvgGX\nhhtt/isiaH+EL1Y9K3X0404ZZndm7iNvILk190Jv2RjDNjASvWZBMcH72MKwaMkP\neYspK33DiA7+IDzE+tVefrC8PFPBepO3VtVfNeKe6xBMa3j2TwlfVgpDqPQBun69\nLuhvG1QMh1GfCnTi0wSeksRSiTS2KlmAuY79I/Y8KRQefLwoPZDDjVD9/3B0Xgly\nC3kKAtmfOSi+U1BUdVf3J58Tj57Yge2QpSBHBPjxwL60iadJoBSgcvBUXg+VgdXg\nqNyENYau5ohyvOjtW4Fhv/d7g/lyi84NBEn6+2ljFzY2xE0tZYM1CLouCU9MeDDu\ngvcHzGOnAgMBAAECggEAUwJbSZ18hvX6iacnLedeOMNgIp4g2J0P/CZDqRIyW2O9\nUMcCP0SOhCyyZ/6k95vYXp/oLmpIiSxVlhhh6BysFMCqFnsFP7Q1VQKDoMct0OD8\n7ea+6s1Gf+C+alD5Kn6NVCWrKCe74Kfa/oOAZNdyRSpwfAc2ddCuScxNM6sUZ+Ik\nK89fM576j8rfBAF1JW4ynQE/oM1y9A+88o1g602tOC5rm4vZH/Jv9tZwnfHMV+S7\n2PMSGDD1N4Mq1z3Gg3HjRHC/zo6SW6Jvk4O6mx9scIaui+u3nn7qQUVCLSMiFV3i\nsPv9zLktadu0h9o+6pCHSDuOFvFQOFIVEMU0fmgkQQKBgQD1Vnitk6X3aRBI/1u9\nJCk1l0GarfvdUed5fb3qRwH85vEhwmWKyglwDtWxxYNSNOIlpxOGKnq9CMdpkF15\nQO1bgbhsrbx75YYT29ulbAhLjm/tIP3OdvokzsH/Lz9qVh0AnEi+lgueRjerALvZ\nfZuLeSFi3FlxEX229a89TN9c0QKBgQC8IP4ea+Zu9hK+8qsF6htQEMqYagAMI63R\nwTVzRHL4H/GGYcs39ZjA9F0y0g65MAtT7+PfR8QifNUAJNRAnu5/CPDyQX9GvBM2\n9uBnwI1pJSdse1m0ez7kigl2TQDYaOADoSvRGALmBAFXXB7CNXgy8l1aQBiRvZjy\nulBQCcn29wKBgQDYQxM5ns9L2lc6oa6Sec/Bp8Vyvi8olsw+sfK5E0LTVhf0tFGi\nGBpdpxYEEKGD044NtstlFwj+nUB684ESI4OXiC+zzSo65MZdtw5VMXfWcoaDNvPE\nDejOjVtAwLtb1vDV2u3c4pL3P9pOaOUuAKUeOvaNGMPXAZ4Zq1R/6sVyIQKBgH93\nCzapfPPpnkHqQZ48RE03U015ZJbVBcx80C5NTmh3nDmkwQAlU15JM2xfjsJCnyo7\n+3UpNub3FYqHaZhvFsDT2g0J+6Z9f7daBinF+QootlF2Mg1rA+3s6QRSoCQAyucq\nqHl/f1dBl3cNX3nOqKY8OKwRiZQVli+/tPLF7yV5AoGBAI8+aLt3R9rioBtPFDUK\nuwxoeide0LH8UWFa5H+LV/13sWk6io/E6gdQOi1A2CrOdR+YFTCUG3VbIszbWCba\ne4Ws8Yy0CAvz0ePVE2aySJE0aRQWZH2BjPIckeGE8AGxViS1JiiX+wruYJiLSzx9\n+ejlxtjjkUsMoDd3trSI/s5d\n-----END PRIVATE KEY-----\n","client_email":"maxun-999@maxun12345.iam.gserviceaccount.com","client_id":"108198347095941384494","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_x509_cert_url":"https://www.googleapis.com/robot/v1/metadata/x509/maxun-999%40maxun12345.iam.gserviceaccount.com","universe_domain":"googleapis.com"} \ No newline at end of file From d9dfd2a07da6bf29ae2ed350edb594eece1dcbc6 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 07:28:57 +0530 Subject: [PATCH 054/109] fix: remove extra ; --- server/src/routes/integration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index d5f1cf9b..36446404 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -1,4 +1,4 @@ -import { Router } from 'express';; +import { Router } from 'express'; import { google, sheets_v4 } from "googleapis"; import fs from 'fs'; import path from 'path'; From b99e6d2b15fc00e03f6ada03567339ff609fd9f5 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 18:27:38 +0530 Subject: [PATCH 055/109] feat(wip): wrap all write logic in one route --- server/src/routes/integration.ts | 93 +++++++++++++++++--------------- 1 file changed, 50 insertions(+), 43 deletions(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index 36446404..ee77f541 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -1,70 +1,77 @@ import { Router } from 'express'; -import { google, sheets_v4 } from "googleapis"; +import { google } from "googleapis"; import fs from 'fs'; import path from 'path'; +import logger from "../logger"; -export const router = Router() +export const router = Router(); -router.post('/upload-credentials', (req, res) => { - const credentials = req.body.credentials; +router.post('/upload-credentials', async (req, res) => { + const { credentials, spreadsheetId, range } = req.body; - if (!credentials) { - return res.status(400).json({ message: 'Credentials are required.' }); + if (!credentials || !spreadsheetId || !range) { + return res.status(400).json({ message: 'Credentials, Spreadsheet ID, and Range are required.' }); } + // Todo: Store the credentials in a secure place (for test, we store them locally) const storedCredentialsPath = path.join(__dirname, 'service_account_credentials.json'); - fs.writeFileSync(storedCredentialsPath, JSON.stringify(credentials)); - - res.status(200).json({ message: 'Service Account credentials saved successfully.' }); -}); - -router.post('/write-to-sheet', async (req, res) => { + try { - const { spreadsheetId, range } = req.body; + fs.writeFileSync(storedCredentialsPath, JSON.stringify(credentials)); + logger.log('info', 'Service account credentials saved successfully.'); + } catch (error) { + logger.log('error', `Error saving credentials: ${error.message}`); + return res.status(500).json({ message: 'Failed to save credentials.', error: error.message }); + } - // Todo: remove this. This is just for testing purposes. - const values = [ - ['Scraped Data 1', 'More Data'], - ['Scraped Data 2', 'More Data'], - ]; + let storedCredentials; + try { + storedCredentials = JSON.parse(fs.readFileSync(storedCredentialsPath, 'utf-8')); + } catch (error) { + logger.log('error', `Error reading credentials: ${error.message}`); + return res.status(500).json({ message: 'Failed to read credentials.', error: error.message }); + } - const resource = { - values, - }; - - // Load the stored credentials - const credentialsPath = path.join(__dirname, 'service_account_credentials.json'); - if (!fs.existsSync(credentialsPath)) { - return res.status(400).json({ message: 'No credentials found. Please provide credentials first.' }); - } - - const credentials = JSON.parse(fs.readFileSync(credentialsPath, 'utf-8')); - - // Authenticate with Google using the service account credentials + let authToken; + try { const auth = new google.auth.GoogleAuth({ credentials: { - client_email: credentials.client_email, - private_key: credentials.private_key, + client_email: storedCredentials.client_email, + private_key: storedCredentials.private_key, }, scopes: ['https://www.googleapis.com/auth/spreadsheets'], }); - const authToken = await auth.getClient(); - console.log('authToken:', authToken); - // return authToken; - const sheets = google.sheets({ version: 'v4', auth }); + authToken = await auth.getClient(); + logger.log('info', 'Authenticated with Google Sheets API successfully.'); + } catch (error) { + logger.log('error', `Google Sheets API Authentication failed: ${error.message}`); + return res.status(500).json({ message: 'Authentication with Google failed.', error: error.message }); + } - // Write data to the provided Google Sheet and range + const sheets = google.sheets({ version: 'v4', auth: authToken }); + + // Data to be written to the sheet + const values = [ + ['Scraped Data 1', 'More Data', 'IDKKKKKKKKK'], + ]; + + const resource = { + values, + }; + + try { await sheets.spreadsheets.values.append({ spreadsheetId, range, valueInputOption: 'USER_ENTERED', requestBody: resource, - }); - - res.status(200).json({ message: 'Data written to Google Sheet successfully.' }); + }); + logger.log('info', `Data written to Google Sheet: ${spreadsheetId}, Range: ${range}`); + return res.status(200).json({ message: 'Data written to Google Sheet successfully.' }); } catch (error) { - console.error('Error writing to sheet:', error); - res.status(500).json({ message: 'Failed to write to Google Sheet.', error }); + logger.log('error', `Failed to write to Google Sheet: ${error.message}`); + return res.status(500).json({ message: 'Failed to write to Google Sheet.', error: error.message }); } }); + From a3818c92f1c94296a2847bccb511cb6e3e6037e7 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 18:28:00 +0530 Subject: [PATCH 056/109] chore: lint --- server/src/routes/integration.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index ee77f541..8798bca9 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -15,7 +15,7 @@ router.post('/upload-credentials', async (req, res) => { // Todo: Store the credentials in a secure place (for test, we store them locally) const storedCredentialsPath = path.join(__dirname, 'service_account_credentials.json'); - + try { fs.writeFileSync(storedCredentialsPath, JSON.stringify(credentials)); logger.log('info', 'Service account credentials saved successfully.'); @@ -51,7 +51,6 @@ router.post('/upload-credentials', async (req, res) => { const sheets = google.sheets({ version: 'v4', auth: authToken }); - // Data to be written to the sheet const values = [ ['Scraped Data 1', 'More Data', 'IDKKKKKKKKK'], ]; From dd838016ba84c5be432b13ef4afcc61f0f216ff3 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 18:30:04 +0530 Subject: [PATCH 057/109] feat: -rm handleWriteToSheet --- src/api/interpretation.ts | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/api/interpretation.ts b/src/api/interpretation.ts index 7d835cec..e2a4dea6 100644 --- a/src/api/interpretation.ts +++ b/src/api/interpretation.ts @@ -9,15 +9,4 @@ export const handleUploadCredentials = async (credentials: any) => { console.error('Error uploading credentials:', error); alert('Failed to upload credentials.'); } -}; - -export const handleWriteToSheet = async (spreadsheetId: any, range: any) => { - try { - await axios.post('http://localhost:8080/integration/write-to-sheet', { spreadsheetId, range }); - alert('Data written to Google Sheet successfully.'); - } catch (error) { - console.error('Error writing to Google Sheet:', error); - alert('Failed to write to Google Sheet.'); - } -}; - +}; \ No newline at end of file From 1421d5a50b24a88fe07884ddb7a047d9af8e03ee Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 18:31:20 +0530 Subject: [PATCH 058/109] feat: replace handleWriteToSheet --- src/pages/MainPage.tsx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/pages/MainPage.tsx b/src/pages/MainPage.tsx index 97eadb4a..0fd67622 100644 --- a/src/pages/MainPage.tsx +++ b/src/pages/MainPage.tsx @@ -5,7 +5,7 @@ import { Recordings } from "../components/organisms/Recordings"; import { Runs } from "../components/organisms/Runs"; import { useGlobalInfoStore } from "../context/globalInfo"; import { createRunForStoredRecording, interpretStoredRecording, notifyAboutAbort, scheduleStoredRecording } from "../api/storage"; -import { handleUploadCredentials, handleWriteToSheet } from "../api/interpretation" +import { handleUploadCredentials } from "../api/interpretation" import { io, Socket } from "socket.io-client"; import { stopRecording } from "../api/recording"; import { RunSettings } from "../components/molecules/RunSettings"; @@ -115,10 +115,9 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => { } const handleIntegrateRecording = (settings: IntegrationSettings) => { - handleUploadCredentials(settings.credentials).then(() => { - handleWriteToSheet(settings.spreadsheetId, settings.range).then(() => { + handleUploadCredentials(settings.credentials, settings.spreadsheetId, settings.range).then(() => { + console.log(settings.spreadsheetId, settings.range) notify('success', `Data written to Google Sheet successfully`); - }); }); } From a32a5620456fdeee0b97eeb39ad6e4487253d347 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 18:32:46 +0530 Subject: [PATCH 059/109] feat: accept spreadsheetId and range --- src/api/interpretation.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/interpretation.ts b/src/api/interpretation.ts index e2a4dea6..9fbfb2f6 100644 --- a/src/api/interpretation.ts +++ b/src/api/interpretation.ts @@ -1,9 +1,9 @@ import { default as axios } from "axios"; // todo: proper typescript types for params -export const handleUploadCredentials = async (credentials: any) => { +export const handleUploadCredentials = async (credentials: any, spreadsheetId: any, range: any) => { try { - await axios.post('http://localhost:8080/integration/upload-credentials', { credentials: JSON.parse(credentials) }); + await axios.post('http://localhost:8080/integration/upload-credentials', { credentials: JSON.parse(credentials), spreadsheetId, range }); alert('Service Account credentials saved successfully.'); } catch (error) { console.error('Error uploading credentials:', error); From 768fcd47f4dc2509b3f04961565051451eab621f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 18:44:04 +0530 Subject: [PATCH 060/109] fix: ts --- server/src/routes/integration.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index 8798bca9..eca5b63f 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -19,7 +19,7 @@ router.post('/upload-credentials', async (req, res) => { try { fs.writeFileSync(storedCredentialsPath, JSON.stringify(credentials)); logger.log('info', 'Service account credentials saved successfully.'); - } catch (error) { + } catch (error: any) { logger.log('error', `Error saving credentials: ${error.message}`); return res.status(500).json({ message: 'Failed to save credentials.', error: error.message }); } @@ -27,7 +27,7 @@ router.post('/upload-credentials', async (req, res) => { let storedCredentials; try { storedCredentials = JSON.parse(fs.readFileSync(storedCredentialsPath, 'utf-8')); - } catch (error) { + } catch (error: any) { logger.log('error', `Error reading credentials: ${error.message}`); return res.status(500).json({ message: 'Failed to read credentials.', error: error.message }); } @@ -44,7 +44,7 @@ router.post('/upload-credentials', async (req, res) => { authToken = await auth.getClient(); logger.log('info', 'Authenticated with Google Sheets API successfully.'); - } catch (error) { + } catch (error: any) { logger.log('error', `Google Sheets API Authentication failed: ${error.message}`); return res.status(500).json({ message: 'Authentication with Google failed.', error: error.message }); } @@ -68,7 +68,7 @@ router.post('/upload-credentials', async (req, res) => { }); logger.log('info', `Data written to Google Sheet: ${spreadsheetId}, Range: ${range}`); return res.status(200).json({ message: 'Data written to Google Sheet successfully.' }); - } catch (error) { + } catch (error: any ) { logger.log('error', `Failed to write to Google Sheet: ${error.message}`); return res.status(500).json({ message: 'Failed to write to Google Sheet.', error: error.message }); } From e1d99fc882ee0448a9274d41ba405750b5bbb0eb Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 18:47:59 +0530 Subject: [PATCH 061/109] fix: cast to avoid type errors --- server/src/routes/integration.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index eca5b63f..0a98e2ed 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -13,7 +13,7 @@ router.post('/upload-credentials', async (req, res) => { return res.status(400).json({ message: 'Credentials, Spreadsheet ID, and Range are required.' }); } - // Todo: Store the credentials in a secure place (for test, we store them locally) + // Store the credentials in a secure place (for test, we store them locally) const storedCredentialsPath = path.join(__dirname, 'service_account_credentials.json'); try { @@ -42,7 +42,7 @@ router.post('/upload-credentials', async (req, res) => { scopes: ['https://www.googleapis.com/auth/spreadsheets'], }); - authToken = await auth.getClient(); + authToken = await auth.getClient() as any; // Casting to avoid type errors logger.log('info', 'Authenticated with Google Sheets API successfully.'); } catch (error: any) { logger.log('error', `Google Sheets API Authentication failed: ${error.message}`); @@ -68,9 +68,8 @@ router.post('/upload-credentials', async (req, res) => { }); logger.log('info', `Data written to Google Sheet: ${spreadsheetId}, Range: ${range}`); return res.status(200).json({ message: 'Data written to Google Sheet successfully.' }); - } catch (error: any ) { + } catch (error: any) { logger.log('error', `Failed to write to Google Sheet: ${error.message}`); return res.status(500).json({ message: 'Failed to write to Google Sheet.', error: error.message }); } }); - From 0f66f36a48410336f1e397570bee795ed7d0bb93 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 19:52:12 +0530 Subject: [PATCH 062/109] feat: separate write & auth logic --- server/src/routes/integration.ts | 49 -------------------------------- 1 file changed, 49 deletions(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index 0a98e2ed..6eddb2a8 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -23,53 +23,4 @@ router.post('/upload-credentials', async (req, res) => { logger.log('error', `Error saving credentials: ${error.message}`); return res.status(500).json({ message: 'Failed to save credentials.', error: error.message }); } - - let storedCredentials; - try { - storedCredentials = JSON.parse(fs.readFileSync(storedCredentialsPath, 'utf-8')); - } catch (error: any) { - logger.log('error', `Error reading credentials: ${error.message}`); - return res.status(500).json({ message: 'Failed to read credentials.', error: error.message }); - } - - let authToken; - try { - const auth = new google.auth.GoogleAuth({ - credentials: { - client_email: storedCredentials.client_email, - private_key: storedCredentials.private_key, - }, - scopes: ['https://www.googleapis.com/auth/spreadsheets'], - }); - - authToken = await auth.getClient() as any; // Casting to avoid type errors - logger.log('info', 'Authenticated with Google Sheets API successfully.'); - } catch (error: any) { - logger.log('error', `Google Sheets API Authentication failed: ${error.message}`); - return res.status(500).json({ message: 'Authentication with Google failed.', error: error.message }); - } - - const sheets = google.sheets({ version: 'v4', auth: authToken }); - - const values = [ - ['Scraped Data 1', 'More Data', 'IDKKKKKKKKK'], - ]; - - const resource = { - values, - }; - - try { - await sheets.spreadsheets.values.append({ - spreadsheetId, - range, - valueInputOption: 'USER_ENTERED', - requestBody: resource, - }); - logger.log('info', `Data written to Google Sheet: ${spreadsheetId}, Range: ${range}`); - return res.status(200).json({ message: 'Data written to Google Sheet successfully.' }); - } catch (error: any) { - logger.log('error', `Failed to write to Google Sheet: ${error.message}`); - return res.status(500).json({ message: 'Failed to write to Google Sheet.', error: error.message }); - } }); From 5de67998b328e16590b992396e53da9f577472b0 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 19:54:31 +0530 Subject: [PATCH 063/109] feat: write data to sheet --- .../integrations/gsheet.ts | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 server/src/workflow-management/integrations/gsheet.ts diff --git a/server/src/workflow-management/integrations/gsheet.ts b/server/src/workflow-management/integrations/gsheet.ts new file mode 100644 index 00000000..ef2d5999 --- /dev/null +++ b/server/src/workflow-management/integrations/gsheet.ts @@ -0,0 +1,35 @@ +import { google } from "googleapis"; +import fs from 'fs'; +import path from 'path'; + +export async function writeDataToSheet(spreadsheetId: string, range: string, data: any[]) { + try { + const credentialsPath = path.join(__dirname, 'service_account_credentials.json'); + const credentials = JSON.parse(fs.readFileSync(credentialsPath, 'utf-8')); + + const auth = new google.auth.GoogleAuth({ + credentials: { + client_email: credentials.client_email, + private_key: credentials.private_key, + }, + scopes: ['https://www.googleapis.com/auth/spreadsheets'], + }); + + const authToken = await auth.getClient(); + const sheets = google.sheets({ version: 'v4', auth: authToken as any }); + + const resource = { values: data }; + + await sheets.spreadsheets.values.append({ + spreadsheetId, + range, + valueInputOption: 'USER_ENTERED', + requestBody: resource, + }); + + console.log(`Data written to Google Sheet: ${spreadsheetId}, Range: ${range}`); + } catch (error) { + console.error(`Error writing data to Google Sheet: ${error.message}`); + throw error; + } +} From 3e0cfda838733e19a739e0f173fe538c987398cd Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 19:59:42 +0530 Subject: [PATCH 064/109] chore: remove unused import --- server/src/routes/integration.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index 6eddb2a8..6df91363 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -1,5 +1,4 @@ import { Router } from 'express'; -import { google } from "googleapis"; import fs from 'fs'; import path from 'path'; import logger from "../logger"; From 1c578bee4f1f03396e71713ee26596d3c4db3849 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 20:00:42 +0530 Subject: [PATCH 065/109] feat: get fileName --- server/src/routes/integration.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index 6df91363..3dfbefe8 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -6,9 +6,9 @@ import logger from "../logger"; export const router = Router(); router.post('/upload-credentials', async (req, res) => { - const { credentials, spreadsheetId, range } = req.body; + const { fileName, credentials, spreadsheetId, range } = req.body; - if (!credentials || !spreadsheetId || !range) { + if (!fileName || !credentials || !spreadsheetId || !range) { return res.status(400).json({ message: 'Credentials, Spreadsheet ID, and Range are required.' }); } From a1d672847f5158018e6f56b0d431f2305d9bfd93 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 20:01:45 +0530 Subject: [PATCH 066/109] feat: pass fileName --- src/api/interpretation.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/interpretation.ts b/src/api/interpretation.ts index 9fbfb2f6..7cfd2561 100644 --- a/src/api/interpretation.ts +++ b/src/api/interpretation.ts @@ -1,9 +1,9 @@ import { default as axios } from "axios"; // todo: proper typescript types for params -export const handleUploadCredentials = async (credentials: any, spreadsheetId: any, range: any) => { +export const handleUploadCredentials = async (fileName: any, credentials: any, spreadsheetId: any, range: any) => { try { - await axios.post('http://localhost:8080/integration/upload-credentials', { credentials: JSON.parse(credentials), spreadsheetId, range }); + await axios.post('http://localhost:8080/integration/upload-credentials', { fileName, credentials: JSON.parse(credentials), spreadsheetId, range }); alert('Service Account credentials saved successfully.'); } catch (error) { console.error('Error uploading credentials:', error); From 7116ac30418745035257cde77829c1c341ed3429 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 20:07:31 +0530 Subject: [PATCH 067/109] feat: pass fileName --- src/pages/MainPage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/MainPage.tsx b/src/pages/MainPage.tsx index 0fd67622..2d813b99 100644 --- a/src/pages/MainPage.tsx +++ b/src/pages/MainPage.tsx @@ -115,7 +115,7 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => { } const handleIntegrateRecording = (settings: IntegrationSettings) => { - handleUploadCredentials(settings.credentials, settings.spreadsheetId, settings.range).then(() => { + handleUploadCredentials(runningRecordingName, settings.credentials, settings.spreadsheetId, settings.range).then(() => { console.log(settings.spreadsheetId, settings.range) notify('success', `Data written to Google Sheet successfully`); }); From 4fc88d50367af6709a9e193e3e8e4ce2f2a5fac3 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 20:29:47 +0530 Subject: [PATCH 068/109] feat(temp): integrations load & save --- server/src/routes/integration.ts | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index 3dfbefe8..9f3b08c1 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -5,18 +5,42 @@ import logger from "../logger"; export const router = Router(); +// Temproary Path to the JSON file that will store the integration details +const integrationsFilePath = path.join(__dirname, 'integrations.json'); + +function loadIntegrations() { + if (fs.existsSync(integrationsFilePath)) { + const data = fs.readFileSync(integrationsFilePath, 'utf-8'); + return JSON.parse(data); + } + return {}; +} + +function saveIntegrations(integrations: any) { + fs.writeFileSync(integrationsFilePath, JSON.stringify(integrations, null, 2)); +} + router.post('/upload-credentials', async (req, res) => { const { fileName, credentials, spreadsheetId, range } = req.body; + console.log(`fileName: ${fileName}, credentials: ${credentials}, spreadsheetId: ${spreadsheetId}, range: ${range}`); + if (!fileName || !credentials || !spreadsheetId || !range) { return res.status(400).json({ message: 'Credentials, Spreadsheet ID, and Range are required.' }); } // Store the credentials in a secure place (for test, we store them locally) - const storedCredentialsPath = path.join(__dirname, 'service_account_credentials.json'); + // Load existing integrations from the JSON file + const integrations = loadIntegrations(); + + // Add or update the integration for the specific task (fileName) + integrations[fileName] = { spreadsheetId, range }; + + // Save the updated integrations back to the file + saveIntegrations(integrations); try { - fs.writeFileSync(storedCredentialsPath, JSON.stringify(credentials)); + //fs.writeFileSync(storedCredentialsPath, JSON.stringify(credentials)); logger.log('info', 'Service account credentials saved successfully.'); } catch (error: any) { logger.log('error', `Error saving credentials: ${error.message}`); From 6f64ad17d7864c74ba67d3f092c5d1efdc2414df Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 21:43:41 +0530 Subject: [PATCH 069/109] feat: add missing export --- server/src/routes/integration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index 9f3b08c1..bda578fa 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -8,7 +8,7 @@ export const router = Router(); // Temproary Path to the JSON file that will store the integration details const integrationsFilePath = path.join(__dirname, 'integrations.json'); -function loadIntegrations() { +export function loadIntegrations() { if (fs.existsSync(integrationsFilePath)) { const data = fs.readFileSync(integrationsFilePath, 'utf-8'); return JSON.parse(data); From 2ab7204e14f5b38fbe10ac873e7e70c335ef3b7b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 22:05:06 +0530 Subject: [PATCH 070/109] fix: set type as any --- server/src/workflow-management/integrations/gsheet.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/workflow-management/integrations/gsheet.ts b/server/src/workflow-management/integrations/gsheet.ts index ef2d5999..70886d0c 100644 --- a/server/src/workflow-management/integrations/gsheet.ts +++ b/server/src/workflow-management/integrations/gsheet.ts @@ -28,7 +28,7 @@ export async function writeDataToSheet(spreadsheetId: string, range: string, dat }); console.log(`Data written to Google Sheet: ${spreadsheetId}, Range: ${range}`); - } catch (error) { + } catch (error: any) { console.error(`Error writing data to Google Sheet: ${error.message}`); throw error; } From e6f2b40c6116f7e6a7d7e00f4731276951ade19e Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 22:30:44 +0530 Subject: [PATCH 071/109] feat: error handling --- src/pages/MainPage.tsx | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/pages/MainPage.tsx b/src/pages/MainPage.tsx index 2d813b99..3f15fa81 100644 --- a/src/pages/MainPage.tsx +++ b/src/pages/MainPage.tsx @@ -57,8 +57,8 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => { setRunningRecordingName(fileName); } - const readyForRunHandler = useCallback( (browserId: string, runId: string) => { - interpretStoredRecording(runningRecordingName, runId).then( async (interpretation: boolean) => { + const readyForRunHandler = useCallback((browserId: string, runId: string) => { + interpretStoredRecording(runningRecordingName, runId).then(async (interpretation: boolean) => { if (!aborted) { if (interpretation) { notify('success', `Interpretation of ${runningRecordingName} succeeded`); @@ -80,8 +80,8 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => { }, [currentInterpretationLog]) const handleRunRecording = useCallback((settings: RunSettings) => { - createRunForStoredRecording(runningRecordingName, settings).then(({browserId, runId}: CreateRunResponse) => { - setIds({browserId, runId}); + createRunForStoredRecording(runningRecordingName, settings).then(({ browserId, runId }: CreateRunResponse) => { + setIds({ browserId, runId }); const socket = io(`http://localhost:8080/${browserId}`, { transports: ["websocket"], @@ -105,7 +105,7 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => { const handleScheduleRecording = (settings: ScheduleSettings) => { scheduleStoredRecording(runningRecordingName, settings) - .then(({message, runId}: ScheduleRunResponse) => { + .then(({ message, runId }: ScheduleRunResponse) => { if (message === 'success') { notify('success', `Recording ${runningRecordingName} scheduled successfully`); } else { @@ -115,10 +115,15 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => { } const handleIntegrateRecording = (settings: IntegrationSettings) => { - handleUploadCredentials(runningRecordingName, settings.credentials, settings.spreadsheetId, settings.range).then(() => { - console.log(settings.spreadsheetId, settings.range) - notify('success', `Data written to Google Sheet successfully`); - }); + handleUploadCredentials(runningRecordingName, settings.credentials, settings.spreadsheetId, settings.range) + .then((response) => { + if (response) { + notify('success', `Service Account credentials saved successfully.`); + } else { + notify('error', `Failed to save credentials.`); + + } + }) } const DisplayContent = () => { @@ -144,9 +149,9 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => { } return ( - - - { DisplayContent() } + + + {DisplayContent()} ); }; From d40ec715b2601b94aa67479c02aec9553881dd3d Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 22:31:48 +0530 Subject: [PATCH 072/109] feat: return promise --- src/api/interpretation.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/api/interpretation.ts b/src/api/interpretation.ts index 7cfd2561..3081caf2 100644 --- a/src/api/interpretation.ts +++ b/src/api/interpretation.ts @@ -1,12 +1,18 @@ import { default as axios } from "axios"; // todo: proper typescript types for params -export const handleUploadCredentials = async (fileName: any, credentials: any, spreadsheetId: any, range: any) => { +export const handleUploadCredentials = async (fileName: any, credentials: any, spreadsheetId: any, range: any): Promise => { try { - await axios.post('http://localhost:8080/integration/upload-credentials', { fileName, credentials: JSON.parse(credentials), spreadsheetId, range }); - alert('Service Account credentials saved successfully.'); + const response = await axios.post('http://localhost:8080/integration/upload-credentials', { fileName, credentials: JSON.parse(credentials), spreadsheetId, range }); + if (response.status === 200) { + alert('Service Account credentials saved successfully.'); + return response.data; + } else { + throw new Error(`Couldn't make gsheet integration for ${fileName}`); + } } catch (error) { console.error('Error uploading credentials:', error); alert('Failed to upload credentials.'); + return false; } }; \ No newline at end of file From 14ef8b9750dcd39d16b51f02422ee1252c13a7e5 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 22:32:16 +0530 Subject: [PATCH 073/109] feat: !alerts --- src/api/interpretation.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/api/interpretation.ts b/src/api/interpretation.ts index 3081caf2..85843c64 100644 --- a/src/api/interpretation.ts +++ b/src/api/interpretation.ts @@ -5,14 +5,12 @@ export const handleUploadCredentials = async (fileName: any, credentials: any, s try { const response = await axios.post('http://localhost:8080/integration/upload-credentials', { fileName, credentials: JSON.parse(credentials), spreadsheetId, range }); if (response.status === 200) { - alert('Service Account credentials saved successfully.'); return response.data; } else { throw new Error(`Couldn't make gsheet integration for ${fileName}`); } } catch (error) { console.error('Error uploading credentials:', error); - alert('Failed to upload credentials.'); return false; } }; \ No newline at end of file From 2702260e32ee107bdb1e97af652763b0b89c75b5 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 18 Sep 2024 22:33:01 +0530 Subject: [PATCH 074/109] feat: return boolean value --- server/src/routes/integration.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index bda578fa..ecb35102 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -21,6 +21,7 @@ function saveIntegrations(integrations: any) { } router.post('/upload-credentials', async (req, res) => { + try { const { fileName, credentials, spreadsheetId, range } = req.body; console.log(`fileName: ${fileName}, credentials: ${credentials}, spreadsheetId: ${spreadsheetId}, range: ${range}`); @@ -38,10 +39,11 @@ router.post('/upload-credentials', async (req, res) => { // Save the updated integrations back to the file saveIntegrations(integrations); + logger.log('info', 'Service account credentials saved successfully.'); - try { - //fs.writeFileSync(storedCredentialsPath, JSON.stringify(credentials)); - logger.log('info', 'Service account credentials saved successfully.'); + return res.send(true); + + //fs.writeFileSync(storedCredentialsPath, JSON.stringify(credentials)); } catch (error: any) { logger.log('error', `Error saving credentials: ${error.message}`); return res.status(500).json({ message: 'Failed to save credentials.', error: error.message }); From bd1270a75b9ab8c3738b3ec2936e6f366d192ecf Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 08:48:49 +0530 Subject: [PATCH 075/109] chore: -rm comment --- server/src/routes/storage.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 08be5b7c..11ccdcfc 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -144,10 +144,9 @@ router.get('/runs/run/:fileName/:runId', async (req, res) => { */ router.post('/runs/run/:fileName/:runId', async (req, res) => { try { - // read the recording from storage const recording = await readFile(`./../storage/recordings/${req.params.fileName}.waw.json`) const parsedRecording = JSON.parse(recording); - // read the run from storage + const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`) const parsedRun = JSON.parse(run); From a7650da5e07b1f7f0c70a9cc677dea9c73182eb1 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 09:00:56 +0530 Subject: [PATCH 076/109] feat: route to write to sheet --- server/src/routes/integration.ts | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index ecb35102..f342624a 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -2,6 +2,9 @@ import { Router } from 'express'; import fs from 'fs'; import path from 'path'; import logger from "../logger"; +import { writeDataToSheet } from "../workflow-management/integrations/gsheet" +import { readFile } from "../workflow-management/storage"; + export const router = Router(); @@ -49,3 +52,35 @@ router.post('/upload-credentials', async (req, res) => { return res.status(500).json({ message: 'Failed to save credentials.', error: error.message }); } }); + +router.post('/update-google-sheet/:fileName/:runId', async (req, res) => { + try { + const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`); + const parsedRun = JSON.parse(run); + + if (parsedRun.status === 'success' && parsedRun.serializableOutput) { + const data = parsedRun.serializableOutput as { [key: string]: any }[]; + const integrationConfig = await loadIntegrations(); + + if (integrationConfig) { + const { spreadsheetId, sheetName, credentials } = integrationConfig; + + if (spreadsheetId && sheetName && credentials) { + // Convert data to Google Sheets format (headers and rows) + const headers = Object.keys(data[0]); + const rows = data.map((row: { [key: string]: any }) => Object.values(row)); + const outputData = [headers, ...rows]; // Include headers + + await writeDataToSheet(spreadsheetId, sheetName, outputData); + logger.log('info', `Data written to Google Sheet successfully for ${req.params.fileName}_${req.params.runId}`); + return res.send({ success: true, message: 'Data updated in Google Sheet' }); + } + } + return res.status(400).send({ success: false, message: 'Google Sheet integration not configured' }); + } + return res.status(400).send({ success: false, message: 'Run not successful or no data to update' }); + } catch (error) { + logger.log('error', `Failed to write data to Google Sheet for ${req.params.fileName}_${req.params.runId}: ${error.message}`); + return res.status(500).send({ success: false, message: 'Failed to update Google Sheet', error: error.message }); + } + }); From 825b1db932353f725eda60618a15e023eaefa85f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 09:14:18 +0530 Subject: [PATCH 077/109] feat: api fe writetogsheet --- src/api/interpretation.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/api/interpretation.ts b/src/api/interpretation.ts index 85843c64..7e2e3f39 100644 --- a/src/api/interpretation.ts +++ b/src/api/interpretation.ts @@ -13,4 +13,21 @@ export const handleUploadCredentials = async (fileName: any, credentials: any, s console.error('Error uploading credentials:', error); return false; } +}; + +export const handleWriteToGsheet = async (fileName: string, runId: string): Promise<{ + success: boolean; + message: string; + }> => { + try { + const response = await axios.post(`http://localhost:8080/integration/update-google-sheet/${fileName}/${runId}`); + if (response.status === 200) { + return response.data; + } else { + throw new Error(`Couldn't make gsheet integration for ${fileName}`); + } + } catch (error) { + console.error('Error uploading credentials:', error); + return { success: false, message: 'Failed to write to Google Sheet' }; + } }; \ No newline at end of file From 3dad5a0c41e688582cc084dd7a1c18bc53a18c20 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 09:14:31 +0530 Subject: [PATCH 078/109] chore: lint --- src/api/interpretation.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/api/interpretation.ts b/src/api/interpretation.ts index 7e2e3f39..768d56d7 100644 --- a/src/api/interpretation.ts +++ b/src/api/interpretation.ts @@ -6,7 +6,7 @@ export const handleUploadCredentials = async (fileName: any, credentials: any, s const response = await axios.post('http://localhost:8080/integration/upload-credentials', { fileName, credentials: JSON.parse(credentials), spreadsheetId, range }); if (response.status === 200) { return response.data; - } else { + } else { throw new Error(`Couldn't make gsheet integration for ${fileName}`); } } catch (error) { @@ -18,12 +18,12 @@ export const handleUploadCredentials = async (fileName: any, credentials: any, s export const handleWriteToGsheet = async (fileName: string, runId: string): Promise<{ success: boolean; message: string; - }> => { +}> => { try { const response = await axios.post(`http://localhost:8080/integration/update-google-sheet/${fileName}/${runId}`); if (response.status === 200) { return response.data; - } else { + } else { throw new Error(`Couldn't make gsheet integration for ${fileName}`); } } catch (error) { From 22047cd81479c8f9dc4ae2b65bbca5863b7889e8 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 09:18:50 +0530 Subject: [PATCH 079/109] feat: set browserID to parsed run browserID --- server/src/routes/storage.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 11ccdcfc..bdecccc3 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -146,7 +146,7 @@ router.post('/runs/run/:fileName/:runId', async (req, res) => { try { const recording = await readFile(`./../storage/recordings/${req.params.fileName}.waw.json`) const parsedRecording = JSON.parse(recording); - + const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`) const parsedRun = JSON.parse(run); @@ -172,7 +172,7 @@ router.post('/runs/run/:fileName/:runId', async (req, res) => { status: interpretationInfo.result, finishedAt: new Date().toLocaleString(), duration: durString, - browserId: null, + browserId: parsedRun.browserId, log: interpretationInfo.log.join('\n'), serializableOutput: interpretationInfo.serializableOutput, binaryOutput: interpretationInfo.binaryOutput, From 8251394b325d9e97a0979a1955fd07acc21025bc Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 09:38:25 +0530 Subject: [PATCH 080/109] feat: emit socket event on succesful run --- server/src/routes/storage.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index bdecccc3..a155b7a5 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -7,7 +7,7 @@ import logger from "../logger"; import { deleteFile, readFile, readFiles, saveFile } from "../workflow-management/storage"; import { createRemoteBrowserForRun, destroyRemoteBrowser } from "../browser-management/controller"; import { chromium } from "playwright"; -import { browserPool } from "../server"; +import { browserPool, io } from "../server"; import fs from "fs"; import { uuid } from "uuidv4"; import { workflowQueue } from '../workflow-management/scheduler'; @@ -182,6 +182,7 @@ router.post('/runs/run/:fileName/:runId', async (req, res) => { `../storage/runs/${parsedRun.name}_${req.params.runId}.json`, JSON.stringify(run_meta, null, 2) ); + io.emit('run-finished', { success: true }); return res.send(true); } else { throw new Error('Could not destroy browser'); From 9e990204ce0eca1430ebf062b6408af70beb91cd Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 12:52:58 +0530 Subject: [PATCH 081/109] feat(temp): standalone gsheet update fxn --- server/src/routes/integration.ts | 34 +++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index f342624a..47069b46 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -79,8 +79,40 @@ router.post('/update-google-sheet/:fileName/:runId', async (req, res) => { return res.status(400).send({ success: false, message: 'Google Sheet integration not configured' }); } return res.status(400).send({ success: false, message: 'Run not successful or no data to update' }); - } catch (error) { + } catch (error: any) { logger.log('error', `Failed to write data to Google Sheet for ${req.params.fileName}_${req.params.runId}: ${error.message}`); return res.status(500).send({ success: false, message: 'Failed to update Google Sheet', error: error.message }); } }); + +export async function updateGoogleSheet(fileName: string, runId: string) { + try { + const run = await readFile(`./../storage/runs/${fileName}_${runId}.json`); + const parsedRun = JSON.parse(run); + + if (parsedRun.status === 'success' && parsedRun.serializableOutput) { + const data = parsedRun.serializableOutput as { [key: string]: any }[]; + const integrationConfig = await loadIntegrations(); // Assume this function loads config + + if (integrationConfig) { + const { spreadsheetId, sheetName } = integrationConfig; + + if (spreadsheetId && sheetName) { + // Convert data to Google Sheets format (headers and rows) + const headers = Object.keys(data[0]); + const rows = data.map((row: { [key: string]: any }) => Object.values(row)); + const outputData = [headers, ...rows]; // Include headers + + await writeDataToSheet(spreadsheetId, sheetName, outputData); + logger.log('info', `Data written to Google Sheet successfully for ${fileName}_${runId}`); + } + } else { + logger.log('error', 'Google Sheet integration not configured'); + } + } else { + logger.log('error', 'Run not successful or no data to update'); + } + } catch (error: any) { + logger.log('error', `Failed to write data to Google Sheet for ${fileName}_${runId}: ${error.message}`); + } +} \ No newline at end of file From 1bc3b1d35fff8c1678c23182959ec28fa13bac99 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 17:38:35 +0530 Subject: [PATCH 082/109] feat: match the serializable outut --- server/src/routes/integration.ts | 38 +++++++++++++++++--------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index 47069b46..10724670 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -38,7 +38,7 @@ router.post('/upload-credentials', async (req, res) => { const integrations = loadIntegrations(); // Add or update the integration for the specific task (fileName) - integrations[fileName] = { spreadsheetId, range }; + integrations[fileName] = { fileName, spreadsheetId, range, credentials }; // Save the updated integrations back to the file saveIntegrations(integrations); @@ -59,19 +59,23 @@ router.post('/update-google-sheet/:fileName/:runId', async (req, res) => { const parsedRun = JSON.parse(run); if (parsedRun.status === 'success' && parsedRun.serializableOutput) { - const data = parsedRun.serializableOutput as { [key: string]: any }[]; + const data = parsedRun.serializableOutput['item-0'] as { [key: string]: any }[]; const integrationConfig = await loadIntegrations(); + + logger.log(`info`, `integration config ${JSON.stringify(integrationConfig)}`) if (integrationConfig) { - const { spreadsheetId, sheetName, credentials } = integrationConfig; - - if (spreadsheetId && sheetName && credentials) { + const { spreadsheetId, range, credentials } = integrationConfig; + + logger.log(`info`, `data from routeeeeeeeeeeeeeeeee: ${JSON.stringify(data)}`) + + if (spreadsheetId && range && credentials) { // Convert data to Google Sheets format (headers and rows) const headers = Object.keys(data[0]); const rows = data.map((row: { [key: string]: any }) => Object.values(row)); const outputData = [headers, ...rows]; // Include headers - await writeDataToSheet(spreadsheetId, sheetName, outputData); + await writeDataToSheet(spreadsheetId, range, outputData); logger.log('info', `Data written to Google Sheet successfully for ${req.params.fileName}_${req.params.runId}`); return res.send({ success: true, message: 'Data updated in Google Sheet' }); } @@ -91,28 +95,26 @@ export async function updateGoogleSheet(fileName: string, runId: string) { const parsedRun = JSON.parse(run); if (parsedRun.status === 'success' && parsedRun.serializableOutput) { - const data = parsedRun.serializableOutput as { [key: string]: any }[]; - const integrationConfig = await loadIntegrations(); // Assume this function loads config - + const data = parsedRun.serializableOutput['item-0'] as { [key: string]: any }[]; + const integrationConfig = await loadIntegrations(); + if (integrationConfig) { - const { spreadsheetId, sheetName } = integrationConfig; + const { spreadsheetId, range, credentials } = integrationConfig; - if (spreadsheetId && sheetName) { + if (spreadsheetId && range && credentials) { // Convert data to Google Sheets format (headers and rows) const headers = Object.keys(data[0]); const rows = data.map((row: { [key: string]: any }) => Object.values(row)); - const outputData = [headers, ...rows]; // Include headers + const outputData = [headers, ...rows]; - await writeDataToSheet(spreadsheetId, sheetName, outputData); + await writeDataToSheet(spreadsheetId, range, outputData); logger.log('info', `Data written to Google Sheet successfully for ${fileName}_${runId}`); } - } else { - logger.log('error', 'Google Sheet integration not configured'); } - } else { - logger.log('error', 'Run not successful or no data to update'); + logger.log('error', `Google Sheet integration not configured for ${fileName}_${runId}`); } + logger.log('error', `Run not successful or no data to update for ${fileName}_${runId}`); } catch (error: any) { logger.log('error', `Failed to write data to Google Sheet for ${fileName}_${runId}: ${error.message}`); } -} \ No newline at end of file +}; \ No newline at end of file From 01295deb33ce55baa59b66d1f5b00ea0bb6b5f91 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 17:39:33 +0530 Subject: [PATCH 083/109] feat(temp): use in-memory store --- server/src/routes/storage.ts | 118 ++++++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 38 deletions(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index a155b7a5..b5263244 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -7,12 +7,13 @@ import logger from "../logger"; import { deleteFile, readFile, readFiles, saveFile } from "../workflow-management/storage"; import { createRemoteBrowserForRun, destroyRemoteBrowser } from "../browser-management/controller"; import { chromium } from "playwright"; -import { browserPool, io } from "../server"; +import { browserPool } from "../server"; import fs from "fs"; import { uuid } from "uuidv4"; import { workflowQueue } from '../workflow-management/scheduler'; import moment from 'moment-timezone'; import cron from 'node-cron'; +import { updateGoogleSheet } from './integration'; export const router = Router(); @@ -20,7 +21,7 @@ export const router = Router(); * Logs information about recordings API. */ router.all('/', (req, res, next) => { - logger.log('debug',`The recordings API was invoked: ${req.url}`) + logger.log('debug', `The recordings API was invoked: ${req.url}`) next() // pass control to the next handler }) @@ -45,7 +46,7 @@ router.delete('/recordings/:fileName', async (req, res) => { await deleteFile(`./../storage/recordings/${req.params.fileName}.waw.json`); return res.send(true); } catch (e) { - const {message} = e as Error; + const { message } = e as Error; logger.log('info', `Error while deleting a recording with name: ${req.params.fileName}.waw.json`); return res.send(false); } @@ -72,7 +73,7 @@ router.delete('/runs/:fileName', async (req, res) => { await deleteFile(`./../storage/runs/${req.params.fileName}.json`); return res.send(true); } catch (e) { - const {message} = e as Error; + const { message } = e as Error; logger.log('info', `Error while deleting a run with name: ${req.params.fileName}.json`); return res.send(false); } @@ -117,7 +118,7 @@ router.put('/runs/:fileName', async (req, res) => { runId: runId, }); } catch (e) { - const {message} = e as Error; + const { message } = e as Error; logger.log('info', `Error while creating a run with name: ${req.params.fileName}.json`); return res.send(''); } @@ -139,6 +140,16 @@ router.get('/runs/run/:fileName/:runId', async (req, res) => { } }); +interface GoogleSheetUpdateTask { + name: string; + runId: string; + status: 'pending' | 'completed'; +} + +let googleSheetUpdateTasks: { [runId: string]: GoogleSheetUpdateTask } = {}; + +console.log('googleSheetUpdateTasks:', googleSheetUpdateTasks); + /** * PUT endpoint for finishing a run and saving it to the storage. */ @@ -167,28 +178,36 @@ router.post('/runs/run/:fileName/:runId', async (req, res) => { } })(); await destroyRemoteBrowser(parsedRun.browserId); - const run_meta = { - ...parsedRun, - status: interpretationInfo.result, - finishedAt: new Date().toLocaleString(), - duration: durString, - browserId: parsedRun.browserId, - log: interpretationInfo.log.join('\n'), - serializableOutput: interpretationInfo.serializableOutput, - binaryOutput: interpretationInfo.binaryOutput, - }; - fs.mkdirSync('../storage/runs', { recursive: true }) - await saveFile( - `../storage/runs/${parsedRun.name}_${req.params.runId}.json`, - JSON.stringify(run_meta, null, 2) - ); - io.emit('run-finished', { success: true }); - return res.send(true); - } else { - throw new Error('Could not destroy browser'); - } + const run_meta = { + ...parsedRun, + status: 'success', + finishedAt: new Date().toLocaleString(), + duration: durString, + browserId: parsedRun.browserId, + log: interpretationInfo.log.join('\n'), + serializableOutput: interpretationInfo.serializableOutput, + binaryOutput: interpretationInfo.binaryOutput, + }; + fs.mkdirSync('../storage/runs', { recursive: true }) + await saveFile( + `../storage/runs/${parsedRun.name}_${req.params.runId}.json`, + JSON.stringify(run_meta, null, 2) + ); + + res.send(true); + + googleSheetUpdateTasks[req.params.runId] = { + name: parsedRun.name, + runId: req.params.runId, + status: 'pending', + }; + + return; + } else { + throw new Error('Could not destroy browser'); + } } catch (e) { - const {message} = e as Error; + const { message } = e as Error; logger.log('info', `Error while running a recording with name: ${req.params.fileName}_${req.params.runId}.json`); return res.send(false); } @@ -198,12 +217,12 @@ router.put('/schedule/:fileName/', async (req, res) => { console.log(req.body); try { const { fileName } = req.params; - const { - runEvery, - runEveryUnit, - startFrom, - atTime, - timezone + const { + runEvery, + runEveryUnit, + startFrom, + atTime, + timezone } = req.body; if (!fileName || !runEvery || !runEveryUnit || !startFrom || !atTime || !timezone) { @@ -258,7 +277,7 @@ router.put('/schedule/:fileName/', async (req, res) => { await workflowQueue.add( 'run workflow', { fileName, runId }, - { + { repeat: { pattern: cronExpression, tz: timezone @@ -266,8 +285,8 @@ router.put('/schedule/:fileName/', async (req, res) => { } ); - res.status(200).json({ - message: 'success', + res.status(200).json({ + message: 'success', runId, // cronExpression, // nextRunTime: getNextRunTime(cronExpression, timezone) @@ -291,11 +310,9 @@ router.put('/schedule/:fileName/', async (req, res) => { */ router.post('/runs/abort/:fileName/:runId', async (req, res) => { try { - // read the run from storage const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`) const parsedRun = JSON.parse(run); - //get current log const browser = browserPool.getRemoteBrowser(parsedRun.browserId); const currentLog = browser?.interpreter.debugMessages.join('/n'); const serializableOutput = browser?.interpreter.serializableData.reduce((reducedObject, item, index) => { @@ -326,8 +343,33 @@ router.post('/runs/abort/:fileName/:runId', async (req, res) => { ); return res.send(true); } catch (e) { - const {message} = e as Error; + const { message } = e as Error; logger.log('info', `Error while running a recording with name: ${req.params.fileName}_${req.params.runId}.json`); return res.send(false); } }); + +const processGoogleSheetUpdates = async () => { + while (true) { + for (const runId in googleSheetUpdateTasks) { + const task = googleSheetUpdateTasks[runId]; + if (task.status === 'pending') { + try { + try { + await updateGoogleSheet(task.name, task.runId); + console.log(`Successfully updated Google Sheets for run ${task.runId}`); + googleSheetUpdateTasks[runId].status = 'completed'; + + } catch (error: any) { + console.error(`update google sheet error ${task.runId}:`, error); + } + } catch (error: any) { + console.error(`Failed to update Google Sheets for run ${task.runId}:`, error); + } + } + } + await new Promise(resolve => setTimeout(resolve, 5000)); + } +}; + +processGoogleSheetUpdates(); \ No newline at end of file From 14baa96a6d8b8bb96ed734a3dcef295ef9d9f304 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 17:39:51 +0530 Subject: [PATCH 084/109] chore: lint --- server/src/routes/storage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index b5263244..6ec98b60 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -199,7 +199,7 @@ router.post('/runs/run/:fileName/:runId', async (req, res) => { googleSheetUpdateTasks[req.params.runId] = { name: parsedRun.name, runId: req.params.runId, - status: 'pending', + status: 'pending', }; return; From 6ce10b84c76835ae6a602a73ae03c6c7c8063964 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 17:42:20 +0530 Subject: [PATCH 085/109] feat: use proper credenetials path --- .../src/workflow-management/integrations/gsheet.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/server/src/workflow-management/integrations/gsheet.ts b/server/src/workflow-management/integrations/gsheet.ts index 70886d0c..9e1a6d95 100644 --- a/server/src/workflow-management/integrations/gsheet.ts +++ b/server/src/workflow-management/integrations/gsheet.ts @@ -1,16 +1,17 @@ import { google } from "googleapis"; import fs from 'fs'; import path from 'path'; +import logger from "../../logger"; export async function writeDataToSheet(spreadsheetId: string, range: string, data: any[]) { try { - const credentialsPath = path.join(__dirname, 'service_account_credentials.json'); - const credentials = JSON.parse(fs.readFileSync(credentialsPath, 'utf-8')); + const integrationCredentialsPath = path.join(__dirname, 'integrations.json'); + const integrationCredentials = JSON.parse(fs.readFileSync(integrationCredentialsPath, 'utf-8')); const auth = new google.auth.GoogleAuth({ credentials: { - client_email: credentials.client_email, - private_key: credentials.private_key, + client_email: integrationCredentials.credentials.client_email, + private_key: integrationCredentials.credentials.private_key, }, scopes: ['https://www.googleapis.com/auth/spreadsheets'], }); @@ -27,9 +28,9 @@ export async function writeDataToSheet(spreadsheetId: string, range: string, dat requestBody: resource, }); - console.log(`Data written to Google Sheet: ${spreadsheetId}, Range: ${range}`); + logger.log(`info`, `Data written to Google Sheet: ${spreadsheetId}, Range: ${range}`); } catch (error: any) { - console.error(`Error writing data to Google Sheet: ${error.message}`); + logger.log(`error`, `Error writing data to Google Sheet: ${error.message}`); throw error; } } From 4158ddf0c458295f877af29afa95ce2b666e0ec3 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 17:43:42 +0530 Subject: [PATCH 086/109] chore: remove stale code --- server/src/routes/integration.ts | 81 ++++++++++++++------------------ 1 file changed, 35 insertions(+), 46 deletions(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index 10724670..323f8805 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -24,70 +24,59 @@ function saveIntegrations(integrations: any) { } router.post('/upload-credentials', async (req, res) => { - try { + try { const { fileName, credentials, spreadsheetId, range } = req.body; - - console.log(`fileName: ${fileName}, credentials: ${credentials}, spreadsheetId: ${spreadsheetId}, range: ${range}`); - if (!fileName || !credentials || !spreadsheetId || !range) { - return res.status(400).json({ message: 'Credentials, Spreadsheet ID, and Range are required.' }); + return res.status(400).json({ message: 'Credentials, Spreadsheet ID, and Range are required.' }); } - // Store the credentials in a secure place (for test, we store them locally) // Load existing integrations from the JSON file const integrations = loadIntegrations(); - - // Add or update the integration for the specific task (fileName) integrations[fileName] = { fileName, spreadsheetId, range, credentials }; - - // Save the updated integrations back to the file saveIntegrations(integrations); logger.log('info', 'Service account credentials saved successfully.'); - return res.send(true); - - //fs.writeFileSync(storedCredentialsPath, JSON.stringify(credentials)); - } catch (error: any) { - logger.log('error', `Error saving credentials: ${error.message}`); - return res.status(500).json({ message: 'Failed to save credentials.', error: error.message }); - } + } catch (error: any) { + logger.log('error', `Error saving credentials: ${error.message}`); + return res.status(500).json({ message: 'Failed to save credentials.', error: error.message }); + } }); router.post('/update-google-sheet/:fileName/:runId', async (req, res) => { - try { - const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`); - const parsedRun = JSON.parse(run); - - if (parsedRun.status === 'success' && parsedRun.serializableOutput) { - const data = parsedRun.serializableOutput['item-0'] as { [key: string]: any }[]; - const integrationConfig = await loadIntegrations(); + try { + const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`); + const parsedRun = JSON.parse(run); - logger.log(`info`, `integration config ${JSON.stringify(integrationConfig)}`) - - if (integrationConfig) { - const { spreadsheetId, range, credentials } = integrationConfig; + if (parsedRun.status === 'success' && parsedRun.serializableOutput) { + const data = parsedRun.serializableOutput['item-0'] as { [key: string]: any }[]; + const integrationConfig = await loadIntegrations(); - logger.log(`info`, `data from routeeeeeeeeeeeeeeeee: ${JSON.stringify(data)}`) + logger.log(`info`, `integration config ${JSON.stringify(integrationConfig)}`) - if (spreadsheetId && range && credentials) { - // Convert data to Google Sheets format (headers and rows) - const headers = Object.keys(data[0]); - const rows = data.map((row: { [key: string]: any }) => Object.values(row)); - const outputData = [headers, ...rows]; // Include headers - - await writeDataToSheet(spreadsheetId, range, outputData); - logger.log('info', `Data written to Google Sheet successfully for ${req.params.fileName}_${req.params.runId}`); - return res.send({ success: true, message: 'Data updated in Google Sheet' }); - } + if (integrationConfig) { + const { spreadsheetId, range, credentials } = integrationConfig; + + logger.log(`info`, `data from routeeeeeeeeeeeeeeeee: ${JSON.stringify(data)}`) + + if (spreadsheetId && range && credentials) { + // Convert data to Google Sheets format (headers and rows) + const headers = Object.keys(data[0]); + const rows = data.map((row: { [key: string]: any }) => Object.values(row)); + const outputData = [headers, ...rows]; // Include headers + + await writeDataToSheet(spreadsheetId, range, outputData); + logger.log('info', `Data written to Google Sheet successfully for ${req.params.fileName}_${req.params.runId}`); + return res.send({ success: true, message: 'Data updated in Google Sheet' }); } - return res.status(400).send({ success: false, message: 'Google Sheet integration not configured' }); } - return res.status(400).send({ success: false, message: 'Run not successful or no data to update' }); - } catch (error: any) { - logger.log('error', `Failed to write data to Google Sheet for ${req.params.fileName}_${req.params.runId}: ${error.message}`); - return res.status(500).send({ success: false, message: 'Failed to update Google Sheet', error: error.message }); + return res.status(400).send({ success: false, message: 'Google Sheet integration not configured' }); } - }); + return res.status(400).send({ success: false, message: 'Run not successful or no data to update' }); + } catch (error: any) { + logger.log('error', `Failed to write data to Google Sheet for ${req.params.fileName}_${req.params.runId}: ${error.message}`); + return res.status(500).send({ success: false, message: 'Failed to update Google Sheet', error: error.message }); + } +}); export async function updateGoogleSheet(fileName: string, runId: string) { try { @@ -97,7 +86,7 @@ export async function updateGoogleSheet(fileName: string, runId: string) { if (parsedRun.status === 'success' && parsedRun.serializableOutput) { const data = parsedRun.serializableOutput['item-0'] as { [key: string]: any }[]; const integrationConfig = await loadIntegrations(); - + if (integrationConfig) { const { spreadsheetId, range, credentials } = integrationConfig; From 98f531b3ec3b8e730db04cb763e1ff0543e90e0a Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 17:44:09 +0530 Subject: [PATCH 087/109] feat: remove post route --- server/src/routes/integration.ts | 36 -------------------------------- 1 file changed, 36 deletions(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index 323f8805..e05c14f4 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -42,42 +42,6 @@ router.post('/upload-credentials', async (req, res) => { } }); -router.post('/update-google-sheet/:fileName/:runId', async (req, res) => { - try { - const run = await readFile(`./../storage/runs/${req.params.fileName}_${req.params.runId}.json`); - const parsedRun = JSON.parse(run); - - if (parsedRun.status === 'success' && parsedRun.serializableOutput) { - const data = parsedRun.serializableOutput['item-0'] as { [key: string]: any }[]; - const integrationConfig = await loadIntegrations(); - - logger.log(`info`, `integration config ${JSON.stringify(integrationConfig)}`) - - if (integrationConfig) { - const { spreadsheetId, range, credentials } = integrationConfig; - - logger.log(`info`, `data from routeeeeeeeeeeeeeeeee: ${JSON.stringify(data)}`) - - if (spreadsheetId && range && credentials) { - // Convert data to Google Sheets format (headers and rows) - const headers = Object.keys(data[0]); - const rows = data.map((row: { [key: string]: any }) => Object.values(row)); - const outputData = [headers, ...rows]; // Include headers - - await writeDataToSheet(spreadsheetId, range, outputData); - logger.log('info', `Data written to Google Sheet successfully for ${req.params.fileName}_${req.params.runId}`); - return res.send({ success: true, message: 'Data updated in Google Sheet' }); - } - } - return res.status(400).send({ success: false, message: 'Google Sheet integration not configured' }); - } - return res.status(400).send({ success: false, message: 'Run not successful or no data to update' }); - } catch (error: any) { - logger.log('error', `Failed to write data to Google Sheet for ${req.params.fileName}_${req.params.runId}: ${error.message}`); - return res.status(500).send({ success: false, message: 'Failed to update Google Sheet', error: error.message }); - } -}); - export async function updateGoogleSheet(fileName: string, runId: string) { try { const run = await readFile(`./../storage/runs/${fileName}_${runId}.json`); From 3c4a78e7f08de5518fa37084fedc827608a93aed Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 17:49:15 +0530 Subject: [PATCH 088/109] feat: import loadIntegrations and saveIntegrations --- server/src/routes/integration.ts | 54 ++------------------------------ 1 file changed, 3 insertions(+), 51 deletions(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index e05c14f4..1be3a85c 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -2,35 +2,17 @@ import { Router } from 'express'; import fs from 'fs'; import path from 'path'; import logger from "../logger"; -import { writeDataToSheet } from "../workflow-management/integrations/gsheet" -import { readFile } from "../workflow-management/storage"; - +import { loadIntegrations, saveIntegrations } from '../workflow-management/integrations/gsheet'; export const router = Router(); -// Temproary Path to the JSON file that will store the integration details -const integrationsFilePath = path.join(__dirname, 'integrations.json'); - -export function loadIntegrations() { - if (fs.existsSync(integrationsFilePath)) { - const data = fs.readFileSync(integrationsFilePath, 'utf-8'); - return JSON.parse(data); - } - return {}; -} - -function saveIntegrations(integrations: any) { - fs.writeFileSync(integrationsFilePath, JSON.stringify(integrations, null, 2)); -} - router.post('/upload-credentials', async (req, res) => { try { const { fileName, credentials, spreadsheetId, range } = req.body; if (!fileName || !credentials || !spreadsheetId || !range) { return res.status(400).json({ message: 'Credentials, Spreadsheet ID, and Range are required.' }); } - // Store the credentials in a secure place (for test, we store them locally) - // Load existing integrations from the JSON file + // *** TEMPORARILY WE STORE CREDENTIALS HERE *** const integrations = loadIntegrations(); integrations[fileName] = { fileName, spreadsheetId, range, credentials }; saveIntegrations(integrations); @@ -40,34 +22,4 @@ router.post('/upload-credentials', async (req, res) => { logger.log('error', `Error saving credentials: ${error.message}`); return res.status(500).json({ message: 'Failed to save credentials.', error: error.message }); } -}); - -export async function updateGoogleSheet(fileName: string, runId: string) { - try { - const run = await readFile(`./../storage/runs/${fileName}_${runId}.json`); - const parsedRun = JSON.parse(run); - - if (parsedRun.status === 'success' && parsedRun.serializableOutput) { - const data = parsedRun.serializableOutput['item-0'] as { [key: string]: any }[]; - const integrationConfig = await loadIntegrations(); - - if (integrationConfig) { - const { spreadsheetId, range, credentials } = integrationConfig; - - if (spreadsheetId && range && credentials) { - // Convert data to Google Sheets format (headers and rows) - const headers = Object.keys(data[0]); - const rows = data.map((row: { [key: string]: any }) => Object.values(row)); - const outputData = [headers, ...rows]; - - await writeDataToSheet(spreadsheetId, range, outputData); - logger.log('info', `Data written to Google Sheet successfully for ${fileName}_${runId}`); - } - } - logger.log('error', `Google Sheet integration not configured for ${fileName}_${runId}`); - } - logger.log('error', `Run not successful or no data to update for ${fileName}_${runId}`); - } catch (error: any) { - logger.log('error', `Failed to write data to Google Sheet for ${fileName}_${runId}: ${error.message}`); - } -}; \ No newline at end of file +}); \ No newline at end of file From 6461c47d65092bbfe0efa0e2b39a11cedf279718 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 17:50:16 +0530 Subject: [PATCH 089/109] chore: remove unused imports --- server/src/routes/integration.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index 1be3a85c..41ad4eda 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -1,6 +1,4 @@ import { Router } from 'express'; -import fs from 'fs'; -import path from 'path'; import logger from "../logger"; import { loadIntegrations, saveIntegrations } from '../workflow-management/integrations/gsheet'; From 46125f8b716b63e53f68195e28c739bfc14c8d8c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 17:50:59 +0530 Subject: [PATCH 090/109] refactor: move gsheet integration logic --- .../integrations/gsheet.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/server/src/workflow-management/integrations/gsheet.ts b/server/src/workflow-management/integrations/gsheet.ts index 9e1a6d95..8f0baaf4 100644 --- a/server/src/workflow-management/integrations/gsheet.ts +++ b/server/src/workflow-management/integrations/gsheet.ts @@ -2,6 +2,52 @@ import { google } from "googleapis"; import fs from 'fs'; import path from 'path'; import logger from "../../logger"; +import { readFile } from "../storage"; + +// *** Temporary Path to the JSON file that will store the integration details *** +const integrationsFilePath = path.join(__dirname, 'integrations.json'); + +export function loadIntegrations() { + if (fs.existsSync(integrationsFilePath)) { + const data = fs.readFileSync(integrationsFilePath, 'utf-8'); + return JSON.parse(data); + } + return {}; +} + +export function saveIntegrations(integrations: any) { + fs.writeFileSync(integrationsFilePath, JSON.stringify(integrations, null, 2)); +} + +export async function updateGoogleSheet(fileName: string, runId: string) { + try { + const run = await readFile(`./../storage/runs/${fileName}_${runId}.json`); + const parsedRun = JSON.parse(run); + + if (parsedRun.status === 'success' && parsedRun.serializableOutput) { + const data = parsedRun.serializableOutput['item-0'] as { [key: string]: any }[]; + const integrationConfig = await loadIntegrations(); + + if (integrationConfig) { + const { spreadsheetId, range, credentials } = integrationConfig; + + if (spreadsheetId && range && credentials) { + // Convert data to Google Sheets format (headers and rows) + const headers = Object.keys(data[0]); + const rows = data.map((row: { [key: string]: any }) => Object.values(row)); + const outputData = [headers, ...rows]; + + await writeDataToSheet(spreadsheetId, range, outputData); + logger.log('info', `Data written to Google Sheet successfully for ${fileName}_${runId}`); + } + } + logger.log('error', `Google Sheet integration not configured for ${fileName}_${runId}`); + } + logger.log('error', `Run not successful or no data to update for ${fileName}_${runId}`); + } catch (error: any) { + logger.log('error', `Failed to write data to Google Sheet for ${fileName}_${runId}: ${error.message}`); + } +}; export async function writeDataToSheet(spreadsheetId: string, range: string, data: any[]) { try { From 0ab33f29e876c4ba5c971a73689670bf924f2b4c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 17:56:30 +0530 Subject: [PATCH 091/109] fix: broken import path for updateGoogleSheet --- server/src/routes/storage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 6ec98b60..912fbcd2 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -13,7 +13,7 @@ import { uuid } from "uuidv4"; import { workflowQueue } from '../workflow-management/scheduler'; import moment from 'moment-timezone'; import cron from 'node-cron'; -import { updateGoogleSheet } from './integration'; +import { updateGoogleSheet } from '../workflow-management/integrations/gsheet'; export const router = Router(); From 683242ae28d30e8f229819eef0405845c843ccae Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 17:58:38 +0530 Subject: [PATCH 092/109] feat: use googleSheetUpdateTasks --- server/src/routes/storage.ts | 39 ++---------------------------------- 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 912fbcd2..4c842bf4 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -13,7 +13,7 @@ import { uuid } from "uuidv4"; import { workflowQueue } from '../workflow-management/scheduler'; import moment from 'moment-timezone'; import cron from 'node-cron'; -import { updateGoogleSheet } from '../workflow-management/integrations/gsheet'; +import { googleSheetUpdateTasks } from '../workflow-management/integrations/gsheet'; export const router = Router(); @@ -140,16 +140,6 @@ router.get('/runs/run/:fileName/:runId', async (req, res) => { } }); -interface GoogleSheetUpdateTask { - name: string; - runId: string; - status: 'pending' | 'completed'; -} - -let googleSheetUpdateTasks: { [runId: string]: GoogleSheetUpdateTask } = {}; - -console.log('googleSheetUpdateTasks:', googleSheetUpdateTasks); - /** * PUT endpoint for finishing a run and saving it to the storage. */ @@ -347,29 +337,4 @@ router.post('/runs/abort/:fileName/:runId', async (req, res) => { logger.log('info', `Error while running a recording with name: ${req.params.fileName}_${req.params.runId}.json`); return res.send(false); } -}); - -const processGoogleSheetUpdates = async () => { - while (true) { - for (const runId in googleSheetUpdateTasks) { - const task = googleSheetUpdateTasks[runId]; - if (task.status === 'pending') { - try { - try { - await updateGoogleSheet(task.name, task.runId); - console.log(`Successfully updated Google Sheets for run ${task.runId}`); - googleSheetUpdateTasks[runId].status = 'completed'; - - } catch (error: any) { - console.error(`update google sheet error ${task.runId}:`, error); - } - } catch (error: any) { - console.error(`Failed to update Google Sheets for run ${task.runId}:`, error); - } - } - } - await new Promise(resolve => setTimeout(resolve, 5000)); - } -}; - -processGoogleSheetUpdates(); \ No newline at end of file +}); \ No newline at end of file From f6a7f5582c9ef3bedf4aca7091fd59d1aa55549b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 18:04:32 +0530 Subject: [PATCH 093/109] feat: handle task deletion + conditional invocation --- .../integrations/gsheet.ts | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/server/src/workflow-management/integrations/gsheet.ts b/server/src/workflow-management/integrations/gsheet.ts index 8f0baaf4..3d4b285a 100644 --- a/server/src/workflow-management/integrations/gsheet.ts +++ b/server/src/workflow-management/integrations/gsheet.ts @@ -4,6 +4,16 @@ import path from 'path'; import logger from "../../logger"; import { readFile } from "../storage"; + +interface GoogleSheetUpdateTask { + name: string; + runId: string; + status: 'pending' | 'completed'; +} + +export let googleSheetUpdateTasks: { [runId: string]: GoogleSheetUpdateTask } = {}; + + // *** Temporary Path to the JSON file that will store the integration details *** const integrationsFilePath = path.join(__dirname, 'integrations.json'); @@ -80,3 +90,32 @@ export async function writeDataToSheet(spreadsheetId: string, range: string, dat throw error; } } + +const processGoogleSheetUpdates = async () => { + while (true) { + let hasPendingTasks = false; + for (const runId in googleSheetUpdateTasks) { + const task = googleSheetUpdateTasks[runId]; + if (task.status === 'pending') { + hasPendingTasks = true; + try { + await updateGoogleSheet(task.name, task.runId); + console.log(`Successfully updated Google Sheets for run ${task.runId}`); + delete googleSheetUpdateTasks[runId]; + } catch (error: any) { + console.error(`Failed to update Google Sheets for run ${task.runId}:`, error); + } + } + } + if (!hasPendingTasks) { + break; + } + await new Promise(resolve => setTimeout(resolve, 5000)); + } +}; + +export const startProcessGoogleSheetUpdates = () => { + if (Object.keys(googleSheetUpdateTasks).length > 0) { + processGoogleSheetUpdates(); + } +}; \ No newline at end of file From 871a9b88bdd4775c81698e5c57eb6921f52f119b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 18:14:56 +0530 Subject: [PATCH 094/109] feat: max retries for failure --- .../workflow-management/integrations/gsheet.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/server/src/workflow-management/integrations/gsheet.ts b/server/src/workflow-management/integrations/gsheet.ts index 3d4b285a..b937b701 100644 --- a/server/src/workflow-management/integrations/gsheet.ts +++ b/server/src/workflow-management/integrations/gsheet.ts @@ -8,9 +8,12 @@ import { readFile } from "../storage"; interface GoogleSheetUpdateTask { name: string; runId: string; - status: 'pending' | 'completed'; + status: 'pending' | 'completed' | 'failed'; + retries: number; } +const MAX_RETRIES = 5; + export let googleSheetUpdateTasks: { [runId: string]: GoogleSheetUpdateTask } = {}; @@ -93,16 +96,22 @@ export async function writeDataToSheet(spreadsheetId: string, range: string, dat const processGoogleSheetUpdates = async () => { while (true) { - let hasPendingTasks = false; + let hasPendingTasks = false; for (const runId in googleSheetUpdateTasks) { const task = googleSheetUpdateTasks[runId]; if (task.status === 'pending') { - hasPendingTasks = true; + hasPendingTasks = true; try { await updateGoogleSheet(task.name, task.runId); console.log(`Successfully updated Google Sheets for run ${task.runId}`); delete googleSheetUpdateTasks[runId]; } catch (error: any) { + if (task.retries < MAX_RETRIES) { + googleSheetUpdateTasks[runId].retries += 1; + } else { + // Mark as failed after maximum retries + googleSheetUpdateTasks[runId].status = 'failed'; + } console.error(`Failed to update Google Sheets for run ${task.runId}:`, error); } } From adb5084e743f0fc73ae1f61627abbd84a03a515f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 18:15:42 +0530 Subject: [PATCH 095/109] feat: pass retries --- server/src/routes/storage.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 4c842bf4..1dea9b9c 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -1,6 +1,6 @@ /** * RESTful API endpoints handling the recording storage. - */ +*/ import { Router } from 'express'; import logger from "../logger"; @@ -190,6 +190,7 @@ router.post('/runs/run/:fileName/:runId', async (req, res) => { name: parsedRun.name, runId: req.params.runId, status: 'pending', + retries: 5, }; return; From 24dca87e4c518ff766c15e996951e2201ae7bb71 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 18:26:12 +0530 Subject: [PATCH 096/109] refactor: rename to integration.tsx --- src/api/{interpretation.ts => integration.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/api/{interpretation.ts => integration.ts} (100%) diff --git a/src/api/interpretation.ts b/src/api/integration.ts similarity index 100% rename from src/api/interpretation.ts rename to src/api/integration.ts From 08e149b4e8bfcc1bcf567feef1dc2ae8cb8bb6f6 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 18:26:51 +0530 Subject: [PATCH 097/109] feat: remove handleWriteToGSheet --- src/api/integration.ts | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/api/integration.ts b/src/api/integration.ts index 768d56d7..5a7432b6 100644 --- a/src/api/integration.ts +++ b/src/api/integration.ts @@ -13,21 +13,4 @@ export const handleUploadCredentials = async (fileName: any, credentials: any, s console.error('Error uploading credentials:', error); return false; } -}; - -export const handleWriteToGsheet = async (fileName: string, runId: string): Promise<{ - success: boolean; - message: string; -}> => { - try { - const response = await axios.post(`http://localhost:8080/integration/update-google-sheet/${fileName}/${runId}`); - if (response.status === 200) { - return response.data; - } else { - throw new Error(`Couldn't make gsheet integration for ${fileName}`); - } - } catch (error) { - console.error('Error uploading credentials:', error); - return { success: false, message: 'Failed to write to Google Sheet' }; - } }; \ No newline at end of file From 99a30e17a866f26b743631915f96e98bc871a046 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 18:28:03 +0530 Subject: [PATCH 098/109] fix: format --- src/pages/MainPage.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pages/MainPage.tsx b/src/pages/MainPage.tsx index 3f15fa81..b3a36349 100644 --- a/src/pages/MainPage.tsx +++ b/src/pages/MainPage.tsx @@ -5,7 +5,7 @@ import { Recordings } from "../components/organisms/Recordings"; import { Runs } from "../components/organisms/Runs"; import { useGlobalInfoStore } from "../context/globalInfo"; import { createRunForStoredRecording, interpretStoredRecording, notifyAboutAbort, scheduleStoredRecording } from "../api/storage"; -import { handleUploadCredentials } from "../api/interpretation" +import { handleUploadCredentials } from "../api/integration" import { io, Socket } from "socket.io-client"; import { stopRecording } from "../api/recording"; import { RunSettings } from "../components/molecules/RunSettings"; @@ -121,7 +121,6 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => { notify('success', `Service Account credentials saved successfully.`); } else { notify('error', `Failed to save credentials.`); - } }) } From 6d17d60b72d5520f919d66da86a01496796b5d8f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 18:30:27 +0530 Subject: [PATCH 099/109] feat: remove fileName for integration json --- server/src/routes/integration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index 41ad4eda..16a5ae50 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -12,7 +12,7 @@ router.post('/upload-credentials', async (req, res) => { } // *** TEMPORARILY WE STORE CREDENTIALS HERE *** const integrations = loadIntegrations(); - integrations[fileName] = { fileName, spreadsheetId, range, credentials }; + integrations = { fileName, spreadsheetId, range, credentials }; saveIntegrations(integrations); logger.log('info', 'Service account credentials saved successfully.'); return res.send(true); From e2432255d09a665cce729a554fa3f1ed8b1efc7c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 18:33:20 +0530 Subject: [PATCH 100/109] fix: format --- server/src/routes/integration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index 16a5ae50..cc2c39df 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -11,7 +11,7 @@ router.post('/upload-credentials', async (req, res) => { return res.status(400).json({ message: 'Credentials, Spreadsheet ID, and Range are required.' }); } // *** TEMPORARILY WE STORE CREDENTIALS HERE *** - const integrations = loadIntegrations(); + let integrations = loadIntegrations(); integrations = { fileName, spreadsheetId, range, credentials }; saveIntegrations(integrations); logger.log('info', 'Service account credentials saved successfully.'); From 444875bda7de0baf25ec84fe0b115af0d2199b69 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 18:39:29 +0530 Subject: [PATCH 101/109] feat: pass fileName to integration credential path --- server/src/routes/integration.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/routes/integration.ts b/server/src/routes/integration.ts index cc2c39df..3c9f8519 100644 --- a/server/src/routes/integration.ts +++ b/server/src/routes/integration.ts @@ -11,9 +11,9 @@ router.post('/upload-credentials', async (req, res) => { return res.status(400).json({ message: 'Credentials, Spreadsheet ID, and Range are required.' }); } // *** TEMPORARILY WE STORE CREDENTIALS HERE *** - let integrations = loadIntegrations(); + let integrations = loadIntegrations(fileName); integrations = { fileName, spreadsheetId, range, credentials }; - saveIntegrations(integrations); + saveIntegrations(fileName, integrations); logger.log('info', 'Service account credentials saved successfully.'); return res.send(true); } catch (error: any) { From 83dc4276687d73931d27e70dcdeb83cabe0a1ace Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 18:40:45 +0530 Subject: [PATCH 102/109] feat: use fileName for integration files --- .../workflow-management/integrations/gsheet.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server/src/workflow-management/integrations/gsheet.ts b/server/src/workflow-management/integrations/gsheet.ts index b937b701..d6ab6c6f 100644 --- a/server/src/workflow-management/integrations/gsheet.ts +++ b/server/src/workflow-management/integrations/gsheet.ts @@ -3,8 +3,6 @@ import fs from 'fs'; import path from 'path'; import logger from "../../logger"; import { readFile } from "../storage"; - - interface GoogleSheetUpdateTask { name: string; runId: string; @@ -18,18 +16,20 @@ export let googleSheetUpdateTasks: { [runId: string]: GoogleSheetUpdateTask } = // *** Temporary Path to the JSON file that will store the integration details *** -const integrationsFilePath = path.join(__dirname, 'integrations.json'); +const getIntegrationsFilePath = (fileName: string) => path.join(__dirname, `integrations-${fileName}.json`); -export function loadIntegrations() { - if (fs.existsSync(integrationsFilePath)) { - const data = fs.readFileSync(integrationsFilePath, 'utf-8'); +export function loadIntegrations(fileName: string) { + const filePath = getIntegrationsFilePath(fileName); + if (fs.existsSync(filePath)) { + const data = fs.readFileSync(filePath, 'utf-8'); return JSON.parse(data); } return {}; } -export function saveIntegrations(integrations: any) { - fs.writeFileSync(integrationsFilePath, JSON.stringify(integrations, null, 2)); +export function saveIntegrations(fileName: string, integrations: any) { + const filePath = getIntegrationsFilePath(fileName); + fs.writeFileSync(filePath, JSON.stringify(integrations, null, 2)); } export async function updateGoogleSheet(fileName: string, runId: string) { @@ -39,7 +39,7 @@ export async function updateGoogleSheet(fileName: string, runId: string) { if (parsedRun.status === 'success' && parsedRun.serializableOutput) { const data = parsedRun.serializableOutput['item-0'] as { [key: string]: any }[]; - const integrationConfig = await loadIntegrations(); + const integrationConfig = await loadIntegrations(fileName); if (integrationConfig) { const { spreadsheetId, range, credentials } = integrationConfig; From 99ceb6d41c632d1ed28037f2e249f536020407bf Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 18:56:15 +0530 Subject: [PATCH 103/109] feat: use getIntegrationsFilePath() for writeDataToSheet --- .../src/workflow-management/integrations/gsheet.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/workflow-management/integrations/gsheet.ts b/server/src/workflow-management/integrations/gsheet.ts index d6ab6c6f..e2660b9b 100644 --- a/server/src/workflow-management/integrations/gsheet.ts +++ b/server/src/workflow-management/integrations/gsheet.ts @@ -42,15 +42,15 @@ export async function updateGoogleSheet(fileName: string, runId: string) { const integrationConfig = await loadIntegrations(fileName); if (integrationConfig) { - const { spreadsheetId, range, credentials } = integrationConfig; + const { fileName, spreadsheetId, range, credentials } = integrationConfig; - if (spreadsheetId && range && credentials) { + if (fileName && spreadsheetId && range && credentials) { // Convert data to Google Sheets format (headers and rows) const headers = Object.keys(data[0]); const rows = data.map((row: { [key: string]: any }) => Object.values(row)); const outputData = [headers, ...rows]; - await writeDataToSheet(spreadsheetId, range, outputData); + await writeDataToSheet(fileName, spreadsheetId, range, outputData); logger.log('info', `Data written to Google Sheet successfully for ${fileName}_${runId}`); } } @@ -62,10 +62,10 @@ export async function updateGoogleSheet(fileName: string, runId: string) { } }; -export async function writeDataToSheet(spreadsheetId: string, range: string, data: any[]) { +export async function writeDataToSheet(fileName: string, spreadsheetId: string, range: string, data: any[]) { try { - const integrationCredentialsPath = path.join(__dirname, 'integrations.json'); - const integrationCredentials = JSON.parse(fs.readFileSync(integrationCredentialsPath, 'utf-8')); + const integrationCredentialsPath = getIntegrationsFilePath(fileName); + const integrationCredentials = JSON.parse(fs.readFileSync(integrationCredentialsPath, 'utf-8'));; const auth = new google.auth.GoogleAuth({ credentials: { From 9d684c4ffa293560916f3921d6594f564de02949 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 19:36:08 +0530 Subject: [PATCH 104/109] feat: export processGoogleSheetUpdates() --- server/src/workflow-management/integrations/gsheet.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/server/src/workflow-management/integrations/gsheet.ts b/server/src/workflow-management/integrations/gsheet.ts index e2660b9b..3ce55341 100644 --- a/server/src/workflow-management/integrations/gsheet.ts +++ b/server/src/workflow-management/integrations/gsheet.ts @@ -94,7 +94,7 @@ export async function writeDataToSheet(fileName: string, spreadsheetId: string, } } -const processGoogleSheetUpdates = async () => { +export const processGoogleSheetUpdates = async () => { while (true) { let hasPendingTasks = false; for (const runId in googleSheetUpdateTasks) { @@ -122,9 +122,4 @@ const processGoogleSheetUpdates = async () => { await new Promise(resolve => setTimeout(resolve, 5000)); } }; - -export const startProcessGoogleSheetUpdates = () => { - if (Object.keys(googleSheetUpdateTasks).length > 0) { - processGoogleSheetUpdates(); - } -}; \ No newline at end of file + \ No newline at end of file From 6ee8cabdca9402cb04067bc2273e8e7b77cb4330 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 19:36:47 +0530 Subject: [PATCH 105/109] feat: call processGoogleSheetUpdates() --- server/src/routes/storage.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 1dea9b9c..3d427169 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -13,7 +13,7 @@ import { uuid } from "uuidv4"; import { workflowQueue } from '../workflow-management/scheduler'; import moment from 'moment-timezone'; import cron from 'node-cron'; -import { googleSheetUpdateTasks } from '../workflow-management/integrations/gsheet'; +import { googleSheetUpdateTasks, processGoogleSheetUpdates } from '../workflow-management/integrations/gsheet'; export const router = Router(); @@ -183,17 +183,14 @@ router.post('/runs/run/:fileName/:runId', async (req, res) => { `../storage/runs/${parsedRun.name}_${req.params.runId}.json`, JSON.stringify(run_meta, null, 2) ); - - res.send(true); - googleSheetUpdateTasks[req.params.runId] = { name: parsedRun.name, runId: req.params.runId, status: 'pending', retries: 5, }; - - return; + processGoogleSheetUpdates(); + return res.send(true); } else { throw new Error('Could not destroy browser'); } From 901c5fc496644693ba4418192d8e1fa3c99d94ed Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 19:38:31 +0530 Subject: [PATCH 106/109] feat: set status to success --- server/src/workflow-management/scheduler/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index d874ac5b..347ed650 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -140,7 +140,7 @@ async function executeRun(fileName: string, runId: string) { const updated_run_meta = { ...parsedRun, - status: interpretationInfo.result, + status: 'success', finishedAt: new Date().toLocaleString(), duration: durString, browserId: null, From dcd3b7c2ffd2f25f143271785107f87e29c06ebf Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 19:39:58 +0530 Subject: [PATCH 107/109] feat: pass parsed run browserid as browserid --- server/src/workflow-management/scheduler/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 347ed650..9b05ddae 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -143,7 +143,7 @@ async function executeRun(fileName: string, runId: string) { status: 'success', finishedAt: new Date().toLocaleString(), duration: durString, - browserId: null, + browserId: parsedRun.browserId, log: interpretationInfo.log.join('\n'), serializableOutput: interpretationInfo.serializableOutput, binaryOutput: interpretationInfo.binaryOutput, From 5175e6eeb7ffda30d752ae1aceb2623c2bc76218 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 19:41:19 +0530 Subject: [PATCH 108/109] feat: call processGoogleSheetUpdates() --- server/src/workflow-management/scheduler/index.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 9b05ddae..2bd8d8cf 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -8,6 +8,7 @@ import { readFile, saveFile } from "../storage"; import { createRemoteBrowserForRun, destroyRemoteBrowser } from '../../browser-management/controller'; import logger from '../../logger'; import { browserPool } from "../../server"; +import { googleSheetUpdateTasks, processGoogleSheetUpdates } from "../integrations/gsheet"; const connection = new IORedis({ host: 'localhost', @@ -153,7 +154,13 @@ async function executeRun(fileName: string, runId: string) { `../storage/runs/${fileName}_${runId}.json`, JSON.stringify(updated_run_meta, null, 2) ); - + googleSheetUpdateTasks[runId] = { + name: parsedRun.name, + runId: runId, + status: 'pending', + retries: 5, + }; + processGoogleSheetUpdates(); return true; } catch (error: any) { logger.log('info', `Error while running a recording with name: ${fileName}_${runId}.json`); From 0ec283f9d6bde1ced6c07ea9a32cef488fa4390e Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 19 Sep 2024 19:41:35 +0530 Subject: [PATCH 109/109] chore: lint --- server/src/workflow-management/scheduler/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 2bd8d8cf..07d338c1 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -53,9 +53,9 @@ worker.on('failed', async (job: any, err) => { logger.log(`info`, `Worker and queue have been closed after failure.`); }); -async function jobCounts () { +async function jobCounts() { const jobCounts = await workflowQueue.getJobCounts(); - console.log('Jobs:', jobCounts); + console.log('Jobs:', jobCounts); } jobCounts();