From bba40fa3beeb128f6b2daac77e55f4eec7e392e3 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Mon, 9 Sep 2024 05:04:18 +0530 Subject: [PATCH 001/108] chore: remove unused var --- server/src/workflow-management/classes/Generator.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 7819fe1a..7e3bdd48 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -483,7 +483,6 @@ export class WorkflowGenerator { private generateSelector = async (page: Page, coordinates: Coordinates, action: ActionType) => { const elementInfo = await getElementInformation(page, coordinates); const generalSelector = await getNonUniqueSelectors(page, coordinates) - const childSelectors = await getChildSelectors(page, generalSelector.generalSelector); console.log(`Get List value while generating selector`, this.getList); From 7866e98864f50bebec45f18fc151f17c9962686b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Mon, 9 Sep 2024 05:13:28 +0530 Subject: [PATCH 002/108] chore: remove unused code --- server/src/workflow-management/classes/Generator.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 7e3bdd48..f9ae1410 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -482,9 +482,6 @@ export class WorkflowGenerator { */ private generateSelector = async (page: Page, coordinates: Coordinates, action: ActionType) => { const elementInfo = await getElementInformation(page, coordinates); - const generalSelector = await getNonUniqueSelectors(page, coordinates) - - console.log(`Get List value while generating selector`, this.getList); const selectorBasedOnCustomAction = (this.getList === true) ? await getNonUniqueSelectors(page, coordinates) From 7813c9849b0a44154bdb9dc76adfe196beb550e0 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Mon, 9 Sep 2024 05:17:37 +0530 Subject: [PATCH 003/108] feat: res.send message --- server/src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/server.ts b/server/src/server.ts index 6b34de08..4d4550ca 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -31,7 +31,7 @@ app.use('/workflow', workflow); app.use('/storage', storage); app.get('/', function (req, res) { - return res.send('Welcome to the BR recorder server :-)'); + return res.send('Maxun server started 🚀'); }); server.listen(SERVER_PORT, () => logger.log('info',`Server listening on port ${SERVER_PORT}`)); From 62c02860a84ae65f226b80bfa1cb120c4ee61b23 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Mon, 9 Sep 2024 05:19:15 +0530 Subject: [PATCH 004/108] feat: separate list & text capture --- src/components/atoms/canvas.tsx | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/components/atoms/canvas.tsx b/src/components/atoms/canvas.tsx index 3e9d89c0..654c782a 100644 --- a/src/components/atoms/canvas.tsx +++ b/src/components/atoms/canvas.tsx @@ -53,10 +53,11 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { switch (event.type) { case 'mousedown': const clickCoordinates = getMappedCoordinates(event, canvasRef.current, width, height); - if (getTextRef.current === true || getListRef.current === true) { - // todo: remove console.log and return - console.log('get text or get list is true'); - } else { + if (getTextRef.current === true) { + console.log('Capturing Text...'); + } else if (getListRef.current === true){ + console.log('Capturing List...'); + }else { socket.emit('input:mousedown', clickCoordinates); } notifyLastAction('click'); From 9af82721c0b58b3220935049b04ced4beb1e9c34 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Mon, 9 Sep 2024 08:00:48 +0530 Subject: [PATCH 005/108] feat: group runs of same recording --- src/components/molecules/RunsTable.tsx | 95 +++++++++++++++----------- 1 file changed, 55 insertions(+), 40 deletions(-) diff --git a/src/components/molecules/RunsTable.tsx b/src/components/molecules/RunsTable.tsx index d6489cff..578f8140 100644 --- a/src/components/molecules/RunsTable.tsx +++ b/src/components/molecules/RunsTable.tsx @@ -12,6 +12,8 @@ import { useGlobalInfoStore } from "../../context/globalInfo"; import { getStoredRuns } from "../../api/storage"; import { RunSettings } from "./RunSettings"; import { CollapsibleRow } from "./ColapsibleRow"; +import { Accordion, AccordionSummary, AccordionDetails, Typography } from '@mui/material'; +import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; interface Column { id: 'status' | 'name' | 'startedAt' | 'finishedAt' | 'duration' | 'task' | 'runId' | 'delete'; @@ -86,68 +88,81 @@ export const RunsTable = ( } else { console.log('No runs found.'); } - } + }; - useEffect( () => { + useEffect(() => { if (rows.length === 0 || rerenderRuns) { fetchRuns(); setRerenderRuns(false); } - }, [rerenderRuns]); - const handleDelete = () => { setRows([]); notify('success', 'Run deleted successfully'); fetchRuns(); - } + }; + + // Group runs by recording name + const groupedRows = rows.reduce((acc, row) => { + if (!acc[row.name]) { + acc[row.name] = []; + } + acc[row.name].push(row); + return acc; + }, {} as Record); return ( - - - - - {columns.map((column) => ( - - {column.label} - - ))} - - - - {rows.length !== 0 ? rows - .slice(page * rowsPerPage, page * rowsPerPage + rowsPerPage) - .map((row, index) => - - ) - : null } - -
+ {Object.entries(groupedRows).map(([name, group]) => ( + + }> + {name} + + + + + + + {columns.map((column) => ( + + {column.label} + + ))} + + + + {group.slice(page * rowsPerPage, page * rowsPerPage + rowsPerPage).map((row) => ( + + ))} + +
+ + + ))} - + ); -} +}; From 15b8590f537a98cc15b840ccd6034daae5d48920 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Mon, 9 Sep 2024 11:24:23 +0530 Subject: [PATCH 006/108] chore(deps): install react-router-dom --- package.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 3bd306a2..da1acd0b 100644 --- a/package.json +++ b/package.json @@ -29,13 +29,14 @@ "joi": "^17.6.0", "loglevel": "^1.8.0", "loglevel-plugin-remote": "^0.6.8", - "playwright": "^1.18.1", + "playwright": "^1.20.1", "playwright-extra": "^4.3.6", "prismjs": "^1.28.0", "puppeteer-extra-plugin-stealth": "^2.11.2", "react": "^18.0.0", "react-dom": "^18.0.0", "react-highlight": "^0.14.0", + "react-router-dom": "^6.26.1", "react-scripts": "5.0.1", "react-simple-code-editor": "^0.11.2", "react-transition-group": "^4.4.2", @@ -84,6 +85,7 @@ "@types/react-highlight": "^0.12.5", "@types/react-transition-group": "^4.4.4", "@types/styled-components": "^5.1.23", + "ajv": "^8.8.2", "concurrently": "^7.0.0", "nodemon": "^2.0.15", "react-app-rewired": "^2.2.1", From 67d52cad0dd266833856138d2386479decb4e4df Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Mon, 9 Sep 2024 11:38:31 +0530 Subject: [PATCH 007/108] feat: wrap app in browser router --- src/App.tsx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/App.tsx b/src/App.tsx index 7f0715a9..4a791830 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,14 +1,16 @@ import React from 'react'; - +import { BrowserRouter } from 'react-router-dom'; import { GlobalInfoProvider } from "./context/globalInfo"; import { PageWrapper } from "./pages/PageWrappper"; function App() { return ( - - - + + + + + ); } From f616a9038daf4ff93ac3f06348112036edb65749 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Mon, 9 Sep 2024 12:16:51 +0530 Subject: [PATCH 008/108] feat: wrap app in browser router --- src/App.tsx | 10 +++------- src/index.tsx | 5 ++++- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/App.tsx b/src/App.tsx index 4a791830..0c060a28 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,16 +1,12 @@ import React from 'react'; -import { BrowserRouter } from 'react-router-dom'; import { GlobalInfoProvider } from "./context/globalInfo"; import { PageWrapper } from "./pages/PageWrappper"; function App() { - return ( - - - - - + + + ); } diff --git a/src/index.tsx b/src/index.tsx index b7688442..8c14f60a 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -1,6 +1,7 @@ import React from 'react'; import ReactDOM from 'react-dom/client'; import './index.css'; +import { BrowserRouter } from 'react-router-dom'; import App from './App'; const root = ReactDOM.createRoot( @@ -8,7 +9,9 @@ const root = ReactDOM.createRoot( ); root.render( - + + + ); From a9b625051fbb30391998f544d16a3e7a7003ebf4 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 02:30:02 +0530 Subject: [PATCH 009/108] feat: route for page wrapper --- src/App.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/App.tsx b/src/App.tsx index 0c060a28..c6458fdc 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,11 +1,14 @@ import React from 'react'; +import { Routes, Route } from 'react-router-dom'; import { GlobalInfoProvider } from "./context/globalInfo"; import { PageWrapper } from "./pages/PageWrappper"; function App() { return ( - + + } /> + ); } From 790693d8470b73c9a5149befb94f3fc1b4308d25 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 02:33:56 +0530 Subject: [PATCH 010/108] feat: move recordingName to global context --- src/context/globalInfo.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/context/globalInfo.tsx b/src/context/globalInfo.tsx index e7eb2ed2..e0efb678 100644 --- a/src/context/globalInfo.tsx +++ b/src/context/globalInfo.tsx @@ -16,6 +16,8 @@ interface GlobalInfo { setRerenderRuns: (rerenderRuns: boolean) => void; recordingLength: number; setRecordingLength: (recordingLength: number) => void; + recordingName: string; + setRecordingName: (recordingName: string) => void; }; class GlobalInfoStore implements Partial { @@ -29,6 +31,7 @@ class GlobalInfoStore implements Partial { }; recordings: string[] = []; rerenderRuns = false; + recordingName = ''; }; const globalInfoStore = new GlobalInfoStore(); @@ -43,6 +46,7 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => { const [recordings, setRecordings] = useState(globalInfoStore.recordings); const [rerenderRuns, setRerenderRuns] = useState(globalInfoStore.rerenderRuns); const [recordingLength, setRecordingLength] = useState(globalInfoStore.recordingLength); + const [recordingName, setRecordingName] = useState(globalInfoStore.recordingName); const notify = (severity: 'error' | 'warning' | 'info' | 'success', message: string) => { setNotification({ severity, message, isOpen: true }); @@ -75,6 +79,8 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => { setRerenderRuns, recordingLength, setRecordingLength, + recordingName, + setRecordingName }} > {children} From c2e12fc9b5f3e14a76d134d112fee29986046f52 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 02:34:28 +0530 Subject: [PATCH 011/108] feat: routing for recording & main page --- src/pages/PageWrappper.tsx | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/pages/PageWrappper.tsx b/src/pages/PageWrappper.tsx index 2cbf3102..50dcf146 100644 --- a/src/pages/PageWrappper.tsx +++ b/src/pages/PageWrappper.tsx @@ -7,21 +7,27 @@ import { MainPage } from "./MainPage"; import { useGlobalInfoStore } from "../context/globalInfo"; import { getActiveBrowserId } from "../api/recording"; import { AlertSnackbar } from "../components/atoms/AlertSnackbar"; +import { Routes, Route, useNavigate } from 'react-router-dom'; export const PageWrapper = () => { - const [recordingName, setRecordingName] = useState(''); const [open, setOpen] = useState(false); - const { browserId, setBrowserId, notification } = useGlobalInfoStore(); + const navigate = useNavigate(); + + const { browserId, setBrowserId, notification, recordingName, setRecordingName } = useGlobalInfoStore(); const handleNewRecording = () => { setBrowserId('new-recording'); setRecordingName(''); + navigate('/recording'); + } const handleEditRecording = (fileName: string) => { setRecordingName(fileName); setBrowserId('new-recording'); + navigate('/recording'); + } const isNotification = (): boolean => { @@ -36,6 +42,7 @@ export const PageWrapper = () => { const id = await getActiveBrowserId(); if (id) { setBrowserId(id); + navigate('/recording'); } } isRecordingInProgress(); @@ -46,18 +53,20 @@ export const PageWrapper = () => { - {browserId - ? ( - - - - - - ) - : - } + + } + /> + + + + } + /> + {isNotification() ? From 83ea37680e3fef2a542c10770b4e6b1b68b23a7d Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 02:34:42 +0530 Subject: [PATCH 012/108] chore: lint --- src/pages/PageWrappper.tsx | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/pages/PageWrappper.tsx b/src/pages/PageWrappper.tsx index 50dcf146..cf43defb 100644 --- a/src/pages/PageWrappper.tsx +++ b/src/pages/PageWrappper.tsx @@ -54,19 +54,19 @@ export const PageWrapper = () => { - } - /> - - - - } - /> - + } + /> + + + + } + /> + {isNotification() ? From e4b4148594eb6517bf6512ae735dd3dea9404c29 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 02:43:44 +0530 Subject: [PATCH 013/108] fix: format --- src/components/molecules/SaveRecording.tsx | 54 +++++++++++----------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/components/molecules/SaveRecording.tsx b/src/components/molecules/SaveRecording.tsx index b4e175bb..b7d0c125 100644 --- a/src/components/molecules/SaveRecording.tsx +++ b/src/components/molecules/SaveRecording.tsx @@ -13,14 +13,14 @@ interface SaveRecordingProps { fileName: string; } -export const SaveRecording = ({fileName}: SaveRecordingProps) => { +export const SaveRecording = ({ fileName }: SaveRecordingProps) => { const [openModal, setOpenModal] = useState(false); const [needConfirm, setNeedConfirm] = useState(false); const [recordingName, setRecordingName] = useState(fileName); const [waitingForSave, setWaitingForSave] = useState(false); - const { browserId, setBrowserId, notify, recordings } = useGlobalInfoStore(); + const { browserId, setBrowserId, notify, recordings } = useGlobalInfoStore(); const { socket } = useSocketStore(); const handleChangeOfTitle = (event: React.ChangeEvent) => { @@ -41,7 +41,7 @@ export const SaveRecording = ({fileName}: SaveRecordingProps) => { } }; - const exitRecording = useCallback(async() => { + const exitRecording = useCallback(async () => { notify('success', 'Recording saved successfully'); if (browserId) { await stopRecording(browserId); @@ -66,45 +66,45 @@ export const SaveRecording = ({fileName}: SaveRecordingProps) => { return (
setOpenModal(false)} modalStyle={modalStyle}> -
+ Save the recording as: - { needConfirm - ? - ( - - - - Recording already exists, please confirm the recording's overwrite. - - ) - : - } - { waitingForSave && - - - - - + {needConfirm + ? + ( + + + + Recording already exists, please confirm the recording's overwrite. + + ) + : + } + {waitingForSave && + + + + + }
@@ -119,7 +119,7 @@ const modalStyle = { width: '20%', backgroundColor: 'background.paper', p: 4, - height:'fit-content', - display:'block', + height: 'fit-content', + display: 'block', padding: '20px', }; From 86837b1986ca24035437a7d38e047b46bdab6e65 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 02:46:54 +0530 Subject: [PATCH 014/108] feat: redirect to home page after saving --- src/components/molecules/SaveRecording.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/components/molecules/SaveRecording.tsx b/src/components/molecules/SaveRecording.tsx index b7d0c125..8088622f 100644 --- a/src/components/molecules/SaveRecording.tsx +++ b/src/components/molecules/SaveRecording.tsx @@ -8,6 +8,7 @@ import { TextField, Typography } from "@mui/material"; import { WarningText } from "../atoms/texts"; import NotificationImportantIcon from "@mui/icons-material/NotificationImportant"; import FlagIcon from '@mui/icons-material/Flag'; +import { useNavigate } from 'react-router-dom'; // Import useNavigate interface SaveRecordingProps { fileName: string; @@ -22,6 +23,7 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => { const { browserId, setBrowserId, notify, recordings } = useGlobalInfoStore(); const { socket } = useSocketStore(); + const navigate = useNavigate(); const handleChangeOfTitle = (event: React.ChangeEvent) => { const { value } = event.target; @@ -47,6 +49,7 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => { await stopRecording(browserId); } setBrowserId(null); + navigate('/'); }, [setBrowserId, browserId, notify]); // notifies backed to save the recording in progress, From 8bc023a5954ccc69331a953172770ca212d183cf Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 02:47:23 +0530 Subject: [PATCH 015/108] chore: -rm comment --- src/components/molecules/SaveRecording.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/molecules/SaveRecording.tsx b/src/components/molecules/SaveRecording.tsx index 8088622f..05ff43af 100644 --- a/src/components/molecules/SaveRecording.tsx +++ b/src/components/molecules/SaveRecording.tsx @@ -8,7 +8,7 @@ import { TextField, Typography } from "@mui/material"; import { WarningText } from "../atoms/texts"; import NotificationImportantIcon from "@mui/icons-material/NotificationImportant"; import FlagIcon from '@mui/icons-material/Flag'; -import { useNavigate } from 'react-router-dom'; // Import useNavigate +import { useNavigate } from 'react-router-dom'; interface SaveRecordingProps { fileName: string; From dce520b0e5b8f1f8441d78f49801f9f951191090 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 02:49:30 +0530 Subject: [PATCH 016/108] fix: format --- src/components/molecules/NavBar.tsx | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/components/molecules/NavBar.tsx b/src/components/molecules/NavBar.tsx index 640f09cb..af357af3 100644 --- a/src/components/molecules/NavBar.tsx +++ b/src/components/molecules/NavBar.tsx @@ -14,13 +14,13 @@ interface NavBarProps { isRecording: boolean; } -export const NavBar = ({newRecording, recordingName, isRecording}:NavBarProps) => { +export const NavBar = ({ newRecording, recordingName, isRecording }: NavBarProps) => { const { notify, browserId, setBrowserId, recordingLength } = useGlobalInfoStore(); // If recording is in progress, the resources and change page view by setting browserId to null // else it won't affect the page - const goToMainMenu = async() => { + const goToMainMenu = async () => { if (browserId) { await stopRecording(browserId); notify('warning', 'Current Recording was terminated'); @@ -43,8 +43,8 @@ export const NavBar = ({newRecording, recordingName, isRecording}:NavBarProps) = display: 'flex', justifyContent: 'flex-start', }}> - -
Maxun
+ +
Maxun
- {isRecording ? 'NEW' : 'RECORD'} + {isRecording ? 'NEW' : 'RECORD'} { recordingLength > 0 - ? - :null + ? + : null } - { isRecording ? - : null } + : null}
From 8353ca7817b628e2381b2d5a783ecc2912fb6c72 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 02:52:07 +0530 Subject: [PATCH 017/108] feat: redirect to home page on exit --- src/components/molecules/NavBar.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/components/molecules/NavBar.tsx b/src/components/molecules/NavBar.tsx index af357af3..15508942 100644 --- a/src/components/molecules/NavBar.tsx +++ b/src/components/molecules/NavBar.tsx @@ -7,6 +7,7 @@ import { RecordingIcon } from "../atoms/RecorderIcon"; import { SaveRecording } from "./SaveRecording"; import { Circle } from "@mui/icons-material"; import MeetingRoomIcon from '@mui/icons-material/MeetingRoom'; +import { useNavigate } from "react-router-dom"; interface NavBarProps { newRecording: () => void; @@ -17,6 +18,7 @@ interface NavBarProps { export const NavBar = ({ newRecording, recordingName, isRecording }: NavBarProps) => { const { notify, browserId, setBrowserId, recordingLength } = useGlobalInfoStore(); + const navigate = useNavigate(); // If recording is in progress, the resources and change page view by setting browserId to null // else it won't affect the page @@ -26,6 +28,7 @@ export const NavBar = ({ newRecording, recordingName, isRecording }: NavBarProps notify('warning', 'Current Recording was terminated'); setBrowserId(null); } + navigate('/'); }; const handleNewRecording = async () => { From 4a455c7768d92449716d347e66f6ed7e80eb72c0 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 03:09:34 +0530 Subject: [PATCH 018/108] chore: lint --- src/components/molecules/RecordingsTable.tsx | 36 ++++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index ee510761..997c4023 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -15,7 +15,7 @@ import { useGlobalInfoStore } from "../../context/globalInfo"; import { deleteRecordingFromStorage, getStoredRecordings } from "../../api/storage"; interface Column { - id: 'interpret' | 'name' | 'create_date' | 'edit' | 'pairs' | 'update_date'| 'delete'; + id: 'interpret' | 'name' | 'create_date' | 'edit' | 'pairs' | 'update_date' | 'delete'; label: string; minWidth?: number; align?: 'right'; @@ -65,8 +65,8 @@ interface Data { } interface RecordingsTableProps { - handleEditRecording: (fileName:string) => void; - handleRunRecording: (fileName:string, params: string[]) => void; + handleEditRecording: (fileName: string) => void; + handleRunRecording: (fileName: string, params: string[]) => void; } export const RecordingsTable = ({ handleEditRecording, handleRunRecording }: RecordingsTableProps) => { @@ -106,7 +106,7 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording }: Rec } } - useEffect( () => { + useEffect(() => { if (rows.length === 0) { fetchRecordings(); } @@ -138,7 +138,7 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording }: Rec {columns.map((column) => { // @ts-ignore - const value : any = row[column.id]; + const value: any = row[column.id]; if (value !== undefined) { return ( @@ -150,23 +150,23 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording }: Rec case 'interpret': return ( - handleRunRecording(row.name, row.params || [])}/> + handleRunRecording(row.name, row.params || [])} /> ); case 'edit': return ( - { + { handleEditRecording(row.name); - }} sx={{'&:hover': { color: '#1976d2', backgroundColor: 'transparent' }}}> - + }} sx={{ '&:hover': { color: '#1976d2', backgroundColor: 'transparent' } }}> + ); case 'delete': return ( - { + { deleteRecordingFromStorage(row.name).then((result: boolean) => { if (result) { setRows([]); @@ -174,20 +174,20 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording }: Rec fetchRecordings(); } }) - }} sx={{'&:hover': { color: '#1976d2', backgroundColor: 'transparent' }}}> - + }} sx={{ '&:hover': { color: '#1976d2', backgroundColor: 'transparent' } }}> + ); default: - return null; + return null; } } })} ); }) - : null } + : null} @@ -208,13 +208,13 @@ interface InterpretButtonProps { handleInterpret: () => void; } -const InterpretButton = ( {handleInterpret}:InterpretButtonProps) => { +const InterpretButton = ({ handleInterpret }: InterpretButtonProps) => { return ( - { + { handleInterpret(); }} - sx={{'&:hover': { color: '#1976d2', backgroundColor: 'transparent' }}}> - + sx={{ '&:hover': { color: '#1976d2', backgroundColor: 'transparent' } }}> + ) } From 6c11825ffcdefcfbb7c3ec35d6a8ef79b6cd7993 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 03:11:49 +0530 Subject: [PATCH 019/108] fix: format --- src/components/organisms/Recordings.tsx | 26 ++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/components/organisms/Recordings.tsx b/src/components/organisms/Recordings.tsx index ae993956..b27c8962 100644 --- a/src/components/organisms/Recordings.tsx +++ b/src/components/organisms/Recordings.tsx @@ -33,20 +33,20 @@ export const Recordings = ({ handleEditRecording, handleRunRecording, setFileNam return ( - handleRunRecording(settings) } - isTask={params.length !== 0} - params={params} - /> - - - + handleRunRecording(settings)} + isTask={params.length !== 0} + params={params} + /> + + + + - ); } From a5f26c7e31270e59fc98ae49fba42c5b776e5613 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 03:15:56 +0530 Subject: [PATCH 020/108] feat: add schedule to column array --- src/components/molecules/RecordingsTable.tsx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index 997c4023..6bd3e7ef 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -15,7 +15,7 @@ import { useGlobalInfoStore } from "../../context/globalInfo"; import { deleteRecordingFromStorage, getStoredRecordings } from "../../api/storage"; interface Column { - id: 'interpret' | 'name' | 'create_date' | 'edit' | 'pairs' | 'update_date' | 'delete'; + id: 'interpret' | 'name' | 'create_date' | 'edit' | 'pairs' | 'update_date' | 'delete' | 'schedule'; label: string; minWidth?: number; align?: 'right'; @@ -41,6 +41,11 @@ const columns: readonly Column[] = [ label: 'Pairs', minWidth: 80, }, + { + id: 'schedule', + label: 'Schedule', + minWidth: 80, + }, { id: 'update_date', label: 'Updated at', From 0ba0c350346cd45bcc0f90f0454152fde6803f2e Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 03:17:11 +0530 Subject: [PATCH 021/108] feat: schedule button --- src/components/molecules/RecordingsTable.tsx | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index 6bd3e7ef..31f77baf 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -223,3 +223,19 @@ const InterpretButton = ({ handleInterpret }: InterpretButtonProps) => { ) } + +interface ScheduleButtonProps { + handleSchedule: () => void; +} + +const ScheduleButton = ({ handleSchedule }: ScheduleButtonProps) => { + return ( + + + + ); +}; From 2071df4b13a823d1e625d306c8633c95e37cd754 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 03:17:47 +0530 Subject: [PATCH 022/108] feat: schedule case --- src/components/molecules/RecordingsTable.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index 31f77baf..d60a2ee6 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -168,6 +168,12 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording }: Rec ); + case 'schedule': + return ( + + {/* todo: schedule logic here */}} /> + + ); case 'delete': return ( From fc56a395b8ddbb832ec501de3c0e7762b220bf36 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 03:18:04 +0530 Subject: [PATCH 023/108] chore: lint --- src/components/molecules/RecordingsTable.tsx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index d60a2ee6..b5842fc9 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -42,7 +42,7 @@ const columns: readonly Column[] = [ minWidth: 80, }, { - id: 'schedule', + id: 'schedule', label: 'Schedule', minWidth: 80, }, @@ -168,12 +168,12 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording }: Rec ); - case 'schedule': - return ( - - {/* todo: schedule logic here */}} /> - - ); + case 'schedule': + return ( + + {/* todo: schedule logic here */ }} /> + + ); case 'delete': return ( From 5107bc2b2b5ab2496c99a89a57f91cb9077018c8 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 06:41:22 +0530 Subject: [PATCH 024/108] feat(wip): scheduler --- src/components/molecules/RecordingsTable.tsx | 21 +++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index b5842fc9..c1a3c025 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -72,9 +72,10 @@ interface Data { interface RecordingsTableProps { handleEditRecording: (fileName: string) => void; handleRunRecording: (fileName: string, params: string[]) => void; + handleScheduleRecording: (fileName: string, params: string[]) => void; } -export const RecordingsTable = ({ handleEditRecording, handleRunRecording }: RecordingsTableProps) => { +export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handleScheduleRecording }: RecordingsTableProps) => { const [page, setPage] = React.useState(0); const [rowsPerPage, setRowsPerPage] = React.useState(10); const [rows, setRows] = React.useState([]); @@ -171,7 +172,7 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording }: Rec case 'schedule': return ( - {/* todo: schedule logic here */ }} /> + handleScheduleRecording(row.name, row.params || []) } /> ); case 'delete': @@ -230,18 +231,20 @@ const InterpretButton = ({ handleInterpret }: InterpretButtonProps) => { ) } + interface ScheduleButtonProps { handleSchedule: () => void; } const ScheduleButton = ({ handleSchedule }: ScheduleButtonProps) => { return ( - { + handleSchedule(); + }} sx={{ '&:hover': { color: '#1976d2', backgroundColor: 'transparent' } }}> - + - ); -}; + ) +} + + From 327bd967a16aff3b51219815ca2b76ab82daf93b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 06:43:04 +0530 Subject: [PATCH 025/108] feat: use schedule icon from mui --- src/components/molecules/RecordingsTable.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index c1a3c025..c7d5e8dc 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -10,7 +10,7 @@ import TableRow from '@mui/material/TableRow'; import { useEffect } from "react"; import { WorkflowFile } from "maxun-core"; import { IconButton } from "@mui/material"; -import { Assignment, DeleteForever, Edit, PlayCircle } from "@mui/icons-material"; +import { Schedule, DeleteForever, Edit, PlayCircle } from "@mui/icons-material"; import { useGlobalInfoStore } from "../../context/globalInfo"; import { deleteRecordingFromStorage, getStoredRecordings } from "../../api/storage"; @@ -242,7 +242,7 @@ const ScheduleButton = ({ handleSchedule }: ScheduleButtonProps) => { handleSchedule(); }} sx={{ '&:hover': { color: '#1976d2', backgroundColor: 'transparent' } }}> - + ) } From 0ed8ef5f860bef7c471e77933b6266b4ef95a201 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 06:43:29 +0530 Subject: [PATCH 026/108] chore: lint --- src/components/molecules/RecordingsTable.tsx | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/components/molecules/RecordingsTable.tsx b/src/components/molecules/RecordingsTable.tsx index c7d5e8dc..429b8234 100644 --- a/src/components/molecules/RecordingsTable.tsx +++ b/src/components/molecules/RecordingsTable.tsx @@ -172,7 +172,7 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl case 'schedule': return ( - handleScheduleRecording(row.name, row.params || []) } /> + handleScheduleRecording(row.name, row.params || [])} /> ); case 'delete': @@ -245,6 +245,4 @@ const ScheduleButton = ({ handleSchedule }: ScheduleButtonProps) => { ) -} - - +} \ No newline at end of file From e722c6308fed0f47725eb036bfbb8af9f83a40de Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 12:07:44 +0530 Subject: [PATCH 027/108] feat(wip): pass handleScheduleRecording as prop (temp) --- src/pages/MainPage.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pages/MainPage.tsx b/src/pages/MainPage.tsx index d358b1d3..2dacd513 100644 --- a/src/pages/MainPage.tsx +++ b/src/pages/MainPage.tsx @@ -102,6 +102,7 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => { handleEditRecording={handleEditRecording} handleRunRecording={handleRunRecording} setFileName={setFileName} + handleScheduleRecording={() => {}} />; case 'runs': return Date: Tue, 10 Sep 2024 12:09:53 +0530 Subject: [PATCH 028/108] feat(wip): handleScheduleRecording --- src/components/organisms/Recordings.tsx | 31 +++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/components/organisms/Recordings.tsx b/src/components/organisms/Recordings.tsx index b27c8962..00dfd6bc 100644 --- a/src/components/organisms/Recordings.tsx +++ b/src/components/organisms/Recordings.tsx @@ -2,16 +2,18 @@ import React, { useState } from 'react'; import { RecordingsTable } from "../molecules/RecordingsTable"; import { Grid } from "@mui/material"; import { RunSettings, RunSettingsModal } from "../molecules/RunSettings"; +import { ScheduleSettings, ScheduleSettingsModal } from "../molecules/ScheduleSettings"; interface RecordingsProps { handleEditRecording: (fileName: string) => void; handleRunRecording: (settings: RunSettings) => void; + handleScheduleRecording: (settings: RunSettings) => void; setFileName: (fileName: string) => void; - } -export const Recordings = ({ handleEditRecording, handleRunRecording, setFileName }: RecordingsProps) => { +export const Recordings = ({ handleEditRecording, handleRunRecording, setFileName, handleScheduleRecording }: RecordingsProps) => { const [runSettingsAreOpen, setRunSettingsAreOpen] = useState(false); + const [scheduleSettingsAreOpen, setScheduleSettingsAreOpen] = useState(false); const [params, setParams] = useState([]); const handleSettingsAndRun = (fileName: string, params: string[]) => { @@ -25,12 +27,29 @@ export const Recordings = ({ handleEditRecording, handleRunRecording, setFileNam } } + const handleSettingsAndSchedule = (fileName: string, params: string[]) => { + if (params.length === 0) { + setScheduleSettingsAreOpen(true); + setFileName(fileName); + } else { + setParams(params); + setScheduleSettingsAreOpen(true); + setFileName(fileName); + } + } + const handleClose = () => { setParams([]); setRunSettingsAreOpen(false); setFileName(''); } + const handleScheduleClose = () => { + setParams([]); + setScheduleSettingsAreOpen(false); + setFileName(''); + } + return ( + handleScheduleRecording(settings)} + isTask={params.length !== 0} + params={params} + /> + From 486a49d2e1254e128cd8c5f520d0ec03dcf416fe Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 12:11:52 +0530 Subject: [PATCH 029/108] chore: lint --- src/components/organisms/Recordings.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/organisms/Recordings.tsx b/src/components/organisms/Recordings.tsx index 00dfd6bc..dd0b7f68 100644 --- a/src/components/organisms/Recordings.tsx +++ b/src/components/organisms/Recordings.tsx @@ -58,7 +58,7 @@ export const Recordings = ({ handleEditRecording, handleRunRecording, setFileNam isTask={params.length !== 0} params={params} /> - handleScheduleRecording(settings)} isTask={params.length !== 0} From 70f87cfa65a86a1a545fcfd80ef8afde9b33efb2 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Tue, 10 Sep 2024 12:13:04 +0530 Subject: [PATCH 030/108] feat(wip): schedule settings --- src/components/molecules/ScheduleSettings.tsx | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 src/components/molecules/ScheduleSettings.tsx diff --git a/src/components/molecules/ScheduleSettings.tsx b/src/components/molecules/ScheduleSettings.tsx new file mode 100644 index 00000000..f7a01a9e --- /dev/null +++ b/src/components/molecules/ScheduleSettings.tsx @@ -0,0 +1,114 @@ +import React, { useState } from 'react'; +import { GenericModal } from "../atoms/GenericModal"; +import { MenuItem, TextField, Typography } from "@mui/material"; +import { Dropdown } from "../atoms/DropdownMui"; +import Button from "@mui/material/Button"; +import { modalStyle } from "./AddWhereCondModal"; + +interface ScheduleSettingsProps { + isOpen: boolean; + handleStart: (settings: ScheduleSettings) => void; + handleClose: () => void; + isTask: boolean; + params?: string[]; +} + +export interface ScheduleSettings { + maxConcurrency: number; + maxRepeats: number; + debug: boolean; + params?: any; +} + +export const ScheduleSettingsModal = ({ isOpen, handleStart, handleClose, isTask, params }: ScheduleSettingsProps) => { + + const [settings, setSettings] = React.useState({ + maxConcurrency: 1, + maxRepeats: 1, + debug: true, + }); + + return ( + +
+ { isTask + ? + ( + + Recording parameters: + { params?.map((item, index) => { + return setSettings( + { + ...settings, + params: settings.params + ? { + ...settings.params, + [item]: e.target.value, + } + : { + [item]: e.target.value, + }, + })} + /> + }) } + ) + : null + } + Interpreter settings: + setSettings( + { + ...settings, + maxConcurrency: parseInt(e.target.value), + })} + defaultValue={settings.maxConcurrency} + /> + setSettings( + { + ...settings, + maxRepeats: parseInt(e.target.value), + })} + defaultValue={settings.maxRepeats} + /> + setSettings( + { + ...settings, + debug: e.target.value === "true", + })} + > + true + false + + +
+ + ); +} From fd5a86f75c9113e101d0e39c496fd097f0b04247 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 11:25:15 +0530 Subject: [PATCH 031/108] chore(deps): install bullmq & io-reids --- package.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/package.json b/package.json index da1acd0b..1cf99bdd 100644 --- a/package.json +++ b/package.json @@ -21,11 +21,13 @@ "@wbr-project/wbr-interpret": "^0.9.3-marketa.1", "axios": "^0.26.0", "buffer": "^6.0.3", + "bullmq": "^5.12.15", "cors": "^2.8.5", "cross-fetch": "^4.0.0", "dotenv": "^16.0.0", "express": "^4.17.2", "fortawesome": "^0.0.1-security", + "ioredis": "^5.4.1", "joi": "^17.6.0", "loglevel": "^1.8.0", "loglevel-plugin-remote": "^0.6.8", From 27c3ed241d289a38cbffb0430c8ced941c4944d8 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 11:30:39 +0530 Subject: [PATCH 032/108] feat: setup bullmq for workflow --- .../workflow-management/scheduler/index.ts | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 server/src/workflow-management/scheduler/index.ts diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts new file mode 100644 index 00000000..587d9fd4 --- /dev/null +++ b/server/src/workflow-management/scheduler/index.ts @@ -0,0 +1,27 @@ +import { Queue, Worker } from 'bullmq'; +import IORedis from 'ioredis'; + +const connection = new IORedis(); + +const workflowQueue = new Queue('workflow', { connection }); + +const worker = new Worker('workflow', async job => { + const { fileName, runId } = job.data; + try { + const result = await runWorkflow(fileName, runId); + return result; + } catch (error) { + console.error('Error running workflow:', error); + throw error; + } +}, { connection }); + +worker.on('completed', (job: any) => { + console.log(`Job ${job.id} completed for ${job.data.fileName}_${job.data.runId}`); +}); + +worker.on('failed', (job: any, err) => { + console.error(`Job ${job.id} failed for ${job.data.fileName}_${job.data.runId}:`, err); +}); + +export { workflowQueue }; \ No newline at end of file From 890f225912ead569fec12e1364af38f71155c8da Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 11:53:12 +0530 Subject: [PATCH 033/108] feat: run workflow --- .../workflow-management/scheduler/index.ts | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 587d9fd4..5d9584ae 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -1,5 +1,10 @@ import { Queue, Worker } from 'bullmq'; import IORedis from 'ioredis'; +import { deleteFile, readFile, readFiles, saveFile } from "../storage"; +import { createRemoteBrowserForRun, destroyRemoteBrowser } from '../../browser-management/controller'; +import logger from '../../logger'; +import { browserPool } from "../../server"; +import fs from "fs"; const connection = new IORedis(); @@ -24,4 +29,58 @@ worker.on('failed', (job: any, err) => { console.error(`Job ${job.id} failed for ${job.data.fileName}_${job.data.runId}:`, err); }); -export { workflowQueue }; \ No newline at end of file +export { workflowQueue }; + +async function runWorkflow(fileName, runId) { + try { + // read the recording from storage + const recording = await readFile(`./../storage/recordings/${fileName}.waw.json`); + const parsedRecording = JSON.parse(recording); + // read the run from storage + const run = await readFile(`./../storage/runs/${fileName}_${runId}.json`); + const parsedRun = JSON.parse(run); + + // interpret the run in active browser + const browser = browserPool.getRemoteBrowser(parsedRun.browserId); + const currentPage = browser?.getCurrentPage(); + if (browser && currentPage) { + const interpretationInfo = await browser.interpreter.InterpretRecording( + parsedRecording.recording, currentPage, parsedRun.interpreterSettings); + const duration = Math.round((new Date().getTime() - new Date(parsedRun.startedAt).getTime()) / 1000); + const durString = (() => { + if (duration < 60) { + return `${duration} s`; + } + else { + const minAndS = (duration / 60).toString().split('.'); + return `${minAndS[0]} m ${minAndS[1]} s`; + } + })(); + await destroyRemoteBrowser(parsedRun.browserId); + const run_meta = { + ...parsedRun, + status: interpretationInfo.result, + finishedAt: new Date().toLocaleString(), + duration: durString, + browserId: null, + log: interpretationInfo.log.join('\n'), + serializableOutput: interpretationInfo.serializableOutput, + binaryOutput: interpretationInfo.binaryOutput, + }; + fs.mkdirSync('../storage/runs', { recursive: true }); + await saveFile( + `../storage/runs/${parsedRun.name}_${runId}.json`, + JSON.stringify(run_meta, null, 2) + ); + return true; + } else { + throw new Error('Could not destroy browser'); + } + } catch (e) { + const { message } = e as Error; + logger.log('info', `Error while running a recording with name: ${fileName}_${runId}.json`); + return false; + } +} + +export { runWorkflow }; \ No newline at end of file From 17fd593170bc01fbb45637fec8cae9971a2a564a Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 11:53:41 +0530 Subject: [PATCH 034/108] feat: combine exports --- server/src/workflow-management/scheduler/index.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 5d9584ae..f3505375 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -29,7 +29,6 @@ worker.on('failed', (job: any, err) => { console.error(`Job ${job.id} failed for ${job.data.fileName}_${job.data.runId}:`, err); }); -export { workflowQueue }; async function runWorkflow(fileName, runId) { try { @@ -83,4 +82,4 @@ async function runWorkflow(fileName, runId) { } } -export { runWorkflow }; \ No newline at end of file +export { workflowQueue, runWorkflow }; \ No newline at end of file From d36374656ffd1a45c8140a3f1381d6d685c58858 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 11:53:57 +0530 Subject: [PATCH 035/108] chore: lint --- server/src/workflow-management/scheduler/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index f3505375..5bddefc3 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -29,7 +29,6 @@ worker.on('failed', (job: any, err) => { console.error(`Job ${job.id} failed for ${job.data.fileName}_${job.data.runId}:`, err); }); - async function runWorkflow(fileName, runId) { try { // read the recording from storage From 9cbd218d00eb9618b1472e6b658cf8f75fcbe5ac Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 12:36:41 +0530 Subject: [PATCH 036/108] chore: todo --- server/src/workflow-management/scheduler/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 5bddefc3..68a5a1ca 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -6,6 +6,7 @@ import logger from '../../logger'; import { browserPool } from "../../server"; import fs from "fs"; +// todo: specify connection config const connection = new IORedis(); const workflowQueue = new Queue('workflow', { connection }); From 2f681deb3fa04ecf79fd85e6589b41c1f77ec01c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 12:53:28 +0530 Subject: [PATCH 037/108] feat: post endpoint to schedule workflow --- server/src/routes/storage.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index beed25bb..3ead9b00 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -10,6 +10,7 @@ import { chromium } from "playwright"; import { browserPool } from "../server"; import fs from "fs"; import { uuid } from "uuidv4"; +import { workflowQueue } from '../workflow-management/scheduler'; export const router = Router(); @@ -188,6 +189,27 @@ router.post('/runs/run/:fileName/:runId', async (req, res) => { } }); +router.post('/schedule/:fileName/:runId', async (req, res) => { + try { + const { fileName, runId } = req.params; + const { scheduleTime } = req.body; + + if (!fileName || !runId || !scheduleTime) { + return res.status(400).json({ error: 'Missing required parameters' }); + } + + const delay = new Date(scheduleTime).getTime() - Date.now(); + + // Add job to the queue with delay + await workflowQueue.add('run workflow', { fileName, runId }, { delay: Math.max(0, delay) }); + + res.status(200).json({ message: 'Workflow scheduled successfully' }); + } catch (error) { + console.error('Error scheduling workflow:', error); + res.status(500).json({ error: 'Failed to schedule workflow' }); + } +}); + /** * POST endpoint for aborting a current interpretation of the run. */ From 458ba1f6dfbe6ccae822b9f4baabc73d132b18f4 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 13:16:45 +0530 Subject: [PATCH 038/108] feat: export worker --- server/src/workflow-management/scheduler/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 68a5a1ca..8714e011 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -11,7 +11,7 @@ const connection = new IORedis(); const workflowQueue = new Queue('workflow', { connection }); -const worker = new Worker('workflow', async job => { +export const worker = new Worker('workflow', async job => { const { fileName, runId } = job.data; try { const result = await runWorkflow(fileName, runId); @@ -30,7 +30,7 @@ worker.on('failed', (job: any, err) => { console.error(`Job ${job.id} failed for ${job.data.fileName}_${job.data.runId}:`, err); }); -async function runWorkflow(fileName, runId) { +async function runWorkflow(fileName:any, runId:any) { try { // read the recording from storage const recording = await readFile(`./../storage/recordings/${fileName}.waw.json`); From f97a2ef607d794a10824f99ac2c1bab46ebcb6a1 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 13:20:48 +0530 Subject: [PATCH 039/108] feat: run worker --- server/src/server.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/server.ts b/server/src/server.ts index 4d4550ca..ad2630c0 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -8,6 +8,7 @@ import { BrowserPool } from "./browser-management/classes/BrowserPool"; import logger from './logger' import { SERVER_PORT } from "./constants/config"; import {Server} from "socket.io"; +import { worker } from './workflow-management/scheduler'; const app = express(); app.use(cors()); @@ -34,4 +35,9 @@ app.get('/', function (req, res) { return res.send('Maxun server started 🚀'); }); +/** + * Starts the worker for the workflow queue. + */ +worker.run(); + server.listen(SERVER_PORT, () => logger.log('info',`Server listening on port ${SERVER_PORT}`)); From e29a65bc016eee8f6cfed7875a774a93289765d2 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 13:21:16 +0530 Subject: [PATCH 040/108] chore: lint --- server/src/server.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/server.ts b/server/src/server.ts index ad2630c0..55c3df92 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -7,7 +7,7 @@ import { record, workflow, storage } from './routes'; import { BrowserPool } from "./browser-management/classes/BrowserPool"; import logger from './logger' import { SERVER_PORT } from "./constants/config"; -import {Server} from "socket.io"; +import { Server } from "socket.io"; import { worker } from './workflow-management/scheduler'; const app = express(); @@ -37,7 +37,7 @@ app.get('/', function (req, res) { /** * Starts the worker for the workflow queue. - */ +*/ worker.run(); -server.listen(SERVER_PORT, () => logger.log('info',`Server listening on port ${SERVER_PORT}`)); +server.listen(SERVER_PORT, () => logger.log('info', `Server listening on port ${SERVER_PORT}`)); From cb6f3e2a293d1e96a15c9ef4eb115078d75c23d3 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 13:45:53 +0530 Subject: [PATCH 041/108] fix: duplicate worker run --- server/src/server.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/server/src/server.ts b/server/src/server.ts index 55c3df92..a81a216c 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -35,9 +35,4 @@ app.get('/', function (req, res) { return res.send('Maxun server started 🚀'); }); -/** - * Starts the worker for the workflow queue. -*/ -worker.run(); - server.listen(SERVER_PORT, () => logger.log('info', `Server listening on port ${SERVER_PORT}`)); From a80cc256fa29ab917106be3f06430141f7e7e059 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 13:46:48 +0530 Subject: [PATCH 042/108] feat: set maxRetriesPerRequest to null --- server/src/workflow-management/scheduler/index.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 8714e011..d8204e80 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -6,8 +6,19 @@ import logger from '../../logger'; import { browserPool } from "../../server"; import fs from "fs"; -// todo: specify connection config -const connection = new IORedis(); +const connection = new IORedis({ + host: 'localhost', + port: 6379, + maxRetriesPerRequest: null, +}); + +connection.on('connect', () => { + console.log('Connected to Redis!'); +}); + +connection.on('error', (err) => { + console.error('Redis connection error:', err); +}); const workflowQueue = new Queue('workflow', { connection }); From 4298be2127678b6de26b53cf8b0decf63c4dec89 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 13:47:03 +0530 Subject: [PATCH 043/108] chore: lint --- server/src/workflow-management/scheduler/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index d8204e80..94c234ab 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -7,8 +7,8 @@ import { browserPool } from "../../server"; import fs from "fs"; const connection = new IORedis({ - host: 'localhost', - port: 6379, + host: 'localhost', + port: 6379, maxRetriesPerRequest: null, }); @@ -41,7 +41,7 @@ worker.on('failed', (job: any, err) => { console.error(`Job ${job.id} failed for ${job.data.fileName}_${job.data.runId}:`, err); }); -async function runWorkflow(fileName:any, runId:any) { +async function runWorkflow(fileName: any, runId: any) { try { // read the recording from storage const recording = await readFile(`./../storage/recordings/${fileName}.waw.json`); From 11ea6c6f67d517e1cd29a8b573c8e463c3ec1cd5 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 14:49:05 +0530 Subject: [PATCH 044/108] feat: run workflow w/o browser interpretation --- .../workflow-management/scheduler/index.ts | 86 +++++++++---------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 94c234ab..77cb91d7 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -5,6 +5,8 @@ import { createRemoteBrowserForRun, destroyRemoteBrowser } from '../../browser-m import logger from '../../logger'; import { browserPool } from "../../server"; import fs from "fs"; +import { uuid } from "uuidv4"; +import { chromium } from "playwright"; const connection = new IORedis({ host: 'localhost', @@ -41,54 +43,52 @@ worker.on('failed', (job: any, err) => { console.error(`Job ${job.id} failed for ${job.data.fileName}_${job.data.runId}:`, err); }); -async function runWorkflow(fileName: any, runId: any) { +async function runWorkflow(fileName: string, runId: string) { try { - // read the recording from storage - const recording = await readFile(`./../storage/recordings/${fileName}.waw.json`); - const parsedRecording = JSON.parse(recording); - // read the run from storage - const run = await readFile(`./../storage/runs/${fileName}_${runId}.json`); - const parsedRun = JSON.parse(run); + // Create a browser for the run + const browserId = createRemoteBrowserForRun({ + browser: chromium, + launchOptions: { headless: true } + }); - // interpret the run in active browser - const browser = browserPool.getRemoteBrowser(parsedRun.browserId); - const currentPage = browser?.getCurrentPage(); - if (browser && currentPage) { - const interpretationInfo = await browser.interpreter.InterpretRecording( - parsedRecording.recording, currentPage, parsedRun.interpreterSettings); - const duration = Math.round((new Date().getTime() - new Date(parsedRun.startedAt).getTime()) / 1000); - const durString = (() => { - if (duration < 60) { - return `${duration} s`; - } - else { - const minAndS = (duration / 60).toString().split('.'); - return `${minAndS[0]} m ${minAndS[1]} s`; - } - })(); - await destroyRemoteBrowser(parsedRun.browserId); - const run_meta = { - ...parsedRun, - status: interpretationInfo.result, - finishedAt: new Date().toLocaleString(), - duration: durString, - browserId: null, - log: interpretationInfo.log.join('\n'), - serializableOutput: interpretationInfo.serializableOutput, - binaryOutput: interpretationInfo.binaryOutput, - }; - fs.mkdirSync('../storage/runs', { recursive: true }); - await saveFile( - `../storage/runs/${parsedRun.name}_${runId}.json`, - JSON.stringify(run_meta, null, 2) - ); - return true; - } else { - throw new Error('Could not destroy browser'); + // Create a unique runId if not provided + if (!runId) { + runId = uuid(); } + + // Set up run metadata + const run_meta = { + status: 'RUNNING', + name: fileName, + startedAt: new Date().toLocaleString(), + finishedAt: '', + duration: '', + task: '', // Optionally set based on workflow + browserId: browserId, + interpreterSettings: {}, // Placeholder for any settings needed + log: '', + runId: runId, + }; + + // Ensure directory exists + fs.mkdirSync('../storage/runs', { recursive: true }); + + // Save the run metadata to a file + await saveFile( + `../storage/runs/${fileName}_${runId}.json`, + JSON.stringify(run_meta, null, 2) + ); + + // Log creation of the run + logger.log('debug', `Created run with name: ${fileName}_${runId}.json`); + + return { + browserId: browserId, + runId: runId, + }; } catch (e) { const { message } = e as Error; - logger.log('info', `Error while running a recording with name: ${fileName}_${runId}.json`); + logger.log('info', `Error while creating a run with name: ${fileName}_${runId}.json`); return false; } } From d2aea597b838c0a8c254b405b3178f323448a057 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 15:12:56 +0530 Subject: [PATCH 045/108] feat: put route for schedule files --- server/src/routes/storage.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 3ead9b00..07db5c19 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -107,6 +107,9 @@ router.put('/runs/:fileName', async (req, res) => { JSON.stringify({ ...run_meta }, null, 2) ); logger.log('debug', `Created run with name: ${req.params.fileName}.json`); + + console.log('Run meta:', run_meta); + return res.send({ browserId: id, runId: runId, @@ -189,9 +192,11 @@ router.post('/runs/run/:fileName/:runId', async (req, res) => { } }); -router.post('/schedule/:fileName/:runId', async (req, res) => { +router.put('/schedule/:fileName/', async (req, res) => { try { - const { fileName, runId } = req.params; + const runId = uuid(); + + const { fileName } = req.params; const { scheduleTime } = req.body; if (!fileName || !runId || !scheduleTime) { From 003fc61777c09aadd77981d33a52831e7b589491 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 16:52:56 +0530 Subject: [PATCH 046/108] fix: log messages --- server/src/workflow-management/scheduler/index.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 77cb91d7..25fbe2a5 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -65,7 +65,7 @@ async function runWorkflow(fileName: string, runId: string) { duration: '', task: '', // Optionally set based on workflow browserId: browserId, - interpreterSettings: {}, // Placeholder for any settings needed + interpreterSettings: { maxConcurrency: 1, maxRepeats: 1, debug: true }, log: '', runId: runId, }; @@ -80,7 +80,7 @@ async function runWorkflow(fileName: string, runId: string) { ); // Log creation of the run - logger.log('debug', `Created run with name: ${fileName}_${runId}.json`); + logger.log('debug', `Scheduled run with name: ${fileName}.json`); return { browserId: browserId, @@ -88,7 +88,8 @@ async function runWorkflow(fileName: string, runId: string) { }; } catch (e) { const { message } = e as Error; - logger.log('info', `Error while creating a run with name: ${fileName}_${runId}.json`); + logger.log('info', `Error while scheduling a run with name: ${fileName}.json`); + console.log(message) return false; } } From 938331a11bc2e5d4fc964619ce9a150ef9ca3b6b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 22:46:45 +0530 Subject: [PATCH 047/108] chore(deps): install moment-timezone & node-cron --- package.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/package.json b/package.json index 1cf99bdd..2fe96bd8 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,8 @@ "joi": "^17.6.0", "loglevel": "^1.8.0", "loglevel-plugin-remote": "^0.6.8", + "moment-timezone": "^0.5.45", + "node-cron": "^3.0.3", "playwright": "^1.20.1", "playwright-extra": "^4.3.6", "prismjs": "^1.28.0", From e762eda985696381e5f77475cbb5bfbae0687db5 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 22:56:55 +0530 Subject: [PATCH 048/108] fix: missing dependencies --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 2fe96bd8..addce564 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,7 @@ "@types/express": "^4.17.13", "@types/loglevel": "^1.6.3", "@types/node": "^17.0.15", + "@types/node-cron": "^3.0.11", "@types/prismjs": "^1.26.0", "@types/react-highlight": "^0.12.5", "@types/react-transition-group": "^4.4.4", From 5b303dd6c62bddb04f1aa6cfe86263267d97952b Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 23:33:04 +0530 Subject: [PATCH 049/108] feat: cron expressions --- server/src/routes/storage.ts | 95 +++++++++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 11 deletions(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 07db5c19..1f90372b 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -11,6 +11,8 @@ import { browserPool } from "../server"; import fs from "fs"; import { uuid } from "uuidv4"; import { workflowQueue } from '../workflow-management/scheduler'; +import moment from 'moment-timezone'; +import cron from 'node-cron'; export const router = Router(); @@ -194,27 +196,98 @@ router.post('/runs/run/:fileName/:runId', async (req, res) => { router.put('/schedule/:fileName/', async (req, res) => { try { - const runId = uuid(); - const { fileName } = req.params; - const { scheduleTime } = req.body; - - if (!fileName || !runId || !scheduleTime) { + const { + frequency, + frequencyUnit, + startDay, + time, + timezone + } = req.body; + + if (!fileName || !frequency || !frequencyUnit || !startDay || !time || !timezone) { return res.status(400).json({ error: 'Missing required parameters' }); } - const delay = new Date(scheduleTime).getTime() - Date.now(); - - // Add job to the queue with delay - await workflowQueue.add('run workflow', { fileName, runId }, { delay: Math.max(0, delay) }); - - res.status(200).json({ message: 'Workflow scheduled successfully' }); + // Validate inputs + if (!['HOURS', 'DAYS', 'WEEKS', 'MONTHS'].includes(frequencyUnit)) { + return res.status(400).json({ error: 'Invalid frequency unit' }); + } + + if (!moment.tz.zone(timezone)) { + return res.status(400).json({ error: 'Invalid timezone' }); + } + + const [hours, minutes] = time.split(':').map(Number); + if (isNaN(hours) || isNaN(minutes) || hours < 0 || hours > 23 || minutes < 0 || minutes > 59) { + return res.status(400).json({ error: 'Invalid time format' }); + } + + const days = ['SUNDAY', 'MONDAY', 'TUESDAY', 'WEDNESDAY', 'THURSDAY', 'FRIDAY', 'SATURDAY']; + if (!days.includes(startDay)) { + return res.status(400).json({ error: 'Invalid start day' }); + } + + // Generate cron expression + let cronExpression; + switch (frequencyUnit) { + case 'HOURS': + cronExpression = `${minutes} */${frequency} * * *`; + break; + case 'DAYS': + cronExpression = `${minutes} ${hours} */${frequency} * *`; + break; + case 'WEEKS': + const dayIndex = days.indexOf(startDay); + cronExpression = `${minutes} ${hours} * * ${dayIndex}/${7 * frequency}`; + break; + case 'MONTHS': + cronExpression = `${minutes} ${hours} 1-7 */${frequency} *`; + if (startDay !== 'SUNDAY') { + const dayIndex = days.indexOf(startDay); + cronExpression += ` ${dayIndex}`; + } + break; + } + + if (!cronExpression || !cron.validate(cronExpression)) { + return res.status(400).json({ error: 'Invalid cron expression generated' }); + } + + const runId = uuid(); + + // Schedule the recurring job + await workflowQueue.add( + 'run workflow', + { fileName, runId }, + { + repeat: { + pattern: cronExpression, + tz: timezone + } + } + ); + + res.status(200).json({ + message: 'Workflow scheduled successfully', + runId, + cronExpression, + // nextRunTime: getNextRunTime(cronExpression, timezone) + }); + } catch (error) { console.error('Error scheduling workflow:', error); res.status(500).json({ error: 'Failed to schedule workflow' }); } }); +// function getNextRunTime(cronExpression, timezone) { +// const schedule = cron.schedule(cronExpression, () => {}, { timezone }); +// const nextDate = schedule.nextDate(); +// schedule.stop(); +// return nextDate.toDate(); +// } + /** * POST endpoint for aborting a current interpretation of the run. */ From 2358b0435bb104701334ee32032c5c8fb538135c Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 23:33:37 +0530 Subject: [PATCH 050/108] chore: -rm comments --- server/src/routes/storage.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 1f90372b..a326ff26 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -209,7 +209,6 @@ router.put('/schedule/:fileName/', async (req, res) => { return res.status(400).json({ error: 'Missing required parameters' }); } - // Validate inputs if (!['HOURS', 'DAYS', 'WEEKS', 'MONTHS'].includes(frequencyUnit)) { return res.status(400).json({ error: 'Invalid frequency unit' }); } @@ -228,7 +227,6 @@ router.put('/schedule/:fileName/', async (req, res) => { return res.status(400).json({ error: 'Invalid start day' }); } - // Generate cron expression let cronExpression; switch (frequencyUnit) { case 'HOURS': @@ -256,7 +254,6 @@ router.put('/schedule/:fileName/', async (req, res) => { const runId = uuid(); - // Schedule the recurring job await workflowQueue.add( 'run workflow', { fileName, runId }, From 0efae27b16f744a31f66011f978f3c9642d0b80f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 23:35:28 +0530 Subject: [PATCH 051/108] chore: -rm comments --- server/src/workflow-management/scheduler/index.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 25fbe2a5..f5e8252c 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -45,18 +45,15 @@ worker.on('failed', (job: any, err) => { async function runWorkflow(fileName: string, runId: string) { try { - // Create a browser for the run const browserId = createRemoteBrowserForRun({ browser: chromium, launchOptions: { headless: true } }); - // Create a unique runId if not provided if (!runId) { runId = uuid(); } - // Set up run metadata const run_meta = { status: 'RUNNING', name: fileName, @@ -70,16 +67,12 @@ async function runWorkflow(fileName: string, runId: string) { runId: runId, }; - // Ensure directory exists fs.mkdirSync('../storage/runs', { recursive: true }); - - // Save the run metadata to a file await saveFile( `../storage/runs/${fileName}_${runId}.json`, JSON.stringify(run_meta, null, 2) ); - // Log creation of the run logger.log('debug', `Scheduled run with name: ${fileName}.json`); return { From 4b846246c61d86a13fa1f3d8256b0177f1ef090f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Wed, 11 Sep 2024 23:35:59 +0530 Subject: [PATCH 052/108] chore: lint --- server/src/workflow-management/scheduler/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index f5e8252c..f5b1c7d6 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -62,7 +62,7 @@ async function runWorkflow(fileName: string, runId: string) { duration: '', task: '', // Optionally set based on workflow browserId: browserId, - interpreterSettings: { maxConcurrency: 1, maxRepeats: 1, debug: true }, + interpreterSettings: { maxConcurrency: 1, maxRepeats: 1, debug: true }, log: '', runId: runId, }; From e30f8d53fcd20f8ffe4a62c900750e3ee4f32bb5 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 00:57:01 +0530 Subject: [PATCH 053/108] feat: execute robot --- .../workflow-management/scheduler/index.ts | 113 ++++++++++++++++-- 1 file changed, 100 insertions(+), 13 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index f5b1c7d6..2272c4c4 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -1,7 +1,7 @@ import { Queue, Worker } from 'bullmq'; import IORedis from 'ioredis'; import { deleteFile, readFile, readFiles, saveFile } from "../storage"; -import { createRemoteBrowserForRun, destroyRemoteBrowser } from '../../browser-management/controller'; +import { createRemoteBrowserForRun, destroyRemoteBrowser, getActiveBrowserId } from '../../browser-management/controller'; import logger from '../../logger'; import { browserPool } from "../../server"; import fs from "fs"; @@ -44,18 +44,21 @@ worker.on('failed', (job: any, err) => { }); async function runWorkflow(fileName: string, runId: string) { + if (!runId) { + runId = uuid(); + } + + // Phase 1: Scheduling try { const browserId = createRemoteBrowserForRun({ browser: chromium, launchOptions: { headless: true } }); + logger.log(`debug`,`Created browser with ID: ${browserId}`); - if (!runId) { - runId = uuid(); - } const run_meta = { - status: 'RUNNING', + status: 'SCHEDULED', name: fileName, startedAt: new Date().toLocaleString(), finishedAt: '', @@ -73,18 +76,102 @@ async function runWorkflow(fileName: string, runId: string) { JSON.stringify(run_meta, null, 2) ); - logger.log('debug', `Scheduled run with name: ${fileName}.json`); + logger.log('debug', `Scheduled run with name: ${fileName}_${runId}.json`); + + logger.log('debug', `Active in run : ${getActiveBrowserId()}`); + + // Phase 2: Running + return await executeRun(fileName, runId); - return { - browserId: browserId, - runId: runId, - }; } catch (e) { const { message } = e as Error; - logger.log('info', `Error while scheduling a run with name: ${fileName}.json`); - console.log(message) - return false; + logger.log('info', `Error while scheduling a run with name: ${fileName}_${runId}.json`); + console.log(message); + return { + success: false, + error: message, + }; } } +async function executeRun(fileName: string, runId: string) { + try { + // Read the recording from storage + const recording = await readFile(`./../storage/recordings/${fileName}.waw.json`); + const parsedRecording = JSON.parse(recording); + + // Read the run from storage + const run = await readFile(`./../storage/runs/${fileName}_${runId}.json`); + const parsedRun = JSON.parse(run); + + // Update status to RUNNING + parsedRun.status = 'RUNNING'; + await saveFile( + `../storage/runs/${fileName}_${runId}.json`, + JSON.stringify(parsedRun, null, 2) + ); + + // Interpret the run in active browser + + logger.log('debug', `Active in exec : ${getActiveBrowserId()}`); + const browser = browserPool.getRemoteBrowser(parsedRun.browserId); + if (!browser) { + throw new Error('Could not access browser'); + } + + const currentPage = await browser.getCurrentPage(); + if (!currentPage) { + throw new Error('Could not create a new page'); + } + + const interpretationInfo = await browser.interpreter.InterpretRecording( + parsedRecording.recording, currentPage, parsedRun.interpreterSettings); + + const duration = Math.round((new Date().getTime() - new Date(parsedRun.startedAt).getTime()) / 1000); + const durString = duration < 60 ? `${duration} s` : `${Math.floor(duration / 60)} m ${duration % 60} s`; + + await destroyRemoteBrowser(parsedRun.browserId); + + const updated_run_meta = { + ...parsedRun, + status: interpretationInfo.result, + finishedAt: new Date().toLocaleString(), + duration: durString, + browserId: null, + log: interpretationInfo.log.join('\n'), + serializableOutput: interpretationInfo.serializableOutput, + binaryOutput: interpretationInfo.binaryOutput, + }; + + await saveFile( + `../storage/runs/${fileName}_${runId}.json`, + JSON.stringify(updated_run_meta, null, 2) + ); + + return { + browserId: parsedRun.browserId, + runId: runId, + success: true, + }; + } catch (error: any) { + logger.log('info', `Error while running a recording with name: ${fileName}_${runId}.json`); + console.log(error.message); + + // Update run status to ERROR + const errorRun = await readFile(`./../storage/runs/${fileName}_${runId}.json`); + const parsedErrorRun = JSON.parse(errorRun); + parsedErrorRun.status = 'ERROR'; + parsedErrorRun.log += `\nError: ${error.message}`; + await saveFile( + `../storage/runs/${fileName}_${runId}.json`, + JSON.stringify(parsedErrorRun, null, 2) + ); + + return { + runId: runId, + success: false, + error: error.message, + }; + } +} export { workflowQueue, runWorkflow }; \ No newline at end of file From aa3b7029d905af2e31e62990efbcf36c0cbf199a Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 00:57:19 +0530 Subject: [PATCH 054/108] chore: lint --- server/src/workflow-management/scheduler/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 2272c4c4..347f1921 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -54,7 +54,7 @@ async function runWorkflow(fileName: string, runId: string) { browser: chromium, launchOptions: { headless: true } }); - logger.log(`debug`,`Created browser with ID: ${browserId}`); + logger.log(`debug`, `Created browser with ID: ${browserId}`); const run_meta = { @@ -112,7 +112,7 @@ async function executeRun(fileName: string, runId: string) { ); // Interpret the run in active browser - + logger.log('debug', `Active in exec : ${getActiveBrowserId()}`); const browser = browserPool.getRemoteBrowser(parsedRun.browserId); if (!browser) { @@ -126,7 +126,7 @@ async function executeRun(fileName: string, runId: string) { const interpretationInfo = await browser.interpreter.InterpretRecording( parsedRecording.recording, currentPage, parsedRun.interpreterSettings); - + const duration = Math.round((new Date().getTime() - new Date(parsedRun.startedAt).getTime()) / 1000); const durString = duration < 60 ? `${duration} s` : `${Math.floor(duration / 60)} m ${duration % 60} s`; @@ -156,7 +156,7 @@ async function executeRun(fileName: string, runId: string) { } catch (error: any) { logger.log('info', `Error while running a recording with name: ${fileName}_${runId}.json`); console.log(error.message); - + // Update run status to ERROR const errorRun = await readFile(`./../storage/runs/${fileName}_${runId}.json`); const parsedErrorRun = JSON.parse(errorRun); From 69a6be756f1804f3e6cd2c203833d35277243daf Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 17:41:49 +0530 Subject: [PATCH 055/108] feat: close worker & queue --- .../workflow-management/scheduler/index.ts | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 347f1921..0975b709 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -2,6 +2,7 @@ import { Queue, Worker } from 'bullmq'; import IORedis from 'ioredis'; import { deleteFile, readFile, readFiles, saveFile } from "../storage"; import { createRemoteBrowserForRun, destroyRemoteBrowser, getActiveBrowserId } from '../../browser-management/controller'; +import { RemoteBrowser } from '../../browser-management/classes/RemoteBrowser'; import logger from '../../logger'; import { browserPool } from "../../server"; import fs from "fs"; @@ -35,12 +36,24 @@ export const worker = new Worker('workflow', async job => { } }, { connection }); -worker.on('completed', (job: any) => { +// Listen for job completion and close worker/queue +worker.on('completed', async (job: any) => { console.log(`Job ${job.id} completed for ${job.data.fileName}_${job.data.runId}`); + + // Gracefully close the worker and queue + await worker.close(); + await workflowQueue.close(); + console.log('Worker and queue have been closed.'); }); -worker.on('failed', (job: any, err) => { +// Listen for job failure and close worker/queue +worker.on('failed', async (job: any, err) => { console.error(`Job ${job.id} failed for ${job.data.fileName}_${job.data.runId}:`, err); + + // Gracefully close the worker and queue + await worker.close(); + await workflowQueue.close(); + console.log('Worker and queue have been closed after failure.'); }); async function runWorkflow(fileName: string, runId: string) { @@ -54,9 +67,6 @@ async function runWorkflow(fileName: string, runId: string) { browser: chromium, launchOptions: { headless: true } }); - logger.log(`debug`, `Created browser with ID: ${browserId}`); - - const run_meta = { status: 'SCHEDULED', name: fileName, @@ -78,8 +88,6 @@ async function runWorkflow(fileName: string, runId: string) { logger.log('debug', `Scheduled run with name: ${fileName}_${runId}.json`); - logger.log('debug', `Active in run : ${getActiveBrowserId()}`); - // Phase 2: Running return await executeRun(fileName, runId); @@ -113,7 +121,6 @@ async function executeRun(fileName: string, runId: string) { // Interpret the run in active browser - logger.log('debug', `Active in exec : ${getActiveBrowserId()}`); const browser = browserPool.getRemoteBrowser(parsedRun.browserId); if (!browser) { throw new Error('Could not access browser'); From 76f7959630ff8ed67ac18a43c90ebf16253252e4 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 19:35:27 +0530 Subject: [PATCH 056/108] feat: return boolean value --- server/src/workflow-management/scheduler/index.ts | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 0975b709..aa5db82e 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -155,11 +155,7 @@ async function executeRun(fileName: string, runId: string) { JSON.stringify(updated_run_meta, null, 2) ); - return { - browserId: parsedRun.browserId, - runId: runId, - success: true, - }; + return true; } catch (error: any) { logger.log('info', `Error while running a recording with name: ${fileName}_${runId}.json`); console.log(error.message); @@ -174,11 +170,7 @@ async function executeRun(fileName: string, runId: string) { JSON.stringify(parsedErrorRun, null, 2) ); - return { - runId: runId, - success: false, - error: error.message, - }; + return false; } } export { workflowQueue, runWorkflow }; \ No newline at end of file From a4dd08b814aeaeb0617a553a48bd32d5f757b145 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 19:37:20 +0530 Subject: [PATCH 057/108] feat: !execute workflow --- server/src/workflow-management/scheduler/index.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index aa5db82e..2f81e7a2 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -4,7 +4,7 @@ import { deleteFile, readFile, readFiles, saveFile } from "../storage"; import { createRemoteBrowserForRun, destroyRemoteBrowser, getActiveBrowserId } from '../../browser-management/controller'; import { RemoteBrowser } from '../../browser-management/classes/RemoteBrowser'; import logger from '../../logger'; -import { browserPool } from "../../server"; +import { browserPool, io } from "../../server"; import fs from "fs"; import { uuid } from "uuidv4"; import { chromium } from "playwright"; @@ -61,7 +61,6 @@ async function runWorkflow(fileName: string, runId: string) { runId = uuid(); } - // Phase 1: Scheduling try { const browserId = createRemoteBrowserForRun({ browser: chromium, @@ -88,9 +87,6 @@ async function runWorkflow(fileName: string, runId: string) { logger.log('debug', `Scheduled run with name: ${fileName}_${runId}.json`); - // Phase 2: Running - return await executeRun(fileName, runId); - } catch (e) { const { message } = e as Error; logger.log('info', `Error while scheduling a run with name: ${fileName}_${runId}.json`); From c997ca7f9a194c4d70f0417342c6aa061c28bcc2 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 19:38:05 +0530 Subject: [PATCH 058/108] feat: rename to scheduleWorfklow --- server/src/workflow-management/scheduler/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 2f81e7a2..5f616b0e 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -28,7 +28,7 @@ const workflowQueue = new Queue('workflow', { connection }); export const worker = new Worker('workflow', async job => { const { fileName, runId } = job.data; try { - const result = await runWorkflow(fileName, runId); + const result = await scheduleWorkflow(fileName, runId); return result; } catch (error) { console.error('Error running workflow:', error); @@ -56,7 +56,7 @@ worker.on('failed', async (job: any, err) => { console.log('Worker and queue have been closed after failure.'); }); -async function runWorkflow(fileName: string, runId: string) { +async function scheduleWorkflow(fileName: string, runId: string) { if (!runId) { runId = uuid(); } @@ -169,4 +169,4 @@ async function executeRun(fileName: string, runId: string) { return false; } } -export { workflowQueue, runWorkflow }; \ No newline at end of file +export { workflowQueue, scheduleWorkflow }; \ No newline at end of file From 7679febbdcfc7c6d551d8db97b94428572148b67 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 19:42:02 +0530 Subject: [PATCH 059/108] fix: revert --- server/src/workflow-management/scheduler/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 5f616b0e..2f81e7a2 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -28,7 +28,7 @@ const workflowQueue = new Queue('workflow', { connection }); export const worker = new Worker('workflow', async job => { const { fileName, runId } = job.data; try { - const result = await scheduleWorkflow(fileName, runId); + const result = await runWorkflow(fileName, runId); return result; } catch (error) { console.error('Error running workflow:', error); @@ -56,7 +56,7 @@ worker.on('failed', async (job: any, err) => { console.log('Worker and queue have been closed after failure.'); }); -async function scheduleWorkflow(fileName: string, runId: string) { +async function runWorkflow(fileName: string, runId: string) { if (!runId) { runId = uuid(); } @@ -169,4 +169,4 @@ async function executeRun(fileName: string, runId: string) { return false; } } -export { workflowQueue, scheduleWorkflow }; \ No newline at end of file +export { workflowQueue, runWorkflow }; \ No newline at end of file From de76e27a4a3512674720d628d46195399265086f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 21:01:46 +0530 Subject: [PATCH 060/108] feat: handle recording --- .../workflow-management/scheduler/index.ts | 52 ++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 2f81e7a2..d2e7745a 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -4,10 +4,12 @@ import { deleteFile, readFile, readFiles, saveFile } from "../storage"; import { createRemoteBrowserForRun, destroyRemoteBrowser, getActiveBrowserId } from '../../browser-management/controller'; import { RemoteBrowser } from '../../browser-management/classes/RemoteBrowser'; import logger from '../../logger'; -import { browserPool, io } from "../../server"; +import { browserPool } from "../../server"; import fs from "fs"; import { uuid } from "uuidv4"; import { chromium } from "playwright"; +import { io, Socket } from "socket.io-client"; + const connection = new IORedis({ host: 'localhost', @@ -28,7 +30,7 @@ const workflowQueue = new Queue('workflow', { connection }); export const worker = new Worker('workflow', async job => { const { fileName, runId } = job.data; try { - const result = await runWorkflow(fileName, runId); + const result = await handleRunRecording(fileName, runId); return result; } catch (error) { console.error('Error running workflow:', error); @@ -56,6 +58,9 @@ worker.on('failed', async (job: any, err) => { console.log('Worker and queue have been closed after failure.'); }); +const existingJobs = workflowQueue.getRepeatableJobs(); +logger.log(`info`, `jobs ${existingJobs}`) + async function runWorkflow(fileName: string, runId: string) { if (!runId) { runId = uuid(); @@ -87,6 +92,11 @@ async function runWorkflow(fileName: string, runId: string) { logger.log('debug', `Scheduled run with name: ${fileName}_${runId}.json`); + return { + browserId, + runId + } + } catch (e) { const { message } = e as Error; logger.log('info', `Error while scheduling a run with name: ${fileName}_${runId}.json`); @@ -169,4 +179,42 @@ async function executeRun(fileName: string, runId: string) { return false; } } + + +async function handleRunRecording(fileName: string, runId: string) { + try { + const result = await runWorkflow(fileName, runId); + const { browserId, runId: newRunId } = result; + + // Type guard to ensure browserId and newRunId are defined + if (!browserId || !newRunId) { + throw new Error('browserId or runId is undefined'); + } + + // Initialize socket connection + const socket = io(`http://localhost:8080/${browserId}`, { + transports: ['websocket'], + rejectUnauthorized: false + }); + + socket.on('ready-for-run', () => readyForRunHandler(browserId, fileName, newRunId)); + + // Log or notify that the recording is running + logger.log('info', `Running recording: ${fileName}`); + + // Cleanup should only happen after the run is completed + socket.on('disconnect', () => { + cleanupSocketListeners(socket, browserId, newRunId); + }); + + } catch (error: any) { + console.error('Error running recording:', error); + } +} + +function cleanupSocketListeners(socket: Socket, browserId: string, runId: string) { + socket.off('ready-for-run', () => readyForRunHandler(browserId, '', runId)); + logger.log('info', `Cleaned up listeners for browserId: ${browserId}, runId: ${runId}`); +} + export { workflowQueue, runWorkflow }; \ No newline at end of file From ce96510b0bc7980a79f5f352bdef599c283546de Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 21:02:07 +0530 Subject: [PATCH 061/108] feat: ready for run handler --- .../workflow-management/scheduler/index.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index d2e7745a..cd343882 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -180,6 +180,25 @@ async function executeRun(fileName: string, runId: string) { } } +async function readyForRunHandler(browserId: string, fileName: string, runId: string) { + try { + const interpretation = await executeRun(fileName, runId); + + if (interpretation) { + logger.log('info', `Interpretation of ${fileName} succeeded`); + } else { + logger.log('error', `Interpretation of ${fileName} failed`); + await destroyRemoteBrowser(browserId); // Destroy browser if failed + } + + resetRecordingState(browserId, fileName, runId); + + } catch (error: any) { + console.error(`Error during readyForRunHandler: ${error.message}`); + await destroyRemoteBrowser(browserId); + } +} + async function handleRunRecording(fileName: string, runId: string) { try { From d1076098381a77f361f60e7189e534b8c338a287 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 21:02:27 +0530 Subject: [PATCH 062/108] feat: reset recording state --- server/src/workflow-management/scheduler/index.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index cd343882..c5388948 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -199,6 +199,13 @@ async function readyForRunHandler(browserId: string, fileName: string, runId: st } } +function resetRecordingState(browserId: string, fileName: string, runId: string) { + // Reset the running recording name, log, and run id to empty strings + browserId = ''; + fileName = ''; + runId = ''; + logger.log(`info`, `reset values for ${browserId}, ${fileName}, and ${runId}`); +} async function handleRunRecording(fileName: string, runId: string) { try { From 0b8b735289b19e3b91dddbf354d34d98f89a7a2f Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 21:05:49 +0530 Subject: [PATCH 063/108] chore: -rm comments --- .../src/workflow-management/scheduler/index.ts | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index c5388948..f8043a70 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -38,21 +38,17 @@ export const worker = new Worker('workflow', async job => { } }, { connection }); -// Listen for job completion and close worker/queue worker.on('completed', async (job: any) => { console.log(`Job ${job.id} completed for ${job.data.fileName}_${job.data.runId}`); - // Gracefully close the worker and queue await worker.close(); await workflowQueue.close(); console.log('Worker and queue have been closed.'); }); -// Listen for job failure and close worker/queue worker.on('failed', async (job: any, err) => { console.error(`Job ${job.id} failed for ${job.data.fileName}_${job.data.runId}:`, err); - // Gracefully close the worker and queue await worker.close(); await workflowQueue.close(); console.log('Worker and queue have been closed after failure.'); @@ -110,23 +106,18 @@ async function runWorkflow(fileName: string, runId: string) { async function executeRun(fileName: string, runId: string) { try { - // Read the recording from storage const recording = await readFile(`./../storage/recordings/${fileName}.waw.json`); const parsedRecording = JSON.parse(recording); - // Read the run from storage const run = await readFile(`./../storage/runs/${fileName}_${runId}.json`); const parsedRun = JSON.parse(run); - // Update status to RUNNING parsedRun.status = 'RUNNING'; await saveFile( `../storage/runs/${fileName}_${runId}.json`, JSON.stringify(parsedRun, null, 2) ); - // Interpret the run in active browser - const browser = browserPool.getRemoteBrowser(parsedRun.browserId); if (!browser) { throw new Error('Could not access browser'); @@ -166,7 +157,6 @@ async function executeRun(fileName: string, runId: string) { logger.log('info', `Error while running a recording with name: ${fileName}_${runId}.json`); console.log(error.message); - // Update run status to ERROR const errorRun = await readFile(`./../storage/runs/${fileName}_${runId}.json`); const parsedErrorRun = JSON.parse(errorRun); parsedErrorRun.status = 'ERROR'; @@ -188,7 +178,7 @@ async function readyForRunHandler(browserId: string, fileName: string, runId: st logger.log('info', `Interpretation of ${fileName} succeeded`); } else { logger.log('error', `Interpretation of ${fileName} failed`); - await destroyRemoteBrowser(browserId); // Destroy browser if failed + await destroyRemoteBrowser(browserId); } resetRecordingState(browserId, fileName, runId); @@ -200,7 +190,6 @@ async function readyForRunHandler(browserId: string, fileName: string, runId: st } function resetRecordingState(browserId: string, fileName: string, runId: string) { - // Reset the running recording name, log, and run id to empty strings browserId = ''; fileName = ''; runId = ''; @@ -212,12 +201,10 @@ async function handleRunRecording(fileName: string, runId: string) { const result = await runWorkflow(fileName, runId); const { browserId, runId: newRunId } = result; - // Type guard to ensure browserId and newRunId are defined if (!browserId || !newRunId) { throw new Error('browserId or runId is undefined'); } - // Initialize socket connection const socket = io(`http://localhost:8080/${browserId}`, { transports: ['websocket'], rejectUnauthorized: false @@ -225,10 +212,8 @@ async function handleRunRecording(fileName: string, runId: string) { socket.on('ready-for-run', () => readyForRunHandler(browserId, fileName, newRunId)); - // Log or notify that the recording is running logger.log('info', `Running recording: ${fileName}`); - // Cleanup should only happen after the run is completed socket.on('disconnect', () => { cleanupSocketListeners(socket, browserId, newRunId); }); From 5482de258d5de5dd29d690193f9108c3a7ffe8e1 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 21:08:36 +0530 Subject: [PATCH 064/108] feat: logging --- server/src/workflow-management/scheduler/index.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index f8043a70..267ae609 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -33,25 +33,25 @@ export const worker = new Worker('workflow', async job => { const result = await handleRunRecording(fileName, runId); return result; } catch (error) { - console.error('Error running workflow:', error); + logger.error('Error running workflow:', error); throw error; } }, { connection }); worker.on('completed', async (job: any) => { - console.log(`Job ${job.id} completed for ${job.data.fileName}_${job.data.runId}`); + logger.log(`info`,`Job ${job.id} completed for ${job.data.fileName}_${job.data.runId}`); await worker.close(); await workflowQueue.close(); - console.log('Worker and queue have been closed.'); + logger.log(`info`,`Worker and queue have been closed.`); }); worker.on('failed', async (job: any, err) => { - console.error(`Job ${job.id} failed for ${job.data.fileName}_${job.data.runId}:`, err); + logger.log(`error`, `Job ${job.id} failed for ${job.data.fileName}_${job.data.runId}:`, err); await worker.close(); await workflowQueue.close(); - console.log('Worker and queue have been closed after failure.'); + logger.log(`info`, `Worker and queue have been closed after failure.`); }); const existingJobs = workflowQueue.getRepeatableJobs(); @@ -184,7 +184,7 @@ async function readyForRunHandler(browserId: string, fileName: string, runId: st resetRecordingState(browserId, fileName, runId); } catch (error: any) { - console.error(`Error during readyForRunHandler: ${error.message}`); + logger.error(`Error during readyForRunHandler: ${error.message}`); await destroyRemoteBrowser(browserId); } } @@ -219,7 +219,7 @@ async function handleRunRecording(fileName: string, runId: string) { }); } catch (error: any) { - console.error('Error running recording:', error); + logger.error('Error running recording:', error); } } From 12f54c09abdea28140d9d8e5cdcbab4b6b4600db Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 21:09:49 +0530 Subject: [PATCH 065/108] chore: lint --- server/src/workflow-management/scheduler/index.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 267ae609..ba30f232 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -10,7 +10,6 @@ import { uuid } from "uuidv4"; import { chromium } from "playwright"; import { io, Socket } from "socket.io-client"; - const connection = new IORedis({ host: 'localhost', port: 6379, @@ -39,11 +38,11 @@ export const worker = new Worker('workflow', async job => { }, { connection }); worker.on('completed', async (job: any) => { - logger.log(`info`,`Job ${job.id} completed for ${job.data.fileName}_${job.data.runId}`); + logger.log(`info`, `Job ${job.id} completed for ${job.data.fileName}_${job.data.runId}`); await worker.close(); await workflowQueue.close(); - logger.log(`info`,`Worker and queue have been closed.`); + logger.log(`info`, `Worker and queue have been closed.`); }); worker.on('failed', async (job: any, err) => { From e2e8c9da4783e4644f35a66952ebe2473b136628 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 22:17:30 +0530 Subject: [PATCH 066/108] feat(wip): schedule settings --- src/components/molecules/ScheduleSettings.tsx | 163 +++++++++--------- 1 file changed, 84 insertions(+), 79 deletions(-) diff --git a/src/components/molecules/ScheduleSettings.tsx b/src/components/molecules/ScheduleSettings.tsx index f7a01a9e..d40a52ff 100644 --- a/src/components/molecules/ScheduleSettings.tsx +++ b/src/components/molecules/ScheduleSettings.tsx @@ -9,25 +9,29 @@ interface ScheduleSettingsProps { isOpen: boolean; handleStart: (settings: ScheduleSettings) => void; handleClose: () => void; - isTask: boolean; - params?: string[]; } export interface ScheduleSettings { - maxConcurrency: number; - maxRepeats: number; - debug: boolean; - params?: any; + runEvery: number; + runEveryUnit: string; + startFrom: string; + atTime: string; + timezone: string; } -export const ScheduleSettingsModal = ({ isOpen, handleStart, handleClose, isTask, params }: ScheduleSettingsProps) => { - - const [settings, setSettings] = React.useState({ - maxConcurrency: 1, - maxRepeats: 1, - debug: true, +export const ScheduleSettingsModal = ({ isOpen, handleStart, handleClose }: ScheduleSettingsProps) => { + const [settings, setSettings] = useState({ + runEvery: 1, + runEveryUnit: 'hours', + startFrom: 'Monday', + atTime: '00:00', + timezone: 'UTC' }); + const handleChange = (field: keyof ScheduleSettings, value: string | number) => { + setSettings(prev => ({ ...prev, [field]: value })); + }; + return ( - { isTask - ? - ( - - Recording parameters: - { params?.map((item, index) => { - return setSettings( - { - ...settings, - params: settings.params - ? { - ...settings.params, - [item]: e.target.value, - } - : { - [item]: e.target.value, - }, - })} - /> - }) } - ) - : null - } - Interpreter settings: - setSettings( - { - ...settings, - maxConcurrency: parseInt(e.target.value), - })} - defaultValue={settings.maxConcurrency} - /> - setSettings( - { - ...settings, - maxRepeats: parseInt(e.target.value), - })} - defaultValue={settings.maxRepeats} - /> - setSettings( - { - ...settings, - debug: e.target.value === "true", - })} - > - true - false - +
+ Run once every + handleChange('runEvery', parseInt(e.target.value))} + style={{ width: '60px', margin: '0 10px' }} + /> + handleChange('runEveryUnit', e.target.value)} + > + minutes + hours + days + weeks + months + +
+ +
+ Start from + handleChange('startFrom', e.target.value)} + > + Monday + Tuesday + Wednesday + Thursday + Friday + Saturday + Sunday + +
+ +
+ At around + handleChange('atTime', e.target.value)} + /> +
+ +
+ Timezone + handleChange('timezone', e.target.value)} + > + UTC + America/New_York + Europe/London + Asia/Tokyo + Asia/Kolkata + {/* Add more timezone options as needed */} + +
+ ); } + +export default ScheduleSettingsModal; \ No newline at end of file From 8da44ce20f52d64bdb599a495cf132034d74aca9 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 22:17:46 +0530 Subject: [PATCH 067/108] feat(wip): scheduler --- src/components/organisms/Recordings.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/components/organisms/Recordings.tsx b/src/components/organisms/Recordings.tsx index dd0b7f68..94ae073e 100644 --- a/src/components/organisms/Recordings.tsx +++ b/src/components/organisms/Recordings.tsx @@ -61,8 +61,6 @@ export const Recordings = ({ handleEditRecording, handleRunRecording, setFileNam handleScheduleRecording(settings)} - isTask={params.length !== 0} - params={params} /> From 6b1d22ce43e169d815d810d38ec37cf465bfca9e Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 22:18:10 +0530 Subject: [PATCH 068/108] fix: pass schedule settings --- src/components/organisms/Recordings.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/organisms/Recordings.tsx b/src/components/organisms/Recordings.tsx index 94ae073e..3023b044 100644 --- a/src/components/organisms/Recordings.tsx +++ b/src/components/organisms/Recordings.tsx @@ -7,7 +7,7 @@ import { ScheduleSettings, ScheduleSettingsModal } from "../molecules/ScheduleSe interface RecordingsProps { handleEditRecording: (fileName: string) => void; handleRunRecording: (settings: RunSettings) => void; - handleScheduleRecording: (settings: RunSettings) => void; + handleScheduleRecording: (settings: ScheduleSettings) => void; setFileName: (fileName: string) => void; } From ce8344a1befc790b66d529240e1388b5733d09d4 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 22:18:56 +0530 Subject: [PATCH 069/108] chore: lint --- src/components/molecules/ScheduleSettings.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/molecules/ScheduleSettings.tsx b/src/components/molecules/ScheduleSettings.tsx index d40a52ff..894884bb 100644 --- a/src/components/molecules/ScheduleSettings.tsx +++ b/src/components/molecules/ScheduleSettings.tsx @@ -69,7 +69,7 @@ export const ScheduleSettingsModal = ({ isOpen, handleStart, handleClose }: Sche
Start from handleChange('startFrom', e.target.value)} @@ -96,7 +96,7 @@ export const ScheduleSettingsModal = ({ isOpen, handleStart, handleClose }: Sche
Timezone handleChange('timezone', e.target.value)} From a09841372be1fb4bd75af210ea46c025756d13b2 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 22:27:32 +0530 Subject: [PATCH 070/108] feat(wip): scheduler styling --- src/components/molecules/ScheduleSettings.tsx | 77 ++++++++++--------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/src/components/molecules/ScheduleSettings.tsx b/src/components/molecules/ScheduleSettings.tsx index 894884bb..6759eae9 100644 --- a/src/components/molecules/ScheduleSettings.tsx +++ b/src/components/molecules/ScheduleSettings.tsx @@ -1,6 +1,6 @@ import React, { useState } from 'react'; import { GenericModal } from "../atoms/GenericModal"; -import { MenuItem, TextField, Typography } from "@mui/material"; +import { MenuItem, TextField, Typography, Box } from "@mui/material"; import { Dropdown } from "../atoms/DropdownMui"; import Button from "@mui/material/Button"; import { modalStyle } from "./AddWhereCondModal"; @@ -38,19 +38,22 @@ export const ScheduleSettingsModal = ({ isOpen, handleStart, handleClose }: Sche onClose={handleClose} modalStyle={modalStyle} > -
*': { marginBottom: '20px' }, }}> -
- Run once every + Schedule Settings + + + Run once every handleChange('runEvery', parseInt(e.target.value))} - style={{ width: '60px', margin: '0 10px' }} + sx={{ width: '60px', marginRight: '10px' }} /> weeks months -
+ -
- Start from + + Start from Saturday Sunday -
+ -
- At around - handleChange('atTime', e.target.value)} - /> -
+ + + At around + handleChange('atTime', e.target.value)} + /> + + + Timezone + handleChange('timezone', e.target.value)} + > + UTC + America/New_York + Europe/London + Asia/Tokyo + Asia/Kolkata + + + -
- Timezone - handleChange('timezone', e.target.value)} - > - UTC - America/New_York - Europe/London - Asia/Tokyo - Asia/Kolkata - {/* Add more timezone options as needed */} - -
- - -
+ + ); } From 182307b67930e0f7d6df04daf6ec0254f34c0b79 Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 22:27:52 +0530 Subject: [PATCH 071/108] chore: lint --- src/components/molecules/ScheduleSettings.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/molecules/ScheduleSettings.tsx b/src/components/molecules/ScheduleSettings.tsx index 6759eae9..449e8f3b 100644 --- a/src/components/molecules/ScheduleSettings.tsx +++ b/src/components/molecules/ScheduleSettings.tsx @@ -46,7 +46,7 @@ export const ScheduleSettingsModal = ({ isOpen, handleStart, handleClose }: Sche '& > *': { marginBottom: '20px' }, }}> Schedule Settings - + Run once every Date: Thu, 12 Sep 2024 22:44:59 +0530 Subject: [PATCH 072/108] feat: sx prop for custom dropdown --- src/components/atoms/DropdownMui.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/components/atoms/DropdownMui.tsx b/src/components/atoms/DropdownMui.tsx index ff97ff71..c041386d 100644 --- a/src/components/atoms/DropdownMui.tsx +++ b/src/components/atoms/DropdownMui.tsx @@ -1,6 +1,7 @@ import React from 'react'; import { FormControl, InputLabel, Select } from "@mui/material"; import { SelectChangeEvent } from "@mui/material/Select/Select"; +import { SxProps } from '@mui/system'; interface DropdownProps { id: string; @@ -8,9 +9,10 @@ interface DropdownProps { value: string | undefined; handleSelect: (event: SelectChangeEvent) => void; children?: React.ReactNode; + sx?: SxProps; }; -export const Dropdown = ({ id, label, value, handleSelect, children }: DropdownProps) => { +export const Dropdown = ({ id, label, value, handleSelect, children, sx }: DropdownProps) => { return ( {label} @@ -21,6 +23,7 @@ export const Dropdown = ({ id, label, value, handleSelect, children }: DropdownP label={label} onChange={handleSelect} size='small' + sx={sx} > {children} From b611bdefc5b9a353183455c4458893b68d55383d Mon Sep 17 00:00:00 2001 From: karishmas6 Date: Thu, 12 Sep 2024 23:00:36 +0530 Subject: [PATCH 073/108] feat: pass sx prop --- src/components/atoms/DropdownMui.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/atoms/DropdownMui.tsx b/src/components/atoms/DropdownMui.tsx index c041386d..c7b3e59f 100644 --- a/src/components/atoms/DropdownMui.tsx +++ b/src/components/atoms/DropdownMui.tsx @@ -14,7 +14,7 @@ interface DropdownProps { export const Dropdown = ({ id, label, value, handleSelect, children, sx }: DropdownProps) => { return ( - + {label}