From 389a1cbdc8b1124a757c590972d9c09fe6104376 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 9 Apr 2025 19:40:48 +0530 Subject: [PATCH 01/11] feat: add robot retraining logic --- src/components/robot/RecordingsTable.tsx | 85 +++++++++++++++++++++++- 1 file changed, 82 insertions(+), 3 deletions(-) diff --git a/src/components/robot/RecordingsTable.tsx b/src/components/robot/RecordingsTable.tsx index 2fc4f26e..878c998f 100644 --- a/src/components/robot/RecordingsTable.tsx +++ b/src/components/robot/RecordingsTable.tsx @@ -35,7 +35,8 @@ import { Settings, Power, ContentCopy, - MoreHoriz + MoreHoriz, + Refresh } from "@mui/icons-material"; import { useGlobalInfoStore } from "../../context/globalInfo"; import { checkRunsForRecording, deleteRecordingFromStorage, getStoredRecordings } from "../../api/storage"; @@ -117,6 +118,7 @@ const TableRowMemoized = memo(({ row, columns, handlers }: any) => { return ( handlers.handleRetrainRobot(row.id, row.name)} handleEdit={() => handlers.handleEditRobot(row.id, row.name, row.params || [])} handleDuplicate={() => handlers.handleDuplicateRobot(row.id, row.name, row.params || [])} handleDelete={() => handlers.handleDelete(row.id)} @@ -198,6 +200,17 @@ export const RecordingsTable = ({ } } } + + if (event.data && event.data.type === 'session-data-clear') { + window.sessionStorage.removeItem('browserId'); + window.sessionStorage.removeItem('robotToRetrain'); + window.sessionStorage.removeItem('robotName'); + window.sessionStorage.removeItem('recordingUrl'); + window.sessionStorage.removeItem('recordingSessionId'); + window.sessionStorage.removeItem('pendingSessionData'); + window.sessionStorage.removeItem('nextTabIsRecording'); + window.sessionStorage.removeItem('initialUrl'); + } }; window.addEventListener('message', handleMessage); @@ -303,6 +316,63 @@ export const RecordingsTable = ({ setModalOpen(true); }; + const handleRetrainRobot = useCallback(async (id: string, name: string) => { + const activeBrowserId = await getActiveBrowserId(); + const robot = rows.find(row => row.id === id); + let targetUrl; + + if (robot?.content?.workflow && robot.content.workflow.length > 0) { + // Get the last workflow item + const lastPair = robot.content.workflow[robot.content.workflow.length - 1]; + + if (lastPair?.what) { + if (Array.isArray(lastPair.what)) { + const gotoAction = lastPair.what.find(action => + action && typeof action === 'object' && 'action' in action && action.action === "goto" + ) as any; + + if (gotoAction?.args?.[0]) { + targetUrl = gotoAction.args[0]; + } + } + } + } + + // Set the URL in state and session storage + if (targetUrl) { + setInitialUrl(targetUrl); + setRecordingUrl(targetUrl); + window.sessionStorage.setItem('initialUrl', targetUrl); + } + + if (activeBrowserId) { + setActiveBrowserId(activeBrowserId); + setWarningModalOpen(true); + } else { + // Pass the URL directly to avoid timing issues with state updates + startRetrainRecording(id, name, targetUrl); + } + }, [rows, setInitialUrl, setRecordingUrl]); + + const startRetrainRecording = (id: string, name: string, url?: string) => { + setBrowserId('new-recording'); + setRecordingName(''); + setRecordingId(''); + + window.sessionStorage.setItem('browserId', 'new-recording'); + window.sessionStorage.setItem('robotToRetrain', id); + window.sessionStorage.setItem('robotName', name); + + window.sessionStorage.setItem('recordingUrl', url || recordingUrl); + + const sessionId = Date.now().toString(); + window.sessionStorage.setItem('recordingSessionId', sessionId); + + window.openedRecordingWindow = window.open(`/recording-setup?session=${sessionId}`, '_blank'); + + window.sessionStorage.setItem('nextTabIsRecording', 'true'); + }; + const startRecording = () => { setModalOpen(false); @@ -381,6 +451,7 @@ export const RecordingsTable = ({ handleSettingsRecording, handleEditRobot, handleDuplicateRobot, + handleRetrainRobot, handleDelete: async (id: string) => { const hasRuns = await checkRunsForRecording(id); if (hasRuns) { @@ -395,7 +466,7 @@ export const RecordingsTable = ({ fetchRecordings(); } } - }), [handleRunRecording, handleScheduleRecording, handleIntegrateRecording, handleSettingsRecording, handleEditRobot, handleDuplicateRobot, notify, t]); + }), [handleRunRecording, handleScheduleRecording, handleIntegrateRecording, handleSettingsRecording, handleEditRobot, handleDuplicateRobot, handleRetrainRobot, notify, t]); return ( @@ -597,12 +668,13 @@ const SettingsButton = ({ handleSettings }: SettingsButtonProps) => { } interface OptionsButtonProps { + handleRetrain: () => void; handleEdit: () => void; handleDelete: () => void; handleDuplicate: () => void; } -const OptionsButton = ({ handleEdit, handleDelete, handleDuplicate }: OptionsButtonProps) => { +const OptionsButton = ({ handleRetrain, handleEdit, handleDelete, handleDuplicate }: OptionsButtonProps) => { const [anchorEl, setAnchorEl] = React.useState(null); const handleClick = (event: React.MouseEvent) => { @@ -629,6 +701,13 @@ const OptionsButton = ({ handleEdit, handleDelete, handleDuplicate }: OptionsBut open={Boolean(anchorEl)} onClose={handleClose} > + { handleRetrain(); handleClose(); }}> + + + + {t('recordingtable.retrain')} + + { handleEdit(); handleClose(); }}> From 962723b87f83c923def7014a404a649d9e034855 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 9 Apr 2025 20:46:38 +0530 Subject: [PATCH 02/11] feat: add logic to update workflow on save --- .../workflow-management/classes/Generator.ts | 74 ++++++++++++------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 004126bd..563053ba 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -139,12 +139,14 @@ export class WorkflowGenerator { */ private registerEventHandlers = (socket: Socket) => { socket.on('save', (data) => { - const { fileName, userId, isLogin } = data; + const { fileName, userId, isLogin, robotId } = data; logger.log('debug', `Saving workflow ${fileName} for user ID ${userId}`); - this.saveNewWorkflow(fileName, userId, isLogin); + this.saveNewWorkflow(fileName, userId, isLogin, robotId); }); - socket.on('new-recording', () => this.workflowRecord = { - workflow: [], + socket.on('new-recording', (data) => { + this.workflowRecord = { + workflow: [], + }; }); socket.on('activeIndex', (data) => this.generatedData.lastIndex = parseInt(data)); socket.on('decision', async ({ pair, actionType, decision, userId }) => { @@ -764,32 +766,50 @@ export class WorkflowGenerator { * @param fileName The name of the file. * @returns {Promise} */ - public saveNewWorkflow = async (fileName: string, userId: number, isLogin: boolean) => { + public saveNewWorkflow = async (fileName: string, userId: number, isLogin: boolean, robotId?: string) => { const recording = this.optimizeWorkflow(this.workflowRecord); try { - this.recordingMeta = { - name: fileName, - id: uuid(), - createdAt: this.recordingMeta.createdAt || new Date().toLocaleString(), - pairs: recording.workflow.length, - updatedAt: new Date().toLocaleString(), - params: this.getParams() || [], - isLogin: isLogin, - } - const robot = await Robot.create({ - userId, - recording_meta: this.recordingMeta, - recording: recording, - }); - capture( - 'maxun-oss-robot-created', - { - robot_meta: robot.recording_meta, - recording: robot.recording, - } - ) + if (robotId) { + const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId }}); - logger.log('info', `Robot saved with id: ${robot.id}`); + if (robot) { + await robot.update({ + recording: recording, + recording_meta: { + ...robot.recording_meta, + pairs: recording.workflow.length, + params: this.getParams() || [], + updatedAt: new Date().toLocaleString(), + }, + }) + + logger.log('info', `Robot retrained with id: ${robot.id}`); + } + } else { + this.recordingMeta = { + name: fileName, + id: uuid(), + createdAt: this.recordingMeta.createdAt || new Date().toLocaleString(), + pairs: recording.workflow.length, + updatedAt: new Date().toLocaleString(), + params: this.getParams() || [], + isLogin: isLogin, + } + const robot = await Robot.create({ + userId, + recording_meta: this.recordingMeta, + recording: recording, + }); + capture( + 'maxun-oss-robot-created', + { + robot_meta: robot.recording_meta, + recording: robot.recording, + } + ) + + logger.log('info', `Robot saved with id: ${robot.id}`); + } } catch (e) { const { message } = e as Error; From 5be644d3b58be6f607375427c2ae488f4d8a34d8 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 9 Apr 2025 20:47:12 +0530 Subject: [PATCH 03/11] feat: add robot retrain id global context --- src/context/globalInfo.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/context/globalInfo.tsx b/src/context/globalInfo.tsx index eaa6ded7..58ee9672 100644 --- a/src/context/globalInfo.tsx +++ b/src/context/globalInfo.tsx @@ -60,6 +60,8 @@ interface GlobalInfo { setRecordingLength: (recordingLength: number) => void; recordingId: string | null; setRecordingId: (newId: string | null) => void; + retrainRobotId: string | null; + setRetrainRobotId: (newId: string | null) => void; recordingName: string; setRecordingName: (recordingName: string) => void; initialUrl: string; @@ -90,6 +92,7 @@ class GlobalInfoStore implements Partial { isOpen: false, }; recordingId = null; + retrainRobotId = null; recordings: string[] = []; rerenderRuns = false; rerenderRobots = false; @@ -119,6 +122,7 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => { const [rerenderRobots, setRerenderRobots] = useState(globalInfoStore.rerenderRobots); const [recordingLength, setRecordingLength] = useState(globalInfoStore.recordingLength); const [recordingId, setRecordingId] = useState(globalInfoStore.recordingId); + const [retrainRobotId, setRetrainRobotId] = useState(globalInfoStore.retrainRobotId); const [recordingName, setRecordingName] = useState(globalInfoStore.recordingName); const [isLogin, setIsLogin] = useState(globalInfoStore.isLogin); const [initialUrl, setInitialUrl] = useState(globalInfoStore.initialUrl); @@ -169,6 +173,8 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => { setRecordingLength, recordingId, setRecordingId, + retrainRobotId, + setRetrainRobotId, recordingName, setRecordingName, initialUrl, From b95d30cda921cab65eefe33e5b193ffefc926f1f Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 9 Apr 2025 20:48:05 +0530 Subject: [PATCH 04/11] feat: save and set retrain robot params --- src/pages/RecordingPage.tsx | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/pages/RecordingPage.tsx b/src/pages/RecordingPage.tsx index 34c2f90d..7dbed8b2 100644 --- a/src/pages/RecordingPage.tsx +++ b/src/pages/RecordingPage.tsx @@ -43,7 +43,7 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => { const { setId, socket } = useSocketStore(); const { setWidth } = useBrowserDimensionsStore(); - const { browserId, setBrowserId, recordingId, recordingUrl, setRecordingUrl } = useGlobalInfoStore(); + const { browserId, setBrowserId, recordingId, recordingUrl, setRecordingUrl, setRecordingName, setRetrainRobotId } = useGlobalInfoStore(); const handleShowOutputData = useCallback(() => { setShowOutputData(true); @@ -80,6 +80,19 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => { const storedUrl = window.sessionStorage.getItem('recordingUrl'); if (storedUrl && !recordingUrl) { setRecordingUrl(storedUrl); + window.sessionStorage.removeItem('recordingUrl'); + } + + const robotName = window.sessionStorage.getItem('robotName'); + if (robotName) { + setRecordingName(robotName); + window.sessionStorage.removeItem('robotName'); + } + + const recordingId = window.sessionStorage.getItem('robotToRetrain'); + if (recordingId) { + setRetrainRobotId(recordingId); + window.sessionStorage.removeItem('robotToRetrain'); } const id = await getActiveBrowserId(); @@ -101,7 +114,7 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => { return () => { isCancelled = true; } - }, [setId, recordingUrl, setRecordingUrl]); + }, [setId, recordingUrl, setRecordingUrl, setRecordingName, setRetrainRobotId]); const changeBrowserDimensions = useCallback(() => { if (browserContentRef.current) { @@ -126,7 +139,7 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => { } setIsLoaded(true); } - }, [socket, browserId, recordingName, recordingId, isLoaded]) + }, [socket, browserId, recordingName, recordingId, isLoaded]); useEffect(() => { socket?.on('loaded', handleLoaded); From 18924d89dfc3e6940f8b186d189a68e30fe43641 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 9 Apr 2025 20:49:01 +0530 Subject: [PATCH 05/11] feat: pass session storage cleanup message --- src/components/browser/BrowserRecordingSave.tsx | 5 +++++ src/components/robot/RecordingsTable.tsx | 7 ++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/components/browser/BrowserRecordingSave.tsx b/src/components/browser/BrowserRecordingSave.tsx index d4fd54fb..2adfcd79 100644 --- a/src/components/browser/BrowserRecordingSave.tsx +++ b/src/components/browser/BrowserRecordingSave.tsx @@ -55,6 +55,11 @@ const BrowserRecordingSave = () => { type: 'recording-notification', notification: notificationData }, '*'); + + window.opener.postMessage({ + type: 'session-data-clear', + timestamp: Date.now() + }, '*'); } setBrowserId(null); diff --git a/src/components/robot/RecordingsTable.tsx b/src/components/robot/RecordingsTable.tsx index 878c998f..91f39415 100644 --- a/src/components/robot/RecordingsTable.tsx +++ b/src/components/robot/RecordingsTable.tsx @@ -322,7 +322,6 @@ export const RecordingsTable = ({ let targetUrl; if (robot?.content?.workflow && robot.content.workflow.length > 0) { - // Get the last workflow item const lastPair = robot.content.workflow[robot.content.workflow.length - 1]; if (lastPair?.what) { @@ -338,7 +337,6 @@ export const RecordingsTable = ({ } } - // Set the URL in state and session storage if (targetUrl) { setInitialUrl(targetUrl); setRecordingUrl(targetUrl); @@ -349,15 +347,14 @@ export const RecordingsTable = ({ setActiveBrowserId(activeBrowserId); setWarningModalOpen(true); } else { - // Pass the URL directly to avoid timing issues with state updates startRetrainRecording(id, name, targetUrl); } }, [rows, setInitialUrl, setRecordingUrl]); const startRetrainRecording = (id: string, name: string, url?: string) => { setBrowserId('new-recording'); - setRecordingName(''); - setRecordingId(''); + setRecordingName(name); + setRecordingId(id); window.sessionStorage.setItem('browserId', 'new-recording'); window.sessionStorage.setItem('robotToRetrain', id); From a2d2bf893acf561244c58b713d4ecdb1da3281a6 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 9 Apr 2025 20:49:55 +0530 Subject: [PATCH 06/11] feat: save robot based on retrain robot params --- src/components/recorder/SaveRecording.tsx | 44 +++++++++++++++++------ 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/src/components/recorder/SaveRecording.tsx b/src/components/recorder/SaveRecording.tsx index f7020b44..87d9bd17 100644 --- a/src/components/recorder/SaveRecording.tsx +++ b/src/components/recorder/SaveRecording.tsx @@ -19,26 +19,32 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => { const { t } = useTranslation(); const [openModal, setOpenModal] = useState(false); const [needConfirm, setNeedConfirm] = useState(false); - const [recordingName, setRecordingName] = useState(fileName); + const [saveRecordingName, setSaveRecordingName] = useState(fileName); const [waitingForSave, setWaitingForSave] = useState(false); - const { browserId, setBrowserId, notify, recordings, isLogin } = useGlobalInfoStore(); + const { browserId, setBrowserId, notify, recordings, isLogin, recordingName, retrainRobotId } = useGlobalInfoStore(); const { socket } = useSocketStore(); const { state, dispatch } = useContext(AuthContext); const { user } = state; const navigate = useNavigate(); + useEffect(() => { + if (recordingName) { + setSaveRecordingName(recordingName); + } + }, [recordingName]); + const handleChangeOfTitle = (event: React.ChangeEvent) => { const { value } = event.target; if (needConfirm) { setNeedConfirm(false); } - setRecordingName(value); + setSaveRecordingName(value); } const handleSaveRecording = async (event: React.SyntheticEvent) => { event.preventDefault(); - if (recordings.includes(recordingName)) { + if (recordings.includes(saveRecordingName)) { if (needConfirm) { return; } setNeedConfirm(true); } else { @@ -46,19 +52,32 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => { } }; + const handleFinishClick = () => { + if (recordingName && !recordings.includes(recordingName)) { + saveRecording(); + } else { + setOpenModal(true); + } + }; + const exitRecording = useCallback(async () => { const notificationData = { type: 'success', message: t('save_recording.notifications.save_success'), timestamp: Date.now() }; - window.sessionStorage.setItem('pendingNotification', JSON.stringify(notificationData)); if (window.opener) { window.opener.postMessage({ type: 'recording-notification', notification: notificationData }, '*'); + + // Also notify about clearing any remaining session data + window.opener.postMessage({ + type: 'session-data-clear', + timestamp: Date.now() + }, '*'); } if (browserId) { @@ -67,16 +86,21 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => { setBrowserId(null); window.close(); - }, [setBrowserId, browserId]); + }, [setBrowserId, browserId, t]); // notifies backed to save the recording in progress, // releases resources and changes the view for main page by clearing the global browserId const saveRecording = async () => { if (user) { - const payload = { fileName: recordingName, userId: user.id, isLogin: isLogin }; + const payload = { + fileName: saveRecordingName || recordingName, + userId: user.id, + isLogin: isLogin, + robotId: retrainRobotId, + }; socket?.emit('save', payload); setWaitingForSave(true); - console.log(`Saving the recording as ${recordingName} for userId ${user.id}`); + console.log(`Saving the recording as ${saveRecordingName || recordingName} for userId ${user.id}`); } else { console.error(t('save_recording.notifications.user_not_logged')); } @@ -92,7 +116,7 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => { return (