Merge pull request #791 from RohitR311/cache-api-v2

feat: faster data fetching v2

package.json

@@ -11,6 +11,7 @@
     "@mui/lab": "^5.0.0-alpha.80",
     "@mui/material": "^5.6.2",
     "@react-oauth/google": "^0.12.1",
+    "@tanstack/react-query": "^5.89.0",
     "@testing-library/react": "^13.1.1",
     "@testing-library/user-event": "^13.5.0",
     "@types/bcrypt": "^5.0.2",

src/components/robot/RecordingsTable.tsx

@@ -36,8 +36,8 @@ import {
   MoreHoriz,
   Refresh
 } from "@mui/icons-material";
-import { useGlobalInfoStore } from "../../context/globalInfo";
+import { useGlobalInfoStore, useCachedRecordings } from "../../context/globalInfo";
-import { checkRunsForRecording, deleteRecordingFromStorage, getStoredRecordings } from "../../api/storage";
+import { checkRunsForRecording, deleteRecordingFromStorage } from "../../api/storage";
 import { Add } from "@mui/icons-material";
 import { useNavigate } from 'react-router-dom';
 import { canCreateBrowserInState, getActiveBrowserId, stopRecording } from "../../api/recording";
@@ -150,12 +150,11 @@ export const RecordingsTable = ({
   const { t } = useTranslation();
   const [page, setPage] = React.useState(0);
   const [rowsPerPage, setRowsPerPage] = React.useState(10);
-  const [rows, setRows] = React.useState<Data[]>([]);
+  const { data: recordingsData = [], isLoading: isFetching, error, refetch } = useCachedRecordings();
   const [isModalOpen, setModalOpen] = React.useState(false);
   const [searchTerm, setSearchTerm] = React.useState('');
   const [isWarningModalOpen, setWarningModalOpen] = React.useState(false);
   const [activeBrowserId, setActiveBrowserId] = React.useState('');
-  const [isFetching, setIsFetching] = React.useState(true);
 
   const columns = useMemo(() => [
     { id: 'interpret', label: t('recordingtable.run'), minWidth: 80 },
@@ -238,44 +237,42 @@ export const RecordingsTable = ({
       if (dateStr.includes('PM') || dateStr.includes('AM')) {
         return new Date(dateStr);
       }
-      
+
       return new Date(dateStr.replace(/(\d+)\/(\d+)\//, '$2/$1/'))
     } catch {
       return new Date(0);
     }
   };
 
-  const fetchRecordings = useCallback(async () => {
+  const rows = useMemo(() => {
-    try {
+    if (!recordingsData) return [];
-      const recordings = await getStoredRecordings();
+
-      if (recordings) {
+    const parsedRows = recordingsData
-        const parsedRows = recordings
+      .map((recording: any, index: number) => {
-        .map((recording: any, index: number) => {
+        if (recording?.recording_meta) {
-          if (recording?.recording_meta) {
+          const parsedDate = parseDateString(recording.recording_meta.updatedAt);
-            const parsedDate = parseDateString(recording.recording_meta.updatedAt);
+
-            
+          return {
-            return {
+            id: index,
-              id: index,
+            ...recording.recording_meta,
-              ...recording.recording_meta,
+            content: recording.recording,
-              content: recording.recording,
+            parsedDate
-              parsedDate
+          };
-            };
+        }
-          }
+        return null;
-          return null;
+      })
-        })
+      .filter(Boolean)
-        .filter(Boolean) 
+      .sort((a, b) => b.parsedDate.getTime() - a.parsedDate.getTime());
-        .sort((a, b) => b.parsedDate.getTime() - a.parsedDate.getTime());
+
-  
+    return parsedRows;
-        setRecordings(parsedRows.map((recording) => recording.name));
+  }, [recordingsData]);
-        setRows(parsedRows);
+
-      }
+  useEffect(() => {
-    } catch (error) {
+    if (rows.length > 0) {
-      console.error('Error fetching recordings:', error);
+      setRecordings(rows.map((recording) => recording.name));
-      notify('error', t('recordingtable.notifications.fetch_error'));
-    } finally {
-      setIsFetching(false);
     }
-  }, [setRecordings, notify, t]);
+  }, [rows, setRecordings]);
+
 
   const handleNewRecording = useCallback(async () => {
     const canCreateRecording = await canCreateBrowserInState("recording");
@@ -331,7 +328,7 @@ export const RecordingsTable = ({
       
       if (lastPair?.what) {
         if (Array.isArray(lastPair.what)) {
-          const gotoAction = lastPair.what.find(action => 
+          const gotoAction = lastPair.what.find((action: any) => 
             action && typeof action === 'object' && 'action' in action && action.action === "goto"
           ) as any;
           
@@ -408,17 +405,12 @@ export const RecordingsTable = ({
     window.sessionStorage.setItem('initialUrl', event.target.value);
   }
 
-  useEffect(() => {
-    fetchRecordings();
-  }, [fetchRecordings]);
-
   useEffect(() => {
     if (rerenderRobots) {
-      fetchRecordings().then(() => {
+      refetch();
-        setRerenderRobots(false);
+      setRerenderRobots(false);
-      });
     }
-  }, [rerenderRobots, fetchRecordings, setRerenderRobots]);
+  }, [rerenderRobots, setRerenderRobots, refetch]);
 
   function useDebounce<T>(value: T, delay: number): T {
     const [debouncedValue, setDebouncedValue] = React.useState<T>(value);
@@ -468,12 +460,11 @@ export const RecordingsTable = ({
 
       const success = await deleteRecordingFromStorage(id);
       if (success) {
-        setRows([]);
         notify('success', t('recordingtable.notifications.delete_success'));
-        fetchRecordings();
+        refetch();
       }
     }
-  }), [handleRunRecording, handleScheduleRecording, handleIntegrateRecording, handleSettingsRecording, handleEditRobot, handleDuplicateRobot, handleRetrainRobot, notify, t]);
+  }), [handleRunRecording, handleScheduleRecording, handleIntegrateRecording, handleSettingsRecording, handleEditRobot, handleDuplicateRobot, handleRetrainRobot, notify, t, refetch]);
 
   return (
     <React.Fragment>

src/components/run/RunsTable.tsx

@@ -13,8 +13,7 @@ import { Accordion, AccordionSummary, AccordionDetails, Typography, Box, TextFie
 import ExpandMoreIcon from '@mui/icons-material/ExpandMore';
 import SearchIcon from '@mui/icons-material/Search';
 import { useLocation, useNavigate } from 'react-router-dom';
-import { useGlobalInfoStore } from "../../context/globalInfo";
+import { useGlobalInfoStore, useCachedRuns } from "../../context/globalInfo";
-import { getStoredRuns } from "../../api/storage";
 import { RunSettings } from "./RunSettings";
 import { CollapsibleRow } from "./ColapsibleRow";
 import { ArrowDownward, ArrowUpward, UnfoldMore } from '@mui/icons-material';
@@ -132,16 +131,14 @@ export const RunsTable: React.FC<RunsTableProps> = ({
     [t]
   );
 
-  const [rows, setRows] = useState<Data[]>([]);
+  const { notify, rerenderRuns, setRerenderRuns } = useGlobalInfoStore();
-  const [searchTerm, setSearchTerm] = useState('');
+  const { data: rows = [], isLoading: isFetching, error, refetch } = useCachedRuns();
-  const [isFetching, setIsFetching] = useState(true);
 
+  const [searchTerm, setSearchTerm] = useState('');
   const [paginationStates, setPaginationStates] = useState<PaginationState>({});
   const [expandedRows, setExpandedRows] = useState<Set<string>>(new Set());
   const [expandedAccordions, setExpandedAccordions] = useState<Set<string>>(new Set());
 
-  const { notify, rerenderRuns, setRerenderRuns } = useGlobalInfoStore();
-
   const handleAccordionChange = useCallback((robotMetaId: string, isExpanded: boolean) => {
     setExpandedAccordions(prev => {
       const newSet = new Set(prev);
@@ -278,47 +275,18 @@ export const RunsTable: React.FC<RunsTableProps> = ({
     debouncedSetSearch(event.target.value);
   }, [debouncedSearch]);
 
-  const fetchRuns = useCallback(async () => {
-    try {
-      const runs = await getStoredRuns();
-      if (runs) {
-        const parsedRows: Data[] = runs.map((run: any, index: number) => ({
-          id: index,
-          ...run,
-        }));
-        setRows(parsedRows);
-      } else {
-        notify('error', t('runstable.notifications.no_runs'));
-      }
-    } catch (error) {
-      notify('error', t('runstable.notifications.fetch_error'));
-    } finally {
-      setIsFetching(false);
-    }
-  }, [notify, t]);
 
   useEffect(() => {
-    let mounted = true;
+    if (rerenderRuns) {
-
+      refetch();
-    if (rows.length === 0 || rerenderRuns) {
+      setRerenderRuns(false);
-      setIsFetching(true);
-      fetchRuns().then(() => {
-        if (mounted) {
-          setRerenderRuns(false);
-        }
-      });
     }
-
+  }, [rerenderRuns, setRerenderRuns, refetch]);
-    return () => {
-      mounted = false;
-    };
-  }, [rerenderRuns, rows.length, setRerenderRuns, fetchRuns]);
 
   const handleDelete = useCallback(() => {
-    setRows([]);
     notify('success', t('runstable.notifications.delete_success'));
-    fetchRuns();
+    refetch();
-  }, [notify, t, fetchRuns]);
+  }, [notify, t, refetch]);
 
   // Filter rows based on search term
   const filteredRows = useMemo(() => {
@@ -350,15 +318,15 @@ export const RunsTable: React.FC<RunsTableProps> = ({
     }, {} as Record<string, Data[]>);
   
     Object.keys(groupedData).forEach(robotId => {
-      groupedData[robotId].sort((a, b) => 
+      groupedData[robotId].sort((a: any, b: any) => 
         parseDateString(b.startedAt).getTime() - parseDateString(a.startedAt).getTime()
       );
     });
   
     const robotEntries = Object.entries(groupedData).map(([robotId, runs]) => ({
       robotId,
-      runs,
+      runs: runs as Data[],
-      latestRunDate: parseDateString(runs[0].startedAt).getTime()
+      latestRunDate: parseDateString((runs as Data[])[0].startedAt).getTime()
     }));
   
     robotEntries.sort((a, b) => b.latestRunDate - a.latestRunDate);

src/context/globalInfo.tsx

@@ -1,6 +1,24 @@
 import React, { createContext, useContext, useState } from "react";
 import { AlertSnackbarProps } from "../components/ui/AlertSnackbar";
 import { WhereWhatPair } from "maxun-core";
+import { QueryClient, QueryClientProvider, useQuery, useQueryClient } from '@tanstack/react-query';
+import { getStoredRuns, getStoredRecordings } from "../api/storage";
+
+const createDataCacheClient = () => new QueryClient({
+  defaultOptions: {
+    queries: {
+      staleTime: 30 * 1000,
+      gcTime: 5 * 60 * 1000,
+      retry: 2,
+      retryDelay: (attemptIndex) => Math.min(1000 * 2 ** attemptIndex, 30000),
+    }
+  }
+});
+
+const dataCacheKeys = {
+  runs: ['cached-runs'] as const,
+  recordings: ['cached-recordings'] as const,
+} as const;
 
 interface RobotMeta {
     name: string;
@@ -164,6 +182,65 @@ const globalInfoContext = createContext<GlobalInfo>(globalInfoStore as GlobalInf
 
 export const useGlobalInfoStore = () => useContext(globalInfoContext);
 
+export const useCachedRuns = () => {
+  return useQuery({
+    queryKey: dataCacheKeys.runs,
+    queryFn: async () => {
+      const runs = await getStoredRuns();
+      if (!runs) throw new Error('Failed to fetch runs data');
+      return runs.map((run: any, index: number) => ({ id: index, ...run }));
+    },
+    staleTime: 30 * 1000,
+    gcTime: 5 * 60 * 1000,
+    retry: 2,
+  });
+};
+
+export const useCacheInvalidation = () => {
+  const queryClient = useQueryClient();
+  
+  const invalidateRuns = () => {
+    queryClient.invalidateQueries({ queryKey: dataCacheKeys.runs });
+  };
+  
+  const invalidateRecordings = () => {
+    queryClient.invalidateQueries({ queryKey: dataCacheKeys.recordings });
+  };
+  
+  const addOptimisticRun = (newRun: any) => {
+    queryClient.setQueryData(dataCacheKeys.runs, (oldData: any) => {
+      if (!oldData) return [{ id: 0, ...newRun }];
+      return [{ id: oldData.length, ...newRun }, ...oldData];
+    });
+  };
+  
+  const invalidateAllCache = () => {
+    invalidateRuns();
+    invalidateRecordings();
+  };
+  
+  return {
+    invalidateRuns,
+    invalidateRecordings, 
+    addOptimisticRun,
+    invalidateAllCache
+  };
+};
+
+export const useCachedRecordings = () => {
+  return useQuery({
+    queryKey: dataCacheKeys.recordings,
+    queryFn: async () => {
+      const recordings = await getStoredRecordings();
+      if (!recordings) throw new Error('Failed to fetch recordings data');
+      return recordings;
+    },
+    staleTime: 30 * 1000,
+    gcTime: 5 * 60 * 1000,
+    retry: 2,
+  });
+};
+
 export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
   const [browserId, setBrowserId] = useState<string | null>(globalInfoStore.browserId);
   const [lastAction, setLastAction] = useState<string>(globalInfoStore.lastAction);
@@ -172,7 +249,29 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
   const [rerenderRuns, setRerenderRuns] = useState<boolean>(globalInfoStore.rerenderRuns);
   const [rerenderRobots, setRerenderRobots] = useState<boolean>(globalInfoStore.rerenderRobots);
   const [recordingLength, setRecordingLength] = useState<number>(globalInfoStore.recordingLength);
-  const [recordingId, setRecordingId] = useState<string | null>(globalInfoStore.recordingId);
+  // const [recordingId, setRecordingId] = useState<string | null>(globalInfoStore.recordingId);
+   const [recordingId, setRecordingId] = useState<string | null>(() => {
+    try {
+      const stored = sessionStorage.getItem('recordingId');
+      return stored ? JSON.parse(stored) : globalInfoStore.recordingId;
+    } catch {
+      return globalInfoStore.recordingId;
+    }
+  });
+
+  // Create a wrapped setter that persists to sessionStorage
+  const setPersistedRecordingId = (newRecordingId: string | null) => {
+    setRecordingId(newRecordingId);
+    try {
+      if (newRecordingId) {
+        sessionStorage.setItem('recordingId', JSON.stringify(newRecordingId));
+      } else {
+        sessionStorage.removeItem('recordingId');
+      }
+    } catch (error) {
+      console.warn('Failed to persist recordingId to sessionStorage:', error);
+    }
+  };
   const [retrainRobotId, setRetrainRobotId] = useState<string | null>(globalInfoStore.retrainRobotId);
   const [recordingName, setRecordingName] = useState<string>(globalInfoStore.recordingName);
   const [isLogin, setIsLogin] = useState<boolean>(globalInfoStore.isLogin);
@@ -221,9 +320,12 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
     }
   }
 
+  const [dataCacheClient] = useState(() => createDataCacheClient());
+
   return (
-    <globalInfoContext.Provider
+    <QueryClientProvider client={dataCacheClient}>
-      value={{
+      <globalInfoContext.Provider
+        value={{
         browserId,
         setBrowserId: setBrowserIdWithValidation,
         lastAction,
@@ -240,7 +342,7 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
         recordingLength,
         setRecordingLength,
         recordingId,
-        setRecordingId,
+        setRecordingId: setPersistedRecordingId,
         retrainRobotId,
         setRetrainRobotId,
         recordingName,
@@ -266,9 +368,10 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
         currentSnapshot,
         setCurrentSnapshot,
         updateDOMMode,
-      }}
+        }}
-    >
+      >
-      {children}
+        {children}
-    </globalInfoContext.Provider>
+      </globalInfoContext.Provider>
+    </QueryClientProvider>
   );
 };

src/pages/MainPage.tsx

@@ -6,7 +6,7 @@ import { Recordings } from "../components/robot/Recordings";
 import { Runs } from "../components/run/Runs";
 import ProxyForm from '../components/proxy/ProxyForm';
 import ApiKey from '../components/api/ApiKey';
-import { useGlobalInfoStore } from "../context/globalInfo";
+import { useGlobalInfoStore, useCacheInvalidation } from "../context/globalInfo";
 import { createAndRunRecording, createRunForStoredRecording, CreateRunResponseWithQueue, interpretStoredRecording, notifyAboutAbort, scheduleStoredRecording } from "../api/storage";
 import { io, Socket } from "socket.io-client";
 import { stopRecording } from "../api/recording";
@@ -50,6 +50,7 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
   let aborted = false;
 
   const { notify, setRerenderRuns, setRecordingId } = useGlobalInfoStore();
+  const { invalidateRuns } = useCacheInvalidation();
   const navigate  = useNavigate();
 
   const { state } = useContext(AuthContext);
@@ -66,12 +67,14 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
       if (!response.success) {
         notify('error', t('main_page.notifications.abort_failed', { name: robotName }));
         setRerenderRuns(true);
+        invalidateRuns();
         return;
       }
       
       if (response.isQueued) {
         setRerenderRuns(true);
-        
+        invalidateRuns();
+
         notify('success', t('main_page.notifications.abort_success', { name: robotName }));
         
         setQueuedRuns(prev => {
@@ -92,6 +95,7 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
         if (abortData.runId === runId) {
           notify('success', t('main_page.notifications.abort_success', { name: abortData.robotName || robotName }));
           setRerenderRuns(true);
+          invalidateRuns();
           abortSocket.disconnect();
         }
       });
@@ -100,6 +104,7 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
         console.log('Abort socket connection error:', error);
         notify('error', t('main_page.notifications.abort_failed', { name: robotName }));
         setRerenderRuns(true);
+        invalidateRuns();
         abortSocket.disconnect();
       });
     });
@@ -125,8 +130,9 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
       setRunningRecordingName('');
       setCurrentInterpretationLog('');
       setRerenderRuns(true);
+      invalidateRuns();
     })
-  }, [runningRecordingName, aborted, currentInterpretationLog, notify, setRerenderRuns]);
+  }, [runningRecordingName, aborted, currentInterpretationLog, notify, setRerenderRuns, invalidateRuns]);
 
   const debugMessageHandler = useCallback((msg: string) => {
     setCurrentInterpretationLog((prevState) =>
@@ -136,8 +142,9 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
   const handleRunRecording = useCallback((settings: RunSettings) => {
     createAndRunRecording(runningRecordingId, settings).then((response: CreateRunResponseWithQueue) => {
       const { browserId, runId, robotMetaId, queued } = response;
-      
+
       setIds({ browserId, runId, robotMetaId });
+      invalidateRuns();
       navigate(`/runs/${robotMetaId}/run/${runId}`);
             
       if (queued) {
@@ -156,6 +163,7 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
           setRunningRecordingName('');
           setCurrentInterpretationLog('');
           setRerenderRuns(true);
+          invalidateRuns();
           
           const robotName = data.robotName;
           
@@ -193,7 +201,7 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
       socket.off('connect_error');
       socket.off('disconnect');
     }
-  }, [runningRecordingName, sockets, ids, debugMessageHandler, user?.id, t, notify, setRerenderRuns, setQueuedRuns, navigate, setContent, setIds]);
+  }, [runningRecordingName, sockets, ids, debugMessageHandler, user?.id, t, notify, setRerenderRuns, setQueuedRuns, navigate, setContent, setIds, invalidateRuns]);
 
   const handleScheduleRecording = async (settings: ScheduleSettings) => {
     const { message, runId }: ScheduleRunResponse = await scheduleStoredRecording(runningRecordingId, settings);
@@ -209,6 +217,7 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
     if (user?.id) {
       const handleRunCompleted = (completionData: any) => {
         setRerenderRuns(true);
+        invalidateRuns();
         
         if (queuedRuns.has(completionData.runId)) {
           setQueuedRuns(prev => {
@@ -233,7 +242,7 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
         disconnectQueueSocket();
       };
     }
-  }, [user?.id, connectToQueueSocket, disconnectQueueSocket, t, setRerenderRuns, queuedRuns, setQueuedRuns]);
+  }, [user?.id, connectToQueueSocket, disconnectQueueSocket, t, setRerenderRuns, queuedRuns, setQueuedRuns, invalidateRuns]);
 
   const DisplayContent = () => {
     switch (content) {