vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #791 from RohitR311/cache-api-v2

Browse Source 
feat: faster data fetching v2
This commit is contained in:
Karishma Shukla
2025-09-29 16:48:31 +05:30
committed by GitHub
parent 6b81aa6235 6c1d6612a4
commit b61a177c2b
5 changed files with 178 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@@ -11,6 +11,7 @@
"@mui/lab": "^5.0.0-alpha.80",
"@mui/material": "^5.6.2",
"@react-oauth/google": "^0.12.1",
"@tanstack/react-query": "^5.89.0",
"@testing-library/react": "^13.1.1",
"@testing-library/user-event": "^13.5.0",
"@types/bcrypt": "^5.0.2",

85
src/components/robot/RecordingsTable.tsx
View File

@@ -36,8 +36,8 @@ import {
MoreHoriz,
Refresh
} from "@mui/icons-material";
import { useGlobalInfoStore } from "../../context/globalInfo";
import { checkRunsForRecording, deleteRecordingFromStorage, getStoredRecordings } from "../../api/storage";
import { useGlobalInfoStore, useCachedRecordings } from "../../context/globalInfo";
import { checkRunsForRecording, deleteRecordingFromStorage } from "../../api/storage";
import { Add } from "@mui/icons-material";
import { useNavigate } from 'react-router-dom';
import { canCreateBrowserInState, getActiveBrowserId, stopRecording } from "../../api/recording";
@@ -150,12 +150,11 @@ export const RecordingsTable = ({
const { t } = useTranslation();
const [page, setPage] = React.useState(0);
const [rowsPerPage, setRowsPerPage] = React.useState(10);
const [rows, setRows] = React.useState<Data[]>([]);
const { data: recordingsData = [], isLoading: isFetching, error, refetch } = useCachedRecordings();
const [isModalOpen, setModalOpen] = React.useState(false);
const [searchTerm, setSearchTerm] = React.useState('');
const [isWarningModalOpen, setWarningModalOpen] = React.useState(false);
const [activeBrowserId, setActiveBrowserId] = React.useState('');
const [isFetching, setIsFetching] = React.useState(true);
const columns = useMemo(() => [
{ id: 'interpret', label: t('recordingtable.run'), minWidth: 80 },
@@ -238,44 +237,42 @@ export const RecordingsTable = ({
if (dateStr.includes('PM') || dateStr.includes('AM')) {
return new Date(dateStr);
}
return new Date(dateStr.replace(/(\d+)\/(\d+)\//, '$2/$1/'))
} catch {
return new Date(0);
}
};
const fetchRecordings = useCallback(async () => {
try {
const recordings = await getStoredRecordings();
if (recordings) {
const parsedRows = recordings
.map((recording: any, index: number) => {
if (recording?.recording_meta) {
const parsedDate = parseDateString(recording.recording_meta.updatedAt);
return {
id: index,
...recording.recording_meta,
content: recording.recording,
parsedDate
};
}
return null;
})
.filter(Boolean)
.sort((a, b) => b.parsedDate.getTime() - a.parsedDate.getTime());
setRecordings(parsedRows.map((recording) => recording.name));
setRows(parsedRows);
}
} catch (error) {
console.error('Error fetching recordings:', error);
notify('error', t('recordingtable.notifications.fetch_error'));
} finally {
setIsFetching(false);
const rows = useMemo(() => {
if (!recordingsData) return [];
const parsedRows = recordingsData
.map((recording: any, index: number) => {
if (recording?.recording_meta) {
const parsedDate = parseDateString(recording.recording_meta.updatedAt);
return {
id: index,
...recording.recording_meta,
content: recording.recording,
parsedDate
};
}
return null;
})
.filter(Boolean)
.sort((a, b) => b.parsedDate.getTime() - a.parsedDate.getTime());
return parsedRows;
}, [recordingsData]);
useEffect(() => {
if (rows.length > 0) {
setRecordings(rows.map((recording) => recording.name));
}
}, [setRecordings, notify, t]);
}, [rows, setRecordings]);
const handleNewRecording = useCallback(async () => {
const canCreateRecording = await canCreateBrowserInState("recording");
@@ -331,7 +328,7 @@ export const RecordingsTable = ({
if (lastPair?.what) {
if (Array.isArray(lastPair.what)) {
const gotoAction = lastPair.what.find(action =>
const gotoAction = lastPair.what.find((action: any) =>
action && typeof action === 'object' && 'action' in action && action.action === "goto"
) as any;
@@ -408,17 +405,12 @@ export const RecordingsTable = ({
window.sessionStorage.setItem('initialUrl', event.target.value);
}
useEffect(() => {
fetchRecordings();
}, [fetchRecordings]);
useEffect(() => {
if (rerenderRobots) {
fetchRecordings().then(() => {
setRerenderRobots(false);
});
refetch();
setRerenderRobots(false);
}
}, [rerenderRobots, fetchRecordings, setRerenderRobots]);
}, [rerenderRobots, setRerenderRobots, refetch]);
function useDebounce<T>(value: T, delay: number): T {
const [debouncedValue, setDebouncedValue] = React.useState<T>(value);
@@ -468,12 +460,11 @@ export const RecordingsTable = ({
const success = await deleteRecordingFromStorage(id);
if (success) {
setRows([]);
notify('success', t('recordingtable.notifications.delete_success'));
fetchRecordings();
refetch();
}
}
}), [handleRunRecording, handleScheduleRecording, handleIntegrateRecording, handleSettingsRecording, handleEditRobot, handleDuplicateRobot, handleRetrainRobot, notify, t]);
}), [handleRunRecording, handleScheduleRecording, handleIntegrateRecording, handleSettingsRecording, handleEditRobot, handleDuplicateRobot, handleRetrainRobot, notify, t, refetch]);
return (
<React.Fragment>

58
src/components/run/RunsTable.tsx
View File

@@ -13,8 +13,7 @@ import { Accordion, AccordionSummary, AccordionDetails, Typography, Box, TextFie
import ExpandMoreIcon from '@mui/icons-material/ExpandMore';
import SearchIcon from '@mui/icons-material/Search';
import { useLocation, useNavigate } from 'react-router-dom';
import { useGlobalInfoStore } from "../../context/globalInfo";
import { getStoredRuns } from "../../api/storage";
import { useGlobalInfoStore, useCachedRuns } from "../../context/globalInfo";
import { RunSettings } from "./RunSettings";
import { CollapsibleRow } from "./ColapsibleRow";
import { ArrowDownward, ArrowUpward, UnfoldMore } from '@mui/icons-material';
@@ -132,16 +131,14 @@ export const RunsTable: React.FC<RunsTableProps> = ({
[t]
);
const [rows, setRows] = useState<Data[]>([]);
const [searchTerm, setSearchTerm] = useState('');
const [isFetching, setIsFetching] = useState(true);
const { notify, rerenderRuns, setRerenderRuns } = useGlobalInfoStore();
const { data: rows = [], isLoading: isFetching, error, refetch } = useCachedRuns();
const [searchTerm, setSearchTerm] = useState('');
const [paginationStates, setPaginationStates] = useState<PaginationState>({});
const [expandedRows, setExpandedRows] = useState<Set<string>>(new Set());
const [expandedAccordions, setExpandedAccordions] = useState<Set<string>>(new Set());
const { notify, rerenderRuns, setRerenderRuns } = useGlobalInfoStore();
const handleAccordionChange = useCallback((robotMetaId: string, isExpanded: boolean) => {
setExpandedAccordions(prev => {
const newSet = new Set(prev);
@@ -278,47 +275,18 @@ export const RunsTable: React.FC<RunsTableProps> = ({
debouncedSetSearch(event.target.value);
}, [debouncedSearch]);
const fetchRuns = useCallback(async () => {
try {
const runs = await getStoredRuns();
if (runs) {
const parsedRows: Data[] = runs.map((run: any, index: number) => ({
id: index,
...run,
}));
setRows(parsedRows);
} else {
notify('error', t('runstable.notifications.no_runs'));
}
} catch (error) {
notify('error', t('runstable.notifications.fetch_error'));
} finally {
setIsFetching(false);
}
}, [notify, t]);
useEffect(() => {
let mounted = true;
if (rows.length === 0 || rerenderRuns) {
setIsFetching(true);
fetchRuns().then(() => {
if (mounted) {
setRerenderRuns(false);
}
});
if (rerenderRuns) {
refetch();
setRerenderRuns(false);
}
return () => {
mounted = false;
};
}, [rerenderRuns, rows.length, setRerenderRuns, fetchRuns]);
}, [rerenderRuns, setRerenderRuns, refetch]);
const handleDelete = useCallback(() => {
setRows([]);
notify('success', t('runstable.notifications.delete_success'));
fetchRuns();
}, [notify, t, fetchRuns]);
refetch();
}, [notify, t, refetch]);
// Filter rows based on search term
const filteredRows = useMemo(() => {
@@ -350,15 +318,15 @@ export const RunsTable: React.FC<RunsTableProps> = ({
}, {} as Record<string, Data[]>);
Object.keys(groupedData).forEach(robotId => {
groupedData[robotId].sort((a, b) =>
groupedData[robotId].sort((a: any, b: any) =>
parseDateString(b.startedAt).getTime() - parseDateString(a.startedAt).getTime()
);
});
const robotEntries = Object.entries(groupedData).map(([robotId, runs]) => ({
robotId,
runs,
latestRunDate: parseDateString(runs[0].startedAt).getTime()
runs: runs as Data[],
latestRunDate: parseDateString((runs as Data[])[0].startedAt).getTime()
}));
robotEntries.sort((a, b) => b.latestRunDate - a.latestRunDate);

119
src/context/globalInfo.tsx
View File

@@ -1,6 +1,24 @@
import React, { createContext, useContext, useState } from "react";
import { AlertSnackbarProps } from "../components/ui/AlertSnackbar";
import { WhereWhatPair } from "maxun-core";
import { QueryClient, QueryClientProvider, useQuery, useQueryClient } from '@tanstack/react-query';
import { getStoredRuns, getStoredRecordings } from "../api/storage";
const createDataCacheClient = () => new QueryClient({
defaultOptions: {
queries: {
staleTime: 30 * 1000,
gcTime: 5 * 60 * 1000,
retry: 2,
retryDelay: (attemptIndex) => Math.min(1000 * 2 ** attemptIndex, 30000),
}
}
});
const dataCacheKeys = {
runs: ['cached-runs'] as const,
recordings: ['cached-recordings'] as const,
} as const;
interface RobotMeta {
name: string;
@@ -164,6 +182,65 @@ const globalInfoContext = createContext<GlobalInfo>(globalInfoStore as GlobalInf
export const useGlobalInfoStore = () => useContext(globalInfoContext);
export const useCachedRuns = () => {
return useQuery({
queryKey: dataCacheKeys.runs,
queryFn: async () => {
const runs = await getStoredRuns();
if (!runs) throw new Error('Failed to fetch runs data');
return runs.map((run: any, index: number) => ({ id: index, ...run }));
},
staleTime: 30 * 1000,
gcTime: 5 * 60 * 1000,
retry: 2,
});
};
export const useCacheInvalidation = () => {
const queryClient = useQueryClient();
const invalidateRuns = () => {
queryClient.invalidateQueries({ queryKey: dataCacheKeys.runs });
};
const invalidateRecordings = () => {
queryClient.invalidateQueries({ queryKey: dataCacheKeys.recordings });
};
const addOptimisticRun = (newRun: any) => {
queryClient.setQueryData(dataCacheKeys.runs, (oldData: any) => {
if (!oldData) return [{ id: 0, ...newRun }];
return [{ id: oldData.length, ...newRun }, ...oldData];
});
};
const invalidateAllCache = () => {
invalidateRuns();
invalidateRecordings();
};
return {
invalidateRuns,
invalidateRecordings,
addOptimisticRun,
invalidateAllCache
};
};
export const useCachedRecordings = () => {
return useQuery({
queryKey: dataCacheKeys.recordings,
queryFn: async () => {
const recordings = await getStoredRecordings();
if (!recordings) throw new Error('Failed to fetch recordings data');
return recordings;
},
staleTime: 30 * 1000,
gcTime: 5 * 60 * 1000,
retry: 2,
});
};
export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
const [browserId, setBrowserId] = useState<string | null>(globalInfoStore.browserId);
const [lastAction, setLastAction] = useState<string>(globalInfoStore.lastAction);
@@ -172,7 +249,29 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
const [rerenderRuns, setRerenderRuns] = useState<boolean>(globalInfoStore.rerenderRuns);
const [rerenderRobots, setRerenderRobots] = useState<boolean>(globalInfoStore.rerenderRobots);
const [recordingLength, setRecordingLength] = useState<number>(globalInfoStore.recordingLength);
const [recordingId, setRecordingId] = useState<string | null>(globalInfoStore.recordingId);
// const [recordingId, setRecordingId] = useState<string | null>(globalInfoStore.recordingId);
const [recordingId, setRecordingId] = useState<string | null>(() => {
try {
const stored = sessionStorage.getItem('recordingId');
return stored ? JSON.parse(stored) : globalInfoStore.recordingId;
} catch {
return globalInfoStore.recordingId;
}
});
// Create a wrapped setter that persists to sessionStorage
const setPersistedRecordingId = (newRecordingId: string | null) => {
setRecordingId(newRecordingId);
try {
if (newRecordingId) {
sessionStorage.setItem('recordingId', JSON.stringify(newRecordingId));
} else {
sessionStorage.removeItem('recordingId');
}
} catch (error) {
console.warn('Failed to persist recordingId to sessionStorage:', error);
}
};
const [retrainRobotId, setRetrainRobotId] = useState<string | null>(globalInfoStore.retrainRobotId);
const [recordingName, setRecordingName] = useState<string>(globalInfoStore.recordingName);
const [isLogin, setIsLogin] = useState<boolean>(globalInfoStore.isLogin);
@@ -221,9 +320,12 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
}
}
const [dataCacheClient] = useState(() => createDataCacheClient());
return (
<globalInfoContext.Provider
value={{
<QueryClientProvider client={dataCacheClient}>
<globalInfoContext.Provider
value={{
browserId,
setBrowserId: setBrowserIdWithValidation,
lastAction,
@@ -240,7 +342,7 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
recordingLength,
setRecordingLength,
recordingId,
setRecordingId,
setRecordingId: setPersistedRecordingId,
retrainRobotId,
setRetrainRobotId,
recordingName,
@@ -266,9 +368,10 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
currentSnapshot,
setCurrentSnapshot,
updateDOMMode,
}}
>
{children}
</globalInfoContext.Provider>
}}
>
{children}
</globalInfoContext.Provider>
</QueryClientProvider>
);
};

21
src/pages/MainPage.tsx
View File

@@ -6,7 +6,7 @@ import { Recordings } from "../components/robot/Recordings";
import { Runs } from "../components/run/Runs";
import ProxyForm from '../components/proxy/ProxyForm';
import ApiKey from '../components/api/ApiKey';
import { useGlobalInfoStore } from "../context/globalInfo";
import { useGlobalInfoStore, useCacheInvalidation } from "../context/globalInfo";
import { createAndRunRecording, createRunForStoredRecording, CreateRunResponseWithQueue, interpretStoredRecording, notifyAboutAbort, scheduleStoredRecording } from "../api/storage";
import { io, Socket } from "socket.io-client";
import { stopRecording } from "../api/recording";
@@ -50,6 +50,7 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
let aborted = false;
const { notify, setRerenderRuns, setRecordingId } = useGlobalInfoStore();
const { invalidateRuns } = useCacheInvalidation();
const navigate = useNavigate();
const { state } = useContext(AuthContext);
@@ -66,12 +67,14 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
if (!response.success) {
notify('error', t('main_page.notifications.abort_failed', { name: robotName }));
setRerenderRuns(true);
invalidateRuns();
return;
}
if (response.isQueued) {
setRerenderRuns(true);
invalidateRuns();
notify('success', t('main_page.notifications.abort_success', { name: robotName }));
setQueuedRuns(prev => {
@@ -92,6 +95,7 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
if (abortData.runId === runId) {
notify('success', t('main_page.notifications.abort_success', { name: abortData.robotName || robotName }));
setRerenderRuns(true);
invalidateRuns();
abortSocket.disconnect();
}
});
@@ -100,6 +104,7 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
console.log('Abort socket connection error:', error);
notify('error', t('main_page.notifications.abort_failed', { name: robotName }));
setRerenderRuns(true);
invalidateRuns();
abortSocket.disconnect();
});
});
@@ -125,8 +130,9 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
setRunningRecordingName('');
setCurrentInterpretationLog('');
setRerenderRuns(true);
invalidateRuns();
})
}, [runningRecordingName, aborted, currentInterpretationLog, notify, setRerenderRuns]);
}, [runningRecordingName, aborted, currentInterpretationLog, notify, setRerenderRuns, invalidateRuns]);
const debugMessageHandler = useCallback((msg: string) => {
setCurrentInterpretationLog((prevState) =>
@@ -136,8 +142,9 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
const handleRunRecording = useCallback((settings: RunSettings) => {
createAndRunRecording(runningRecordingId, settings).then((response: CreateRunResponseWithQueue) => {
const { browserId, runId, robotMetaId, queued } = response;
setIds({ browserId, runId, robotMetaId });
invalidateRuns();
navigate(`/runs/${robotMetaId}/run/${runId}`);
if (queued) {
@@ -156,6 +163,7 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
setRunningRecordingName('');
setCurrentInterpretationLog('');
setRerenderRuns(true);
invalidateRuns();
const robotName = data.robotName;
@@ -193,7 +201,7 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
socket.off('connect_error');
socket.off('disconnect');
}
}, [runningRecordingName, sockets, ids, debugMessageHandler, user?.id, t, notify, setRerenderRuns, setQueuedRuns, navigate, setContent, setIds]);
}, [runningRecordingName, sockets, ids, debugMessageHandler, user?.id, t, notify, setRerenderRuns, setQueuedRuns, navigate, setContent, setIds, invalidateRuns]);
const handleScheduleRecording = async (settings: ScheduleSettings) => {
const { message, runId }: ScheduleRunResponse = await scheduleStoredRecording(runningRecordingId, settings);
@@ -209,6 +217,7 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
if (user?.id) {
const handleRunCompleted = (completionData: any) => {
setRerenderRuns(true);
invalidateRuns();
if (queuedRuns.has(completionData.runId)) {
setQueuedRuns(prev => {
@@ -233,7 +242,7 @@ export const MainPage = ({ handleEditRecording, initialContent }: MainPageProps)
disconnectQueueSocket();
};
}
}, [user?.id, connectToQueueSocket, disconnectQueueSocket, t, setRerenderRuns, queuedRuns, setQueuedRuns]);
}, [user?.id, connectToQueueSocket, disconnectQueueSocket, t, setRerenderRuns, queuedRuns, setQueuedRuns, invalidateRuns]);
const DisplayContent = () => {
switch (content) {