From 09064e9533d84450462d3675a2b0a50370b83dbe Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 25 Jun 2025 12:44:27 +0530 Subject: [PATCH 1/7] feat: add update screenshot step data --- src/context/browserSteps.tsx | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/context/browserSteps.tsx b/src/context/browserSteps.tsx index 6d8b8f07..fa8358be 100644 --- a/src/context/browserSteps.tsx +++ b/src/context/browserSteps.tsx @@ -6,14 +6,15 @@ export interface TextStep { label: string; data: string; selectorObj: SelectorObject; - actionId?: string; // Add actionId to track which action created this step + actionId?: string; } interface ScreenshotStep { id: number; type: 'screenshot'; fullPage: boolean; - actionId?: string; // Add actionId to track which action created this step + actionId?: string; + screenshotData?: string; } export interface ListStep { @@ -26,7 +27,7 @@ export interface ListStep { selector: string; }; limit?: number; - actionId?: string; // Add actionId to track which action created this step + actionId?: string; } export type BrowserStep = TextStep | ScreenshotStep | ListStep; @@ -50,7 +51,8 @@ interface BrowserStepsContextType { updateListStepLimit: (listId: number, limit: number) => void; updateListStepData: (listId: number, extractedData: any[]) => void; removeListTextField: (listId: number, fieldKey: string) => void; - deleteStepsByActionId: (actionId: string) => void; // New function to delete steps by actionId + deleteStepsByActionId: (actionId: string) => void; + updateScreenshotStepData: (id: number, screenshotData: string) => void; } const BrowserStepsContext = createContext(undefined); @@ -179,6 +181,20 @@ export const BrowserStepsProvider: React.FC<{ children: React.ReactNode }> = ({ }); }; + const updateScreenshotStepData = (id: number, screenshotData: string) => { + setBrowserSteps(prevSteps => { + return prevSteps.map(step => { + if (step.type === 'screenshot' && step.id === id) { + return { + ...step, + screenshotData: screenshotData + }; + } + return step; + }); + }); + }; + const updateListStepLimit = (listId: number, limit: number) => { setBrowserSteps(prevSteps => prevSteps.map(step => { @@ -220,7 +236,8 @@ export const BrowserStepsProvider: React.FC<{ children: React.ReactNode }> = ({ updateListStepLimit, updateListStepData, removeListTextField, - deleteStepsByActionId, // Export the new function + deleteStepsByActionId, + updateScreenshotStepData, }}> {children} From 1a6dc3221aa8764755598362b82459e8611a2416 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 25 Jun 2025 12:50:03 +0530 Subject: [PATCH 2/7] feat: emit direct ss capture socket --- src/components/recorder/RightSidePanel.tsx | 36 ++++++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/src/components/recorder/RightSidePanel.tsx b/src/components/recorder/RightSidePanel.tsx index e7268b09..d405e58c 100644 --- a/src/components/recorder/RightSidePanel.tsx +++ b/src/components/recorder/RightSidePanel.tsx @@ -72,7 +72,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture startAction, finishAction } = useActionContext(); - const { browserSteps, updateBrowserTextStepLabel, deleteBrowserStep, addScreenshotStep, updateListTextFieldLabel, removeListTextField, updateListStepLimit, deleteStepsByActionId, updateListStepData } = useBrowserSteps(); + const { browserSteps, updateBrowserTextStepLabel, deleteBrowserStep, addScreenshotStep, updateListTextFieldLabel, removeListTextField, updateListStepLimit, deleteStepsByActionId, updateListStepData, updateScreenshotStepData } = useBrowserSteps(); const { id, socket } = useSocketStore(); const { t } = useTranslation(); @@ -183,6 +183,29 @@ export const RightSidePanel: React.FC = ({ onFinishCapture }; }, [socket, updateListStepData, isDOMMode]); + useEffect(() => { + if (socket) { + const handleDirectScreenshot = (data: any) => { + const screenshotSteps = browserSteps.filter(step => + step.type === 'screenshot' && step.actionId === currentScreenshotActionId + ); + + if (screenshotSteps.length > 0) { + const latestStep = screenshotSteps[screenshotSteps.length - 1]; + updateScreenshotStepData(latestStep.id, data.screenshot); + } + + setCurrentScreenshotActionId(''); + }; + + socket.on('directScreenshotCaptured', handleDirectScreenshot); + + return () => { + socket.off('directScreenshotCaptured', handleDirectScreenshot); + }; + } + }, [socket, id, notify, t, currentScreenshotActionId, updateScreenshotStepData, setCurrentScreenshotActionId]); + const extractDataClientSide = useCallback( ( listSelector: string, @@ -649,14 +672,15 @@ export const RightSidePanel: React.FC = ({ onFinishCapture }, [currentListActionId, browserSteps, stopGetList, deleteStepsByActionId, resetListState, setShowPaginationOptions, setShowLimitOptions, setCaptureStage, notify, t]); const captureScreenshot = (fullPage: boolean) => { - const screenshotSettings: ScreenshotSettings = { + const screenshotSettings = { fullPage, - type: 'png', + type: 'png' as const, timeout: 30000, - animations: 'allow', - caret: 'hide', - scale: 'device', + animations: 'allow' as const, + caret: 'hide' as const, + scale: 'device' as const, }; + socket?.emit('captureDirectScreenshot', screenshotSettings); socket?.emit('action', { action: 'screenshot', settings: screenshotSettings }); addScreenshotStep(fullPage, currentScreenshotActionId); stopGetScreenshot(); From 6768dda86398014b7294243fe773258947bd7dbd Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 25 Jun 2025 12:53:47 +0530 Subject: [PATCH 3/7] feat: rm socket logic, add ss data --- src/components/run/InterpretationLog.tsx | 121 +++++++---------------- 1 file changed, 33 insertions(+), 88 deletions(-) diff --git a/src/components/run/InterpretationLog.tsx b/src/components/run/InterpretationLog.tsx index e11e0151..0f1f1185 100644 --- a/src/components/run/InterpretationLog.tsx +++ b/src/components/run/InterpretationLog.tsx @@ -3,8 +3,6 @@ import SwipeableDrawer from '@mui/material/SwipeableDrawer'; import Typography from '@mui/material/Typography'; import { Button, Grid, Box } from '@mui/material'; import { useCallback, useEffect, useRef, useState } from "react"; -import { useSocketStore } from "../../context/socket"; -import { Buffer } from 'buffer'; import { useBrowserDimensionsStore } from "../../context/browserDimensions"; import Table from '@mui/material/Table'; import TableBody from '@mui/material/TableBody'; @@ -28,8 +26,6 @@ interface InterpretationLogProps { export const InterpretationLog: React.FC = ({ isOpen, setIsOpen }) => { const { t } = useTranslation(); - const [log, setLog] = useState(''); - const [customValue, setCustomValue] = useState(''); const [captureListData, setCaptureListData] = useState([]); const [captureTextData, setCaptureTextData] = useState([]); @@ -44,11 +40,10 @@ export const InterpretationLog: React.FC = ({ isOpen, se const { browserSteps } = useBrowserSteps(); - const [activeActionId, setActiveActionId] = useState(null); - const { browserWidth, outputPreviewHeight, outputPreviewWidth } = useBrowserDimensionsStore(); - const { socket } = useSocketStore(); - const { currentWorkflowActionsState, shouldResetInterpretationLog, notify } = useGlobalInfoStore(); + const { currentWorkflowActionsState, shouldResetInterpretationLog } = useGlobalInfoStore(); + + const [showPreviewData, setShowPreviewData] = useState(false); const toggleDrawer = (newOpen: boolean) => (event: React.KeyboardEvent | React.MouseEvent) => { if ( @@ -61,43 +56,6 @@ export const InterpretationLog: React.FC = ({ isOpen, se setIsOpen(newOpen); }; - const scrollLogToBottom = () => { - if (logEndRef.current) { - logEndRef.current.scrollIntoView({ behavior: "smooth" }); - } - }; - - const handleLog = useCallback((msg: string, date: boolean = true) => { - if (!date) { - setLog((prevState) => prevState + '\n' + msg); - } else { - setLog((prevState) => prevState + '\n' + `[${new Date().toLocaleString()}] ` + msg); - } - scrollLogToBottom(); - }, []); - - useEffect(() => { - if (activeActionId !== null) { - const textSteps = browserSteps.filter(step => step.type === 'text'); - if (textSteps.length > 0) { - const textDataRow: Record = {}; - - textSteps.forEach(step => { - textDataRow[step.label] = step.data; - }); - - setCaptureTextData([textDataRow]); - } - - const listSteps = browserSteps.filter(step => step.type === 'list'); - if (listSteps.length > 0) { - setCaptureListData(listSteps); - } - - updateActiveTab(); - } - }, [activeActionId, browserSteps, t]); - const updateActiveTab = useCallback(() => { const availableTabs = getAvailableTabs(); @@ -109,62 +67,49 @@ export const InterpretationLog: React.FC = ({ isOpen, se setActiveTab(availableTabs.findIndex(tab => tab.id === 'captureScreenshot')); } }, [captureListData.length, captureTextData.length, screenshotData.length]); - - const handleBinaryCallback = useCallback(({ data, mimetype, type }: { data: any, mimetype: string, type: string }) => { - const base64String = Buffer.from(data).toString('base64'); - const imageSrc = `data:${mimetype};base64,${base64String}`; - - setLog((prevState) => - prevState + '\n' + t('interpretation_log.data_sections.binary_received') + '\n' - + t('interpretation_log.data_sections.mimetype') + mimetype + '\n' - + t('interpretation_log.data_sections.image_below') + '\n' - + t('interpretation_log.data_sections.separator')); - - if (type === 'captureScreenshot') { - setScreenshotData(prev => [...prev, imageSrc]); - if (screenshotData.length === 0) { - const availableTabs = getAvailableTabs(); - const tabIndex = availableTabs.findIndex(tab => tab.id === 'captureScreenshot'); - if (tabIndex !== -1) setActiveTab(tabIndex); - } + + useEffect(() => { + const textSteps = browserSteps.filter(step => step.type === 'text'); + if (textSteps.length > 0) { + const textDataRow: Record = {}; + + textSteps.forEach(step => { + textDataRow[step.label] = step.data; + }); + + setCaptureTextData([textDataRow]); } - scrollLogToBottom(); - }, [screenshotData.length, t]); + const listSteps = browserSteps.filter(step => step.type === 'list'); + if (listSteps.length > 0) { + setCaptureListData(listSteps); + } - const handleActivePairId = useCallback((id: number) => { - setActiveActionId(id); - }, []); + const screenshotSteps = browserSteps.filter(step => + step.type === 'screenshot' + ) as Array<{ type: 'screenshot'; id: number; fullPage: boolean; actionId?: string; screenshotData?: string }>; - const handleCustomValueChange = (event: React.ChangeEvent) => { - setCustomValue(event.target.value); - }; + const screenshotsWithData = screenshotSteps.filter(step => step.screenshotData); + if (screenshotsWithData.length > 0) { + const screenshots = screenshotsWithData.map(step => step.screenshotData!); + setScreenshotData(screenshots); + } + + updateActiveTab(); + }, [browserSteps, updateActiveTab]); useEffect(() => { if (shouldResetInterpretationLog) { - setLog(''); setCaptureListData([]); setCaptureTextData([]); setScreenshotData([]); setActiveTab(0); setCaptureListPage(0); setScreenshotPage(0); - setActiveActionId(null); + setShowPreviewData(false); } }, [shouldResetInterpretationLog]); - useEffect(() => { - socket?.on('log', handleLog); - socket?.on('binaryCallback', handleBinaryCallback); - socket?.on('activePairId', handleActivePairId); - - return () => { - socket?.off('log', handleLog); - socket?.off('binaryCallback', handleBinaryCallback); - socket?.off('activePairId', handleActivePairId); - }; - }, [socket, handleLog, handleBinaryCallback, handleActivePairId]); - const getAvailableTabs = useCallback(() => { const tabs = []; @@ -181,7 +126,7 @@ export const InterpretationLog: React.FC = ({ isOpen, se } return tabs; - }, [captureListData.length, captureTextData.length, screenshotData.length]); + }, [captureListData.length, captureTextData.length, screenshotData.length, showPreviewData]); const availableTabs = getAvailableTabs(); @@ -264,7 +209,7 @@ export const InterpretationLog: React.FC = ({ isOpen, se {t('interpretation_log.titles.output_preview')} - {availableTabs.length > 0 ? ( + {showPreviewData && availableTabs.length > 0 ? ( <> {shouldShowTabs && ( = ({ isOpen, se {t('interpretation_log.messages.successful_training')} - + setShowPreviewData(true)} /> ) : ( From d5f54408b536a5a4b357cc39db5dfb8c04d4bf43 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 25 Jun 2025 12:56:51 +0530 Subject: [PATCH 4/7] feat: add on preview click --- src/components/recorder/SidePanelHeader.tsx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/components/recorder/SidePanelHeader.tsx b/src/components/recorder/SidePanelHeader.tsx index 41b33ae7..889a51e3 100644 --- a/src/components/recorder/SidePanelHeader.tsx +++ b/src/components/recorder/SidePanelHeader.tsx @@ -2,7 +2,11 @@ import React, { FC, useState } from 'react'; import { InterpretationButtons } from "../run/InterpretationButtons"; import { useSocketStore } from "../../context/socket"; -export const SidePanelHeader = () => { +interface SidePanelHeaderProps { + onPreviewClick?: () => void; +} + +export const SidePanelHeader = ({ onPreviewClick }: SidePanelHeaderProps) => { const [steppingIsDisabled, setSteppingIsDisabled] = useState(true); @@ -14,7 +18,10 @@ export const SidePanelHeader = () => { return (
- setSteppingIsDisabled(!isPaused)} /> + setSteppingIsDisabled(!isPaused)} + onPreviewComplete={onPreviewClick} + /> {/*