Merge pull request #478 from getmaxun/remote-resize

Feat: Resized Browser
2025-03-18 19:58:58 +05:30
parent 58da9dabc6 29706ba36d
commit 10ecd2db91
14 changed files with 395 additions and 196 deletions
--- a/src/components/browser/BrowserContent.tsx
+++ b/src/components/browser/BrowserContent.tsx
@@ -11,12 +11,12 @@ import {

 // TODO: Tab !show currentUrl after recordingUrl global state
 export const BrowserContent = () => {
-  const { width } = useBrowserDimensionsStore();
  const { socket } = useSocketStore();

  const [tabs, setTabs] = useState<string[]>(["current"]);
  const [tabIndex, setTabIndex] = React.useState(0);
  const [showOutputData, setShowOutputData] = useState(false);
+  const { browserWidth } = useBrowserDimensionsStore();

  const handleChangeIndex = useCallback(
    (index: number) => {
@@ -146,7 +146,7 @@ export const BrowserContent = () => {
      />
      <BrowserNavBar
        // todo: use width from browser dimension once fixed
-        browserWidth={900}
+        browserWidth={browserWidth}
        handleUrlChanged={handleUrlChanged}

      />
--- a/src/components/browser/BrowserTabs.tsx
+++ b/src/components/browser/BrowserTabs.tsx
@@ -32,7 +32,8 @@ export const BrowserTabs = (

  return (
    <Box sx={{
-      width: 800, // Fixed width
+      width: '100%',
+      maxWidth: '100%',
      display: 'flex',
      overflow: 'auto',
      alignItems: 'center',
--- a/src/components/browser/BrowserWindow.tsx
+++ b/src/components/browser/BrowserWindow.tsx
@@ -9,6 +9,8 @@ import { useBrowserSteps, TextStep } from '../../context/browserSteps';
 import { useGlobalInfoStore } from '../../context/globalInfo';
 import { useTranslation } from 'react-i18next';
 import { AuthContext } from '../../context/auth';
+import { coordinateMapper } from '../../helpers/coordinateMapper';
+import { useBrowserDimensionsStore } from '../../context/browserDimensions';

 interface ElementInfo {
    tagName: string;
@@ -31,6 +33,12 @@ interface AttributeOption {
 interface ScreencastData {
    image: string;
    userId: string;
+    viewport?: ViewportInfo | null;
+}
+
+interface ViewportInfo {
+    width: number;
+    height: number;
 }


@@ -62,6 +70,7 @@ const getAttributeOptions = (tagName: string, elementInfo: ElementInfo | null):

 export const BrowserWindow = () => {
    const { t } = useTranslation();
+    const { browserWidth, browserHeight } = useBrowserDimensionsStore();
    const [canvasRef, setCanvasReference] = useState<React.RefObject<HTMLCanvasElement> | undefined>(undefined);
    const [screenShot, setScreenShot] = useState<string>("");
    const [highlighterData, setHighlighterData] = useState<{ rect: DOMRect, selector: string, elementInfo: ElementInfo | null, childSelectors?: string[] } | null>(null);
@@ -69,6 +78,7 @@ export const BrowserWindow = () => {
    const [attributeOptions, setAttributeOptions] = useState<AttributeOption[]>([]);
    const [selectedElement, setSelectedElement] = useState<{ selector: string, info: ElementInfo | null } | null>(null);
    const [currentListId, setCurrentListId] = useState<number | null>(null);
+    const [viewportInfo, setViewportInfo] = useState<ViewportInfo>({ width: browserWidth, height: browserHeight });

    const [listSelector, setListSelector] = useState<string | null>(null);
    const [fields, setFields] = useState<Record<string, TextStep>>({});
@@ -82,6 +92,15 @@ export const BrowserWindow = () => {
    const { state } = useContext(AuthContext);
    const { user } = state;

+    const dimensions = {
+        width: browserWidth,
+        height: browserHeight
+    };
+
+    useEffect(() => {
+        coordinateMapper.updateDimensions(dimensions.width, dimensions.height, viewportInfo.width, viewportInfo.height);
+    }, [viewportInfo, dimensions.width, dimensions.height]);
+
    useEffect(() => {
        if (listSelector) {
          window.sessionStorage.setItem('recordingListSelector', listSelector);
@@ -130,6 +149,10 @@ export const BrowserWindow = () => {
        } else if (data && typeof data === 'object' && 'image' in data) {
            if (!data.userId || data.userId === user?.id) {
                setScreenShot(data.image);
+                
+                if (data.viewport) {
+                    setViewportInfo(data.viewport);
+                }
            }
        }
    }, [screenShot, user?.id]);
@@ -149,78 +172,85 @@ export const BrowserWindow = () => {
    }, [screenShot, canvasRef, socket, screencastHandler]);

    const highlighterHandler = useCallback((data: { rect: DOMRect, selector: string, elementInfo: ElementInfo | null, childSelectors?: string[] }) => {
+        // Map the incoming DOMRect from browser coordinates to canvas coordinates
+        const mappedRect = new DOMRect(
+            data.rect.x,
+            data.rect.y,
+            data.rect.width,
+            data.rect.height
+        );
+        
+        const mappedData = {
+            ...data,
+            rect: mappedRect
+        };
+        
        if (getList === true) {
            if (listSelector) {
                socket?.emit('listSelector', { selector: listSelector });
-                const hasValidChildSelectors = Array.isArray(data.childSelectors) && data.childSelectors.length > 0;
+                const hasValidChildSelectors = Array.isArray(mappedData.childSelectors) && mappedData.childSelectors.length > 0;

                if (limitMode) {
                    setHighlighterData(null);
                } else if (paginationMode) {
                    // Only set highlighterData if type is not empty, 'none', 'scrollDown', or 'scrollUp'
                    if (paginationType !== '' && !['none', 'scrollDown', 'scrollUp'].includes(paginationType)) {
-                        setHighlighterData(data);
+                        setHighlighterData(mappedData);
                    } else {
                        setHighlighterData(null);
                    }
-                } else if (data.childSelectors && data.childSelectors.includes(data.selector)) {
+                } else if (mappedData.childSelectors && mappedData.childSelectors.includes(mappedData.selector)) {
                    // Highlight only valid child elements within the listSelector
-                    setHighlighterData(data);
-                } else if (data.elementInfo?.isIframeContent && data.childSelectors) {
-                    // Handle pure iframe elements - similar to previous shadow DOM logic but using iframe syntax
-                    // Check if the selector matches any iframe child selectors
-                    const isIframeChild = data.childSelectors.some(childSelector =>
-                        data.selector.includes(':>>') && // Iframe uses :>> for traversal
+                    setHighlighterData(mappedData);
+                } else if (mappedData.elementInfo?.isIframeContent && mappedData.childSelectors) {
+                    // Handle iframe elements
+                    const isIframeChild = mappedData.childSelectors.some(childSelector =>
+                        mappedData.selector.includes(':>>') && 
                        childSelector.split(':>>').some(part =>
-                            data.selector.includes(part.trim())
+                            mappedData.selector.includes(part.trim())
                        )
                    );
-                    setHighlighterData(isIframeChild ? data : null);
-                } else if (data.selector.includes(':>>') && hasValidChildSelectors) {
+                    setHighlighterData(isIframeChild ? mappedData : null);
+                } else if (mappedData.selector.includes(':>>') && hasValidChildSelectors) {
                    // Handle mixed DOM cases with iframes
-                    // Split the selector into parts and check each against child selectors
-                    const selectorParts = data.selector.split(':>>').map(part => part.trim());
+                    const selectorParts = mappedData.selector.split(':>>').map(part => part.trim());
                    const isValidMixedSelector = selectorParts.some(part =>
-                        // We know data.childSelectors is defined due to hasValidChildSelectors check
-                        data.childSelectors!.some(childSelector =>
+                        mappedData.childSelectors!.some(childSelector =>
                            childSelector.includes(part)
                        )
                    );
-                    setHighlighterData(isValidMixedSelector ? data : null);
-                } else if (data.elementInfo?.isShadowRoot && data.childSelectors) {
-                    // New case: Handle pure Shadow DOM elements
-                    // Check if the selector matches any shadow root child selectors
-                    const isShadowChild = data.childSelectors.some(childSelector =>
-                        data.selector.includes('>>') && // Shadow DOM uses >> for piercing
+                    setHighlighterData(isValidMixedSelector ? mappedData : null);
+                } else if (mappedData.elementInfo?.isShadowRoot && mappedData.childSelectors) {
+                    // Handle Shadow DOM elements
+                    const isShadowChild = mappedData.childSelectors.some(childSelector =>
+                        mappedData.selector.includes('>>') &&
                        childSelector.split('>>').some(part =>
-                            data.selector.includes(part.trim())
+                            mappedData.selector.includes(part.trim())
                        )
                    );
-                    setHighlighterData(isShadowChild ? data : null);
-                } else if (data.selector.includes('>>') && hasValidChildSelectors) {
-                    // New case: Handle mixed DOM cases
-                    // Split the selector into parts and check each against child selectors
-                    const selectorParts = data.selector.split('>>').map(part => part.trim());
+                    setHighlighterData(isShadowChild ? mappedData : null);
+                } else if (mappedData.selector.includes('>>') && hasValidChildSelectors) {
+                    // Handle mixed DOM cases
+                    const selectorParts = mappedData.selector.split('>>').map(part => part.trim());
                    const isValidMixedSelector = selectorParts.some(part =>
-                        // Now we know data.childSelectors is defined
-                        data.childSelectors!.some(childSelector =>
+                        mappedData.childSelectors!.some(childSelector =>
                            childSelector.includes(part)
                        )
                    );
-                    setHighlighterData(isValidMixedSelector ? data : null);
+                    setHighlighterData(isValidMixedSelector ? mappedData : null);
                } else {
-                    // if !valid child in normal mode, clear the highlighter
+                    // If not a valid child in normal mode, clear the highlighter
                    setHighlighterData(null);
                }
            } else {
                // Set highlighterData for the initial listSelector selection
-                setHighlighterData(data);
+                setHighlighterData(mappedData);
            }
        } else {
            // For non-list steps
-            setHighlighterData(data);
+            setHighlighterData(mappedData);
        }
-    }, [highlighterData, getList, socket, listSelector, paginationMode, paginationType, captureStage]);
+    }, [getList, socket, listSelector, paginationMode, paginationType, limitMode]);


    useEffect(() => {
@@ -260,11 +290,13 @@ export const BrowserWindow = () => {
            const clickY = e.clientY - canvasRect.top;

            const highlightRect = highlighterData.rect;
+
+            const mappedRect = coordinateMapper.mapBrowserRectToCanvas(highlightRect);
            if (
-                clickX >= highlightRect.left &&
-                clickX <= highlightRect.right &&
-                clickY >= highlightRect.top &&
-                clickY <= highlightRect.bottom
+                clickX >= mappedRect.left &&
+                clickX <= mappedRect.right &&
+                clickY >= mappedRect.top &&
+                clickY <= mappedRect.bottom
            ) {

                const options = getAttributeOptions(highlighterData.elementInfo?.tagName || '', highlighterData.elementInfo);
@@ -437,7 +469,7 @@ export const BrowserWindow = () => {
    }, [paginationMode, resetPaginationSelector]);

    return (
-        <div onClick={handleClick} style={{ width: '900px' }} id="browser-window">
+        <div onClick={handleClick} style={{ width: browserWidth }} id="browser-window">
            {
                getText === true || getList === true ? (
                    <GenericModal
@@ -483,20 +515,20 @@ export const BrowserWindow = () => {
                    </GenericModal>
                ) : null
            }
-            <div style={{ height: '400px', overflow: 'hidden' }}>
+            <div style={{ height: dimensions.height, overflow: 'hidden' }}>
                {((getText === true || getList === true) && !showAttributeModal && highlighterData?.rect != null && highlighterData?.rect.top != null) && canvasRef?.current ?
                    <Highlighter
                        unmodifiedRect={highlighterData?.rect}
                        displayedSelector={highlighterData?.selector}
-                        width={900}
-                        height={400}
+                        width={dimensions.width}
+                        height={dimensions.height}
                        canvasRect={canvasRef.current.getBoundingClientRect()}
                    />
                    : null}
                <Canvas
                    onCreateRef={setCanvasReference}
-                    width={900}
-                    height={400}
+                    width={dimensions.width}
+                    height={dimensions.height}
                />
            </div>
        </div>
@@ -512,7 +544,7 @@ const drawImage = (image: string, canvas: HTMLCanvasElement): void => {
    img.src = image;
    img.onload = () => {
        URL.revokeObjectURL(img.src);
-        ctx?.drawImage(img, 0, 0, 900, 400);
+        ctx?.drawImage(img, 0, 0, canvas.width, canvas.height);
    };

 };
--- a/src/components/recorder/Highlighter.tsx
+++ b/src/components/recorder/Highlighter.tsx
@@ -1,5 +1,6 @@
 import React from 'react';
 import styled from "styled-components";
+import { coordinateMapper } from '../../helpers/coordinateMapper';

 interface HighlighterProps {
  unmodifiedRect: DOMRect;
@@ -13,13 +14,15 @@ export const Highlighter = ({ unmodifiedRect, displayedSelector = '', width, hei
  if (!unmodifiedRect) {
    return null;
  } else {
+    const mappedRect = coordinateMapper.mapBrowserRectToCanvas(unmodifiedRect);
+
    const rect = {
-      top: unmodifiedRect.top + canvasRect.top + window.scrollY,
-      left: unmodifiedRect.left + canvasRect.left + window.scrollX,
-      right: unmodifiedRect.right + canvasRect.left,
-      bottom: unmodifiedRect.bottom + canvasRect.top,
-      width: unmodifiedRect.width,
-      height: unmodifiedRect.height,
+      top: mappedRect.top + canvasRect.top + window.scrollY,
+      left: mappedRect.left + canvasRect.left + window.scrollX,
+      right: mappedRect.right + canvasRect.left,
+      bottom: mappedRect.bottom + canvasRect.top,
+      width: mappedRect.width,
+      height: mappedRect.height,
    };


--- a/src/components/recorder/RightSidePanel.tsx
+++ b/src/components/recorder/RightSidePanel.tsx
@@ -20,6 +20,7 @@ import { getActiveWorkflow } from "../../api/workflow";
 import ActionDescriptionBox from '../action/ActionDescriptionBox';
 import { useThemeMode } from '../../context/theme-provider';
 import { useTranslation } from 'react-i18next';
+import { useBrowserDimensionsStore } from '../../context/browserDimensions';

 const fetchWorkflow = (id: string, callback: (response: WorkflowFile) => void) => {
  getActiveWorkflow(id).then(
@@ -54,6 +55,7 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
  const [browserStepIdList, setBrowserStepIdList] = useState<number[]>([]);
  const [isCaptureTextConfirmed, setIsCaptureTextConfirmed] = useState(false);
  const [isCaptureListConfirmed, setIsCaptureListConfirmed] = useState(false);
+  const { panelHeight } = useBrowserDimensionsStore();

  const { lastAction, notify, currentWorkflowActionsState, setCurrentWorkflowActionsState, resetInterpretationLog } = useGlobalInfoStore();
  const { getText, startGetText, stopGetText, getScreenshot, startGetScreenshot, stopGetScreenshot, getList, startGetList, stopGetList, startPaginationMode, stopPaginationMode, paginationType, updatePaginationType, limitType, customLimit, updateLimitType, updateCustomLimit, stopLimitMode, startLimitMode, captureStage, setCaptureStage, showPaginationOptions, setShowPaginationOptions, showLimitOptions, setShowLimitOptions, workflow, setWorkflow } = useActionContext();
@@ -460,8 +462,9 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture

  const theme = useThemeMode();
  const isDarkMode = theme.darkMode;
+
  return (
-    <Paper sx={{ height: '520px', width: 'auto', alignItems: "center", background: 'inherit' }} id="browser-actions" elevation={0}>
+    <Paper sx={{ height: panelHeight, width: 'auto', alignItems: "center", background: 'inherit' }} id="browser-actions" elevation={0}>
      {/* <SimpleBox height={60} width='100%' background='lightGray' radius='0%'>
        <Typography sx={{ padding: '10px' }}>Last action: {` ${lastAction}`}</Typography>
      </SimpleBox> */}
--- a/src/components/recorder/canvas.tsx
+++ b/src/components/recorder/canvas.tsx
@@ -6,6 +6,7 @@ import DatePicker from '../pickers/DatePicker';
 import Dropdown from '../pickers/Dropdown';
 import TimePicker from '../pickers/TimePicker';
 import DateTimeLocalPicker from '../pickers/DateTimeLocalPicker';
+import { coordinateMapper } from '../../helpers/coordinateMapper';

 interface CreateRefCallback {
    (ref: React.RefObject<HTMLCanvasElement>): void;
@@ -79,7 +80,11 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
    useEffect(() => {
        if (socket) {
            socket.on('showDatePicker', (info: { coordinates: Coordinates, selector: string }) => {
-                setDatePickerInfo(info);
+                const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates);
+                setDatePickerInfo({
+                    ...info,
+                    coordinates: canvasCoords
+                });
            });

            socket.on('showDropdown', (info: {
@@ -92,15 +97,27 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
                    selected: boolean;
                }>;
            }) => {
-                setDropdownInfo(info);
+                const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates);
+                setDropdownInfo({
+                    ...info,
+                    coordinates: canvasCoords
+                });
            });

            socket.on('showTimePicker', (info: { coordinates: Coordinates, selector: string }) => {
-                setTimePickerInfo(info);
+                const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates);
+                setTimePickerInfo({
+                    ...info,
+                    coordinates: canvasCoords
+                });
            });

            socket.on('showDateTimePicker', (info: { coordinates: Coordinates, selector: string }) => {
-                setDateTimeLocalInfo(info);
+                const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates);
+                setDateTimeLocalInfo({
+                    ...info,
+                    coordinates: canvasCoords
+                });
            });

            return () => {
@@ -114,13 +131,14 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {

    const onMouseEvent = useCallback((event: MouseEvent) => {
        if (socket && canvasRef.current) {
-            // Get the canvas bounding rectangle
            const rect = canvasRef.current.getBoundingClientRect();
            const clickCoordinates = {
                x: event.clientX - rect.left, // Use relative x coordinate
                y: event.clientY - rect.top, // Use relative y coordinate
            };

+            const browserCoordinates = coordinateMapper.mapCanvasToBrowser(clickCoordinates);
+
            switch (event.type) {
                case 'mousedown':
                    if (getTextRef.current === true) {
@@ -128,7 +146,7 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
                    } else if (getListRef.current === true) {
                        console.log('Capturing List...');
                    } else {
-                        socket.emit('input:mousedown', clickCoordinates);
+                        socket.emit('input:mousedown', browserCoordinates);
                    }
                    notifyLastAction('click');
                    break;
@@ -146,7 +164,7 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {
                            x: clickCoordinates.x,
                            y: clickCoordinates.y,
                        };
-                        socket.emit('input:mousemove', clickCoordinates);
+                        socket.emit('input:mousemove', browserCoordinates);
                        notifyLastAction('move');
                    }
                    break;
@@ -173,9 +191,11 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => {

    const onKeyboardEvent = useCallback((event: KeyboardEvent) => {
        if (socket) {
+            const browserCoordinates = coordinateMapper.mapCanvasToBrowser(lastMousePosition.current);
+
            switch (event.type) {
                case 'keydown':
-                    socket.emit('input:keydown', { key: event.key, coordinates: lastMousePosition.current });
+                    socket.emit('input:keydown', { key: event.key, coordinates: browserCoordinates });
                    notifyLastAction(`${event.key} pressed`);
                    break;
                case 'keyup':
--- a/src/components/run/InterpretationLog.tsx
+++ b/src/components/run/InterpretationLog.tsx
@@ -34,7 +34,7 @@ export const InterpretationLog: React.FC<InterpretationLogProps> = ({ isOpen, se

  const logEndRef = useRef<HTMLDivElement | null>(null);

-  const { width } = useBrowserDimensionsStore();
+  const { browserWidth, outputPreviewHeight, outputPreviewWidth } = useBrowserDimensionsStore();
  const { socket } = useSocketStore();
  const { currentWorkflowActionsState, shouldResetInterpretationLog, notify } = useGlobalInfoStore();

@@ -130,6 +130,7 @@ export const InterpretationLog: React.FC<InterpretationLogProps> = ({ isOpen, se
  return (
    <Grid container>
      <Grid item xs={12} md={9} lg={9}>
+        <div style={{ height: '20px' }}></div>
        <Button
          onClick={toggleDrawer(true)}
          variant="contained"
@@ -141,7 +142,7 @@ export const InterpretationLog: React.FC<InterpretationLogProps> = ({ isOpen, se
            background: '#ff00c3',
            border: 'none',
            padding: '10px 20px',
-            width: '900px',
+            width: browserWidth,
            overflow: 'hidden',
            textAlign: 'left',
            justifyContent: 'flex-start',
@@ -163,8 +164,8 @@ export const InterpretationLog: React.FC<InterpretationLogProps> = ({ isOpen, se
              background: `${darkMode ? '#1e2124' : 'white'}`,
              color: `${darkMode ? 'white' : 'black'}`,
              padding: '10px',
-              height: 500,
-              width: width - 10,
+              height: outputPreviewHeight,
+              width: outputPreviewWidth,
              display: 'flex',
              borderRadius: '10px 10px 0 0',
            },