diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 85d1928c..ee2dccf8 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -36,8 +36,8 @@ const SCREENCAST_CONFIG: { maxQueueSize: number; } = { format: 'jpeg', - maxWidth: 900, - maxHeight: 400, + maxWidth: 1280, + maxHeight: 720, targetFPS: 30, compressionQuality: 0.8, maxQueueSize: 2 @@ -269,7 +269,7 @@ export class RemoteBrowser { }; } const contextOptions: any = { - viewport: { height: 400, width: 900 }, + // viewport: { height: 400, width: 900 }, // recordVideo: { dir: 'videos/' } // Force reduced motion to prevent animation issues reducedMotion: 'reduce', @@ -322,6 +322,15 @@ export class RemoteBrowser { await this.setupPageEventListeners(this.currentPage); + const viewportSize = await this.currentPage.viewportSize(); + if (viewportSize) { + this.socket.emit('viewportInfo', { + width: viewportSize.width, + height: viewportSize.height, + userId: this.userId + }); + } + try { const blocker = await PlaywrightBlocker.fromLists(fetch, ['https://easylist.to/easylist/easylist.txt']); await blocker.enableBlockingInPage(this.currentPage); @@ -335,6 +344,19 @@ export class RemoteBrowser { } }; + public updateViewportInfo = async (): Promise => { + if (this.currentPage) { + const viewportSize = await this.currentPage.viewportSize(); + if (viewportSize) { + this.socket.emit('viewportInfo', { + width: viewportSize.width, + height: viewportSize.height, + userId: this.userId + }); + } + } + }; + /** * Registers all event listeners needed for the recording editor session. * Should be called only once after the full initialization of the remote browser. @@ -452,6 +474,8 @@ export class RemoteBrowser { // Set flag to indicate screencast is active this.isScreencastActive = true; + await this.updateViewportInfo(); + this.client.on('Page.screencastFrame', ({ data: base64, sessionId }) => { // Only process if screencast is still active for this user if (!this.isScreencastActive) { @@ -563,7 +587,7 @@ export class RemoteBrowser { const workflow = this.generator.AddGeneratedFlags(this.generator.getWorkflowFile()); await this.initializeNewPage(); if (this.currentPage) { - this.currentPage.setViewportSize({ height: 400, width: 900 }); + // this.currentPage.setViewportSize({ height: 400, width: 900 }); const params = this.generator.getParams(); if (params) { this.interpreterSettings.params = params.reduce((acc, param) => { @@ -721,7 +745,7 @@ export class RemoteBrowser { * @param payload the screenshot binary data * @returns void */ - private emitScreenshot = async (payload: Buffer): Promise => { + private emitScreenshot = async (payload: Buffer, viewportSize?: { width: number, height: number }): Promise => { if (this.isProcessingScreenshot) { if (this.screenshotQueue.length < SCREENCAST_CONFIG.maxQueueSize) { this.screenshotQueue.push(payload); @@ -736,11 +760,14 @@ export class RemoteBrowser { const base64Data = optimizedScreenshot.toString('base64'); const dataWithMimeType = `data:image/jpeg;base64,${base64Data}`; -// Emit with user context to ensure the frontend can identify which browser's screenshot this is -this.socket.emit('screencast', { - image: dataWithMimeType, - userId: this.userId -}); logger.debug('Screenshot emitted'); + // Emit with user context to ensure the frontend can identify which browser's screenshot this is + this.socket.emit('screencast', { + image: dataWithMimeType, + userId: this.userId, + viewport: viewportSize || await this.currentPage?.viewportSize() || null + }); + + logger.debug('Screenshot emitted'); } catch (error) { logger.error('Screenshot emission failed:', error); } finally { diff --git a/server/src/pgboss-worker.ts b/server/src/pgboss-worker.ts index 64bfb351..b775c99c 100644 --- a/server/src/pgboss-worker.ts +++ b/server/src/pgboss-worker.ts @@ -47,7 +47,7 @@ interface ExecuteRunData { browserId: string; } -const pgBoss = new PgBoss({connectionString: pgBossConnectionString, schema: 'public'}); +const pgBoss = new PgBoss({connectionString: pgBossConnectionString }); /** * Extract data safely from a job (single job or job array) diff --git a/src/components/browser/BrowserContent.tsx b/src/components/browser/BrowserContent.tsx index ec882965..a73ed6e7 100644 --- a/src/components/browser/BrowserContent.tsx +++ b/src/components/browser/BrowserContent.tsx @@ -11,12 +11,12 @@ import { // TODO: Tab !show currentUrl after recordingUrl global state export const BrowserContent = () => { - const { width } = useBrowserDimensionsStore(); const { socket } = useSocketStore(); const [tabs, setTabs] = useState(["current"]); const [tabIndex, setTabIndex] = React.useState(0); const [showOutputData, setShowOutputData] = useState(false); + const { browserWidth } = useBrowserDimensionsStore(); const handleChangeIndex = useCallback( (index: number) => { @@ -146,7 +146,7 @@ export const BrowserContent = () => { /> diff --git a/src/components/browser/BrowserTabs.tsx b/src/components/browser/BrowserTabs.tsx index 0b3f7051..4aefaa6c 100644 --- a/src/components/browser/BrowserTabs.tsx +++ b/src/components/browser/BrowserTabs.tsx @@ -32,7 +32,8 @@ export const BrowserTabs = ( return ( { const { t } = useTranslation(); + const { browserWidth, browserHeight } = useBrowserDimensionsStore(); const [canvasRef, setCanvasReference] = useState | undefined>(undefined); const [screenShot, setScreenShot] = useState(""); const [highlighterData, setHighlighterData] = useState<{ rect: DOMRect, selector: string, elementInfo: ElementInfo | null, childSelectors?: string[] } | null>(null); @@ -69,6 +78,7 @@ export const BrowserWindow = () => { const [attributeOptions, setAttributeOptions] = useState([]); const [selectedElement, setSelectedElement] = useState<{ selector: string, info: ElementInfo | null } | null>(null); const [currentListId, setCurrentListId] = useState(null); + const [viewportInfo, setViewportInfo] = useState({ width: browserWidth, height: browserHeight }); const [listSelector, setListSelector] = useState(null); const [fields, setFields] = useState>({}); @@ -82,6 +92,15 @@ export const BrowserWindow = () => { const { state } = useContext(AuthContext); const { user } = state; + const dimensions = { + width: browserWidth, + height: browserHeight + }; + + useEffect(() => { + coordinateMapper.updateDimensions(dimensions.width, dimensions.height, viewportInfo.width, viewportInfo.height); + }, [viewportInfo, dimensions.width, dimensions.height]); + useEffect(() => { if (listSelector) { window.sessionStorage.setItem('recordingListSelector', listSelector); @@ -130,6 +149,10 @@ export const BrowserWindow = () => { } else if (data && typeof data === 'object' && 'image' in data) { if (!data.userId || data.userId === user?.id) { setScreenShot(data.image); + + if (data.viewport) { + setViewportInfo(data.viewport); + } } } }, [screenShot, user?.id]); @@ -149,78 +172,85 @@ export const BrowserWindow = () => { }, [screenShot, canvasRef, socket, screencastHandler]); const highlighterHandler = useCallback((data: { rect: DOMRect, selector: string, elementInfo: ElementInfo | null, childSelectors?: string[] }) => { + // Map the incoming DOMRect from browser coordinates to canvas coordinates + const mappedRect = new DOMRect( + data.rect.x, + data.rect.y, + data.rect.width, + data.rect.height + ); + + const mappedData = { + ...data, + rect: mappedRect + }; + if (getList === true) { if (listSelector) { socket?.emit('listSelector', { selector: listSelector }); - const hasValidChildSelectors = Array.isArray(data.childSelectors) && data.childSelectors.length > 0; + const hasValidChildSelectors = Array.isArray(mappedData.childSelectors) && mappedData.childSelectors.length > 0; if (limitMode) { setHighlighterData(null); } else if (paginationMode) { // Only set highlighterData if type is not empty, 'none', 'scrollDown', or 'scrollUp' if (paginationType !== '' && !['none', 'scrollDown', 'scrollUp'].includes(paginationType)) { - setHighlighterData(data); + setHighlighterData(mappedData); } else { setHighlighterData(null); } - } else if (data.childSelectors && data.childSelectors.includes(data.selector)) { + } else if (mappedData.childSelectors && mappedData.childSelectors.includes(mappedData.selector)) { // Highlight only valid child elements within the listSelector - setHighlighterData(data); - } else if (data.elementInfo?.isIframeContent && data.childSelectors) { - // Handle pure iframe elements - similar to previous shadow DOM logic but using iframe syntax - // Check if the selector matches any iframe child selectors - const isIframeChild = data.childSelectors.some(childSelector => - data.selector.includes(':>>') && // Iframe uses :>> for traversal + setHighlighterData(mappedData); + } else if (mappedData.elementInfo?.isIframeContent && mappedData.childSelectors) { + // Handle iframe elements + const isIframeChild = mappedData.childSelectors.some(childSelector => + mappedData.selector.includes(':>>') && childSelector.split(':>>').some(part => - data.selector.includes(part.trim()) + mappedData.selector.includes(part.trim()) ) ); - setHighlighterData(isIframeChild ? data : null); - } else if (data.selector.includes(':>>') && hasValidChildSelectors) { + setHighlighterData(isIframeChild ? mappedData : null); + } else if (mappedData.selector.includes(':>>') && hasValidChildSelectors) { // Handle mixed DOM cases with iframes - // Split the selector into parts and check each against child selectors - const selectorParts = data.selector.split(':>>').map(part => part.trim()); + const selectorParts = mappedData.selector.split(':>>').map(part => part.trim()); const isValidMixedSelector = selectorParts.some(part => - // We know data.childSelectors is defined due to hasValidChildSelectors check - data.childSelectors!.some(childSelector => + mappedData.childSelectors!.some(childSelector => childSelector.includes(part) ) ); - setHighlighterData(isValidMixedSelector ? data : null); - } else if (data.elementInfo?.isShadowRoot && data.childSelectors) { - // New case: Handle pure Shadow DOM elements - // Check if the selector matches any shadow root child selectors - const isShadowChild = data.childSelectors.some(childSelector => - data.selector.includes('>>') && // Shadow DOM uses >> for piercing + setHighlighterData(isValidMixedSelector ? mappedData : null); + } else if (mappedData.elementInfo?.isShadowRoot && mappedData.childSelectors) { + // Handle Shadow DOM elements + const isShadowChild = mappedData.childSelectors.some(childSelector => + mappedData.selector.includes('>>') && childSelector.split('>>').some(part => - data.selector.includes(part.trim()) + mappedData.selector.includes(part.trim()) ) ); - setHighlighterData(isShadowChild ? data : null); - } else if (data.selector.includes('>>') && hasValidChildSelectors) { - // New case: Handle mixed DOM cases - // Split the selector into parts and check each against child selectors - const selectorParts = data.selector.split('>>').map(part => part.trim()); + setHighlighterData(isShadowChild ? mappedData : null); + } else if (mappedData.selector.includes('>>') && hasValidChildSelectors) { + // Handle mixed DOM cases + const selectorParts = mappedData.selector.split('>>').map(part => part.trim()); const isValidMixedSelector = selectorParts.some(part => - // Now we know data.childSelectors is defined - data.childSelectors!.some(childSelector => + mappedData.childSelectors!.some(childSelector => childSelector.includes(part) ) ); - setHighlighterData(isValidMixedSelector ? data : null); + setHighlighterData(isValidMixedSelector ? mappedData : null); } else { - // if !valid child in normal mode, clear the highlighter + // If not a valid child in normal mode, clear the highlighter setHighlighterData(null); } } else { // Set highlighterData for the initial listSelector selection - setHighlighterData(data); + setHighlighterData(mappedData); } } else { // For non-list steps - setHighlighterData(data); + setHighlighterData(mappedData); } - }, [highlighterData, getList, socket, listSelector, paginationMode, paginationType, captureStage]); + }, [getList, socket, listSelector, paginationMode, paginationType, limitMode]); useEffect(() => { @@ -260,11 +290,13 @@ export const BrowserWindow = () => { const clickY = e.clientY - canvasRect.top; const highlightRect = highlighterData.rect; + + const mappedRect = coordinateMapper.mapBrowserRectToCanvas(highlightRect); if ( - clickX >= highlightRect.left && - clickX <= highlightRect.right && - clickY >= highlightRect.top && - clickY <= highlightRect.bottom + clickX >= mappedRect.left && + clickX <= mappedRect.right && + clickY >= mappedRect.top && + clickY <= mappedRect.bottom ) { const options = getAttributeOptions(highlighterData.elementInfo?.tagName || '', highlighterData.elementInfo); @@ -437,7 +469,7 @@ export const BrowserWindow = () => { }, [paginationMode, resetPaginationSelector]); return ( -
+
{ getText === true || getList === true ? ( { ) : null } -
+
{((getText === true || getList === true) && !showAttributeModal && highlighterData?.rect != null && highlighterData?.rect.top != null) && canvasRef?.current ? : null}
@@ -512,7 +544,7 @@ const drawImage = (image: string, canvas: HTMLCanvasElement): void => { img.src = image; img.onload = () => { URL.revokeObjectURL(img.src); - ctx?.drawImage(img, 0, 0, 900, 400); + ctx?.drawImage(img, 0, 0, canvas.width, canvas.height); }; }; diff --git a/src/components/recorder/Highlighter.tsx b/src/components/recorder/Highlighter.tsx index 562fd81e..a63c3c23 100644 --- a/src/components/recorder/Highlighter.tsx +++ b/src/components/recorder/Highlighter.tsx @@ -1,5 +1,6 @@ import React from 'react'; import styled from "styled-components"; +import { coordinateMapper } from '../../helpers/coordinateMapper'; interface HighlighterProps { unmodifiedRect: DOMRect; @@ -13,13 +14,15 @@ export const Highlighter = ({ unmodifiedRect, displayedSelector = '', width, hei if (!unmodifiedRect) { return null; } else { + const mappedRect = coordinateMapper.mapBrowserRectToCanvas(unmodifiedRect); + const rect = { - top: unmodifiedRect.top + canvasRect.top + window.scrollY, - left: unmodifiedRect.left + canvasRect.left + window.scrollX, - right: unmodifiedRect.right + canvasRect.left, - bottom: unmodifiedRect.bottom + canvasRect.top, - width: unmodifiedRect.width, - height: unmodifiedRect.height, + top: mappedRect.top + canvasRect.top + window.scrollY, + left: mappedRect.left + canvasRect.left + window.scrollX, + right: mappedRect.right + canvasRect.left, + bottom: mappedRect.bottom + canvasRect.top, + width: mappedRect.width, + height: mappedRect.height, }; diff --git a/src/components/recorder/RightSidePanel.tsx b/src/components/recorder/RightSidePanel.tsx index f324183d..6027e26d 100644 --- a/src/components/recorder/RightSidePanel.tsx +++ b/src/components/recorder/RightSidePanel.tsx @@ -20,6 +20,7 @@ import { getActiveWorkflow } from "../../api/workflow"; import ActionDescriptionBox from '../action/ActionDescriptionBox'; import { useThemeMode } from '../../context/theme-provider'; import { useTranslation } from 'react-i18next'; +import { useBrowserDimensionsStore } from '../../context/browserDimensions'; const fetchWorkflow = (id: string, callback: (response: WorkflowFile) => void) => { getActiveWorkflow(id).then( @@ -54,6 +55,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture const [browserStepIdList, setBrowserStepIdList] = useState([]); const [isCaptureTextConfirmed, setIsCaptureTextConfirmed] = useState(false); const [isCaptureListConfirmed, setIsCaptureListConfirmed] = useState(false); + const { panelHeight } = useBrowserDimensionsStore(); const { lastAction, notify, currentWorkflowActionsState, setCurrentWorkflowActionsState, resetInterpretationLog } = useGlobalInfoStore(); const { getText, startGetText, stopGetText, getScreenshot, startGetScreenshot, stopGetScreenshot, getList, startGetList, stopGetList, startPaginationMode, stopPaginationMode, paginationType, updatePaginationType, limitType, customLimit, updateLimitType, updateCustomLimit, stopLimitMode, startLimitMode, captureStage, setCaptureStage, showPaginationOptions, setShowPaginationOptions, showLimitOptions, setShowLimitOptions, workflow, setWorkflow } = useActionContext(); @@ -460,8 +462,9 @@ export const RightSidePanel: React.FC = ({ onFinishCapture const theme = useThemeMode(); const isDarkMode = theme.darkMode; + return ( - + {/* Last action: {` ${lastAction}`} */} diff --git a/src/components/recorder/canvas.tsx b/src/components/recorder/canvas.tsx index 9640c061..52d29b00 100644 --- a/src/components/recorder/canvas.tsx +++ b/src/components/recorder/canvas.tsx @@ -6,6 +6,7 @@ import DatePicker from '../pickers/DatePicker'; import Dropdown from '../pickers/Dropdown'; import TimePicker from '../pickers/TimePicker'; import DateTimeLocalPicker from '../pickers/DateTimeLocalPicker'; +import { coordinateMapper } from '../../helpers/coordinateMapper'; interface CreateRefCallback { (ref: React.RefObject): void; @@ -79,7 +80,11 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { useEffect(() => { if (socket) { socket.on('showDatePicker', (info: { coordinates: Coordinates, selector: string }) => { - setDatePickerInfo(info); + const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates); + setDatePickerInfo({ + ...info, + coordinates: canvasCoords + }); }); socket.on('showDropdown', (info: { @@ -92,15 +97,27 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { selected: boolean; }>; }) => { - setDropdownInfo(info); + const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates); + setDropdownInfo({ + ...info, + coordinates: canvasCoords + }); }); socket.on('showTimePicker', (info: { coordinates: Coordinates, selector: string }) => { - setTimePickerInfo(info); + const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates); + setTimePickerInfo({ + ...info, + coordinates: canvasCoords + }); }); socket.on('showDateTimePicker', (info: { coordinates: Coordinates, selector: string }) => { - setDateTimeLocalInfo(info); + const canvasCoords = coordinateMapper.mapBrowserToCanvas(info.coordinates); + setDateTimeLocalInfo({ + ...info, + coordinates: canvasCoords + }); }); return () => { @@ -114,13 +131,14 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { const onMouseEvent = useCallback((event: MouseEvent) => { if (socket && canvasRef.current) { - // Get the canvas bounding rectangle const rect = canvasRef.current.getBoundingClientRect(); const clickCoordinates = { x: event.clientX - rect.left, // Use relative x coordinate y: event.clientY - rect.top, // Use relative y coordinate }; + const browserCoordinates = coordinateMapper.mapCanvasToBrowser(clickCoordinates); + switch (event.type) { case 'mousedown': if (getTextRef.current === true) { @@ -128,7 +146,7 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { } else if (getListRef.current === true) { console.log('Capturing List...'); } else { - socket.emit('input:mousedown', clickCoordinates); + socket.emit('input:mousedown', browserCoordinates); } notifyLastAction('click'); break; @@ -146,7 +164,7 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { x: clickCoordinates.x, y: clickCoordinates.y, }; - socket.emit('input:mousemove', clickCoordinates); + socket.emit('input:mousemove', browserCoordinates); notifyLastAction('move'); } break; @@ -173,9 +191,11 @@ const Canvas = ({ width, height, onCreateRef }: CanvasProps) => { const onKeyboardEvent = useCallback((event: KeyboardEvent) => { if (socket) { + const browserCoordinates = coordinateMapper.mapCanvasToBrowser(lastMousePosition.current); + switch (event.type) { case 'keydown': - socket.emit('input:keydown', { key: event.key, coordinates: lastMousePosition.current }); + socket.emit('input:keydown', { key: event.key, coordinates: browserCoordinates }); notifyLastAction(`${event.key} pressed`); break; case 'keyup': diff --git a/src/components/run/InterpretationLog.tsx b/src/components/run/InterpretationLog.tsx index 925a95ba..09025c99 100644 --- a/src/components/run/InterpretationLog.tsx +++ b/src/components/run/InterpretationLog.tsx @@ -34,7 +34,7 @@ export const InterpretationLog: React.FC = ({ isOpen, se const logEndRef = useRef(null); - const { width } = useBrowserDimensionsStore(); + const { browserWidth, outputPreviewHeight, outputPreviewWidth } = useBrowserDimensionsStore(); const { socket } = useSocketStore(); const { currentWorkflowActionsState, shouldResetInterpretationLog, notify } = useGlobalInfoStore(); @@ -130,6 +130,7 @@ export const InterpretationLog: React.FC = ({ isOpen, se return ( +