import React, { useCallback, useContext, useEffect, useState, useRef, } from "react"; import { useSocketStore } from "../../context/socket"; import { useGlobalInfoStore } from "../../context/globalInfo"; import { useTranslation } from "react-i18next"; import { AuthContext } from "../../context/auth"; import { rebuild, createMirror } from "rrweb-snapshot"; import { ActionType, clientSelectorGenerator, } from "../../helpers/clientSelectorGenerator"; interface ElementInfo { tagName: string; hasOnlyText?: boolean; isIframeContent?: boolean; isShadowRoot?: boolean; innerText?: string; url?: string; imageUrl?: string; attributes?: Record; innerHTML?: string; outerHTML?: string; isDOMMode?: boolean; } interface ProcessedSnapshot { snapshot: RRWebSnapshot; resources: { stylesheets: Array<{ href: string; content: string; media?: string; }>; images: Array<{ src: string; dataUrl: string; alt?: string; }>; fonts: Array<{ url: string; dataUrl: string; format?: string; }>; scripts: Array<{ src: string; content: string; type?: string; }>; media: Array<{ src: string; dataUrl: string; type: string; }>; }; baseUrl: string; viewport: { width: number; height: number }; timestamp: number; processingStats: { totalReplacements: number; discoveredResources: { images: number; stylesheets: number; scripts: number; fonts: number; media: number; }; cachedResources: { stylesheets: number; images: number; fonts: number; scripts: number; media: number; }; totalCacheSize: number; }; } interface RRWebSnapshot { type: number; childNodes?: RRWebSnapshot[]; tagName?: string; attributes?: Record; textContent: string; id: number; [key: string]: any; } interface RRWebDOMBrowserRendererProps { width: number; height: number; snapshot: ProcessedSnapshot; getList?: boolean; getText?: boolean; listSelector?: string | null; paginationMode?: boolean; paginationType?: string; limitMode?: boolean; onHighlight?: (data: { rect: DOMRect; selector: string; elementInfo: ElementInfo | null; childSelectors?: string[]; }) => void; onElementSelect?: (data: { rect: DOMRect; selector: string; elementInfo: ElementInfo | null; childSelectors?: string[]; }) => void; onShowDatePicker?: (info: { coordinates: { x: number; y: number }; selector: string; }) => void; onShowDropdown?: (info: { coordinates: { x: number; y: number }; selector: string; options: Array<{ value: string; text: string; disabled: boolean; selected: boolean; }>; }) => void; onShowTimePicker?: (info: { coordinates: { x: number; y: number }; selector: string; }) => void; onShowDateTimePicker?: (info: { coordinates: { x: number; y: number }; selector: string; }) => void; } export const DOMBrowserRenderer: React.FC = ({ width, height, snapshot, getList = false, getText = false, listSelector = null, paginationMode = false, paginationType = "", limitMode = false, onHighlight, onElementSelect, onShowDatePicker, onShowDropdown, onShowTimePicker, onShowDateTimePicker, }) => { const { t } = useTranslation(); const containerRef = useRef(null); const iframeRef = useRef(null); const [isRendered, setIsRendered] = useState(false); const [renderError, setRenderError] = useState(null); const [lastMousePosition, setLastMousePosition] = useState({ x: 0, y: 0 }); const [currentHighlight, setCurrentHighlight] = useState<{ element: Element; rect: DOMRect; selector: string; elementInfo: ElementInfo; childSelectors?: string[]; } | null>(null); const { socket } = useSocketStore(); const { setLastAction, lastAction } = useGlobalInfoStore(); const { state } = useContext(AuthContext); const { user } = state; const MOUSE_MOVE_THROTTLE = 16; // ~60fps const lastMouseMoveTime = useRef(0); const notifyLastAction = (action: string) => { if (lastAction !== action) { setLastAction(action); } }; const isInCaptureMode = getText || getList; useEffect(() => { clientSelectorGenerator.setGetList(getList); clientSelectorGenerator.setListSelector(listSelector || ""); clientSelectorGenerator.setPaginationMode(paginationMode); }, [getList, listSelector, paginationMode]); useEffect(() => { if (listSelector) { clientSelectorGenerator.setListSelector(listSelector); clientSelectorGenerator.setGetList(getList); clientSelectorGenerator.setPaginationMode(paginationMode); } }, [listSelector, getList, paginationMode]); /** * Handle client-side highlighting for DOM mode using complete backend logic */ const handleDOMHighlighting = useCallback( (x: number, y: number, iframeDoc: Document) => { try { const highlighterData = clientSelectorGenerator.generateDataForHighlighter( { x, y }, iframeDoc, true ); if (!highlighterData) { setCurrentHighlight(null); if (onHighlight) { onHighlight({ rect: new DOMRect(0, 0, 0, 0), selector: "", elementInfo: null, }); } return; } const { rect, selector, elementInfo, childSelectors } = highlighterData; let shouldHighlight = false; if (getList) { if (listSelector) { const hasValidChildSelectors = Array.isArray(childSelectors) && childSelectors.length > 0; if (limitMode) { shouldHighlight = false; } else if (paginationMode) { if ( paginationType !== "" && !["none", "scrollDown", "scrollUp"].includes(paginationType) ) { shouldHighlight = true; } else { shouldHighlight = false; } } else if (childSelectors && childSelectors.includes(selector)) { shouldHighlight = true; } else if (elementInfo?.isIframeContent && childSelectors) { const isIframeChild = childSelectors.some( (childSelector: string) => selector.includes(":>>") && childSelector .split(":>>") .some((part) => selector.includes(part.trim())) ); shouldHighlight = isIframeChild; } else if (selector.includes(":>>") && hasValidChildSelectors) { const selectorParts = selector .split(":>>") .map((part: string) => part.trim()); const isValidMixedSelector = selectorParts.some((part: any) => childSelectors!.some((childSelector) => childSelector.includes(part) ) ); } else if (elementInfo?.isShadowRoot && childSelectors) { const isShadowChild = childSelectors.some( (childSelector: string) => selector.includes(">>") && childSelector .split(">>") .some((part) => selector.includes(part.trim())) ); } else if (selector.includes(">>") && hasValidChildSelectors) { const selectorParts = selector .split(">>") .map((part: string) => part.trim()); const isValidMixedSelector = selectorParts.some((part: any) => childSelectors!.some((childSelector) => childSelector.includes(part) ) ); } else { shouldHighlight = false; } } else { shouldHighlight = true; } } else { shouldHighlight = true; } if (shouldHighlight) { const element = iframeDoc.elementFromPoint(x, y); if (element) { setCurrentHighlight({ element, rect: rect, selector, elementInfo: { ...elementInfo, tagName: elementInfo?.tagName ?? "", isDOMMode: true, }, childSelectors, }); if (onHighlight) { onHighlight({ rect: rect, elementInfo: { ...elementInfo, tagName: elementInfo?.tagName ?? "", isDOMMode: true, }, selector, childSelectors, }); } } } else { setCurrentHighlight(null); if (onHighlight) { onHighlight({ rect: new DOMRect(0, 0, 0, 0), selector: "", elementInfo: null, }); } } } catch (error) { console.error("Error in DOM highlighting:", error); setCurrentHighlight(null); } }, [ getList, listSelector, paginationMode, paginationType, limitMode, onHighlight, ] ); /** * Set up enhanced interaction handlers for DOM mode */ const setupIframeInteractions = useCallback( (iframeDoc: Document) => { const existingHandlers = (iframeDoc as any)._domRendererHandlers; if (existingHandlers) { Object.entries(existingHandlers).forEach(([event, handler]) => { iframeDoc.removeEventListener(event, handler as EventListener, false); // Changed to false }); } const handlers: { [key: string]: EventListener } = {}; const mouseMoveHandler: EventListener = (e: Event) => { if (e.target && !iframeDoc.contains(e.target as Node)) { return; } const now = performance.now(); if (now - lastMouseMoveTime.current < MOUSE_MOVE_THROTTLE) { return; } lastMouseMoveTime.current = now; const mouseEvent = e as MouseEvent; const iframeX = mouseEvent.clientX; const iframeY = mouseEvent.clientY; const iframe = iframeRef.current; if (iframe) { const iframeRect = iframe.getBoundingClientRect(); setLastMousePosition({ x: iframeX + iframeRect.left, y: iframeY + iframeRect.top, }); } handleDOMHighlighting(iframeX, iframeY, iframeDoc); notifyLastAction("move"); }; const mouseDownHandler: EventListener = (e: Event) => { if (e.target && !iframeDoc.contains(e.target as Node)) { return; } const mouseEvent = e as MouseEvent; const target = mouseEvent.target as Element; const iframeX = mouseEvent.clientX; const iframeY = mouseEvent.clientY; if (isInCaptureMode) { e.preventDefault(); e.stopPropagation(); if (currentHighlight && onElementSelect) { onElementSelect({ rect: currentHighlight.rect, selector: currentHighlight.selector, elementInfo: currentHighlight.elementInfo, childSelectors: currentHighlight.childSelectors || [], }); } notifyLastAction("select element"); return; } const linkElement = target.closest("a[href]") as HTMLAnchorElement; if (linkElement && linkElement.href && socket) { e.preventDefault(); e.stopPropagation(); const href = linkElement.href; if (linkElement.target) { linkElement.target = ""; } const originalHref = linkElement.href; linkElement.removeAttribute("href"); setTimeout(() => { linkElement.setAttribute("href", originalHref); }, 100); const isSPALink = href.endsWith("#") || (href.includes("#") && new URL(href).hash !== ""); const selector = clientSelectorGenerator.generateSelector( iframeDoc, { x: iframeX, y: iframeY }, ActionType.Click ); const elementInfo = clientSelectorGenerator.getElementInformation( iframeDoc, { x: iframeX, y: iframeY }, clientSelectorGenerator.getCurrentState().listSelector, clientSelectorGenerator.getCurrentState().getList ); if (selector && socket) { socket.emit("dom:click", { selector, url: snapshot.baseUrl, userId: user?.id || "unknown", elementInfo, coordinates: undefined, isSPA: isSPALink, }); notifyLastAction( isSPALink ? `SPA navigation to ${href}` : `navigate to ${href}` ); } return; } const selector = clientSelectorGenerator.generateSelector( iframeDoc, { x: iframeX, y: iframeY }, ActionType.Click ); const elementInfo = clientSelectorGenerator.getElementInformation( iframeDoc, { x: iframeX, y: iframeY }, clientSelectorGenerator.getCurrentState().listSelector, clientSelectorGenerator.getCurrentState().getList ); if (selector && elementInfo && socket) { if (elementInfo?.tagName === "SELECT" && elementInfo.innerHTML) { const inputElement = target as HTMLInputElement; inputElement.blur(); const wasDisabled = inputElement.disabled; inputElement.disabled = true; setTimeout(() => { inputElement.disabled = wasDisabled; }, 100); const options = elementInfo.innerHTML .split(" { const valueMatch = optionHtml.match(/value="([^"]*)"/); const textMatch = optionHtml.match(/>([^<]*) { inputElement.disabled = wasDisabled; }, 100); const pickerInfo = { coordinates: { x: iframeX, y: iframeY }, selector, }; switch (inputType) { case "date": case "month": case "week": if (onShowDatePicker) { onShowDatePicker(pickerInfo); } break; case "time": if (onShowTimePicker) { onShowTimePicker(pickerInfo); } break; case "datetime-local": if (onShowDateTimePicker) { onShowDateTimePicker(pickerInfo); } break; } notifyLastAction(`${inputType} picker opened`); return; } } if ( elementInfo?.tagName !== "INPUT" && elementInfo?.tagName !== "SELECT" ) { socket.emit("dom:click", { selector, url: snapshot.baseUrl, userId: user?.id || "unknown", elementInfo, coordinates: { x: iframeX, y: iframeY }, isSPA: false, }); } } notifyLastAction("click"); }; const mouseUpHandler: EventListener = (e: Event) => { if (e.target && !iframeDoc.contains(e.target as Node)) { return; } if (!isInCaptureMode) { notifyLastAction("release"); } }; const keyDownHandler: EventListener = (e: Event) => { if (e.target && !iframeDoc.contains(e.target as Node)) { return; } const keyboardEvent = e as KeyboardEvent; const target = keyboardEvent.target as HTMLElement; if (!isInCaptureMode && socket && snapshot?.baseUrl) { const iframe = iframeRef.current; if (iframe) { const iframeRect = iframe.getBoundingClientRect(); const iframeX = lastMousePosition.x - iframeRect.left; const iframeY = lastMousePosition.y - iframeRect.top; const selector = clientSelectorGenerator.generateSelector( iframeDoc, { x: iframeX, y: iframeY }, ActionType.Keydown ); const elementInfo = clientSelectorGenerator.getElementInformation( iframeDoc, { x: iframeX, y: iframeY }, clientSelectorGenerator.getCurrentState().listSelector, clientSelectorGenerator.getCurrentState().getList ); if (selector) { socket.emit("dom:keypress", { selector, key: keyboardEvent.key, url: snapshot.baseUrl, userId: user?.id || "unknown", inputType: elementInfo?.attributes?.type || "text", }); } } notifyLastAction(`${keyboardEvent.key} typed`); } if ( ["INPUT", "TEXTAREA"].includes(target.tagName) && !isInCaptureMode ) { return; } }; const keyUpHandler: EventListener = (e: Event) => { if (e.target && !iframeDoc.contains(e.target as Node)) { return; } const keyboardEvent = e as KeyboardEvent; if (!isInCaptureMode && socket) { socket.emit("input:keyup", { key: keyboardEvent.key }); } }; const wheelHandler: EventListener = (e: Event) => { if (e.target && !iframeDoc.contains(e.target as Node)) { return; } e.preventDefault(); if (!isInCaptureMode) { const wheelEvent = e as WheelEvent; const deltaX = Math.round(wheelEvent.deltaX / 10) * 10; const deltaY = Math.round(wheelEvent.deltaY / 10) * 10; if (Math.abs(deltaX) > 5 || Math.abs(deltaY) > 5) { if (socket) { socket.emit("dom:scroll", { deltaX, deltaY }) } notifyLastAction("scroll"); } } }; const clickHandler: EventListener = (e: Event) => { if (e.target && !iframeDoc.contains(e.target as Node)) { return; } if (isInCaptureMode) { e.preventDefault(); e.stopPropagation(); return; } }; const preventDefaults = (e: Event) => { if (e.target && !iframeDoc.contains(e.target as Node)) { return; } e.preventDefault(); e.stopPropagation(); return false; }; handlers.mousedown = mouseDownHandler; handlers.mouseup = mouseUpHandler; handlers.mousemove = mouseMoveHandler; handlers.wheel = wheelHandler; handlers.keydown = keyDownHandler; handlers.keyup = keyUpHandler; handlers.click = clickHandler; handlers.submit = preventDefaults; handlers.beforeunload = preventDefaults; Object.entries(handlers).forEach(([event, handler]) => { iframeDoc.addEventListener(event, handler, false); }); // Store handlers for cleanup (iframeDoc as any)._domRendererHandlers = handlers; // Make iframe focusable for keyboard events if (iframeRef.current) { iframeRef.current.tabIndex = 0; } }, [ socket, lastMousePosition, notifyLastAction, handleDOMHighlighting, currentHighlight, onElementSelect, isInCaptureMode, snapshot, user?.id, onShowDatePicker, onShowDropdown, onShowTimePicker, onShowDateTimePicker, ] ); /** * Render DOM snapshot using rrweb */ const renderRRWebSnapshot = useCallback( (snapshotData: ProcessedSnapshot) => { if (!iframeRef.current) { console.warn("No iframe reference available"); return; } const iframe = iframeRef.current; try { setRenderError(null); setIsRendered(false); const tempDoc = document.implementation.createHTMLDocument("RRWeb Snapshot"); const mirror = createMirror(); try { rebuild(snapshotData.snapshot, { doc: tempDoc, mirror: mirror, cache: { stylesWithHoverClass: new Map() }, afterAppend: (node) => { if (node.nodeType === Node.TEXT_NODE && node.textContent) { const text = node.textContent.trim(); if ( text.startsWith("<") && text.includes(">") && text.length > 50 ) { if (node.parentNode) { node.parentNode.removeChild(node); } } } }, }); } catch (rebuildError) { console.error("rrweb rebuild failed:", rebuildError); throw new Error(`rrweb rebuild failed: ${rebuildError}`); } let rebuiltHTML = tempDoc.documentElement.outerHTML; rebuiltHTML = "\n" + rebuiltHTML; const additionalCSS = []; if (snapshotData.resources.fonts?.length > 0) { const fontCSS = snapshotData.resources.fonts .map((font) => { const format = font.format || "woff2"; return ` @font-face { font-family: 'ProxiedFont-${ font.url.split("/").pop()?.split(".")[0] || "unknown" }'; src: url("${font.dataUrl}") format("${format}"); font-display: swap; } `; }) .join("\n"); additionalCSS.push(fontCSS); } if (snapshotData.resources.stylesheets?.length > 0) { const externalCSS = snapshotData.resources.stylesheets .map((stylesheet) => stylesheet.content) .join("\n\n"); additionalCSS.push(externalCSS); } const enhancedCSS = ` /* rrweb rebuilt content styles */ html, body { margin: 0 !important; padding: 8px !important; font-family: system-ui, -apple-system, BlinkMacSystemFont, sans-serif !important; background: white !important; overflow-x: hidden !important; } html::-webkit-scrollbar, body::-webkit-scrollbar { display: none !important; width: 0 !important; height: 0 !important; background: transparent !important; } /* Hide scrollbars for all elements */ *::-webkit-scrollbar { display: none !important; width: 0 !important; height: 0 !important; background: transparent !important; } * { scrollbar-width: none !important; /* Firefox */ -ms-overflow-style: none !important; /* Internet Explorer 10+ */ } img { max-width: 100% !important; height: auto !important; } /* Make everything interactive */ * { cursor: ${isInCaptureMode ? "crosshair" : "pointer"} !important; } /* Additional CSS from resources */ ${additionalCSS.join("\n\n")} `; const headTagRegex = /]*>/i; const cssInjection = ` `; if (headTagRegex.test(rebuiltHTML)) { rebuiltHTML = rebuiltHTML.replace( headTagRegex, `${cssInjection}` ); } else { rebuiltHTML = rebuiltHTML.replace( /]*>/i, `${cssInjection}` ); } rebuiltHTML = rebuiltHTML .replace(/)<[^<]*)*<\/script>/gi, "") .replace(/\s*on\w+\s*=\s*"[^"]*"/gi, "") .replace(/\s*on\w+\s*=\s*'[^']*'/gi, "") .replace(/javascript:/gi, "void:") .replace(/ { setIsRendered(true); setupIframeInteractions(iframeDoc); }; } catch (error) { console.error("Error rendering rrweb snapshot:", error); setRenderError(error instanceof Error ? error.message : String(error)); showErrorInIframe(error); } }, [setupIframeInteractions, isInCaptureMode] ); useEffect(() => { if (snapshot && iframeRef.current) { renderRRWebSnapshot(snapshot); } }, [snapshot]); useEffect(() => { if (isRendered && iframeRef.current) { const iframeDoc = iframeRef.current.contentDocument; if (iframeDoc) { setupIframeInteractions(iframeDoc); } } }, [getText, getList, listSelector, isRendered, setupIframeInteractions]); /** * Show error message in iframe */ const showErrorInIframe = (error: any) => { if (!iframeRef.current) return; const iframe = iframeRef.current; const iframeDoc = iframe.contentDocument || iframe.contentWindow?.document; if (iframeDoc) { try { iframeDoc.open(); iframeDoc.write(`

Error Loading DOM Content

Failed to render the page in DOM mode.

Common causes:

  • Page is still loading or navigating
  • Resource proxy timeouts or failures
  • Network connectivity issues
  • Invalid HTML structure

Solutions:

  • Try switching back to Screenshot mode
  • Wait for the page to fully load and try again
  • Check your network connection
  • Refresh the browser page
Technical details
${error.toString()}
`); iframeDoc.close(); window.addEventListener("message", (event) => { if (event.data === "retry-dom-mode") { if (socket) { socket.emit("enable-dom-streaming"); } } }); } catch (e) { console.error("Failed to write error message to iframe:", e); } } }; useEffect(() => { return () => { if (iframeRef.current) { const iframeDoc = iframeRef.current.contentDocument; if (iframeDoc) { const handlers = (iframeDoc as any)._domRendererHandlers; if (handlers) { Object.entries(handlers).forEach(([event, handler]) => { iframeDoc.removeEventListener( event, handler as EventListener, true ); }); } } } }; }, []); return (