From 7d615c27a7993497e5adbc05b7de2d7e0df32483 Mon Sep 17 00:00:00 2001 From: Rohit Date: Mon, 23 Jun 2025 14:28:25 +0530 Subject: [PATCH] feat: add dom browser renderer --- .../recorder/DOMBrowserRenderer.tsx | 1140 +++++++++++++++++ 1 file changed, 1140 insertions(+) create mode 100644 src/components/recorder/DOMBrowserRenderer.tsx diff --git a/src/components/recorder/DOMBrowserRenderer.tsx b/src/components/recorder/DOMBrowserRenderer.tsx new file mode 100644 index 00000000..94543f0c --- /dev/null +++ b/src/components/recorder/DOMBrowserRenderer.tsx @@ -0,0 +1,1140 @@ +import React, { + useCallback, + useContext, + useEffect, + useState, + useRef, +} from "react"; +import { useSocketStore } from "../../context/socket"; +import { useGlobalInfoStore } from "../../context/globalInfo"; +import { useTranslation } from "react-i18next"; +import { AuthContext } from "../../context/auth"; +import { rebuild, createMirror } from "rrweb-snapshot"; +import { + ActionType, + clientSelectorGenerator, +} from "../../helpers/clientSelectorGenerator"; + +interface ElementInfo { + tagName: string; + hasOnlyText?: boolean; + isIframeContent?: boolean; + isShadowRoot?: boolean; + innerText?: string; + url?: string; + imageUrl?: string; + attributes?: Record; + innerHTML?: string; + outerHTML?: string; + isDOMMode?: boolean; +} + +interface ProcessedSnapshot { + snapshot: RRWebSnapshot; + resources: { + stylesheets: Array<{ + href: string; + content: string; + media?: string; + }>; + images: Array<{ + src: string; + dataUrl: string; + alt?: string; + }>; + fonts: Array<{ + url: string; + dataUrl: string; + format?: string; + }>; + scripts: Array<{ + src: string; + content: string; + type?: string; + }>; + media: Array<{ + src: string; + dataUrl: string; + type: string; + }>; + }; + baseUrl: string; + viewport: { width: number; height: number }; + timestamp: number; + processingStats: { + totalReplacements: number; + discoveredResources: { + images: number; + stylesheets: number; + scripts: number; + fonts: number; + media: number; + }; + cachedResources: { + stylesheets: number; + images: number; + fonts: number; + scripts: number; + media: number; + }; + totalCacheSize: number; + }; +} + +interface RRWebSnapshot { + type: number; + childNodes?: RRWebSnapshot[]; + tagName?: string; + attributes?: Record; + textContent: string; + id: number; + [key: string]: any; +} + +interface RRWebDOMBrowserRendererProps { + width: number; + height: number; + snapshot: ProcessedSnapshot; + getList?: boolean; + getText?: boolean; + listSelector?: string | null; + paginationMode?: boolean; + paginationType?: string; + limitMode?: boolean; + onHighlight?: (data: { + rect: DOMRect; + selector: string; + elementInfo: ElementInfo | null; + childSelectors?: string[]; + }) => void; + onElementSelect?: (data: { + rect: DOMRect; + selector: string; + elementInfo: ElementInfo | null; + childSelectors?: string[]; + }) => void; + onShowDatePicker?: (info: { + coordinates: { x: number; y: number }; + selector: string; + }) => void; + onShowDropdown?: (info: { + coordinates: { x: number; y: number }; + selector: string; + options: Array<{ + value: string; + text: string; + disabled: boolean; + selected: boolean; + }>; + }) => void; + onShowTimePicker?: (info: { + coordinates: { x: number; y: number }; + selector: string; + }) => void; + onShowDateTimePicker?: (info: { + coordinates: { x: number; y: number }; + selector: string; + }) => void; +} + +export const DOMBrowserRenderer: React.FC = ({ + width, + height, + snapshot, + getList = false, + getText = false, + listSelector = null, + paginationMode = false, + paginationType = "", + limitMode = false, + onHighlight, + onElementSelect, + onShowDatePicker, + onShowDropdown, + onShowTimePicker, + onShowDateTimePicker, +}) => { + const { t } = useTranslation(); + const containerRef = useRef(null); + const iframeRef = useRef(null); + const [isRendered, setIsRendered] = useState(false); + const [renderError, setRenderError] = useState(null); + const [lastMousePosition, setLastMousePosition] = useState({ x: 0, y: 0 }); + const [currentHighlight, setCurrentHighlight] = useState<{ + element: Element; + rect: DOMRect; + selector: string; + elementInfo: ElementInfo; + childSelectors?: string[]; + } | null>(null); + + const { socket } = useSocketStore(); + const { setLastAction, lastAction } = useGlobalInfoStore(); + + const { state } = useContext(AuthContext); + const { user } = state; + + const MOUSE_MOVE_THROTTLE = 16; // ~60fps + const lastMouseMoveTime = useRef(0); + + const notifyLastAction = (action: string) => { + if (lastAction !== action) { + setLastAction(action); + } + }; + + const isInCaptureMode = getText || getList; + + useEffect(() => { + clientSelectorGenerator.setGetList(getList); + clientSelectorGenerator.setListSelector(listSelector || ""); + clientSelectorGenerator.setPaginationMode(paginationMode); + }, [getList, listSelector, paginationMode]); + + useEffect(() => { + if (listSelector) { + clientSelectorGenerator.setListSelector(listSelector); + clientSelectorGenerator.setGetList(getList); + clientSelectorGenerator.setPaginationMode(paginationMode); + } + }, [listSelector, getList, paginationMode]); + + /** + * Handle client-side highlighting for DOM mode using complete backend logic + */ + const handleDOMHighlighting = useCallback( + (x: number, y: number, iframeDoc: Document) => { + try { + const highlighterData = + clientSelectorGenerator.generateDataForHighlighter( + { x, y }, + iframeDoc, + true + ); + + if (!highlighterData) { + setCurrentHighlight(null); + if (onHighlight) { + onHighlight({ + rect: new DOMRect(0, 0, 0, 0), + selector: "", + elementInfo: null, + }); + } + return; + } + + const { rect, selector, elementInfo, childSelectors } = highlighterData; + + let shouldHighlight = false; + + if (getList) { + if (listSelector) { + const hasValidChildSelectors = + Array.isArray(childSelectors) && childSelectors.length > 0; + + if (limitMode) { + shouldHighlight = false; + } else if (paginationMode) { + if ( + paginationType !== "" && + !["none", "scrollDown", "scrollUp"].includes(paginationType) + ) { + shouldHighlight = true; + } else { + shouldHighlight = false; + } + } else if (childSelectors && childSelectors.includes(selector)) { + shouldHighlight = true; + } else if (elementInfo?.isIframeContent && childSelectors) { + const isIframeChild = childSelectors.some( + (childSelector: string) => + selector.includes(":>>") && + childSelector + .split(":>>") + .some((part) => selector.includes(part.trim())) + ); + shouldHighlight = isIframeChild; + } else if (selector.includes(":>>") && hasValidChildSelectors) { + const selectorParts = selector + .split(":>>") + .map((part: string) => part.trim()); + const isValidMixedSelector = selectorParts.some((part: any) => + childSelectors!.some((childSelector) => + childSelector.includes(part) + ) + ); + } else if (elementInfo?.isShadowRoot && childSelectors) { + const isShadowChild = childSelectors.some( + (childSelector: string) => + selector.includes(">>") && + childSelector + .split(">>") + .some((part) => selector.includes(part.trim())) + ); + } else if (selector.includes(">>") && hasValidChildSelectors) { + const selectorParts = selector + .split(">>") + .map((part: string) => part.trim()); + const isValidMixedSelector = selectorParts.some((part: any) => + childSelectors!.some((childSelector) => + childSelector.includes(part) + ) + ); + } else { + shouldHighlight = false; + } + } else { + shouldHighlight = true; + } + } else { + shouldHighlight = true; + } + + if (shouldHighlight) { + const element = iframeDoc.elementFromPoint(x, y); + if (element) { + setCurrentHighlight({ + element, + rect: rect, + selector, + elementInfo: { + ...elementInfo, + tagName: elementInfo?.tagName ?? "", + isDOMMode: true, + }, + childSelectors, + }); + + if (onHighlight) { + onHighlight({ + rect: rect, + elementInfo: { + ...elementInfo, + tagName: elementInfo?.tagName ?? "", + isDOMMode: true, + }, + selector, + childSelectors, + }); + } + } + } else { + setCurrentHighlight(null); + if (onHighlight) { + onHighlight({ + rect: new DOMRect(0, 0, 0, 0), + selector: "", + elementInfo: null, + }); + } + } + } catch (error) { + console.error("Error in DOM highlighting:", error); + setCurrentHighlight(null); + } + }, + [ + getList, + listSelector, + paginationMode, + paginationType, + limitMode, + onHighlight, + ] + ); + + /** + * Set up enhanced interaction handlers for DOM mode + */ + const setupIframeInteractions = useCallback( + (iframeDoc: Document) => { + const existingHandlers = (iframeDoc as any)._domRendererHandlers; + if (existingHandlers) { + Object.entries(existingHandlers).forEach(([event, handler]) => { + iframeDoc.removeEventListener(event, handler as EventListener, false); // Changed to false + }); + } + + const handlers: { [key: string]: EventListener } = {}; + + const mouseMoveHandler: EventListener = (e: Event) => { + if (e.target && !iframeDoc.contains(e.target as Node)) { + return; + } + + const now = performance.now(); + if (now - lastMouseMoveTime.current < MOUSE_MOVE_THROTTLE) { + return; + } + lastMouseMoveTime.current = now; + + const mouseEvent = e as MouseEvent; + const iframeX = mouseEvent.clientX; + const iframeY = mouseEvent.clientY; + + const iframe = iframeRef.current; + if (iframe) { + const iframeRect = iframe.getBoundingClientRect(); + setLastMousePosition({ + x: iframeX + iframeRect.left, + y: iframeY + iframeRect.top, + }); + } + + handleDOMHighlighting(iframeX, iframeY, iframeDoc); + notifyLastAction("move"); + }; + + const mouseDownHandler: EventListener = (e: Event) => { + if (e.target && !iframeDoc.contains(e.target as Node)) { + return; + } + + const mouseEvent = e as MouseEvent; + const target = mouseEvent.target as Element; + const iframeX = mouseEvent.clientX; + const iframeY = mouseEvent.clientY; + + if (isInCaptureMode) { + e.preventDefault(); + e.stopPropagation(); + + if (currentHighlight && onElementSelect) { + onElementSelect({ + rect: currentHighlight.rect, + selector: currentHighlight.selector, + elementInfo: currentHighlight.elementInfo, + childSelectors: currentHighlight.childSelectors || [], + }); + } + notifyLastAction("select element"); + return; + } + + const linkElement = target.closest("a[href]") as HTMLAnchorElement; + if (linkElement && linkElement.href && socket) { + e.preventDefault(); + e.stopPropagation(); + + const href = linkElement.href; + + if (linkElement.target) { + linkElement.target = ""; + } + + const originalHref = linkElement.href; + linkElement.removeAttribute("href"); + + setTimeout(() => { + linkElement.setAttribute("href", originalHref); + }, 100); + + const isSPALink = + href.endsWith("#") || + (href.includes("#") && new URL(href).hash !== ""); + + const selector = clientSelectorGenerator.generateSelector( + iframeDoc, + { x: iframeX, y: iframeY }, + ActionType.Click + ); + + const elementInfo = clientSelectorGenerator.getElementInformation( + iframeDoc, + { x: iframeX, y: iframeY }, + clientSelectorGenerator.getCurrentState().listSelector, + clientSelectorGenerator.getCurrentState().getList + ); + + if (selector && socket) { + socket.emit("dom:click", { + selector, + url: snapshot.baseUrl, + userId: user?.id || "unknown", + elementInfo, + coordinates: undefined, + isSPA: isSPALink, + }); + + notifyLastAction( + isSPALink ? `SPA navigation to ${href}` : `navigate to ${href}` + ); + } + return; + } + + const selector = clientSelectorGenerator.generateSelector( + iframeDoc, + { x: iframeX, y: iframeY }, + ActionType.Click + ); + + const elementInfo = clientSelectorGenerator.getElementInformation( + iframeDoc, + { x: iframeX, y: iframeY }, + clientSelectorGenerator.getCurrentState().listSelector, + clientSelectorGenerator.getCurrentState().getList + ); + + if (selector && elementInfo && socket) { + if (elementInfo?.tagName === "SELECT" && elementInfo.innerHTML) { + const inputElement = target as HTMLInputElement; + inputElement.blur(); + + const wasDisabled = inputElement.disabled; + inputElement.disabled = true; + + setTimeout(() => { + inputElement.disabled = wasDisabled; + }, 100); + + const options = elementInfo.innerHTML + .split(" { + const valueMatch = optionHtml.match(/value="([^"]*)"/); + const textMatch = optionHtml.match(/>([^<]*) { + inputElement.disabled = wasDisabled; + }, 100); + + const pickerInfo = { + coordinates: { x: iframeX, y: iframeY }, + selector, + }; + + switch (inputType) { + case "date": + case "month": + case "week": + if (onShowDatePicker) { + onShowDatePicker(pickerInfo); + } + break; + case "time": + if (onShowTimePicker) { + onShowTimePicker(pickerInfo); + } + break; + case "datetime-local": + if (onShowDateTimePicker) { + onShowDateTimePicker(pickerInfo); + } + break; + } + + notifyLastAction(`${inputType} picker opened`); + return; + } + } + + if ( + elementInfo?.tagName !== "INPUT" && + elementInfo?.tagName !== "SELECT" + ) { + socket.emit("dom:click", { + selector, + url: snapshot.baseUrl, + userId: user?.id || "unknown", + elementInfo, + coordinates: { x: iframeX, y: iframeY }, + isSPA: false, + }); + } + } + + notifyLastAction("click"); + }; + + const mouseUpHandler: EventListener = (e: Event) => { + if (e.target && !iframeDoc.contains(e.target as Node)) { + return; + } + + if (!isInCaptureMode) { + notifyLastAction("release"); + } + }; + + const keyDownHandler: EventListener = (e: Event) => { + if (e.target && !iframeDoc.contains(e.target as Node)) { + return; + } + + const keyboardEvent = e as KeyboardEvent; + const target = keyboardEvent.target as HTMLElement; + + if (!isInCaptureMode && socket && snapshot?.baseUrl) { + const iframe = iframeRef.current; + if (iframe) { + const iframeRect = iframe.getBoundingClientRect(); + const iframeX = lastMousePosition.x - iframeRect.left; + const iframeY = lastMousePosition.y - iframeRect.top; + + const selector = clientSelectorGenerator.generateSelector( + iframeDoc, + { x: iframeX, y: iframeY }, + ActionType.Keydown + ); + + const elementInfo = clientSelectorGenerator.getElementInformation( + iframeDoc, + { x: iframeX, y: iframeY }, + clientSelectorGenerator.getCurrentState().listSelector, + clientSelectorGenerator.getCurrentState().getList + ); + + if (selector) { + socket.emit("dom:keypress", { + selector, + key: keyboardEvent.key, + url: snapshot.baseUrl, + userId: user?.id || "unknown", + inputType: elementInfo?.attributes?.type || "text", + }); + } + } + + notifyLastAction(`${keyboardEvent.key} typed`); + } + + if ( + ["INPUT", "TEXTAREA"].includes(target.tagName) && + !isInCaptureMode + ) { + return; + } + }; + + const keyUpHandler: EventListener = (e: Event) => { + if (e.target && !iframeDoc.contains(e.target as Node)) { + return; + } + + const keyboardEvent = e as KeyboardEvent; + + if (!isInCaptureMode && socket) { + socket.emit("input:keyup", { key: keyboardEvent.key }); + } + }; + + const wheelHandler: EventListener = (e: Event) => { + if (e.target && !iframeDoc.contains(e.target as Node)) { + return; + } + + e.preventDefault(); + + if (!isInCaptureMode) { + const wheelEvent = e as WheelEvent; + const deltaX = Math.round(wheelEvent.deltaX / 10) * 10; + const deltaY = Math.round(wheelEvent.deltaY / 10) * 10; + + if (Math.abs(deltaX) > 5 || Math.abs(deltaY) > 5) { + if (socket) { + socket.emit("dom:scroll", { + deltaX, + deltaY + }) + } + notifyLastAction("scroll"); + } + } + }; + + const clickHandler: EventListener = (e: Event) => { + if (e.target && !iframeDoc.contains(e.target as Node)) { + return; + } + + if (isInCaptureMode) { + e.preventDefault(); + e.stopPropagation(); + return; + } + }; + + const preventDefaults = (e: Event) => { + if (e.target && !iframeDoc.contains(e.target as Node)) { + return; + } + + e.preventDefault(); + e.stopPropagation(); + return false; + }; + + handlers.mousedown = mouseDownHandler; + handlers.mouseup = mouseUpHandler; + handlers.mousemove = mouseMoveHandler; + handlers.wheel = wheelHandler; + handlers.keydown = keyDownHandler; + handlers.keyup = keyUpHandler; + handlers.click = clickHandler; + handlers.submit = preventDefaults; + handlers.beforeunload = preventDefaults; + + Object.entries(handlers).forEach(([event, handler]) => { + iframeDoc.addEventListener(event, handler, false); + }); + + // Store handlers for cleanup + (iframeDoc as any)._domRendererHandlers = handlers; + + // Make iframe focusable for keyboard events + if (iframeRef.current) { + iframeRef.current.tabIndex = 0; + } + }, + [ + socket, + lastMousePosition, + notifyLastAction, + handleDOMHighlighting, + currentHighlight, + onElementSelect, + isInCaptureMode, + snapshot, + user?.id, + onShowDatePicker, + onShowDropdown, + onShowTimePicker, + onShowDateTimePicker, + ] + ); + + /** + * Render DOM snapshot using rrweb + */ + const renderRRWebSnapshot = useCallback( + (snapshotData: ProcessedSnapshot) => { + if (!iframeRef.current) { + console.warn("No iframe reference available"); + return; + } + + const iframe = iframeRef.current; + + try { + setRenderError(null); + setIsRendered(false); + + const tempDoc = + document.implementation.createHTMLDocument("RRWeb Snapshot"); + + const mirror = createMirror(); + + try { + rebuild(snapshotData.snapshot, { + doc: tempDoc, + mirror: mirror, + cache: { stylesWithHoverClass: new Map() }, + afterAppend: (node) => { + if (node.nodeType === Node.TEXT_NODE && node.textContent) { + const text = node.textContent.trim(); + + if ( + text.startsWith("<") && + text.includes(">") && + text.length > 50 + ) { + if (node.parentNode) { + node.parentNode.removeChild(node); + } + } + } + }, + }); + } catch (rebuildError) { + console.error("rrweb rebuild failed:", rebuildError); + throw new Error(`rrweb rebuild failed: ${rebuildError}`); + } + + let rebuiltHTML = tempDoc.documentElement.outerHTML; + + rebuiltHTML = "\n" + rebuiltHTML; + + const additionalCSS = []; + + if (snapshotData.resources.fonts?.length > 0) { + const fontCSS = snapshotData.resources.fonts + .map((font) => { + const format = font.format || "woff2"; + return ` + @font-face { + font-family: 'ProxiedFont-${ + font.url.split("/").pop()?.split(".")[0] || + "unknown" + }'; + src: url("${font.dataUrl}") format("${format}"); + font-display: swap; + } + `; + }) + .join("\n"); + additionalCSS.push(fontCSS); + } + + if (snapshotData.resources.stylesheets?.length > 0) { + const externalCSS = snapshotData.resources.stylesheets + .map((stylesheet) => stylesheet.content) + .join("\n\n"); + additionalCSS.push(externalCSS); + } + + const enhancedCSS = ` + /* rrweb rebuilt content styles */ + html, body { + margin: 0 !important; + padding: 8px !important; + font-family: system-ui, -apple-system, BlinkMacSystemFont, sans-serif !important; + background: white !important; + overflow-x: hidden !important; + } + + html::-webkit-scrollbar, + body::-webkit-scrollbar { + display: none !important; + width: 0 !important; + height: 0 !important; + background: transparent !important; + } + + /* Hide scrollbars for all elements */ + *::-webkit-scrollbar { + display: none !important; + width: 0 !important; + height: 0 !important; + background: transparent !important; + } + + * { + scrollbar-width: none !important; /* Firefox */ + -ms-overflow-style: none !important; /* Internet Explorer 10+ */ + } + + img { + max-width: 100% !important; + height: auto !important; + } + + /* Make everything interactive */ + * { + cursor: ${isInCaptureMode ? "crosshair" : "pointer"} !important; + } + + /* Additional CSS from resources */ + ${additionalCSS.join("\n\n")} + `; + + const headTagRegex = /]*>/i; + const cssInjection = ` + + + + + `; + + if (headTagRegex.test(rebuiltHTML)) { + rebuiltHTML = rebuiltHTML.replace( + headTagRegex, + `${cssInjection}` + ); + } else { + rebuiltHTML = rebuiltHTML.replace( + /]*>/i, + `${cssInjection}` + ); + } + + rebuiltHTML = rebuiltHTML + .replace(/)<[^<]*)*<\/script>/gi, "") + .replace(/\s*on\w+\s*=\s*"[^"]*"/gi, "") + .replace(/\s*on\w+\s*=\s*'[^']*'/gi, "") + .replace(/javascript:/gi, "void:") + .replace(/ { + setIsRendered(true); + setupIframeInteractions(iframeDoc); + }; + } catch (error) { + console.error("Error rendering rrweb snapshot:", error); + setRenderError(error instanceof Error ? error.message : String(error)); + showErrorInIframe(error); + } + }, + [setupIframeInteractions, isInCaptureMode] + ); + + useEffect(() => { + if (snapshot && iframeRef.current) { + renderRRWebSnapshot(snapshot); + } + }, [snapshot]); + + useEffect(() => { + if (isRendered && iframeRef.current) { + const iframeDoc = iframeRef.current.contentDocument; + if (iframeDoc) { + setupIframeInteractions(iframeDoc); + } + } + }, [getText, getList, listSelector, isRendered, setupIframeInteractions]); + + /** + * Show error message in iframe + */ + const showErrorInIframe = (error: any) => { + if (!iframeRef.current) return; + + const iframe = iframeRef.current; + const iframeDoc = iframe.contentDocument || iframe.contentWindow?.document; + + if (iframeDoc) { + try { + iframeDoc.open(); + iframeDoc.write(` + + + + + +
+

Error Loading DOM Content

+

Failed to render the page in DOM mode.

+

Common causes:

+
    +
  • Page is still loading or navigating
    • +
  • Resource proxy timeouts or failures
    • +
  • Network connectivity issues
    • +
  • Invalid HTML structure
    • +
+

Solutions:

+
    +
  • Try switching back to Screenshot mode
    • +
  • Wait for the page to fully load and try again
    • +
  • Check your network connection
    • +
  • Refresh the browser page
    • +
+ +
+ Technical details + 
${error.toString()}
+
+
+ + + `); + iframeDoc.close(); + + window.addEventListener("message", (event) => { + if (event.data === "retry-dom-mode") { + if (socket) { + socket.emit("enable-dom-streaming"); + } + } + }); + } catch (e) { + console.error("Failed to write error message to iframe:", e); + } + } + }; + + useEffect(() => { + return () => { + if (iframeRef.current) { + const iframeDoc = iframeRef.current.contentDocument; + if (iframeDoc) { + const handlers = (iframeDoc as any)._domRendererHandlers; + if (handlers) { + Object.entries(handlers).forEach(([event, handler]) => { + iframeDoc.removeEventListener( + event, + handler as EventListener, + true + ); + }); + } + } + } + }; + }, []); + + return ( +
+