vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add replayer func client side

Browse Source
This commit is contained in:
Rohit Rajan
2026-01-09 17:59:18 +05:30
parent d7830ee6ef
commit dddac2d966
15 changed files with 308 additions and 732 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
src/components/recorder/Canvas.tsx → legacy/src/Canvas.tsx
View File

0
src/components/recorder/Highlighter.tsx → legacy/src/Highlighter.tsx
View File

0
src/helpers/coordinateMapper.ts → legacy/src/coordinateMapper.ts
View File

0
src/helpers/inputHelpers.ts → legacy/src/inputHelpers.ts
View File

1
server/src/browser-management/classes/RemoteBrowser.ts
View File

@@ -640,7 +640,6 @@ export class RemoteBrowser {
private removeAllSocketListeners(): void {
try {
this.socket.removeAllListeners('captureDirectScreenshot');
this.socket.removeAllListeners('rerender');
this.socket.removeAllListeners('settings');
this.socket.removeAllListeners('changeTab');
this.socket.removeAllListeners('addTab');

246
src/components/browser/BrowserWindow.tsx
View File

@@ -7,7 +7,6 @@ import { useBrowserSteps, TextStep, ListStep } from '../../context/browserSteps'
import { useGlobalInfoStore } from '../../context/globalInfo';
import { useTranslation } from 'react-i18next';
import { AuthContext } from '../../context/auth';
import { coordinateMapper } from '../../helpers/coordinateMapper';
import { useBrowserDimensionsStore } from '../../context/browserDimensions';
import { clientSelectorGenerator, ElementFingerprint } from "../../helpers/clientSelectorGenerator";
import { capturedElementHighlighter } from "../../helpers/capturedElementHighlighter";
@@ -36,79 +35,6 @@ interface AttributeOption {
value: string;
}
interface ViewportInfo {
width: number;
height: number;
}
interface RRWebSnapshot {
type: number;
childNodes?: RRWebSnapshot[];
tagName?: string;
attributes?: Record<string, string>;
textContent: string;
id: number;
[key: string]: any;
}
interface ProcessedSnapshot {
snapshot: RRWebSnapshot;
resources: {
stylesheets: Array<{
href: string;
content: string;
media?: string;
}>;
images: Array<{
src: string;
dataUrl: string;
alt?: string;
}>;
fonts: Array<{
url: string;
dataUrl: string;
format?: string;
}>;
scripts: Array<{
src: string;
content: string;
type?: string;
}>;
media: Array<{
src: string;
dataUrl: string;
type: string;
}>;
};
baseUrl: string;
viewport: { width: number; height: number };
timestamp: number;
processingStats: {
totalReplacements: number;
discoveredResources: {
images: number;
stylesheets: number;
scripts: number;
fonts: number;
media: number;
};
cachedResources: {
stylesheets: number;
images: number;
fonts: number;
scripts: number;
media: number;
};
totalCacheSize: number;
};
}
interface RRWebDOMCastData {
snapshotData: ProcessedSnapshot;
userId: string;
timestamp: number;
}
const getAttributeOptions = (tagName: string, elementInfo: ElementInfo | null): AttributeOption[] => {
if (!elementInfo) return [];
switch (tagName.toLowerCase()) {
@@ -154,8 +80,6 @@ export const BrowserWindow = () => {
const [attributeOptions, setAttributeOptions] = useState<AttributeOption[]>([]);
const [selectedElement, setSelectedElement] = useState<{ selector: string, info: ElementInfo | null } | null>(null);
const [currentListId, setCurrentListId] = useState<number | null>(null);
const [viewportInfo, setViewportInfo] = useState<ViewportInfo>({ width: browserWidth, height: browserHeight });
const [isLoading, setIsLoading] = useState(false);
const [cachedChildSelectors, setCachedChildSelectors] = useState<string[]>([]);
const [processingGroupCoordinates, setProcessingGroupCoordinates] = useState<Array<{ element: HTMLElement; rect: DOMRect }>>([]);
const [listSelector, setListSelector] = useState<string | null>(null);
@@ -177,7 +101,7 @@ export const BrowserWindow = () => {
const [manuallyAddedFieldIds, setManuallyAddedFieldIds] = useState<Set<number>>(new Set());
const { socket } = useSocketStore();
const { notify, currentTextActionId, currentListActionId, updateDOMMode, isDOMMode, currentSnapshot } = useGlobalInfoStore();
const { notify, currentTextActionId, currentListActionId, updateDOMMode, isDOMMode } = useGlobalInfoStore();
const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext();
const { addTextStep, addListStep, browserSteps } = useBrowserSteps();
@@ -258,27 +182,11 @@ export const BrowserWindow = () => {
[]
);
const rrwebSnapshotHandler = useCallback(
(data: RRWebDOMCastData) => {
if (!data.userId || data.userId === user?.id) {
if (data.snapshotData && data.snapshotData.snapshot) {
updateDOMMode(true, data.snapshotData);
socket?.emit("dom-mode-enabled");
setIsLoading(false);
} else {
setIsLoading(false);
}
}
},
[user?.id, socket, updateDOMMode]
);
const domModeHandler = useCallback(
(data: any) => {
if (!data.userId || data.userId === user?.id) {
updateDOMMode(true);
socket?.emit("dom-mode-enabled");
setIsLoading(false);
}
},
[user?.id, socket, updateDOMMode]
@@ -288,10 +196,13 @@ export const BrowserWindow = () => {
(data: any) => {
if (!data.userId || data.userId === user?.id) {
updateDOMMode(false);
setIsLoading(false);
if (data.error) {
notify("error", data.error);
}
}
},
[user?.id, updateDOMMode]
[user?.id, updateDOMMode, notify]
);
useEffect(() => {
@@ -698,7 +609,7 @@ export const BrowserWindow = () => {
return finalFields;
},
[currentSnapshot]
[]
);
const removeParentChildDuplicates = (
@@ -798,7 +709,7 @@ export const BrowserWindow = () => {
clientSelectorGenerator.setListSelector(listSelector);
if (currentSnapshot && cachedListSelector !== listSelector) {
if (cachedListSelector !== listSelector) {
setCachedChildSelectors([]);
setIsCachingChildSelectors(true);
setCachedListSelector(listSelector);
@@ -878,7 +789,6 @@ export const BrowserWindow = () => {
listSelector,
socket,
getList,
currentSnapshot,
cachedListSelector,
pendingNotification,
notify,
@@ -989,10 +899,6 @@ export const BrowserWindow = () => {
}
}, [browserSteps, getText, getList, listSelector, currentTextActionId, currentListActionId, isDOMMode, manuallyAddedFieldIds]);
useEffect(() => {
coordinateMapper.updateDimensions(dimensions.width, dimensions.height, viewportInfo.width, viewportInfo.height);
}, [viewportInfo, dimensions.width, dimensions.height]);
useEffect(() => {
if (listSelector) {
sessionStorage.setItem('recordingListSelector', listSelector);
@@ -1027,21 +933,18 @@ export const BrowserWindow = () => {
useEffect(() => {
if (socket) {
socket.on("domcast", rrwebSnapshotHandler);
socket.on("dom-mode-enabled", domModeHandler);
socket.on("dom-mode-error", domModeErrorHandler);
}
return () => {
if (socket) {
socket.off("domcast", rrwebSnapshotHandler);
socket.off("dom-mode-enabled", domModeHandler);
socket.off("dom-mode-error", domModeErrorHandler);
}
};
}, [
socket,
rrwebSnapshotHandler,
domModeHandler,
domModeErrorHandler,
]);
@@ -1068,12 +971,13 @@ export const BrowserWindow = () => {
if (paginationMode && paginationSelector) {
return;
}
if (!getText && !getList) {
setHighlighterData(null);
return;
}
if (!isDOMMode || !currentSnapshot) {
if (!isDOMMode) {
return;
}
@@ -1187,7 +1091,6 @@ export const BrowserWindow = () => {
},
[
isDOMMode,
currentSnapshot,
getText,
getList,
socket,
@@ -1200,105 +1103,6 @@ export const BrowserWindow = () => {
]
);
const highlighterHandler = useCallback((data: { rect: DOMRect, selector: string, elementInfo: ElementInfo | null, childSelectors?: string[], isDOMMode?: boolean; }) => {
if (paginationMode && paginationSelector) {
return;
}
if (isDOMMode || data.isDOMMode) {
domHighlighterHandler(data);
return;
}
const now = performance.now();
if (now - highlighterUpdateRef.current < 16) {
return;
}
highlighterUpdateRef.current = now;
const mappedRect = new DOMRect(
data.rect.x,
data.rect.y,
data.rect.width,
data.rect.height
);
const mappedData = {
...data,
rect: mappedRect
};
if (getList === true) {
if (listSelector) {
socket?.emit('listSelector', { selector: listSelector });
const hasValidChildSelectors = Array.isArray(mappedData.childSelectors) && mappedData.childSelectors.length > 0;
if (limitMode) {
setHighlighterData(null);
} else if (paginationMode) {
if (paginationType !== '' && !['none', 'scrollDown', 'scrollUp'].includes(paginationType)) {
setHighlighterData(mappedData);
} else {
setHighlighterData(null);
}
} else if (mappedData.childSelectors && mappedData.childSelectors.includes(mappedData.selector)) {
setHighlighterData(mappedData);
} else if (mappedData.elementInfo?.isIframeContent && mappedData.childSelectors) {
const isIframeChild = mappedData.childSelectors.some(childSelector =>
mappedData.selector.includes(':>>') &&
childSelector.split(':>>').some(part =>
mappedData.selector.includes(part.trim())
)
);
setHighlighterData(isIframeChild ? mappedData : null);
} else if (mappedData.selector.includes(':>>') && hasValidChildSelectors) {
const selectorParts = mappedData.selector.split(':>>').map(part => part.trim());
const isValidMixedSelector = selectorParts.some(part =>
mappedData.childSelectors!.some(childSelector =>
childSelector.includes(part)
)
);
setHighlighterData(isValidMixedSelector ? mappedData : null);
} else if (mappedData.elementInfo?.isShadowRoot && mappedData.childSelectors) {
const isShadowChild = mappedData.childSelectors.some(childSelector =>
mappedData.selector.includes('>>') &&
childSelector.split('>>').some(part =>
mappedData.selector.includes(part.trim())
)
);
setHighlighterData(isShadowChild ? mappedData : null);
} else if (mappedData.selector.includes('>>') && hasValidChildSelectors) {
const selectorParts = mappedData.selector.split('>>').map(part => part.trim());
const isValidMixedSelector = selectorParts.some(part =>
mappedData.childSelectors!.some(childSelector =>
childSelector.includes(part)
)
);
setHighlighterData(isValidMixedSelector ? mappedData : null);
} else {
setHighlighterData(null);
}
} else {
setHighlighterData(mappedData);
}
} else {
setHighlighterData(mappedData);
}
}, [getList, socket, listSelector, paginationMode, paginationType, limitMode]);
useEffect(() => {
document.addEventListener("mousemove", onMouseMove, false);
if (socket) {
socket.off("highlighter", highlighterHandler);
socket.on("highlighter", highlighterHandler);
}
return () => {
document.removeEventListener("mousemove", onMouseMove);
if (socket) {
socket.off("highlighter", highlighterHandler);
}
};
}, [socket, highlighterHandler, getList, listSelector]);
useEffect(() => {
if (socket && listSelector) {
socket.emit('setGetList', { getList: true });
@@ -1533,7 +1337,6 @@ export const BrowserWindow = () => {
]
);
const handleClick = (e: React.MouseEvent<HTMLDivElement>) => {
if (highlighterData) {
const shouldProcessClick = true;
@@ -1720,7 +1523,7 @@ export const BrowserWindow = () => {
default:
data = selectedElement.info?.innerText || '';
}
{
if (getText === true) {
addTextStep('', data, {
selector: selectedElement.selector,
@@ -1729,6 +1532,7 @@ export const BrowserWindow = () => {
attribute: attribute
}, currentTextActionId || `text-${crypto.randomUUID()}`);
}
if (getList === true && listSelector && currentListId) {
const newField: TextStep = {
id: Date.now(),
@@ -1762,7 +1566,7 @@ export const BrowserWindow = () => {
);
}
}
}
}
setShowAttributeModal(false);
@@ -1817,7 +1621,6 @@ export const BrowserWindow = () => {
style={{ width: browserWidth }}
id="browser-window"
>
{/* Attribute selection modal */}
{(getText === true || getList === true) && (
<GenericModal
isOpen={showAttributeModal}
@@ -1906,7 +1709,6 @@ export const BrowserWindow = () => {
/>
)}
{/* Main content area */}
<div
style={{
position: "relative",
@@ -1916,7 +1718,6 @@ export const BrowserWindow = () => {
borderRadius: "0px 0px 5px 5px",
}}
>
{/* Add CSS for the spinner animation */}
<style>{`
@keyframes spin {
0% { transform: rotate(0deg); }
@@ -1933,13 +1734,12 @@ export const BrowserWindow = () => {
id="dom-highlight-overlay"
style={{
position: "absolute",
inset: 0, // top:0; right:0; bottom:0; left:0
overflow: "hidden", // clip everything within iframe area
inset: 0,
overflow: "hidden",
pointerEvents: "none",
zIndex: 1000,
}}
>
{/* Individual element highlight (for non-group or hovered element) */}
{((getText && !listSelector) ||
(getList && paginationMode && !paginationSelector && paginationType !== "" &&
!["none", "scrollDown", "scrollUp"].includes(paginationType))) && (
@@ -1960,7 +1760,6 @@ export const BrowserWindow = () => {
/>
)}
{/* Grouped list element highlights */}
{getList &&
!listSelector &&
currentGroupInfo?.isGroupElement &&
@@ -2029,7 +1828,6 @@ export const BrowserWindow = () => {
}}
/>
{/* Label for similar element */}
<div
style={{
position: "absolute",
@@ -2054,23 +1852,22 @@ export const BrowserWindow = () => {
)}
</>
)}
{/* --- Main DOM Renderer Section --- */}
<div
id="iframe-wrapper"
style={{
position: "relative",
width: "100%",
height: "100%",
overflow: "hidden", // key: confine everything below
overflow: "hidden",
borderRadius: "0px 0px 5px 5px",
}}
>
{currentSnapshot ? (
{isDOMMode ? (
<>
<DOMBrowserRenderer
width={dimensions.width}
height={dimensions.height}
snapshot={currentSnapshot}
getList={getList}
getText={getText}
listSelector={listSelector}
@@ -2088,7 +1885,6 @@ export const BrowserWindow = () => {
onShowDateTimePicker={handleShowDateTimePicker}
/>
{/* --- Loading overlay --- */}
{isCachingChildSelectors && (
<>
<div
@@ -2209,7 +2005,6 @@ export const BrowserWindow = () => {
const DOMLoadingIndicator: React.FC = () => {
const [progress, setProgress] = useState(0);
const [pendingRequests, setPendingRequests] = useState(0);
const [hasStartedLoading, setHasStartedLoading] = useState(false);
const { socket } = useSocketStore();
const { state } = useContext(AuthContext);
@@ -2225,15 +2020,12 @@ const DOMLoadingIndicator: React.FC = () => {
userId: string;
}) => {
if (!data.userId || data.userId === user?.id) {
// Once loading has started, never reset progress to 0
if (!hasStartedLoading && data.progress > 0) {
setHasStartedLoading(true);
}
// Only update progress if we haven't started or if new progress is higher
if (!hasStartedLoading || data.progress >= progress) {
setProgress(data.progress);
setPendingRequests(data.pendingRequests);
}
}
};
@@ -2259,7 +2051,6 @@ const DOMLoadingIndicator: React.FC = () => {
gap: "15px",
}}
>
{/* Loading text with percentage */}
<div
style={{
fontSize: "18px",
@@ -2270,7 +2061,6 @@ const DOMLoadingIndicator: React.FC = () => {
Loading {progress}%
</div>
{/* Progress bar */}
<div
style={{
width: "240px",

6
src/components/pickers/DatePicker.tsx
View File

@@ -1,6 +1,10 @@
import React, { useState } from 'react';
import { useSocketStore } from '../../context/socket';
import { Coordinates } from '../recorder/Canvas';
interface Coordinates {
x: number;
y: number;
};
interface DatePickerProps {
coordinates: Coordinates;

6
src/components/pickers/DateTimeLocalPicker.tsx
View File

@@ -1,6 +1,10 @@
import React, { useState } from 'react';
import { useSocketStore } from '../../context/socket';
import { Coordinates } from '../recorder/Canvas';
interface Coordinates {
x: number;
y: number;
};
interface DateTimeLocalPickerProps {
coordinates: Coordinates;

6
src/components/pickers/Dropdown.tsx
View File

@@ -1,6 +1,10 @@
import React, { useState } from 'react';
import { useSocketStore } from '../../context/socket';
import { Coordinates } from '../recorder/Canvas';
interface Coordinates {
x: number;
y: number;
};
interface DropdownProps {
coordinates: Coordinates;

6
src/components/pickers/TimePicker.tsx
View File

@@ -1,6 +1,10 @@
import React, { useState } from 'react';
import { useSocketStore } from '../../context/socket';
import { Coordinates } from '../recorder/Canvas';
interface Coordinates {
x: number;
y: number;
};
interface TimePickerProps {
coordinates: Coordinates;

549
src/components/recorder/DOMBrowserRenderer.tsx
View File

@@ -7,9 +7,8 @@ import React, {
} from "react";
import { useSocketStore } from "../../context/socket";
import { useGlobalInfoStore } from "../../context/globalInfo";
import { useTranslation } from "react-i18next";
import { AuthContext } from "../../context/auth";
import { rebuild, createMirror } from "rrweb-snapshot";
import { Replayer } from "rrweb"
import {
ActionType,
clientSelectorGenerator,
@@ -29,72 +28,9 @@ interface ElementInfo {
isDOMMode?: boolean;
}
interface ProcessedSnapshot {
snapshot: RRWebSnapshot;
resources: {
stylesheets: Array<{
href: string;
content: string;
media?: string;
}>;
images: Array<{
src: string;
dataUrl: string;
alt?: string;
}>;
fonts: Array<{
url: string;
dataUrl: string;
format?: string;
}>;
scripts: Array<{
src: string;
content: string;
type?: string;
}>;
media: Array<{
src: string;
dataUrl: string;
type: string;
}>;
};
baseUrl: string;
viewport: { width: number; height: number };
timestamp: number;
processingStats: {
totalReplacements: number;
discoveredResources: {
images: number;
stylesheets: number;
scripts: number;
fonts: number;
media: number;
};
cachedResources: {
stylesheets: number;
images: number;
fonts: number;
scripts: number;
media: number;
};
totalCacheSize: number;
};
}
interface RRWebSnapshot {
type: number;
childNodes?: RRWebSnapshot[];
tagName?: string;
attributes?: Record<string, string>;
textContent: string;
id: number;
[key: string]: any;
}
interface RRWebDOMBrowserRendererProps {
width: number;
height: number;
snapshot: ProcessedSnapshot;
getList?: boolean;
getText?: boolean;
listSelector?: string | null;
@@ -148,7 +84,6 @@ interface RRWebDOMBrowserRendererProps {
export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
width,
height,
snapshot,
getList = false,
getText = false,
listSelector = null,
@@ -165,8 +100,9 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
onShowTimePicker,
onShowDateTimePicker,
}) => {
const { t } = useTranslation();
const containerRef = useRef<HTMLDivElement>(null);
const replayerIframeRef = useRef<HTMLIFrameElement | null>(null);
const replayerRef = useRef<any>(null);
const iframeRef = useRef<HTMLIFrameElement>(null);
const [isRendered, setIsRendered] = useState(false);
const [lastMousePosition, setLastMousePosition] = useState({ x: 0, y: 0 });
@@ -184,7 +120,7 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
const { state } = useContext(AuthContext);
const { user } = state;
const MOUSE_MOVE_THROTTLE = 16; // ~60fps
const MOUSE_MOVE_THROTTLE = 16;
const lastMouseMoveTime = useRef(0);
const notifyLastAction = (action: string) => {
@@ -313,7 +249,7 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
isShadow,
childSelectors,
groupInfo,
similarElements, // Pass similar elements data
similarElements,
});
}
}
@@ -337,12 +273,14 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
getList,
listSelector,
paginationMode,
paginationSelector,
cachedChildSelectors,
paginationType,
limitMode,
onHighlight,
]
);
/**
* Set up enhanced interaction handlers for DOM mode
*/
@@ -379,7 +317,7 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
const iframeX = mouseEvent.clientX;
const iframeY = mouseEvent.clientY;
const iframe = iframeRef.current;
const iframe = replayerIframeRef.current;
if (iframe) {
const iframeRect = iframe.getBoundingClientRect();
setLastMousePosition({
@@ -407,7 +345,6 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
e.stopPropagation();
if (currentHighlight && onElementSelect) {
// Get the group info for the current highlight
const highlighterData =
clientSelectorGenerator.generateDataForHighlighter(
{ x: iframeX, y: iframeY },
@@ -438,6 +375,7 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
e.stopPropagation();
const href = linkElement.href;
const originalTarget = linkElement.target;
if (linkElement.target) {
linkElement.target = "";
@@ -447,12 +385,17 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
linkElement.removeAttribute("href");
setTimeout(() => {
try {
linkElement.setAttribute("href", originalHref);
if (originalTarget) {
linkElement.setAttribute("target", originalTarget);
}
} catch (error) {
console.warn("Could not restore link attributes:", error);
}
}, 100);
const isSPALink =
href.endsWith("#") ||
(href.includes("#") && new URL(href).hash !== "");
const isSPALink = href.startsWith('#');
const selector = clientSelectorGenerator.generateSelector(
iframeDoc,
@@ -470,7 +413,6 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
if (selector && socket) {
socket.emit("dom:click", {
selector,
url: snapshot.baseUrl,
userId: user?.id || "unknown",
elementInfo,
coordinates: undefined,
@@ -592,17 +534,14 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
socket.emit("dom:click", {
selector,
url: snapshot.baseUrl,
userId: user?.id || "unknown",
elementInfo,
coordinates: { x: relativeX, y: relativeY },
isSPA: false,
});
} else if (elementInfo?.tagName !== "SELECT") {
// Handle other elements normally
socket.emit("dom:click", {
selector,
url: snapshot.baseUrl,
userId: user?.id || "unknown",
elementInfo,
coordinates: { x: iframeX, y: iframeY },
@@ -632,21 +571,19 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
const keyboardEvent = e as KeyboardEvent;
const target = keyboardEvent.target as HTMLElement;
if (!isInCaptureMode && socket && snapshot?.baseUrl) {
const iframe = iframeRef.current;
if (!isInCaptureMode && socket) {
const iframe = replayerIframeRef.current;
if (iframe) {
const focusedElement = iframeDoc.activeElement as HTMLElement;
let coordinates = { x: 0, y: 0 };
if (focusedElement && focusedElement !== iframeDoc.body) {
// Get coordinates from the focused element
const rect = focusedElement.getBoundingClientRect();
coordinates = {
x: rect.left + rect.width / 2,
y: rect.top + rect.height / 2
};
} else {
// Fallback to last mouse position if no focused element
const iframeRect = iframe.getBoundingClientRect();
coordinates = {
x: lastMousePosition.x - iframeRect.left,
@@ -671,7 +608,6 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
socket.emit("dom:keypress", {
selector,
key: keyboardEvent.key,
url: snapshot.baseUrl,
userId: user?.id || "unknown",
inputType: elementInfo?.attributes?.type || "text",
});
@@ -768,12 +704,11 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
iframeDoc.addEventListener(event, handler, options);
});
// Store handlers for cleanup
(iframeDoc as any)._domRendererHandlers = handlers;
// Make iframe focusable for keyboard events
if (iframeRef.current) {
iframeRef.current.tabIndex = 0;
const iframe = replayerIframeRef.current;
if (iframe) {
iframe.tabIndex = 0;
}
},
[
@@ -784,287 +719,219 @@ export const DOMBrowserRenderer: React.FC<RRWebDOMBrowserRendererProps> = ({
currentHighlight,
onElementSelect,
isInCaptureMode,
snapshot,
user?.id,
onShowDatePicker,
onShowDropdown,
onShowTimePicker,
onShowDateTimePicker,
cachedChildSelectors
]
);
/**
* Render DOM snapshot using rrweb
* Cleanup replayer on unmount
*/
const renderRRWebSnapshot = useCallback(
(snapshotData: ProcessedSnapshot) => {
if (!iframeRef.current) {
console.warn("No iframe reference available");
return;
}
if (isInCaptureMode || isCachingChildSelectors) {
return; // Skip rendering in capture mode
}
try {
setIsRendered(false);
const iframe = iframeRef.current!;
let iframeDoc: Document;
try {
iframeDoc = iframe.contentDocument!;
if (!iframeDoc) {
throw new Error("Cannot access iframe document");
}
} catch (crossOriginError) {
console.warn("Cross-origin iframe access blocked, recreating iframe");
const newIframe = document.createElement('iframe');
newIframe.style.cssText = iframe.style.cssText;
newIframe.sandbox = iframe.sandbox.value;
newIframe.title = iframe.title;
newIframe.tabIndex = iframe.tabIndex;
newIframe.id = iframe.id;
iframe.parentNode?.replaceChild(newIframe, iframe);
Object.defineProperty(iframeRef, 'current', {
value: newIframe,
writable: false,
enumerable: true,
configurable: true
});
iframeDoc = newIframe.contentDocument!;
if (!iframeDoc) {
throw new Error("Cannot access new iframe document");
}
}
const styleTags = Array.from(
document.querySelectorAll('link[rel="stylesheet"], style')
)
.map((tag) => tag.outerHTML)
.join("\n");
const enhancedCSS = `
/* rrweb rebuilt content styles */
html, body {
margin: 0 !important;
padding: 8px !important;
overflow-x: hidden !important;
}
html::-webkit-scrollbar,
body::-webkit-scrollbar {
display: none !important;
width: 0 !important;
height: 0 !important;
background: transparent !important;
}
/* Hide scrollbars for all elements */
*::-webkit-scrollbar {
display: none !important;
width: 0 !important;
height: 0 !important;
background: transparent !important;
}
* {
scrollbar-width: none !important; /* Firefox */
-ms-overflow-style: none !important; /* Internet Explorer 10+ */
}
/* Make everything interactive */
* {
cursor: "pointer" !important;
}
`;
const skeleton = `
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<base href="${snapshotData.baseUrl}">
${styleTags}
<style>${enhancedCSS}</style>
</head>
<body></body>
</html>
`;
if (!iframeDoc) {
throw new Error("Cannot access iframe document");
}
// Write the skeleton into the iframe
iframeDoc.open();
iframeDoc.write(skeleton);
iframeDoc.close();
const mirror = createMirror();
try {
rebuild(snapshotData.snapshot, {
doc: iframeDoc,
mirror: mirror,
hackCss: false,
cache: { stylesWithHoverClass: new Map() },
afterAppend: (node) => {
if (node.nodeType === Node.TEXT_NODE && node.textContent) {
const text = node.textContent.trim();
if (
text.startsWith("<") &&
text.includes(">") &&
text.length > 50
) {
if (node.parentNode) {
node.parentNode.removeChild(node);
}
}
}
},
});
} catch (rebuildError) {
console.error("rrweb rebuild failed:", rebuildError);
throw new Error(`rrweb rebuild failed: ${rebuildError}`);
}
setIsRendered(true);
setupIframeInteractions(iframeDoc);
} catch (error) {
console.error("Error rendering rrweb snapshot:", error);
}
},
[setupIframeInteractions, isInCaptureMode, isCachingChildSelectors]
);
useEffect(() => {
if (snapshot && iframeRef.current) {
renderRRWebSnapshot(snapshot);
}
}, [snapshot]);
useEffect(() => {
if (isRendered && iframeRef.current) {
const iframeDoc = iframeRef.current.contentDocument;
if (iframeDoc) {
setupIframeInteractions(iframeDoc);
}
}
}, [getText, getList, listSelector, isRendered, setupIframeInteractions]);
useEffect(() => {
return () => {
if (iframeRef.current) {
const iframeDoc = iframeRef.current.contentDocument;
if (iframeDoc) {
const handlers = (iframeDoc as any)._domRendererHandlers;
if (handlers) {
Object.entries(handlers).forEach(([event, handler]) => {
const options: boolean | AddEventListenerOptions = ['wheel', 'touchstart', 'touchmove'].includes(event)
? { passive: false }
: false;
iframeDoc.removeEventListener(
event,
handler as EventListener,
options
);
});
}
}
if (replayerRef.current) {
replayerRef.current.pause();
replayerRef.current = null;
}
};
}, []);
/**
* Listen for rrweb events from backend and add to replayer
*/
useEffect(() => {
if (!socket) {
console.warn('No socket available, skipping event listener setup');
return;
}
const handleRRWebEvent = (event: any) => {
if (!replayerRef.current && event.type === 2) {
const container = document.getElementById('mirror-container');
if (!container) {
console.warn('Container #mirror-container not found');
return;
}
const replayer = new Replayer([], {
root: container,
liveMode: true,
mouseTail: false
});
replayer.startLive();
replayer.addEvent(event);
replayerRef.current = replayer;
setTimeout(() => {
const replayerWrapper = container.querySelector('.replayer-wrapper');
const replayerIframe = replayerWrapper?.querySelector('iframe') as HTMLIFrameElement;
if (replayerIframe) {
replayerIframe.style.width = '100%';
replayerIframe.style.height = '100%';
replayerIframe.style.border = 'none';
replayerIframe.style.position = 'absolute';
replayerIframe.style.top = '0';
replayerIframe.style.left = '0';
replayerIframe.style.backgroundColor = '#ffffff';
replayerIframe.style.display = 'block';
replayerIframe.style.pointerEvents = 'auto';
replayerIframe.id = 'dom-browser-iframe';
replayerIframeRef.current = replayerIframe;
try {
const iframeDoc = replayerIframe.contentDocument;
if (iframeDoc) {
setupIframeInteractions(iframeDoc);
}
} catch (err) {
console.warn('Error accessing iframe:', err);
}
replayer.on('fullsnapshot-rebuilded', () => {
const iframe = replayerIframeRef.current;
if (iframe && iframe.contentDocument) {
setupIframeInteractions(iframe.contentDocument);
iframe.style.pointerEvents = 'auto';
const wrapper = container.querySelector('.replayer-wrapper') as HTMLElement;
if(wrapper) wrapper.style.pointerEvents = 'auto';
setIsRendered(true);
}
});
} else {
console.warn('Could not find iframe in replayer-wrapper');
}
}, 150);
} else if (replayerRef.current) {
replayerRef.current.addEvent(event);
}
};
socket.on('rrweb-event', handleRRWebEvent);
socket.emit('request-refresh');
return () => {
socket.off('rrweb-event', handleRRWebEvent);
};
}, [socket, setupIframeInteractions]);
useEffect(() => {
const iframe = replayerIframeRef.current;
if (iframe && iframe.contentDocument) {
setupIframeInteractions(iframe.contentDocument);
}
}, [setupIframeInteractions]);
return (
<div
id="mirror-container"
ref={containerRef}
style={{
width: width,
height: height,
overflow: "hidden !important",
position: "relative",
borderRadius: "0px 0px 5px 5px",
backgroundColor: "white",
backgroundColor: "#ffffff",
overflow: "hidden",
isolation: "isolate",
}}
>
<iframe
ref={iframeRef}
id="dom-browser-iframe"
style={{
width: "100%",
height: "100%",
border: "none",
display: "block",
overflow: isCachingChildSelectors ? "hidden !important" : "hidden !important",
pointerEvents: isCachingChildSelectors ? "none" : "auto",
}}
sandbox="allow-same-origin allow-forms allow-scripts"
title="DOM Browser Content"
tabIndex={0}
/>
{/* Loading indicator */}
{!isRendered && (
<div
style={{
position: "absolute",
top: 0,
left: 0,
right: 0,
bottom: 0,
background: "rgba(255, 255, 255, 0.9)",
display: "flex",
alignItems: "center",
justifyContent: "center",
fontSize: "18px",
color: "#666",
flexDirection: "column",
gap: "10px",
}}
>
<div
style={{
width: "40px",
height: "40px",
border: "3px solid #ff00c3",
borderTop: "3px solid transparent",
borderRadius: "50%",
animation: "spin 1s linear infinite",
}}
/>
<div>Loading website...</div>
<style>{`
@keyframes spin {
0% { transform: rotate(0deg); }
100% { transform: rotate(360deg); }
}
`}</style>
</div>
)}
{/* Capture mode overlay */}
{isInCaptureMode && (
<div
style={{
position: "absolute",
top: 0,
left: 0,
right: 0,
bottom: 0,
cursor: "pointer",
pointerEvents: "none",
zIndex: 999,
borderRadius: "0px 0px 5px 5px",
}}
/>
<DOMLoadingIndicator />
)}
</div>
);
};
const DOMLoadingIndicator: React.FC = () => {
const [progress, setProgress] = useState(0);
const [hasStartedLoading, setHasStartedLoading] = useState(false);
const { socket } = useSocketStore();
const { state } = useContext(AuthContext);
const { user } = state;
useEffect(() => {
if (!socket) return;
const handleLoadingProgress = (data: {
progress: number;
pendingRequests: number;
userId: string;
}) => {
if (!data.userId || data.userId === user?.id) {
if (!hasStartedLoading && data.progress > 0) {
setHasStartedLoading(true);
}
if (!hasStartedLoading || data.progress >= progress) {
setProgress(data.progress);
}
}
};
socket.on("domLoadingProgress", handleLoadingProgress);
return () => {
socket.off("domLoadingProgress", handleLoadingProgress);
};
}, [socket, user?.id, hasStartedLoading, progress]);
return (
<div
style={{
position: "absolute",
top: 0,
left: 0,
right: 0,
bottom: 0,
background: "#f5f5f5",
display: "flex",
alignItems: "center",
justifyContent: "center",
flexDirection: "column",
gap: "15px",
zIndex: 9999,
}}
>
<div
style={{
fontSize: "18px",
fontWeight: "500",
color: "#333",
}}
>
Loading {progress}%
</div>
<div
style={{
width: "240px",
height: "6px",
background: "#e0e0e0",
borderRadius: "3px",
overflow: "hidden",
}}
>
<div
style={{
width: `${progress}%`,
height: "100%",
background: "linear-gradient(90deg, #ff00c3, #ff66d9)",
borderRadius: "3px",
transition: "width 0.3s ease-out",
}}
/>
</div>
</div>
);
};

41
src/components/recorder/RightSidePanel.tsx
View File

@@ -36,10 +36,6 @@ interface RightSidePanelProps {
}
export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture }) => {
const [textLabels, setTextLabels] = useState<{ [id: string]: string }>({});
const [errors, setErrors] = useState<{ [id: string]: string }>({});
const [confirmedTextSteps, setConfirmedTextSteps] = useState<{ [id: string]: boolean }>({});
const [confirmedListTextFields, setConfirmedListTextFields] = useState<{ [listId: string]: { [fieldKey: string]: boolean } }>({});
const [showCaptureList, setShowCaptureList] = useState(true);
const [showCaptureScreenshot, setShowCaptureScreenshot] = useState(true);
const [showCaptureText, setShowCaptureText] = useState(true);
@@ -52,7 +48,7 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
} | null>(null);
const autoDetectionRunRef = useRef<string | null>(null);
const { lastAction, notify, currentWorkflowActionsState, setCurrentWorkflowActionsState, resetInterpretationLog, currentListActionId, setCurrentListActionId, currentTextActionId, setCurrentTextActionId, currentScreenshotActionId, setCurrentScreenshotActionId, isDOMMode, setIsDOMMode, currentSnapshot, setCurrentSnapshot, updateDOMMode, initialUrl, setRecordingUrl, currentTextGroupName } = useGlobalInfoStore();
const { notify, currentWorkflowActionsState, setCurrentWorkflowActionsState, resetInterpretationLog, currentListActionId, setCurrentListActionId, currentTextActionId, setCurrentTextActionId, currentScreenshotActionId, setCurrentScreenshotActionId, isDOMMode, updateDOMMode, currentTextGroupName } = useGlobalInfoStore();
const {
getText, startGetText, stopGetText,
getList, startGetList, stopGetList,
@@ -65,8 +61,7 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
showPaginationOptions, setShowPaginationOptions,
showLimitOptions, setShowLimitOptions,
workflow, setWorkflow,
activeAction, setActiveAction,
startAction, finishAction
activeAction, setActiveAction, finishAction
} = useActionContext();
const { browserSteps, addScreenshotStep, updateListStepLimit, updateListStepPagination, deleteStepsByActionId, updateListStepData, updateScreenshotStepData, emitActionForStep } = useBrowserSteps();
@@ -154,20 +149,10 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
}
};
const domcastHandler = (data: any) => {
if (!data.userId || data.userId === id) {
if (data.snapshotData && data.snapshotData.snapshot) {
updateDOMMode(true, data.snapshotData);
}
}
};
socket.on("dom-mode-enabled", domModeHandler);
socket.on("domcast", domcastHandler);
return () => {
socket.off("dom-mode-enabled", domModeHandler);
socket.off("domcast", domcastHandler);
};
}
}, [socket, id, updateDOMMode]);
@@ -176,11 +161,9 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
if (socket) {
socket.on("workflow", workflowHandler);
}
// fetch the workflow every time the id changes
if (id) {
fetchWorkflow(id, workflowHandler);
}
// fetch workflow in 15min intervals
let interval = setInterval(() => {
if (id) {
fetchWorkflow(id, workflowHandler);
@@ -267,7 +250,7 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
fields: Record<string, any>,
currentListId: number
) => {
if (isDOMMode && currentSnapshot) {
if (isDOMMode) {
try {
let iframeElement = document.querySelector(
"#dom-browser-iframe"
@@ -318,25 +301,9 @@ export const RightSidePanel: React.FC<RightSidePanelProps> = ({ onFinishCapture
console.error("Error in client-side data extraction:", error);
notify("error", "Failed to extract data client-side");
}
} else {
if (!socket) {
console.error("Socket not available for backend extraction");
return;
}
try {
socket.emit("extractListData", {
listSelector,
fields,
currentListId,
pagination: { type: "", selector: "" },
});
} catch (error) {
console.error("Error in backend data extraction:", error);
}
}
},
[isDOMMode, currentSnapshot, updateListStepData, socket, notify, currentWorkflowActionsState]
[isDOMMode, updateListStepData, socket, notify, currentWorkflowActionsState]
);
useEffect(() => {

58
src/context/globalInfo.tsx
View File

@@ -1,4 +1,4 @@
import React, { createContext, useContext, useState } from "react";
import { createContext, useContext, useState } from "react";
import { AlertSnackbarProps } from "../components/ui/AlertSnackbar";
import { WhereWhatPair } from "maxun-core";
import { QueryClient, QueryClientProvider, useQuery, useQueryClient } from '@tanstack/react-query';
@@ -49,41 +49,6 @@ interface ScheduleConfig {
cronExpression?: string;
}
interface ProcessedSnapshot {
snapshot: any;
resources: {
stylesheets: Array<{
href: string;
content: string;
media?: string;
}>;
images: Array<{
src: string;
dataUrl: string;
alt?: string;
}>;
fonts: Array<{
url: string;
dataUrl: string;
format?: string;
}>;
scripts: Array<{
src: string;
content: string;
type?: string;
}>;
media: Array<{
src: string;
dataUrl: string;
type: string;
}>;
};
baseUrl: string;
viewport: { width: number; height: number };
timestamp: number;
processingStats: any;
}
export interface RobotSettings {
id: string;
userId?: number;
@@ -147,9 +112,7 @@ interface GlobalInfo {
setCurrentTextGroupName: (name: string) => void;
isDOMMode: boolean;
setIsDOMMode: (isDOMMode: boolean) => void;
currentSnapshot: ProcessedSnapshot | null;
setCurrentSnapshot: (snapshot: ProcessedSnapshot | null) => void;
updateDOMMode: (isDOMMode: boolean, snapshot?: ProcessedSnapshot | null) => void;
updateDOMMode: (isDOMMode: boolean) => void;
};
class GlobalInfoStore implements Partial<GlobalInfo> {
@@ -181,7 +144,6 @@ class GlobalInfoStore implements Partial<GlobalInfo> {
currentScreenshotActionId = '';
currentTextGroupName = 'Text Data';
isDOMMode = false;
currentSnapshot = null;
};
const globalInfoStore = new GlobalInfoStore();
@@ -272,7 +234,6 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
const [rerenderRuns, setRerenderRuns] = useState<boolean>(globalInfoStore.rerenderRuns);
const [rerenderRobots, setRerenderRobots] = useState<boolean>(globalInfoStore.rerenderRobots);
const [recordingLength, setRecordingLength] = useState<number>(globalInfoStore.recordingLength);
// const [recordingId, setRecordingId] = useState<string | null>(globalInfoStore.recordingId);
const [recordingId, setRecordingId] = useState<string | null>(() => {
try {
const stored = sessionStorage.getItem('recordingId');
@@ -282,7 +243,6 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
}
});
// Create a wrapped setter that persists to sessionStorage
const setPersistedRecordingId = (newRecordingId: string | null) => {
setRecordingId(newRecordingId);
try {
@@ -307,7 +267,6 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
const [currentScreenshotActionId, setCurrentScreenshotActionId] = useState<string>('');
const [currentTextGroupName, setCurrentTextGroupName] = useState<string>('Text Data');
const [isDOMMode, setIsDOMMode] = useState<boolean>(globalInfoStore.isDOMMode);
const [currentSnapshot, setCurrentSnapshot] = useState<ProcessedSnapshot | null>(globalInfoStore.currentSnapshot);
const notify = (severity: 'error' | 'warning' | 'info' | 'success', message: string) => {
setNotification({ severity, message, isOpen: true });
@@ -326,22 +285,13 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
const resetInterpretationLog = () => {
setShouldResetInterpretationLog(true);
// Reset the flag after a short delay to allow components to respond
setTimeout(() => {
setShouldResetInterpretationLog(false);
}, 100);
}
const updateDOMMode = (mode: boolean, snapshot?: ProcessedSnapshot | null) => {
const updateDOMMode = (mode: boolean) => {
setIsDOMMode(mode);
if (snapshot !== undefined) {
setCurrentSnapshot(snapshot);
}
if (!mode) {
setCurrentSnapshot(null);
}
}
const [dataCacheClient] = useState(() => createDataCacheClient());
@@ -391,8 +341,6 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
setCurrentTextGroupName,
isDOMMode,
setIsDOMMode,
currentSnapshot,
setCurrentSnapshot,
updateDOMMode,
}}
>

7
src/helpers/capturedElementHighlighter.ts
View File

@@ -59,7 +59,12 @@ class CapturedElementHighlighter {
* Get the iframe document
*/
private getIframeDocument(): Document | null {
const iframeElement = document.querySelector('#dom-browser-iframe') as HTMLIFrameElement;
let iframeElement = document.querySelector('#dom-browser-iframe') as HTMLIFrameElement;
if (!iframeElement) {
iframeElement = document.querySelector('.replayer-wrapper iframe') as HTMLIFrameElement;
}
return iframeElement?.contentDocument || null;
}

16
src/pages/RecordingPage.tsx
View File

@@ -56,8 +56,6 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
});
};
useEffect(() => changeBrowserDimensions(), [isLoaded])
useEffect(() => {
if (darkMode) {
@@ -118,20 +116,6 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
}
}, [setId, recordingUrl, setRecordingUrl, setRecordingName, setRetrainRobotId]);
const changeBrowserDimensions = useCallback(() => {
if (browserContentRef.current) {
const currentWidth = Math.floor(browserContentRef.current.getBoundingClientRect().width);
const innerHeightWithoutNavBar = window.innerHeight - 54.5;
if (innerHeightWithoutNavBar <= (currentWidth / 1.6)) {
setWidth(currentWidth - 10);
setHasScrollbar(true);
} else {
setWidth(currentWidth);
}
socket?.emit("rerender");
}
}, [socket]);
const handleLoaded = useCallback(() => {
if (recordingName && browserId && recordingId) {
editRecordingFromStorage(browserId, recordingId).then(() => setIsLoaded(true));