feat: add iframeSelector generation logic for capture text
This commit is contained in:
RohitR311
@@ -23,247 +23,110 @@ export const getElementInformation = async (
|
|||||||
if (!getList || listSelector !== '') {
|
if (!getList || listSelector !== '') {
|
||||||
const elementInfo = await page.evaluate(
|
const elementInfo = await page.evaluate(
|
||||||
async ({ x, y }) => {
|
async ({ x, y }) => {
|
||||||
// Helper function to get element info
|
// Helper function to find elements within iframes, handling nested cases
|
||||||
const getElementInfo = (element: HTMLElement) => {
|
const getElementFromIframePoint = (
|
||||||
let info: {
|
x: number,
|
||||||
tagName: string;
|
y: number,
|
||||||
hasOnlyText?: boolean;
|
context: Document = document,
|
||||||
innerText?: string;
|
iframePath: string[] = []
|
||||||
url?: string;
|
): { element: HTMLElement | null; iframePath: string[] } => {
|
||||||
imageUrl?: string;
|
// First try to get element at the given coordinates
|
||||||
attributes?: Record<string, string>;
|
let element = context.elementFromPoint(x, y) as HTMLElement;
|
||||||
innerHTML?: string;
|
if (!element) return { element: null, iframePath };
|
||||||
outerHTML?: string;
|
|
||||||
fromIframe?: boolean;
|
|
||||||
iframePath?: string[];
|
|
||||||
} = {
|
|
||||||
tagName: element?.tagName ?? '',
|
|
||||||
};
|
|
||||||
|
|
||||||
if (element) {
|
// Check if we found an iframe
|
||||||
info.attributes = Array.from(element.attributes).reduce(
|
if (element.tagName === 'IFRAME') {
|
||||||
(acc, attr) => {
|
const iframe = element as HTMLIFrameElement;
|
||||||
acc[attr.name] = attr.value;
|
try {
|
||||||
return acc;
|
// Make sure we can access the iframe's content
|
||||||
},
|
if (!iframe.contentDocument) {
|
||||||
{} as Record<string, string>
|
return { element, iframePath };
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (element?.tagName === 'A') {
|
|
||||||
info.url = (element as HTMLAnchorElement).href;
|
|
||||||
info.innerText = element.innerText ?? '';
|
|
||||||
} else if (element?.tagName === 'IMG') {
|
|
||||||
info.imageUrl = (element as HTMLImageElement).src;
|
|
||||||
} else if (element?.tagName === 'SELECT') {
|
|
||||||
const selectElement = element as HTMLSelectElement;
|
|
||||||
info.innerText = selectElement.options[selectElement.selectedIndex]?.text ?? '';
|
|
||||||
info.attributes = {
|
|
||||||
...info.attributes,
|
|
||||||
selectedValue: selectElement.value,
|
|
||||||
};
|
|
||||||
} else if (element?.tagName === 'INPUT' &&
|
|
||||||
((element as HTMLInputElement).type === 'time' ||
|
|
||||||
(element as HTMLInputElement).type === 'date')) {
|
|
||||||
info.innerText = (element as HTMLInputElement).value;
|
|
||||||
} else {
|
|
||||||
info.hasOnlyText = element?.children?.length === 0 &&
|
|
||||||
element?.innerText?.length > 0;
|
|
||||||
info.innerText = element?.innerText ?? '';
|
|
||||||
}
|
|
||||||
|
|
||||||
info.innerHTML = element.innerHTML;
|
|
||||||
info.outerHTML = element.outerHTML;
|
|
||||||
return info;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Helper function to search in iframe
|
|
||||||
const searchInIframe = (
|
|
||||||
iframe: HTMLIFrameElement,
|
|
||||||
relativeX: number,
|
|
||||||
relativeY: number,
|
|
||||||
iframePath: string[]
|
|
||||||
) => {
|
|
||||||
try {
|
|
||||||
if (!iframe.contentDocument) return null;
|
|
||||||
|
|
||||||
const el = iframe.contentDocument.elementFromPoint(relativeX, relativeY) as HTMLElement;
|
|
||||||
if (!el) return null;
|
|
||||||
|
|
||||||
const { parentElement } = el;
|
|
||||||
const element = parentElement?.tagName === 'A' ? parentElement : el;
|
|
||||||
|
|
||||||
const info = getElementInfo(element);
|
|
||||||
info.fromIframe = true;
|
|
||||||
info.iframePath = iframePath;
|
|
||||||
|
|
||||||
return info;
|
|
||||||
} catch (e) {
|
|
||||||
console.warn('Cannot access iframe content:', e);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const el = document.elementFromPoint(x, y) as HTMLElement;
|
|
||||||
if (el) {
|
|
||||||
// Check if the element is an iframe
|
|
||||||
if (el.tagName === 'IFRAME') {
|
|
||||||
const iframe = el as HTMLIFrameElement;
|
|
||||||
const rect = iframe.getBoundingClientRect();
|
|
||||||
const relativeX = x - rect.left;
|
|
||||||
const relativeY = y - rect.top;
|
|
||||||
|
|
||||||
const iframeResult = searchInIframe(
|
|
||||||
iframe,
|
|
||||||
relativeX,
|
|
||||||
relativeY,
|
|
||||||
[iframe.id || 'unnamed-iframe']
|
|
||||||
);
|
|
||||||
if (iframeResult) return iframeResult;
|
|
||||||
}
|
|
||||||
|
|
||||||
const { parentElement } = el;
|
|
||||||
const element = parentElement?.tagName === 'A' ? parentElement : el;
|
|
||||||
return getElementInfo(element);
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
},
|
|
||||||
{ x: coordinates.x, y: coordinates.y }
|
|
||||||
);
|
|
||||||
return elementInfo;
|
|
||||||
} else {
|
|
||||||
const elementInfo = await page.evaluate(
|
|
||||||
async ({ x, y }) => {
|
|
||||||
// Helper function to get element info (same as above)
|
|
||||||
const getElementInfo = (element: HTMLElement) => {
|
|
||||||
let info: {
|
|
||||||
tagName: string;
|
|
||||||
hasOnlyText?: boolean;
|
|
||||||
innerText?: string;
|
|
||||||
url?: string;
|
|
||||||
imageUrl?: string;
|
|
||||||
attributes?: Record<string, string>;
|
|
||||||
innerHTML?: string;
|
|
||||||
outerHTML?: string;
|
|
||||||
fromIframe?: boolean;
|
|
||||||
iframePath?: string[];
|
|
||||||
} = {
|
|
||||||
tagName: element?.tagName ?? '',
|
|
||||||
};
|
|
||||||
|
|
||||||
if (element) {
|
|
||||||
info.attributes = Array.from(element.attributes).reduce(
|
|
||||||
(acc, attr) => {
|
|
||||||
acc[attr.name] = attr.value;
|
|
||||||
return acc;
|
|
||||||
},
|
|
||||||
{} as Record<string, string>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (element?.tagName === 'A') {
|
|
||||||
info.url = (element as HTMLAnchorElement).href;
|
|
||||||
info.innerText = element.innerText ?? '';
|
|
||||||
} else if (element?.tagName === 'IMG') {
|
|
||||||
info.imageUrl = (element as HTMLImageElement).src;
|
|
||||||
} else {
|
|
||||||
info.hasOnlyText = element?.children?.length === 0 &&
|
|
||||||
element?.innerText?.length > 0;
|
|
||||||
info.innerText = element?.innerText ?? '';
|
|
||||||
}
|
|
||||||
|
|
||||||
info.innerHTML = element.innerHTML;
|
|
||||||
info.outerHTML = element.outerHTML;
|
|
||||||
return info;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Helper function to search in iframe (same as above)
|
|
||||||
const searchInIframe = (
|
|
||||||
iframe: HTMLIFrameElement,
|
|
||||||
relativeX: number,
|
|
||||||
relativeY: number,
|
|
||||||
iframePath: string[]
|
|
||||||
) => {
|
|
||||||
try {
|
|
||||||
if (!iframe.contentDocument) return null;
|
|
||||||
|
|
||||||
const el = iframe.contentDocument.elementFromPoint(relativeX, relativeY) as HTMLElement;
|
|
||||||
if (!el) return null;
|
|
||||||
|
|
||||||
let element = el;
|
|
||||||
while (element.parentElement) {
|
|
||||||
const parentRect = element.parentElement.getBoundingClientRect();
|
|
||||||
const childRect = element.getBoundingClientRect();
|
|
||||||
|
|
||||||
const fullyContained =
|
|
||||||
parentRect.left <= childRect.left &&
|
|
||||||
parentRect.right >= childRect.right &&
|
|
||||||
parentRect.top <= childRect.top &&
|
|
||||||
parentRect.bottom >= childRect.bottom;
|
|
||||||
|
|
||||||
const significantOverlap =
|
|
||||||
(childRect.width * childRect.height) /
|
|
||||||
(parentRect.width * parentRect.height) > 0.5;
|
|
||||||
|
|
||||||
if (fullyContained && significantOverlap) {
|
|
||||||
element = element.parentElement;
|
|
||||||
} else {
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Transform coordinates to iframe's space
|
||||||
|
const rect = iframe.getBoundingClientRect();
|
||||||
|
const relativeX = x - rect.left;
|
||||||
|
const relativeY = y - rect.top;
|
||||||
|
|
||||||
|
// Add this iframe to the path
|
||||||
|
const updatedPath = [...iframePath, iframe.id || 'unnamed-iframe'];
|
||||||
|
|
||||||
|
// Recursively search within the iframe
|
||||||
|
const iframeResult = getElementFromIframePoint(
|
||||||
|
relativeX,
|
||||||
|
relativeY,
|
||||||
|
iframe.contentDocument,
|
||||||
|
updatedPath
|
||||||
|
);
|
||||||
|
|
||||||
|
// If we found an element in the iframe, return it
|
||||||
|
if (iframeResult.element) {
|
||||||
|
return iframeResult;
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.warn('Cannot access iframe content:', e);
|
||||||
}
|
}
|
||||||
|
|
||||||
const info = getElementInfo(element);
|
|
||||||
info.fromIframe = true;
|
|
||||||
info.iframePath = iframePath;
|
|
||||||
|
|
||||||
return info;
|
|
||||||
} catch (e) {
|
|
||||||
console.warn('Cannot access iframe content:', e);
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Return the element we found (either in main document or iframe)
|
||||||
|
return { element, iframePath };
|
||||||
};
|
};
|
||||||
|
|
||||||
const originalEl = document.elementFromPoint(x, y) as HTMLElement;
|
// Get the element and its iframe path
|
||||||
if (originalEl) {
|
const { element: el, iframePath } = getElementFromIframePoint(x, y);
|
||||||
// Check if the element is an iframe
|
|
||||||
if (originalEl.tagName === 'IFRAME') {
|
if (el) {
|
||||||
const iframe = originalEl as HTMLIFrameElement;
|
// Handle potential anchor parent
|
||||||
const rect = iframe.getBoundingClientRect();
|
const { parentElement } = el;
|
||||||
const relativeX = x - rect.left;
|
const targetElement = parentElement?.tagName === 'A' ? parentElement : el;
|
||||||
const relativeY = y - rect.top;
|
|
||||||
|
|
||||||
const iframeResult = searchInIframe(
|
// Build the element information object
|
||||||
iframe,
|
let info: {
|
||||||
relativeX,
|
tagName: string;
|
||||||
relativeY,
|
hasOnlyText?: boolean;
|
||||||
[iframe.id || 'unnamed-iframe']
|
innerText?: string;
|
||||||
|
url?: string;
|
||||||
|
imageUrl?: string;
|
||||||
|
attributes?: Record<string, string>;
|
||||||
|
innerHTML?: string;
|
||||||
|
outerHTML?: string;
|
||||||
|
fromIframe?: boolean;
|
||||||
|
iframePath?: string[];
|
||||||
|
} = {
|
||||||
|
tagName: targetElement?.tagName ?? '',
|
||||||
|
fromIframe: iframePath.length > 0,
|
||||||
|
iframePath: iframePath.length > 0 ? iframePath : undefined
|
||||||
|
};
|
||||||
|
|
||||||
|
// Collect element attributes and properties
|
||||||
|
if (targetElement) {
|
||||||
|
// Get all attributes
|
||||||
|
info.attributes = Array.from(targetElement.attributes).reduce(
|
||||||
|
(acc, attr) => {
|
||||||
|
acc[attr.name] = attr.value;
|
||||||
|
return acc;
|
||||||
|
},
|
||||||
|
{} as Record<string, string>
|
||||||
);
|
);
|
||||||
if (iframeResult) return iframeResult;
|
|
||||||
}
|
|
||||||
|
|
||||||
let element = originalEl;
|
// Handle specific element types
|
||||||
while (element.parentElement) {
|
if (targetElement.tagName === 'A') {
|
||||||
const parentRect = element.parentElement.getBoundingClientRect();
|
info.url = (targetElement as HTMLAnchorElement).href;
|
||||||
const childRect = element.getBoundingClientRect();
|
info.innerText = targetElement.textContent ?? '';
|
||||||
|
} else if (targetElement.tagName === 'IMG') {
|
||||||
const fullyContained =
|
info.imageUrl = (targetElement as HTMLImageElement).src;
|
||||||
parentRect.left <= childRect.left &&
|
|
||||||
parentRect.right >= childRect.right &&
|
|
||||||
parentRect.top <= childRect.top &&
|
|
||||||
parentRect.bottom >= childRect.bottom;
|
|
||||||
|
|
||||||
const significantOverlap =
|
|
||||||
(childRect.width * childRect.height) /
|
|
||||||
(parentRect.width * parentRect.height) > 0.5;
|
|
||||||
|
|
||||||
if (fullyContained && significantOverlap) {
|
|
||||||
element = element.parentElement;
|
|
||||||
} else {
|
} else {
|
||||||
break;
|
info.hasOnlyText = targetElement.children.length === 0 &&
|
||||||
|
(targetElement.textContent !== null &&
|
||||||
|
targetElement.textContent.trim().length > 0);
|
||||||
|
info.innerText = targetElement.textContent ?? '';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
info.innerHTML = targetElement.innerHTML;
|
||||||
|
info.outerHTML = targetElement.outerHTML;
|
||||||
}
|
}
|
||||||
|
|
||||||
return getElementInfo(element);
|
return info;
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
},
|
},
|
||||||
@@ -271,6 +134,7 @@ export const getElementInformation = async (
|
|||||||
);
|
);
|
||||||
return elementInfo;
|
return elementInfo;
|
||||||
}
|
}
|
||||||
|
// ... rest of the code remains same
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const { message, stack } = error as Error;
|
const { message, stack } = error as Error;
|
||||||
console.error('Error while retrieving selector:', message);
|
console.error('Error while retrieving selector:', message);
|
||||||
@@ -984,6 +848,148 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
|
|||||||
}
|
}
|
||||||
return output;
|
return output;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const getIframeOffset = (iframe: HTMLIFrameElement): { x: number; y: number } => {
|
||||||
|
const rect = iframe.getBoundingClientRect();
|
||||||
|
return {
|
||||||
|
x: rect.left,
|
||||||
|
y: rect.top
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const isAccessibleIframe = (iframe: HTMLIFrameElement): boolean => {
|
||||||
|
try {
|
||||||
|
return !!iframe.contentDocument;
|
||||||
|
} catch (e) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => {
|
||||||
|
// Get the initial element at the specified coordinates
|
||||||
|
let currentElement = document.elementFromPoint(x, y) as HTMLElement;
|
||||||
|
if (!currentElement) return null;
|
||||||
|
|
||||||
|
let deepestElement = currentElement;
|
||||||
|
let current = currentElement;
|
||||||
|
let currentX = x;
|
||||||
|
let currentY = y;
|
||||||
|
let depth = 0;
|
||||||
|
const MAX_DEPTH = 20; // Prevent infinite loops with deeply nested iframes
|
||||||
|
|
||||||
|
// Continue traversing while we find nested iframes
|
||||||
|
while (current && depth < MAX_DEPTH) {
|
||||||
|
// Check if the current element is an iframe and if we can access it
|
||||||
|
if (current instanceof HTMLIFrameElement && isAccessibleIframe(current)) {
|
||||||
|
// Calculate the offset of the iframe
|
||||||
|
const iframeOffset = getIframeOffset(current);
|
||||||
|
|
||||||
|
// Transform coordinates to be relative to the iframe's content window
|
||||||
|
const relativeX = currentX - iframeOffset.x;
|
||||||
|
const relativeY = currentY - iframeOffset.y;
|
||||||
|
|
||||||
|
// Find the element at these coordinates within the iframe
|
||||||
|
const iframeElement = current.contentDocument?.elementFromPoint(relativeX, relativeY) as HTMLElement;
|
||||||
|
|
||||||
|
// If we don't find an element or we get the same element, stop traversing
|
||||||
|
if (!iframeElement || iframeElement === current) break;
|
||||||
|
|
||||||
|
// Update our tracking variables
|
||||||
|
deepestElement = iframeElement;
|
||||||
|
current = iframeElement;
|
||||||
|
currentX = relativeX;
|
||||||
|
currentY = relativeY;
|
||||||
|
depth++;
|
||||||
|
} else {
|
||||||
|
// If the current element is not an iframe, we're done traversing
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return deepestElement;
|
||||||
|
};
|
||||||
|
|
||||||
|
interface IframeContext {
|
||||||
|
frame: HTMLIFrameElement;
|
||||||
|
document: Document;
|
||||||
|
element: HTMLElement;
|
||||||
|
}
|
||||||
|
|
||||||
|
const genSelectorForIframe = (element: HTMLElement) => {
|
||||||
|
// Helper function to check if we can access an iframe's content
|
||||||
|
const isAccessibleIframe = (iframe: HTMLIFrameElement): boolean => {
|
||||||
|
try {
|
||||||
|
return !!iframe.contentDocument;
|
||||||
|
} catch (e) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Get complete path up through nested iframes to document root
|
||||||
|
const getIframePath = (el: HTMLElement) => {
|
||||||
|
const path: IframeContext[] = [];
|
||||||
|
let current = el;
|
||||||
|
let currentDoc = el.ownerDocument;
|
||||||
|
let depth = 0;
|
||||||
|
const MAX_DEPTH = 20; // Limit depth to prevent infinite loops
|
||||||
|
|
||||||
|
while (current && depth < MAX_DEPTH) {
|
||||||
|
// If we're in an iframe, get its parent document
|
||||||
|
const frameElement = currentDoc.defaultView?.frameElement as HTMLIFrameElement;
|
||||||
|
if (frameElement && isAccessibleIframe(frameElement)) {
|
||||||
|
path.unshift({
|
||||||
|
frame: frameElement,
|
||||||
|
document: currentDoc,
|
||||||
|
element: current
|
||||||
|
});
|
||||||
|
current = frameElement;
|
||||||
|
currentDoc = frameElement.ownerDocument;
|
||||||
|
depth++;
|
||||||
|
} else {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return path;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Get the iframe path for our target element
|
||||||
|
const iframePath = getIframePath(element);
|
||||||
|
if (iframePath.length === 0) return null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const selectorParts: string[] = [];
|
||||||
|
|
||||||
|
// Generate selector for each iframe boundary
|
||||||
|
iframePath.forEach((context, index) => {
|
||||||
|
// Get selector for the iframe element in its parent document
|
||||||
|
const frameSelector = finder(context.frame, {
|
||||||
|
root: index === 0 ? document.body : (iframePath[index - 1].document.body as Element)
|
||||||
|
});
|
||||||
|
|
||||||
|
// For the last context, get selector for target element
|
||||||
|
if (index === iframePath.length - 1) {
|
||||||
|
const elementSelector = finder(element, {
|
||||||
|
root: context.document.body as Element
|
||||||
|
});
|
||||||
|
// Use :>> for iframe traversal in the selector
|
||||||
|
selectorParts.push(`${frameSelector} :>> ${elementSelector}`);
|
||||||
|
} else {
|
||||||
|
selectorParts.push(frameSelector);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
// Join all parts with :>> to indicate iframe traversal
|
||||||
|
fullSelector: selectorParts.join(' :>> '),
|
||||||
|
// Include additional metadata about the frames if needed
|
||||||
|
frameCount: iframePath.length,
|
||||||
|
isAccessible: true
|
||||||
|
};
|
||||||
|
} catch (e) {
|
||||||
|
console.warn('Error generating iframe selector:', e);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const genSelectors = (element: HTMLElement | null) => {
|
const genSelectors = (element: HTMLElement | null) => {
|
||||||
if (element == null) {
|
if (element == null) {
|
||||||
@@ -1004,6 +1010,8 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
|
|||||||
} catch (e) {
|
} catch (e) {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const iframeSelector = genSelectorForIframe(element);
|
||||||
|
|
||||||
const hrefSelector = genSelectorForAttributes(element, ['href']);
|
const hrefSelector = genSelectorForAttributes(element, ['href']);
|
||||||
const formSelector = genSelectorForAttributes(element, [
|
const formSelector = genSelectorForAttributes(element, [
|
||||||
'name',
|
'name',
|
||||||
@@ -1050,6 +1058,11 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
|
|||||||
hrefSelector,
|
hrefSelector,
|
||||||
accessibilitySelector,
|
accessibilitySelector,
|
||||||
formSelector,
|
formSelector,
|
||||||
|
iframeSelector: iframeSelector ? {
|
||||||
|
full: iframeSelector.fullSelector,
|
||||||
|
frame: iframeSelector.frameCount,
|
||||||
|
accesible: iframeSelector.isAccessible
|
||||||
|
} : null
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1092,7 +1105,7 @@ export const getSelectors = async (page: Page, coordinates: Coordinates) => {
|
|||||||
return char.length === 1 && char.match(/[0-9]/);
|
return char.length === 1 && char.match(/[0-9]/);
|
||||||
}
|
}
|
||||||
|
|
||||||
const hoveredElement = document.elementFromPoint(x, y) as HTMLElement;
|
const hoveredElement = getDeepestElementFromPoint(x, y) as HTMLElement;
|
||||||
if (
|
if (
|
||||||
hoveredElement != null &&
|
hoveredElement != null &&
|
||||||
!hoveredElement.closest('#overlay-controls') != null
|
!hoveredElement.closest('#overlay-controls') != null
|
||||||
|
|||||||
Reference in New Issue
Block a user