From a131ce6c04d1d9f5b1982493f4200b972e989555 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Tue, 24 Dec 2024 23:44:20 +0530 Subject: [PATCH] feat: shadow dom selection --- server/src/workflow-management/selector.ts | 105 +++++++++++++++++---- 1 file changed, 89 insertions(+), 16 deletions(-) diff --git a/server/src/workflow-management/selector.ts b/server/src/workflow-management/selector.ts index c0fa21f1..5a7273df 100644 --- a/server/src/workflow-management/selector.ts +++ b/server/src/workflow-management/selector.ts @@ -23,7 +23,28 @@ export const getElementInformation = async ( if (!getList || listSelector !== '') { const elementInfo = await page.evaluate( async ({ x, y }) => { - const el = document.elementFromPoint(x, y) as HTMLElement; + // Helper function to get element from point including shadow DOM + const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { + let element = document.elementFromPoint(x, y) as HTMLElement; + if (!element) return null; + + // Traverse through shadow roots + let current = element; + while (current) { + // Check if element has shadow root + const shadowRoot = current.shadowRoot; + if (!shadowRoot) break; + + // Try to find deeper element in shadow DOM + const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; + if (!shadowElement || shadowElement === current) break; + + current = shadowElement; + } + return current; + }; + + const el = getDeepestElementFromPoint(x, y); if (el) { const { parentElement } = el; const element = parentElement?.tagName === 'A' ? parentElement : el; @@ -36,9 +57,12 @@ export const getElementInformation = async ( attributes?: Record; innerHTML?: string; outerHTML?: string; + isShadowRoot?: boolean; } = { tagName: element?.tagName ?? '', + isShadowRoot: !!element?.shadowRoot }; + if (element) { info.attributes = Array.from(element.attributes).reduce( (acc, attr) => { @@ -48,6 +72,7 @@ export const getElementInformation = async ( {} as Record ); } + // Gather specific information based on the tag if (element?.tagName === 'A') { info.url = (element as HTMLAnchorElement).href; @@ -61,7 +86,7 @@ export const getElementInformation = async ( ...info.attributes, selectedValue: selectElement.value, }; - } else if (element?.tagName === 'INPUT' && (element as HTMLInputElement).type === 'time' || (element as HTMLInputElement).type === 'date') { + } else if (element?.tagName === 'INPUT' && ((element as HTMLInputElement).type === 'time' || (element as HTMLInputElement).type === 'date')) { info.innerText = (element as HTMLInputElement).value; } else { info.hasOnlyText = element?.children?.length === 0 && @@ -80,7 +105,26 @@ export const getElementInformation = async ( } else { const elementInfo = await page.evaluate( async ({ x, y }) => { - const originalEl = document.elementFromPoint(x, y) as HTMLElement; + // Helper function to get element from point including shadow DOM + const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { + let element = document.elementFromPoint(x, y) as HTMLElement; + if (!element) return null; + + // Traverse through shadow roots + let current = element; + while (current) { + const shadowRoot = current.shadowRoot; + if (!shadowRoot) break; + + const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; + if (!shadowElement || shadowElement === current) break; + + current = shadowElement; + } + return current; + }; + + const originalEl = getDeepestElementFromPoint(x, y); if (originalEl) { let element = originalEl; @@ -114,8 +158,10 @@ export const getElementInformation = async ( attributes?: Record; innerHTML?: string; outerHTML?: string; + isShadowRoot?: boolean; } = { tagName: element?.tagName ?? '', + isShadowRoot: !!element?.shadowRoot }; if (element) { @@ -156,24 +202,33 @@ export const getElementInformation = async ( } }; -/** - * Returns a {@link Rectangle} object representing - * the coordinates, width, height and corner points of the element. - * If an element is not found, returns null. - * @param page The page instance. - * @param coordinates Coordinates of an element. - * @category WorkflowManagement-Selectors - * @returns {Promise} - */ export const getRect = async (page: Page, coordinates: Coordinates, listSelector: string, getList: boolean) => { try { if (!getList || listSelector !== '') { const rect = await page.evaluate( async ({ x, y }) => { - const el = document.elementFromPoint(x, y) as HTMLElement; + // Helper function to get element from point including shadow DOM + const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { + let element = document.elementFromPoint(x, y) as HTMLElement; + if (!element) return null; + + // Traverse through shadow roots + let current = element; + while (current) { + const shadowRoot = current.shadowRoot; + if (!shadowRoot) break; + + const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; + if (!shadowElement || shadowElement === current) break; + + current = shadowElement; + } + return current; + }; + + const el = getDeepestElementFromPoint(x, y); if (el) { const { parentElement } = el; - // Match the logic in recorder.ts for link clicks const element = parentElement?.tagName === 'A' ? parentElement : el; const rectangle = element?.getBoundingClientRect(); if (rectangle) { @@ -196,7 +251,26 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector } else { const rect = await page.evaluate( async ({ x, y }) => { - const originalEl = document.elementFromPoint(x, y) as HTMLElement; + // Helper function to get element from point including shadow DOM + const getDeepestElementFromPoint = (x: number, y: number): HTMLElement | null => { + let element = document.elementFromPoint(x, y) as HTMLElement; + if (!element) return null; + + // Traverse through shadow roots + let current = element; + while (current) { + const shadowRoot = current.shadowRoot; + if (!shadowRoot) break; + + const shadowElement = shadowRoot.elementFromPoint(x, y) as HTMLElement; + if (!shadowElement || shadowElement === current) break; + + current = shadowElement; + } + return current; + }; + + const originalEl = getDeepestElementFromPoint(x, y); if (originalEl) { let element = originalEl; @@ -249,7 +323,6 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector } }; - /** * Returns the best and unique css {@link Selectors} for the element on the page. * Internally uses a finder function from https://github.com/antonmedv/finder/blob/master/finder.ts