feat: iframe support for getRect

server/src/workflow-management/selector.ts

@@ -292,14 +292,9 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
     if (!getList || listSelector !== '') {
       const rect = await page.evaluate(
         async ({ x, y }) => {
-          const el = document.elementFromPoint(x, y) as HTMLElement;
+          // Helper function to convert rectangle to plain object
-          if (el) {
+          const getRectangleInfo = (rectangle: DOMRect) => {
-            const { parentElement } = el;
+            const info = {
-            // Match the logic in recorder.ts for link clicks
-            const element = parentElement?.tagName === 'A' ? parentElement : el;
-            const rectangle = element?.getBoundingClientRect();
-            if (rectangle) {
-              return {
               x: rectangle.x,
               y: rectangle.y,
               width: rectangle.width,
@@ -308,9 +303,79 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
               right: rectangle.right,
               bottom: rectangle.bottom,
               left: rectangle.left,
+              fromIframe: false,
+              iframePath: [] as string[]
             };
+            return info;
+          };
+
+          // Helper function to search in iframe
+          const searchInIframe = (
+            iframe: HTMLIFrameElement, 
+            relativeX: number, 
+            relativeY: number,
+            iframePath: string[]
+          ) => {
+            try {
+              if (!iframe.contentDocument) return null;
+              
+              const el = iframe.contentDocument.elementFromPoint(relativeX, relativeY) as HTMLElement;
+              if (!el) return null;
+
+              const { parentElement } = el;
+              const element = parentElement?.tagName === 'A' ? parentElement : el;
+              const rectangle = element?.getBoundingClientRect();
+              
+              if (rectangle) {
+                const iframeRect = iframe.getBoundingClientRect();
+                const rectInfo = getRectangleInfo(rectangle);
+                
+                // Adjust coordinates relative to the main document
+                rectInfo.x += iframeRect.x;
+                rectInfo.y += iframeRect.y;
+                rectInfo.top += iframeRect.top;
+                rectInfo.right += iframeRect.left;
+                rectInfo.bottom += iframeRect.top;
+                rectInfo.left += iframeRect.left;
+                rectInfo.fromIframe = true;
+                rectInfo.iframePath = iframePath;
+                
+                return rectInfo;
+              }
+              return null;
+            } catch (e) {
+              console.warn('Cannot access iframe content:', e);
+              return null;
+            }
+          };
+
+          const el = document.elementFromPoint(x, y) as HTMLElement;
+          if (el) {
+            // Check if the element is an iframe
+            if (el.tagName === 'IFRAME') {
+              const iframe = el as HTMLIFrameElement;
+              const rect = iframe.getBoundingClientRect();
+              const relativeX = x - rect.left;
+              const relativeY = y - rect.top;
+              
+              const iframeResult = searchInIframe(
+                iframe, 
+                relativeX, 
+                relativeY, 
+                [iframe.id || 'unnamed-iframe']
+              );
+              if (iframeResult) return iframeResult;
+            }
+
+            const { parentElement } = el;
+            const element = parentElement?.tagName === 'A' ? parentElement : el;
+            const rectangle = element?.getBoundingClientRect();
+            
+            if (rectangle) {
+              return getRectangleInfo(rectangle);
             }
           }
+          return null;
         },
         { x: coordinates.x, y: coordinates.y },
       );
@@ -318,10 +383,34 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
     } else {
       const rect = await page.evaluate(
         async ({ x, y }) => {
-          const originalEl = document.elementFromPoint(x, y) as HTMLElement;
+          // Helper function to convert rectangle to plain object (same as above)
-          if (originalEl) {
+          const getRectangleInfo = (rectangle: DOMRect) => ({
-            let element = originalEl;
+            x: rectangle.x,
+            y: rectangle.y,
+            width: rectangle.width,
+            height: rectangle.height,
+            top: rectangle.top,
+            right: rectangle.right,
+            bottom: rectangle.bottom,
+            left: rectangle.left,
+            fromIframe: false,
+            iframePath: [] as string[]
+          });
 
+          // Helper function to search in iframe (same as above)
+          const searchInIframe = (
+            iframe: HTMLIFrameElement, 
+            relativeX: number, 
+            relativeY: number,
+            iframePath: string[]
+          ) => {
+            try {
+              if (!iframe.contentDocument) return null;
+              
+              const el = iframe.contentDocument.elementFromPoint(relativeX, relativeY) as HTMLElement;
+              if (!el) return null;
+
+              let element = el;
               while (element.parentElement) {
                 const parentRect = element.parentElement.getBoundingClientRect();
                 const childRect = element.getBoundingClientRect();
@@ -344,18 +433,72 @@ export const getRect = async (page: Page, coordinates: Coordinates, listSelector
               }
 
               const rectangle = element?.getBoundingClientRect();
-
               if (rectangle) {
-              return {
+                const iframeRect = iframe.getBoundingClientRect();
-                x: rectangle.x,
+                const rectInfo = getRectangleInfo(rectangle);
-                y: rectangle.y,
+                
-                width: rectangle.width,
+                // Adjust coordinates relative to the main document
-                height: rectangle.height,
+                rectInfo.x += iframeRect.x;
-                top: rectangle.top,
+                rectInfo.y += iframeRect.y;
-                right: rectangle.right,
+                rectInfo.top += iframeRect.top;
-                bottom: rectangle.bottom,
+                rectInfo.right += iframeRect.left;
-                left: rectangle.left,
+                rectInfo.bottom += iframeRect.top;
+                rectInfo.left += iframeRect.left;
+                rectInfo.fromIframe = true;
+                rectInfo.iframePath = iframePath;
+                
+                return rectInfo;
+              }
+              return null;
+            } catch (e) {
+              console.warn('Cannot access iframe content:', e);
+              return null;
+            }
           };
+
+          const originalEl = document.elementFromPoint(x, y) as HTMLElement;
+          if (originalEl) {
+            // Check if the element is an iframe
+            if (originalEl.tagName === 'IFRAME') {
+              const iframe = originalEl as HTMLIFrameElement;
+              const rect = iframe.getBoundingClientRect();
+              const relativeX = x - rect.left;
+              const relativeY = y - rect.top;
+              
+              const iframeResult = searchInIframe(
+                iframe, 
+                relativeX, 
+                relativeY, 
+                [iframe.id || 'unnamed-iframe']
+              );
+              if (iframeResult) return iframeResult;
+            }
+
+            let element = originalEl;
+            while (element.parentElement) {
+              const parentRect = element.parentElement.getBoundingClientRect();
+              const childRect = element.getBoundingClientRect();
+
+              const fullyContained =
+                parentRect.left <= childRect.left &&
+                parentRect.right >= childRect.right &&
+                parentRect.top <= childRect.top &&
+                parentRect.bottom >= childRect.bottom;
+
+              const significantOverlap =
+                (childRect.width * childRect.height) /
+                (parentRect.width * parentRect.height) > 0.5;
+
+              if (fullyContained && significantOverlap) {
+                element = element.parentElement;
+              } else {
+                break;
+              }
+            }
+
+            const rectangle = element?.getBoundingClientRect();
+            if (rectangle) {
+              return getRectangleInfo(rectangle);
             }
           }
           return null;