feat: proper rect and element info

server/src/workflow-management/selector.ts

@@ -27,15 +27,33 @@ export const getElementInformation = async (
   try {
     const elementInfo = await page.evaluate(
       async ({ x, y }) => {
-        // Find the initial element at the point
+        const originalEl = document.elementFromPoint(x, y) as HTMLElement;
-        const initialElement = document.elementFromPoint(x, y) as HTMLElement;
+        if (originalEl) {
-        
+          let element = originalEl;
-        if (initialElement) {
+          
-          // Simply use the direct parent, no complex logic
+          // Generic parent finding logic based on visual containment
-          const parentElement = initialElement.parentElement;
+          while (element.parentElement) {
+            const parentRect = element.parentElement.getBoundingClientRect();
+            const childRect = element.getBoundingClientRect();
 
-          // Use the parent if it exists, otherwise use the initial element
+            // Check if parent visually contains the child
-          const element = parentElement || initialElement;
+            const fullyContained = 
+              parentRect.left <= childRect.left &&
+              parentRect.right >= childRect.right &&
+              parentRect.top <= childRect.top &&
+              parentRect.bottom >= childRect.bottom;
+
+            // Additional checks for more comprehensive containment
+            const significantOverlap = 
+              (childRect.width * childRect.height) / 
+              (parentRect.width * parentRect.height) > 0.5;
+
+            if (fullyContained && significantOverlap) {
+              element = element.parentElement;
+            } else {
+              break;
+            }
+          }
 
           let info: {
             tagName: string;
@@ -46,44 +64,34 @@ export const getElementInformation = async (
             attributes?: Record<string, string>;
             innerHTML?: string;
             outerHTML?: string;
-            parentTagName?: string;
-            parentClasses?: string[];
           } = {
-            tagName: element.tagName,
+            tagName: element?.tagName ?? '',
-            parentTagName: element.parentElement?.tagName,
-            parentClasses: element.parentElement 
-              ? Array.from(element.parentElement.classList) 
-              : []
           };
 
-          // Collect attributes
+          if (element) {
-          info.attributes = Array.from(element.attributes).reduce(
+            info.attributes = Array.from(element.attributes).reduce(
-            (acc, attr) => {
+              (acc, attr) => {
-              acc[attr.name] = attr.value;
+                acc[attr.name] = attr.value;
-              return acc;
+                return acc;
-            },
+              },
-            {} as Record<string, string>
+              {} as Record<string, string>
-          );
+            );
-
+          }
-          // Specific handling for different element types
+
-          if (element.tagName === 'A') {
+          // Existing tag-specific logic
-            const anchorElement = element as HTMLAnchorElement;
+          if (element?.tagName === 'A') {
-            info.url = anchorElement.href;
+            info.url = (element as HTMLAnchorElement).href;
-            info.innerText = anchorElement.innerText ?? '';
+            info.innerText = element.innerText ?? '';
-          } else if (element.tagName === 'IMG') {
+          } else if (element?.tagName === 'IMG') {
-            const imgElement = element as HTMLImageElement;
+            info.imageUrl = (element as HTMLImageElement).src;
-            info.imageUrl = imgElement.src;
+          } else {
-          } else {
+            info.hasOnlyText = element?.children?.length === 0 &&
-            // Check if element contains only text
+              element?.innerText?.length > 0;
-            info.hasOnlyText = element.children.length === 0 &&
+            info.innerText = element?.innerText ?? '';
-              (element.innerText?.length ?? 0) > 0;
-            info.innerText = element.innerText ?? '';
           }
 
-          // HTML content
           info.innerHTML = element.innerHTML;
           info.outerHTML = element.outerHTML;
-
           return info;
         }
         return null;
@@ -102,17 +110,32 @@ export const getRect = async (page: Page, coordinates: Coordinates) => {
   try {
     const rect = await page.evaluate(
       async ({ x, y }) => {
-        // Find the initial element at the point
+        const originalEl = document.elementFromPoint(x, y) as HTMLElement;
-        const initialElement = document.elementFromPoint(x, y) as HTMLElement;
+        if (originalEl) {
-        
+          let element = originalEl;
-        if (initialElement) {
+          
-          // Simply use the direct parent, no complex logic
+          // Same parent-finding logic as in getElementInformation
-          const parentElement = initialElement.parentElement;
+          while (element.parentElement) {
+            const parentRect = element.parentElement.getBoundingClientRect();
+            const childRect = element.getBoundingClientRect();
 
-          // Use the parent if it exists, otherwise use the initial element
+            const fullyContained = 
-          const element = parentElement || initialElement;
+              parentRect.left <= childRect.left &&
+              parentRect.right >= childRect.right &&
+              parentRect.top <= childRect.top &&
+              parentRect.bottom >= childRect.bottom;
+
+            const significantOverlap = 
+              (childRect.width * childRect.height) / 
+              (parentRect.width * parentRect.height) > 0.5;
+
+            if (fullyContained && significantOverlap) {
+              element = element.parentElement;
+            } else {
+              break;
+            }
+          }
 
-          // Get bounding rectangle
           const rectangle = element?.getBoundingClientRect();
           
           if (rectangle) {
@@ -134,10 +157,11 @@ export const getRect = async (page: Page, coordinates: Coordinates) => {
     return rect;
   } catch (error) {
     const { message, stack } = error as Error;
-    console.error('Error while retrieving selector:', message);
+    logger.log('error', `Error while retrieving selector: ${message}`);
-    console.error('Stack:', stack);
+    logger.log('error', `Stack: ${stack}`);
   }
-};
+}
+
 
 /**
  * Returns the best and unique css {@link Selectors} for the element on the page.
@@ -774,25 +798,42 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates
         let depth = 0;
         const maxDepth = 2;
 
-        // Ensure we start with a valid element
+        while (element && element !== document.body && depth < maxDepth) {
-        let currentElement = element;
+          const selector = getNonUniqueSelector(element);
-        while (currentElement && currentElement !== document.body && depth < maxDepth) {
-          const selector = getNonUniqueSelector(currentElement);
           path.unshift(selector);
-          currentElement = currentElement.parentElement;
+          element = element.parentElement;
           depth++;
         }
 
         return path.join(' > ');
       }
 
-      // Find the initial element at the point
+      const originalEl = document.elementFromPoint(x, y) as HTMLElement;
-      const initialElement = document.elementFromPoint(x, y) as HTMLElement;
+      if (!originalEl) return null;
-      
-      if (!initialElement) return null;
 
-      // Prefer parent if exists, otherwise use initial element
+      let element = originalEl;
-      const element = initialElement.parentElement || initialElement;
+      
+      // Find the most appropriate parent element
+      while (element.parentElement) {
+        const parentRect = element.parentElement.getBoundingClientRect();
+        const childRect = element.getBoundingClientRect();
+
+        const fullyContained = 
+          parentRect.left <= childRect.left &&
+          parentRect.right >= childRect.right &&
+          parentRect.top <= childRect.top &&
+          parentRect.bottom >= childRect.bottom;
+
+        const significantOverlap = 
+          (childRect.width * childRect.height) / 
+          (parentRect.width * parentRect.height) > 0.5;
+
+        if (fullyContained && significantOverlap) {
+          element = element.parentElement;
+        } else {
+          break;
+        }
+      }
 
       const generalSelector = getSelectorPath(element);
       return {
@@ -807,7 +848,6 @@ export const getNonUniqueSelectors = async (page: Page, coordinates: Coordinates
   }
 };
 
-
 export const getChildSelectors = async (page: Page, parentSelector: string): Promise<string[]> => {
   try {
     const childSelectors = await page.evaluate((parentSelector: string) => {