feat: traverse dom tree for parent element selection

server/src/workflow-management/selector.ts

@@ -20,49 +20,6 @@ type Workflow = WorkflowFile["workflow"];
  * @category WorkflowManagement-Selectors
  * @returns {Promise<Rectangle|undefined|null>}
  */
-export const getRect = async (page: Page, coordinates: Coordinates) => {
-  try {
-    const rect = await page.evaluate(
-      async ({ x, y }) => {
-        const el = document.elementFromPoint(x, y) as HTMLElement;
-        if (el) {
-          const { parentElement } = el;
-          // Match the logic in recorder.ts for link clicks
-          const element = parentElement?.tagName === 'A' ? parentElement : el;
-          const rectangle = element?.getBoundingClientRect();
-          // @ts-ignore
-          if (rectangle) {
-            return {
-              x: rectangle.x,
-              y: rectangle.y,
-              width: rectangle.width,
-              height: rectangle.height,
-              top: rectangle.top,
-              right: rectangle.right,
-              bottom: rectangle.bottom,
-              left: rectangle.left,
-            };
-          }
-        }
-      },
-      { x: coordinates.x, y: coordinates.y },
-    );
-    return rect;
-  } catch (error) {
-    const { message, stack } = error as Error;
-    logger.log('error', `Error while retrieving selector: ${message}`);
-    logger.log('error', `Stack: ${stack}`);
-  }
-}
-
-/**
- * Checks the basic info about an element and returns a {@link BaseActionInfo} object.
- * If the element is not found, returns undefined.
- * @param page The page instance.
- * @param coordinates Coordinates of an element.
- * @category WorkflowManagement-Selectors
- * @returns {Promise<BaseActionInfo|undefined>}
- */
 export const getElementInformation = async (
   page: Page,
   coordinates: Coordinates
@@ -70,10 +27,15 @@ export const getElementInformation = async (
   try {
     const elementInfo = await page.evaluate(
       async ({ x, y }) => {
-        const el = document.elementFromPoint(x, y) as HTMLElement;
+        // Find the initial element at the point
-        if (el) {
+        const initialElement = document.elementFromPoint(x, y) as HTMLElement;
-          const { parentElement } = el;
+        
-          const element = parentElement?.tagName === 'A' ? parentElement : el;
+        if (initialElement) {
+          // Simply use the direct parent, no complex logic
+          const parentElement = initialElement.parentElement;
+
+          // Use the parent if it exists, otherwise use the initial element
+          const element = parentElement || initialElement;
 
           let info: {
             tagName: string;
@@ -84,32 +46,41 @@ export const getElementInformation = async (
             attributes?: Record<string, string>;
             innerHTML?: string;
             outerHTML?: string;
+            parentTagName?: string;
+            parentClasses?: string[];
           } = {
-            tagName: element?.tagName ?? '',
+            tagName: element.tagName,
+            parentTagName: element.parentElement?.tagName,
+            parentClasses: element.parentElement 
+              ? Array.from(element.parentElement.classList) 
+              : []
           };
 
-          if (element) {
+          // Collect attributes
-            info.attributes = Array.from(element.attributes).reduce(
+          info.attributes = Array.from(element.attributes).reduce(
-              (acc, attr) => {
+            (acc, attr) => {
-                acc[attr.name] = attr.value;
+              acc[attr.name] = attr.value;
-                return acc;
+              return acc;
-              },
+            },
-              {} as Record<string, string>
+            {} as Record<string, string>
-            );
+          );
-          }
 
-          // Gather specific information based on the tag
+          // Specific handling for different element types
-          if (element?.tagName === 'A') {
+          if (element.tagName === 'A') {
-            info.url = (element as HTMLAnchorElement).href;
+            const anchorElement = element as HTMLAnchorElement;
-            info.innerText = element.innerText ?? '';
+            info.url = anchorElement.href;
-          } else if (element?.tagName === 'IMG') {
+            info.innerText = anchorElement.innerText ?? '';
-            info.imageUrl = (element as HTMLImageElement).src;
+          } else if (element.tagName === 'IMG') {
+            const imgElement = element as HTMLImageElement;
+            info.imageUrl = imgElement.src;
           } else {
-            info.hasOnlyText = element?.children?.length === 0 &&
+            // Check if element contains only text
-              element?.innerText?.length > 0;
+            info.hasOnlyText = element.children.length === 0 &&
-            info.innerText = element?.innerText ?? '';
+              (element.innerText?.length ?? 0) > 0;
+            info.innerText = element.innerText ?? '';
           }
 
+          // HTML content
           info.innerHTML = element.innerHTML;
           info.outerHTML = element.outerHTML;
 
@@ -127,6 +98,47 @@ export const getElementInformation = async (
   }
 };
 
+export const getRect = async (page: Page, coordinates: Coordinates) => {
+  try {
+    const rect = await page.evaluate(
+      async ({ x, y }) => {
+        // Find the initial element at the point
+        const initialElement = document.elementFromPoint(x, y) as HTMLElement;
+        
+        if (initialElement) {
+          // Simply use the direct parent, no complex logic
+          const parentElement = initialElement.parentElement;
+
+          // Use the parent if it exists, otherwise use the initial element
+          const element = parentElement || initialElement;
+
+          // Get bounding rectangle
+          const rectangle = element?.getBoundingClientRect();
+          
+          if (rectangle) {
+            return {
+              x: rectangle.x,
+              y: rectangle.y,
+              width: rectangle.width,
+              height: rectangle.height,
+              top: rectangle.top,
+              right: rectangle.right,
+              bottom: rectangle.bottom,
+              left: rectangle.left,
+            };
+          }
+        }
+        return null;
+      },
+      { x: coordinates.x, y: coordinates.y },
+    );
+    return rect;
+  } catch (error) {
+    const { message, stack } = error as Error;
+    console.error('Error while retrieving selector:', message);
+    console.error('Stack:', stack);
+  }
+};
 
 /**
  * Returns the best and unique css {@link Selectors} for the element on the page.