Merge branch 'develop' into iframe

server/src/types/index.ts

@@ -129,11 +129,17 @@ export interface BaseActionInfo {
     hasOnlyText: boolean;
 }
 
+
 interface IframeSelector {
     full: string;
     isIframe: boolean;
 }
 
+interface ShadowSelector {
+    full: string;
+    mode: string;
+}
+
 /**
  * Holds all the possible css selectors that has been found for an element.
  * @category Types
@@ -149,6 +155,7 @@ export interface Selectors {
     accessibilitySelector: string|null;
     formSelector: string|null;
     iframeSelector: IframeSelector|null;
+    shadowSelector: ShadowSelector|null;
 }
 
 /**

server/src/workflow-management/classes/Generator.ts

@@ -730,15 +730,26 @@ export class WorkflowGenerator {
     const displaySelector = await this.generateSelector(page, coordinates, ActionType.Click);
     const elementInfo = await getElementInformation(page, coordinates, this.listSelector, this.getList);
     if (rect) {
+      const highlighterData = {
+        rect,
+        selector: displaySelector,
+        elementInfo,
+        // Include shadow DOM specific information
+        shadowInfo: elementInfo?.isShadowRoot ? {
+          mode: elementInfo.shadowRootMode,
+          content: elementInfo.shadowRootContent
+        } : null
+      };
+
       if (this.getList === true) {
         if (this.listSelector !== '') {
           const childSelectors = await getChildSelectors(page, this.listSelector || '');
-          this.socket.emit('highlighter', { rect, selector: displaySelector, elementInfo, childSelectors })
+          this.socket.emit('highlighter', { ...highlighterData, childSelectors })
         } else {
-          this.socket.emit('highlighter', { rect, selector: displaySelector, elementInfo });
+          this.socket.emit('highlighter', { ...highlighterData });
         }
       } else {
-        this.socket.emit('highlighter', { rect, selector: displaySelector, elementInfo });
+        this.socket.emit('highlighter', { ...highlighterData });
       }
     }
   }

server/src/workflow-management/utils.ts

@@ -13,8 +13,12 @@ export const getBestSelectorForAction = (action: Action) => {
     case ActionType.DragAndDrop: {
       const selectors = action.selectors;
 
+
       if (selectors?.iframeSelector?.full) {
         return selectors.iframeSelector.full;
+
+      if (selectors?.shadowSelector?.full) {
+        return selectors.shadowSelector.full;
       }
 
       // less than 25 characters, and element only has text inside
@@ -80,6 +84,11 @@ export const getBestSelectorForAction = (action: Action) => {
     case ActionType.Input:
     case ActionType.Keydown: {
       const selectors = action.selectors;
+
+      if (selectors?.shadowSelector?.full) {
+        return selectors.shadowSelector.full;
+      }
+
       return (
         selectors.testIdSelector ??
         selectors?.id ??