feat: add cursor position on click action

server/src/workflow-management/classes/Generator.ts

@@ -429,25 +429,38 @@ export class WorkflowGenerator {
 
     if ((elementInfo?.tagName === 'INPUT' || elementInfo?.tagName === 'TEXTAREA') && selector) {
       // Calculate the exact position within the element
-      const elementPos = await page.evaluate((selector) => {
+      const positionAndCursor = await page.evaluate((selector) => {
         const element = document.querySelector(selector);
         if (!element) return null;
+    
+        // Get element position for the relative click coordinates
         const rect = element.getBoundingClientRect();
+        
+        // Get cursor position index
+        let cursorIndex = 0;
+        if ('selectionStart' in element) {
+          // selectionStart gives us the exact index where the cursor is
+          cursorIndex = (element as HTMLInputElement | HTMLTextAreaElement).selectionStart || 0;
+        }
+        
         return {
+          rect: {
             x: rect.left,
             y: rect.top
+          },
+          cursorIndex  
         };
       }, selector);
 
-      if (elementPos) {
+      if (positionAndCursor) {
-        const relativeX = coordinates.x - elementPos.x;
+        const relativeX = coordinates.x - positionAndCursor.rect.x;
-        const relativeY = coordinates.y - elementPos.y;
+        const relativeY = coordinates.y - positionAndCursor.rect.y;
 
         const pair: WhereWhatPair = {
             where,
             what: [{
                 action: 'click',
-                args: [selector, { position: { x: relativeX, y: relativeY } }]
+                args: [selector, { position: { x: relativeX, y: relativeY } }, { cursorIndex: positionAndCursor.cursorIndex }],
             }]
         };
 
@@ -1027,24 +1040,24 @@ export class WorkflowGenerator {
    * @param workflow The workflow to be optimized.
    */
   private optimizeWorkflow = (workflow: WorkflowFile) => {
-
+    // Enhanced input state to include cursor position
-    // replace a sequence of press actions by a single fill action
     let input = {
       selector: '',
       value: '',
       type: '',
       actionCounter: 0,
+      cursorPosition: -1  // Track cursor position, -1 means end of text
     };
   
     const pushTheOptimizedAction = (pair: WhereWhatPair, index: number) => {
       if (input.value.length === 1) {
-        // when only one press action is present, keep it and add a waitForLoadState action
+        // Single character - keep as is with waitForLoadState
         pair.what.splice(index + 1, 0, {
           action: 'waitForLoadState',
           args: ['networkidle'],
-        })
+        });
       } else {
-        // when more than one press action is present, add a type action
+        // Multiple characters - optimize to type action
         pair.what.splice(index - input.actionCounter, input.actionCounter, {
           action: 'type',
           args: [input.selector, encrypt(input.value), input.type],
@@ -1053,55 +1066,122 @@ export class WorkflowGenerator {
           args: ['networkidle'],
         });
       }
-    }
+    };
-
   
     for (const pair of workflow.workflow) {
-      pair.what.forEach((condition, index) => {
+      for (let i = 0; i < pair.what.length; i++) {
-        if (condition.action === 'press') {
+        const condition = pair.what[i];
-          if (condition.args && condition.args[1]) {
+  
-            if (!input.selector) {
+        // Handle click actions that set cursor position
-              input.selector = condition.args[0];
+        if (condition.action === 'click' && condition.args?.[1]) {
-              input.type = condition.args[2]
+          const cursorIndex = condition.args[1].cursorIndex;
+  
+          // If we have pending input, commit it before processing the click
+          if (input.value.length > 0) {
+            pushTheOptimizedAction(pair, i);
+            input = {
+              selector: '',
+              value: '',
+              type: '',
+              actionCounter: 0,
+              cursorPosition: -1
+            };
+          }
+  
+          // Update cursor position for next operations
+          input.cursorPosition = cursorIndex;
+          continue;
+        }
+  
+        // Handle text input and editing
+        if (condition.action === 'press' && condition.args?.[1]) {
+          const [selector, encryptedKey, type] = condition.args;
+          const key = decrypt(encryptedKey);
+  
+          // Initialize new input state if selector changes
+          if (!input.selector || input.selector !== selector) {
+            if (input.value.length > 0) {
+              pushTheOptimizedAction(pair, i);
             }
+            input = {
+              selector,
+              value: '',
+              type: type || 'text',
+              actionCounter: 0,
+              cursorPosition: -1
+            };
+          }
   
-            if (input.selector === condition.args[0]) {
+          input.actionCounter++;
-              input.actionCounter++;
-              const decryptedKey = decrypt(condition.args[1]);
   
-              if (decryptedKey.length === 1) {
+          // Handle different key types with cursor awareness
-                input.value += decryptedKey;
+          if (key.length === 1) {
-              } else if (decryptedKey === 'Backspace') {
+            // Insert character at cursor position or append if no cursor set
-                input.value = input.value.slice(0, -1);
+            if (input.cursorPosition === -1) {
-              } else if (decryptedKey !== 'Shift') {
+              // No cursor position set, append to end
-                pushTheOptimizedAction(pair, index);
+              input.value += key;
-                pair.what.splice(index + 1, 0, {
-                  action: 'waitForLoadState',
-                  args: ['networkidle'],
-                })
-                input = { selector: '', value: '', type: '', actionCounter: 0 };
-              }
             } else {
-              pushTheOptimizedAction(pair, index);
+              // Insert at cursor position
+              input.value = 
+                input.value.slice(0, input.cursorPosition) +
+                key +
+                input.value.slice(input.cursorPosition);
+              input.cursorPosition++;
+            }
+          } else if (key === 'Backspace') {
+            if (input.cursorPosition > 0) {
+              // Delete character before cursor
+              input.value = 
+                input.value.slice(0, input.cursorPosition - 1) +
+                input.value.slice(input.cursorPosition);
+              input.cursorPosition--;
+            } else if (input.cursorPosition === -1 && input.value.length > 0) {
+              // No cursor position set, delete from end
+              input.value = input.value.slice(0, -1);
+            }
+          } else if (key !== 'Shift') {
+            // Handle other special keys
+            if (input.value.length > 0) {
+              pushTheOptimizedAction(pair, i);
               input = {
-                selector: condition.args[0],
+                selector: '',
-                value: decrypt(condition.args[1]),
+                value: '',
-                type: condition.args[2],
+                type: '',
-                actionCounter: 1,
+                actionCounter: 0,
+                cursorPosition: -1
               };
             }
           }
         } else {
-          if (input.value.length !== 0) {
+          // Handle non-text actions
-            pushTheOptimizedAction(pair, index);
+          if (input.value.length > 0) {
-            // clear the input
+            pushTheOptimizedAction(pair, i);
-            input = { selector: '', value: '', type: '', actionCounter: 0 };
+            input = {
+              selector: '',
+              value: '',
+              type: '',
+              actionCounter: 0,
+              cursorPosition: -1
+            };
           }
         }
-      });
+      }
+  
+      // Clean up any remaining input state
+      if (input.value.length > 0) {
+        pushTheOptimizedAction(pair, pair.what.length);
+        input = {
+          selector: '',
+          value: '',
+          type: '',
+          actionCounter: 0,
+          cursorPosition: -1
+        };
+      }
     }
+  
     return workflow;
-  }
+  };
 
   /**
    * Returns workflow params from the stored metadata.