Merge pull request #435 from getmaxun/key-fix

fix: handle displayed input texts

server/src/routes/storage.ts

@@ -171,54 +171,82 @@ interface Credentials {
   [key: string]: CredentialInfo;
 }
 
-function updateTypeActionsInWorkflow(workflow: any[], credentials: Credentials) {
+function handleWorkflowActions(workflow: any[], credentials: Credentials) {
   return workflow.map(step => {
       if (!step.what) return step;
 
-      const indicesToRemove = new Set<number>();
+      const newWhat: any[] = [];
-      step.what.forEach((action: any, index: number) => {
+      const processedSelectors = new Set<string>();
-          if (!action.action || !action.args?.[0]) return;
+      
+      for (let i = 0; i < step.what.length; i++) {
+          const action = step.what[i];
           
-          if ((action.action === 'type' || action.action === 'press') && credentials[action.args[0]]) {
+          if (!action?.action || !action?.args?.[0]) {
-              indicesToRemove.add(index);
+              newWhat.push(action);
-
+              continue;
-              if (step.what[index + 1]?.action === 'waitForLoadState') {
-                  indicesToRemove.add(index + 1);
-              }
           }
-      });
 
-      const filteredWhat = step.what.filter((_: any, index: number) => !indicesToRemove.has(index));
+          const selector = action.args[0];
+          const credential = credentials[selector];
 
-      Object.entries(credentials).forEach(([selector, credentialInfo]) => {
+          if (!credential) {
-          const clickIndex = filteredWhat.findIndex((action: any) => 
+              newWhat.push(action);
-              action.action === 'click' && action.args?.[0] === selector
+              continue;
-          );
+          }
 
-          if (clickIndex !== -1) {
+          if (action.action === 'click') {
-              const chars = credentialInfo.value.split('');
+              newWhat.push(action);
               
-              chars.forEach((char, i) => {
+              if (!processedSelectors.has(selector) && 
-                  filteredWhat.splice(clickIndex + 1 + (i * 2), 0, {
+                  i + 1 < step.what.length && 
+                  (step.what[i + 1].action === 'type' || step.what[i + 1].action === 'press')) {
+                  
+                  newWhat.push({
                       action: 'type',
-                      args: [
+                      args: [selector, encrypt(credential.value), credential.type]
-                          selector,
-                          encrypt(char),
-                          credentialInfo.type 
-                      ]
                   });
                   
-                  filteredWhat.splice(clickIndex + 2 + (i * 2), 0, {
+                  newWhat.push({
                       action: 'waitForLoadState',
                       args: ['networkidle']
                   });
+                  
+                  processedSelectors.add(selector);
+                  
+                  while (i + 1 < step.what.length && 
+                         (step.what[i + 1].action === 'type' || 
+                          step.what[i + 1].action === 'press' || 
+                          step.what[i + 1].action === 'waitForLoadState')) {
+                      i++;
+                  }
+              }
+          } else if ((action.action === 'type' || action.action === 'press') && 
+                     !processedSelectors.has(selector)) {
+              newWhat.push({
+                  action: 'type',
+                  args: [selector, encrypt(credential.value), credential.type]
               });
+              
+              newWhat.push({
+                  action: 'waitForLoadState',
+                  args: ['networkidle']
+              });
+              
+              processedSelectors.add(selector);
+              
+              // Skip subsequent type/press/waitForLoadState actions for this selector
+              while (i + 1 < step.what.length && 
+                     (step.what[i + 1].action === 'type' || 
+                      step.what[i + 1].action === 'press' || 
+                      step.what[i + 1].action === 'waitForLoadState')) {
+                  i++;
+              }
           }
-      });
+      }
 
       return {
           ...step,
-          what: filteredWhat
+          what: newWhat
       };
   });
 }
@@ -251,7 +279,7 @@ router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, r
     let workflow = [...robot.recording.workflow]; // Create a copy of the workflow
 
     if (credentials) {
-      workflow = updateTypeActionsInWorkflow(workflow, credentials);
+      workflow = handleWorkflowActions(workflow, credentials);
     }
 
     // Update the limit
@@ -289,9 +317,23 @@ router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, r
       }
     }
 
-    robot.set('recording', { ...robot.recording, workflow });
+    const updates: any = {
+      recording: {
+        ...robot.recording,
+        workflow
+      }
+    };
 
-    await robot.save();
+    if (name) {
+      updates.recording_meta = {
+        ...robot.recording_meta,
+        name
+      };
+    }
+
+    await Robot.update(updates, {
+      where: { 'recording_meta.id': id }
+    });
 
     const updatedRobot = await Robot.findOne({ where: { 'recording_meta.id': id } });
 

server/src/workflow-management/classes/Generator.ts

@@ -22,7 +22,7 @@ import { getBestSelectorForAction } from "../utils";
 import { browserPool } from "../../server";
 import { uuid } from "uuidv4";
 import { capture } from "../../utils/analytics"
-import { encrypt } from "../../utils/auth";
+import { decrypt, encrypt } from "../../utils/auth";
 
 interface PersistedGeneratedData {
   lastUsedSelector: string;
@@ -42,6 +42,13 @@ interface MetaData {
   isLogin?: boolean;
 }
 
+interface InputState {
+  selector: string;
+  value: string;
+  type: string;
+  cursorPosition: number;
+}
+
 /**
  * Workflow generator is used to transform the user's interactions into an automatically
  * generated correct workflows, using the ability of internal state persistence and
@@ -428,26 +435,86 @@ export class WorkflowGenerator {
     }
 
     if ((elementInfo?.tagName === 'INPUT' || elementInfo?.tagName === 'TEXTAREA') && selector) {
-      // Calculate the exact position within the element
+      const positionAndCursor = await page.evaluate(
-      const elementPos = await page.evaluate((selector) => {
+        ({ selector, coords }) => {
-        const element = document.querySelector(selector);
+          const getCursorPosition = (element: any, clickX: any) => {
-        if (!element) return null;
+            const text = element.value;
-        const rect = element.getBoundingClientRect();
+            
-        return {
+            const mirror = document.createElement('div');
-            x: rect.left,
+            
-            y: rect.top
+            const style = window.getComputedStyle(element);
-        };
+            mirror.style.cssText = `
-      }, selector);
+              font: ${style.font};
+              line-height: ${style.lineHeight};
+              padding: ${style.padding};
+              border: ${style.border};
+              box-sizing: ${style.boxSizing};
+              white-space: ${style.whiteSpace};
+              overflow-wrap: ${style.overflowWrap};
+              position: absolute;
+              top: -9999px;
+              left: -9999px;
+              width: ${element.offsetWidth}px;
+            `;
+            
+            document.body.appendChild(mirror);
+          
+            const paddingLeft = parseFloat(style.paddingLeft);
+            const borderLeft = parseFloat(style.borderLeftWidth);
+            
+            const adjustedClickX = clickX - (paddingLeft + borderLeft);
+            
+            let bestIndex = 0;
+            let bestDiff = Infinity;
+          
+            for (let i = 0; i <= text.length; i++) {
+              const textBeforeCursor = text.substring(0, i);
+              const span = document.createElement('span');
+              span.textContent = textBeforeCursor;
+              mirror.innerHTML = '';
+              mirror.appendChild(span);
+              
+              const textWidth = span.getBoundingClientRect().width;
+              
+              const diff = Math.abs(adjustedClickX - textWidth);
+              
+              if (diff < bestDiff) {
+                bestIndex = i;
+                bestDiff = diff;
+              }
+            }
+            
+            document.body.removeChild(mirror);
+            
+            return bestIndex;
+          };
 
-      if (elementPos) {
+            const element = document.querySelector(selector) as HTMLInputElement | HTMLTextAreaElement;
-        const relativeX = coordinates.x - elementPos.x;
+            if (!element) return null;
-        const relativeY = coordinates.y - elementPos.y;
+    
+            const rect = element.getBoundingClientRect();
+            const relativeX = coords.x - rect.left;
+            
+            return {
+              rect: {
+                x: rect.left,
+                y: rect.top
+              },
+              cursorIndex: getCursorPosition(element, relativeX)
+            };
+        },
+        { selector, coords: coordinates } 
+      );
+
+      if (positionAndCursor) {
+        const relativeX = coordinates.x - positionAndCursor.rect.x;
+        const relativeY = coordinates.y - positionAndCursor.rect.y;
 
         const pair: WhereWhatPair = {
             where,
             what: [{
                 action: 'click',
-                args: [selector, { position: { x: relativeX, y: relativeY } }]
+                args: [selector, { position: { x: relativeX, y: relativeY } }, { cursorIndex: positionAndCursor.cursorIndex }],
             }]
         };
 
@@ -1026,77 +1093,107 @@ export class WorkflowGenerator {
    * @param workflow The workflow to be optimized.
    */
   private optimizeWorkflow = (workflow: WorkflowFile) => {
-
+    const inputStates = new Map<string, InputState>();
-    // replace a sequence of press actions by a single fill action
+  
-    let input = {
+    for (const pair of workflow.workflow) {
-      selector: '',
+      let currentIndex = 0;
-      value: '',
+      
-      type: '',
+      while (currentIndex < pair.what.length) {
-      actionCounter: 0,
+        const condition = pair.what[currentIndex];
-    };
+  
-
+        if (condition.action === 'click' && condition.args?.[2]?.cursorIndex !== undefined) {
-    const pushTheOptimizedAction = (pair: WhereWhatPair, index: number) => {
+          const selector = condition.args[0];
-      if (input.value.length === 1) {
+          const cursorIndex = condition.args[2].cursorIndex;
-        // when only one press action is present, keep it and add a waitForLoadState action
+  
-        pair.what.splice(index + 1, 0, {
+          let state = inputStates.get(selector) || {
-          action: 'waitForLoadState',
+            selector,
-          args: ['networkidle'],
+            value: '',
-        })
+            type: 'text',
-      } else {
+            cursorPosition: -1
-        // when more than one press action is present, add a type action
+          };
-        pair.what.splice(index - input.actionCounter, input.actionCounter, {
+  
-          action: 'type',
+          state.cursorPosition = cursorIndex;
-          args: [input.selector, encrypt(input.value), input.type],
+          inputStates.set(selector, state);
-        }, {
+          
-          action: 'waitForLoadState',
+          pair.what.splice(currentIndex, 1);
-          args: ['networkidle'],
+          continue;
-        });
+        }
+  
+        if (condition.action === 'press' && condition.args?.[1]) {
+          const [selector, encryptedKey, type] = condition.args;
+          const key = decrypt(encryptedKey);
+          
+          let state = inputStates.get(selector);
+          if (!state) {
+            state = {
+              selector,
+              value: '',
+              type: type || 'text', 
+              cursorPosition: -1
+            };
+          } else {
+            state.type = type || state.type;
+          }
+  
+          if (key.length === 1) {
+            if (state.cursorPosition === -1) {
+              state.value += key;
+            } else {
+              state.value = 
+                state.value.slice(0, state.cursorPosition) +
+                key +
+                state.value.slice(state.cursorPosition);
+              state.cursorPosition++;
+            }
+          } else if (key === 'Backspace') {
+            if (state.cursorPosition > 0) {
+              state.value = 
+                state.value.slice(0, state.cursorPosition - 1) +
+                state.value.slice(state.cursorPosition);
+              state.cursorPosition--;
+            } else if (state.cursorPosition === -1 && state.value.length > 0) {
+              state.value = state.value.slice(0, -1);
+            }
+          } else if (key === 'Delete') {
+            if (state.cursorPosition >= 0 && state.cursorPosition < state.value.length) {
+              state.value = 
+                state.value.slice(0, state.cursorPosition) +
+                state.value.slice(state.cursorPosition + 1);
+            } else if (state.cursorPosition === -1 && state.value.length > 0) {
+              state.value = state.value.slice(0, -1);
+            }
+          }
+  
+          inputStates.set(selector, state);
+          
+          pair.what.splice(currentIndex, 1);
+          continue;
+        }
+  
+        currentIndex++;
       }
     }
-
+  
-
+    for (const [selector, state] of inputStates.entries()) {
-    for (const pair of workflow.workflow) {
+      if (state.value) {
-      pair.what.forEach((condition, index) => {
+        for (let i = workflow.workflow.length - 1; i >= 0; i--) {
-        if (condition.action === 'press') {
+          const pair = workflow.workflow[i];
-          if (condition.args && condition.args[1]) {
+          
-            if (!input.selector) {
+          pair.what.push({
-              input.selector = condition.args[0];
+            action: 'type',
-            }
+            args: [selector, encrypt(state.value), state.type]
-            if (input.selector === condition.args[0]) {
+          }, {
-              input.actionCounter++;
+            action: 'waitForLoadState',
-              if (condition.args[1].length === 1) {
+            args: ['networkidle']
-                input.value = input.value + condition.args[1];
+          });
-              } else if (condition.args[1] === 'Backspace') {
+          
-                input.value = input.value.slice(0, -1);
+          break; 
-              } else if (condition.args[1] !== 'Shift') {
-                pushTheOptimizedAction(pair, index);
-                pair.what.splice(index + 1, 0, {
-                  action: 'waitForLoadState',
-                  args: ['networkidle'],
-                })
-                input = { selector: '', value: '', type: '', actionCounter: 0 };
-              }
-            } else {
-              pushTheOptimizedAction(pair, index);
-              input = {
-                selector: condition.args[0],
-                value: condition.args[1],
-                type: condition.args[2],
-                actionCounter: 1,
-              };
-            }
-          }
-        } else {
-          if (input.value.length !== 0) {
-            pushTheOptimizedAction(pair, index);
-            // clear the input
-            input = { selector: '', value: '', type: '', actionCounter: 0 };
-          }
         }
-      });
+      }
     }
+  
     return workflow;
-  }
+  };
 
   /**
    * Returns workflow params from the stored metadata.

src/components/robot/RobotEdit.tsx

@@ -124,81 +124,113 @@ export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettin
         }
     }, [robot]);
 
-    const extractInitialCredentials = (workflow: any[]): Credentials => {
+    function extractInitialCredentials(workflow: any[]): Credentials {
         const credentials: Credentials = {};
-
+    
-        // Helper function to check if a character is printable
         const isPrintableCharacter = (char: string): boolean => {
             return char.length === 1 && !!char.match(/^[\x20-\x7E]$/);
         };
-
+    
-        // Process each step in the workflow
         workflow.forEach(step => {
             if (!step.what) return;
-
+    
-            // Keep track of the current input field being processed
             let currentSelector = '';
             let currentValue = '';
             let currentType = '';
-
+            let i = 0;
-            // Process actions in sequence to maintain correct text state
+    
-            step.what.forEach((action: any) => {
+            while (i < step.what.length) {
-                if (
+                const action = step.what[i];
-                    (action.action === 'type' || action.action === 'press') &&
+    
-                    action.args?.length >= 2 &&
+                if (!action.action || !action.args?.[0]) {
-                    typeof action.args[1] === 'string'
+                    i++;
-                ) {
+                    continue;
-                    const selector: string = action.args[0];
+                }
-                    const character: string = action.args[1];
+    
-                    const inputType: string = action.args[2] || '';
+                const selector = action.args[0];
-
+    
-                    // Detect `input[type="password"]`
+                // Handle full word type actions first
-                    if (!currentType && inputType.toLowerCase() === 'password') {
+                if (action.action === 'type' && 
-                        currentType = 'password';
+                    action.args?.length >= 2 && 
+                    typeof action.args[1] === 'string' &&
+                    action.args[1].length > 1) {  
+                    
+                    if (!credentials[selector]) {
+                        credentials[selector] = {
+                            value: action.args[1],
+                            type: action.args[2] || 'text'
+                        };
                     }
-
+                    i++;
-                    // If we're dealing with a new selector, store the previous one
+                    continue;
-                    if (currentSelector && selector !== currentSelector) {
+                }
-                        if (!credentials[currentSelector]) {
+    
+                // Handle character-by-character sequences (both type and press)
+                if ((action.action === 'type' || action.action === 'press') &&
+                    action.args?.length >= 2 &&
+                    typeof action.args[1] === 'string') {
+    
+                    if (selector !== currentSelector) {
+                        if (currentSelector && currentValue) {
                             credentials[currentSelector] = {
                                 value: currentValue,
-                                type: currentType
+                                type: currentType || 'text'
                             };
-                        } else {
-                            credentials[currentSelector].value = currentValue;
                         }
-                    }
-
-                    // Update current tracking variables
-                    if (selector !== currentSelector) {
                         currentSelector = selector;
                         currentValue = credentials[selector]?.value || '';
-                        currentType = inputType || credentials[selector]?.type || '';
+                        currentType = action.args[2] || credentials[selector]?.type || 'text';
                     }
-
+    
-                    // Handle different types of key actions
+                    const character = action.args[1];
-                    if (character === 'Backspace') {
+    
-                        // Remove the last character when backspace is pressed
+                    if (isPrintableCharacter(character)) {
-                        currentValue = currentValue.slice(0, -1);
-                    } else if (isPrintableCharacter(character)) {
-                        // Add the character to the current value
                         currentValue += character;
+                    } else if (character === 'Backspace') {
+                        currentValue = currentValue.slice(0, -1);
                     }
-                    // Note: We ignore other special keys like 'Shift', 'Enter', etc.
+    
+                    if (!currentType && action.args[2]?.toLowerCase() === 'password') {
+                        currentType = 'password';
+                    }
+    
+                    let j = i + 1;
+                    while (j < step.what.length) {
+                        const nextAction = step.what[j];
+                        if (!nextAction.action || !nextAction.args?.[0] || 
+                            nextAction.args[0] !== selector ||
+                            (nextAction.action !== 'type' && nextAction.action !== 'press')) {
+                            break;
+                        }
+                        if (nextAction.args[1] === 'Backspace') {
+                            currentValue = currentValue.slice(0, -1);
+                        } else if (isPrintableCharacter(nextAction.args[1])) {
+                            currentValue += nextAction.args[1];
+                        }
+                        j++;
+                    }
+    
+                    credentials[currentSelector] = {
+                        value: currentValue,
+                        type: currentType
+                    };
+    
+                    i = j; 
+                } else {
+                    i++;
                 }
-            });
+            }
-
+    
-            // Store the final state of the last processed selector
+            if (currentSelector && currentValue) {
-            if (currentSelector) {
                 credentials[currentSelector] = {
                     value: currentValue,
-                    type: currentType
+                    type: currentType || 'text'
                 };
             }
         });
-
+    
         return credentials;
-    };
+    }
 
     const groupCredentialsByType = (credentials: Credentials): GroupedCredentials => {
         return Object.entries(credentials).reduce((acc: GroupedCredentials, [selector, info]) => {