Merge branch 'develop' of https://github.com/getmaxun/maxun into develop

maxun-core/src/interpret.ts

@@ -490,6 +490,14 @@ export default class Interpreter extends EventEmitter {
 
     const executeAction = async (invokee: any, methodName: string, args: any) => {
       console.log("Executing action:", methodName, args);
+
+      if (methodName === 'press' || methodName === 'type') {
+        // Extract only the first two arguments for these methods
+        const limitedArgs = Array.isArray(args) ? args.slice(0, 2) : [args];
+        await (<any>invokee[methodName])(...limitedArgs);
+        return;
+      }
+
       if (!args || Array.isArray(args)) {
         await (<any>invokee[methodName])(...(args ?? []));
       } else {

server/src/routes/storage.ts

@@ -154,48 +154,52 @@ function formatRunResponse(run: any) {
   return formattedRun;
 }
 
-interface CredentialUpdate {
+interface CredentialInfo {
-  [selector: string]: string;
+  value: string;
+  type: string;
 }
 
-function updateTypeActionsInWorkflow(workflow: any[], credentials: CredentialUpdate) {
+interface Credentials {
+  [key: string]: CredentialInfo;
+}
+
+function updateTypeActionsInWorkflow(workflow: any[], credentials: Credentials) {
   return workflow.map(step => {
       if (!step.what) return step;
 
-      // First pass: mark indices to remove
       const indicesToRemove = new Set<number>();
-      step.what.forEach((action: any, index: any) => {
+      step.what.forEach((action: any, index: number) => {
           if (!action.action || !action.args?.[0]) return;
           
-          // If it's a type/press action for a credential
           if ((action.action === 'type' || action.action === 'press') && credentials[action.args[0]]) {
               indicesToRemove.add(index);
-              // Check if next action is waitForLoadState
+
               if (step.what[index + 1]?.action === 'waitForLoadState') {
                   indicesToRemove.add(index + 1);
               }
           }
       });
 
-      // Filter out marked indices and create new what array
+      const filteredWhat = step.what.filter((_: any, index: number) => !indicesToRemove.has(index));
-      const filteredWhat = step.what.filter((_: any, index: any) => !indicesToRemove.has(index));
 
-      // Add new type actions after click actions
+      Object.entries(credentials).forEach(([selector, credentialInfo]) => {
-      Object.entries(credentials).forEach(([selector, credential]) => {
           const clickIndex = filteredWhat.findIndex((action: any) => 
               action.action === 'click' && action.args?.[0] === selector
           );
 
           if (clickIndex !== -1) {
-              const chars = credential.split('');
+              const chars = credentialInfo.value.split('');
+              
               chars.forEach((char, i) => {
-                  // Add type action
                   filteredWhat.splice(clickIndex + 1 + (i * 2), 0, {
                       action: 'type',
-                      args: [selector, encrypt(char)]  
+                      args: [
+                          selector,
+                          encrypt(char),
+                          credentialInfo.type 
+                      ]
                   });
                   
-                  // Add waitForLoadState
                   filteredWhat.splice(clickIndex + 2 + (i * 2), 0, {
                       action: 'waitForLoadState',
                       args: ['networkidle']

server/src/workflow-management/classes/Generator.ts

@@ -354,6 +354,40 @@ export class WorkflowGenerator {
     const elementInfo = await getElementInformation(page, coordinates, '', false);
     console.log("Element info: ", elementInfo);
 
+    if ((elementInfo?.tagName === 'INPUT' || elementInfo?.tagName === 'TEXTAREA') && selector) {
+      // Calculate the exact position within the element
+      const elementPos = await page.evaluate((selector) => {
+        const element = document.querySelector(selector);
+        if (!element) return null;
+        const rect = element.getBoundingClientRect();
+        return {
+            x: rect.left,
+            y: rect.top
+        };
+      }, selector);
+
+      if (elementPos) {
+        const relativeX = coordinates.x - elementPos.x;
+        const relativeY = coordinates.y - elementPos.y;
+
+        const pair: WhereWhatPair = {
+            where,
+            what: [{
+                action: 'click',
+                args: [selector, { position: { x: relativeX, y: relativeY } }]
+            }]
+        };
+
+        if (selector) {
+            this.generatedData.lastUsedSelector = selector;
+            this.generatedData.lastAction = 'click';
+        }
+
+        await this.addPairToWorkflowAndNotifyClient(pair, page);
+        return;
+      }
+    }
+
     // Check if clicked element is a select dropdown
     const isDropdown = elementInfo?.tagName === 'SELECT';
     
@@ -474,6 +508,10 @@ export class WorkflowGenerator {
   public onKeyboardInput = async (key: string, coordinates: Coordinates, page: Page) => {
     let where: WhereWhatPair["where"] = { url: this.getBestUrl(page.url()) };
     const selector = await this.generateSelector(page, coordinates, ActionType.Keydown);
+
+    const elementInfo = await getElementInformation(page, coordinates, '', false);
+    const inputType = elementInfo?.attributes?.type || "text";
+
     if (selector) {
       where.selectors = [selector];
     }
@@ -481,7 +519,7 @@ export class WorkflowGenerator {
       where,
       what: [{
         action: 'press',
-        args: [selector, encrypt(key)],
+        args: [selector, encrypt(key), inputType],
       }],
     }
     if (selector) {
@@ -992,6 +1030,7 @@ export class WorkflowGenerator {
     let input = {
       selector: '',
       value: '',
+      type: '',
       actionCounter: 0,
     };
 
@@ -1006,7 +1045,7 @@ export class WorkflowGenerator {
         // when more than one press action is present, add a type action
         pair.what.splice(index - input.actionCounter, input.actionCounter, {
           action: 'type',
-          args: [input.selector, encrypt(input.value)],
+          args: [input.selector, encrypt(input.value), input.type],
         }, {
           action: 'waitForLoadState',
           args: ['networkidle'],
@@ -1034,13 +1073,14 @@ export class WorkflowGenerator {
                   action: 'waitForLoadState',
                   args: ['networkidle'],
                 })
-                input = { selector: '', value: '', actionCounter: 0 };
+                input = { selector: '', value: '', type: '', actionCounter: 0 };
               }
             } else {
               pushTheOptimizedAction(pair, index);
               input = {
                 selector: condition.args[0],
                 value: condition.args[1],
+                type: condition.args[2],
                 actionCounter: 1,
               };
             }
@@ -1049,7 +1089,7 @@ export class WorkflowGenerator {
           if (input.value.length !== 0) {
             pushTheOptimizedAction(pair, index);
             // clear the input
-            input = { selector: '', value: '', actionCounter: 0 };
+            input = { selector: '', value: '', type: '', actionCounter: 0 };
           }
         }
       });

src/api/storage.ts

@@ -5,8 +5,13 @@ import { ScheduleSettings } from "../components/robot/ScheduleSettings";
 import { CreateRunResponse, ScheduleRunResponse } from "../pages/MainPage";
 import { apiUrl } from "../apiConfig";
 
+interface CredentialInfo {
+  value: string;
+  type: string;
+}
+
 interface Credentials {
-  [key: string]: string;
+  [key: string]: CredentialInfo;
 }
 
 export const getStoredRecordings = async (): Promise<string[] | null> => {

src/components/robot/RobotEdit.tsx

@@ -8,6 +8,7 @@ import { useGlobalInfoStore } from '../../context/globalInfo';
 import { getStoredRecording, updateRecording } from '../../api/storage';
 import { WhereWhatPair } from 'maxun-core';
 
+// Base interfaces for robot data structure
 interface RobotMeta {
     name: string;
     id: string;
@@ -21,19 +22,6 @@ interface RobotWorkflow {
     workflow: WhereWhatPair[];
 }
 
-interface RobotEditOptions {
-    name: string;
-    limit?: number;
-}
-
-interface Credentials {
-    [key: string]: string;
-}
-
-interface CredentialVisibility {
-    [key: string]: boolean;
-}
-
 interface ScheduleConfig {
     runEvery: number;
     runEveryUnit: 'MINUTES' | 'HOURS' | 'DAYS' | 'WEEKS' | 'MONTHS';
@@ -67,21 +55,70 @@ interface RobotSettingsProps {
     initialSettings?: RobotSettings | null;
 }
 
+// Enhanced interfaces for credential handling
+interface CredentialInfo {
+    value: string;
+    type: string;
+}
+
+interface Credentials {
+    [key: string]: CredentialInfo;
+}
+
+interface CredentialVisibility {
+    [key: string]: boolean;
+}
+
+interface GroupedCredentials {
+    passwords: string[];
+    emails: string[];
+    usernames: string[];
+    others: string[];
+}
+
 export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettings }: RobotSettingsProps) => {
     const { t } = useTranslation();
     const [robot, setRobot] = useState<RobotSettings | null>(null);
     const [credentials, setCredentials] = useState<Credentials>({});
     const { recordingId, notify } = useGlobalInfoStore();
-    const [credentialSelectors, setCredentialSelectors] = useState<string[]>([]);
+    const [credentialGroups, setCredentialGroups] = useState<GroupedCredentials>({ 
+        passwords: [], 
+        emails: [],
+        usernames: [],
+        others: [] 
+    });
     const [showPasswords, setShowPasswords] = useState<CredentialVisibility>({});
 
-    const handleClickShowPassword = (selector: string) => {
+    const isEmailPattern = (value: string): boolean => {
-        setShowPasswords(prev => ({
+        return value.includes('@');
-            ...prev,
+    };
-            [selector]: !prev[selector]
+    
-        }));
+    const isUsernameSelector = (selector: string): boolean => {
+        return selector.toLowerCase().includes('username') || 
+               selector.toLowerCase().includes('user') ||
+               selector.toLowerCase().includes('email');
     };
 
+    const determineCredentialType = (selector: string, info: CredentialInfo): 'password' | 'email' | 'username' | 'other' => {
+        // Check for password type first
+        if (info.type === 'password') {
+            return 'password';
+        }
+        
+        // Check for email patterns in the value or selector
+        if (isEmailPattern(info.value) || selector.toLowerCase().includes('email')) {
+            return 'email';
+        }
+        
+        // Check for username patterns in the selector
+        if (isUsernameSelector(selector)) {
+            return 'username';
+        }
+        
+        // If no specific pattern is matched, classify as other
+        return 'other';
+    };
+    
     useEffect(() => {
         if (isOpen) {
             getRobot();
@@ -90,66 +127,105 @@ export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettin
 
     useEffect(() => {
         if (robot?.recording?.workflow) {
-          const selectors = findCredentialSelectors(robot.recording.workflow);
+            const extractedCredentials = extractInitialCredentials(robot.recording.workflow);
-          setCredentialSelectors(selectors);
+            setCredentials(extractedCredentials);
-          
+            setCredentialGroups(groupCredentialsByType(extractedCredentials));
-          const initialCredentials = extractInitialCredentials(robot.recording.workflow);
-          setCredentials(initialCredentials);
         }
     }, [robot]);
 
-    const findCredentialSelectors = (workflow: WhereWhatPair[]): string[] => {
+    const extractInitialCredentials = (workflow: any[]): Credentials => {
-        const selectors = new Set<string>();
+        const credentials: Credentials = {};
-        
+    
-        workflow?.forEach(step => {
+        // Helper function to check if a character is printable
-          step.what?.forEach(action => {
-            if (
-              (action.action === 'type' || action.action === 'press') && 
-              action.args && 
-              action.args[0] && 
-              typeof action.args[0] === 'string'
-            ) {
-              selectors.add(action.args[0]);
-            }
-          });
-        });
-        
-        return Array.from(selectors);
-    };
-
-    const extractInitialCredentials = (workflow: any[]): Record<string, string> => {
-        const credentials: Record<string, string> = {};
-
         const isPrintableCharacter = (char: string): boolean => {
             return char.length === 1 && !!char.match(/^[\x20-\x7E]$/);
         };
-        
+    
+        // Process each step in the workflow
         workflow.forEach(step => {
             if (!step.what) return;
-            
+    
+            // Keep track of the current input field being processed
+            let currentSelector = '';
+            let currentValue = '';
+            let currentType = '';
+    
+            // Process actions in sequence to maintain correct text state
             step.what.forEach((action: any) => {
                 if (
                     (action.action === 'type' || action.action === 'press') && 
                     action.args?.length >= 2 && 
                     typeof action.args[1] === 'string'
                 ) {
-                    let currentSelector: string = action.args[0];
+                    const selector: string = action.args[0];
-                    let character: string = action.args[1];
+                    const character: string = action.args[1];
-                    
+                    const inputType: string = action.args[2] || '';
-                    if (!credentials.hasOwnProperty(currentSelector)) {
+    
-                        credentials[currentSelector] = '';
+                    // If we're dealing with a new selector, store the previous one
+                    if (currentSelector && selector !== currentSelector) {
+                        if (!credentials[currentSelector]) {
+                            credentials[currentSelector] = {
+                                value: currentValue,
+                                type: currentType
+                            };
+                        } else {
+                            credentials[currentSelector].value = currentValue;
+                        }
                     }
-                    
+    
-                    if (isPrintableCharacter(character)) {
+                    // Update current tracking variables
-                        credentials[currentSelector] += character;
+                    if (selector !== currentSelector) {
+                        currentSelector = selector;
+                        currentValue = credentials[selector]?.value || '';
+                        currentType = inputType || credentials[selector]?.type || '';
                     }
+    
+                    // Handle different types of key actions
+                    if (character === 'Backspace') {
+                        // Remove the last character when backspace is pressed
+                        currentValue = currentValue.slice(0, -1);
+                    } else if (isPrintableCharacter(character)) {
+                        // Add the character to the current value
+                        currentValue += character;
+                    }
+                    // Note: We ignore other special keys like 'Shift', 'Enter', etc.
                 }
             });
+    
+            // Store the final state of the last processed selector
+            if (currentSelector) {
+                credentials[currentSelector] = {
+                    value: currentValue,
+                    type: currentType
+                };
+            }
         });
-        
+    
         return credentials;
     };
 
+    const groupCredentialsByType = (credentials: Credentials): GroupedCredentials => {
+        return Object.entries(credentials).reduce((acc: GroupedCredentials, [selector, info]) => {
+            const credentialType = determineCredentialType(selector, info);
+            
+            switch (credentialType) {
+                case 'password':
+                    acc.passwords.push(selector);
+                    break;
+                case 'email':
+                    acc.emails.push(selector);
+                    break;
+                case 'username':
+                    acc.usernames.push(selector);
+                    break;
+                default:
+                    acc.others.push(selector);
+            }
+            
+            return acc;
+        }, { passwords: [], emails: [], usernames: [], others: [] });
+    };
+
     const getRobot = async () => {
         if (recordingId) {
             const robot = await getStoredRecording(recordingId);
@@ -157,7 +233,14 @@ export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettin
         } else {
             notify('error', t('robot_edit.notifications.update_failed'));
         }
-    }
+    };
+
+    const handleClickShowPassword = (selector: string) => {
+        setShowPasswords(prev => ({
+            ...prev,
+            [selector]: !prev[selector]
+        }));
+    };
 
     const handleRobotNameChange = (newName: string) => {
         setRobot((prev) =>
@@ -167,8 +250,11 @@ export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettin
 
     const handleCredentialChange = (selector: string, value: string) => {
         setCredentials(prev => ({
-          ...prev,
+            ...prev,
-          [selector]: value
+            [selector]: {
+                ...prev[selector],
+                value
+            }
         }));
     };
 
@@ -177,7 +263,6 @@ export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettin
             if (!prev) return prev;
 
             const updatedWorkflow = [...prev.recording.workflow];
-
             if (
                 updatedWorkflow.length > 0 &&
                 updatedWorkflow[0]?.what &&
@@ -193,21 +278,101 @@ export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettin
         });
     };
 
+    const renderAllCredentialFields = () => {
+        return (
+            <>
+                {/* Render username credentials */}
+                {renderCredentialFields(
+                    credentialGroups.usernames, 
+                    t('Username Credentials'),
+                    'text'  // Always show usernames as text
+                )}
+                
+                {/* Render email credentials */}
+                {renderCredentialFields(
+                    credentialGroups.emails, 
+                    t('Email Credentials'),
+                    'text'  // Always show emails as text
+                )}
+                
+                {/* Render password credentials */}
+                {renderCredentialFields(
+                    credentialGroups.passwords, 
+                    t('Password Credentials'),
+                    'password'  // Use password masking
+                )}
+                
+                {/* Render other credentials */}
+                {renderCredentialFields(
+                    credentialGroups.others, 
+                    t('Other Credentials'),
+                    'text'  // Show other credentials as text
+                )}
+            </>
+        );
+    };
+
+    const renderCredentialFields = (selectors: string[], headerText: string, defaultType: 'text' | 'password' = 'text') => {
+        if (selectors.length === 0) return null;
+    
+        return (
+            <>
+                <Typography variant="h6" style={{ marginBottom: '20px'}}>
+                    {headerText}
+                </Typography>
+                {selectors.map((selector) => (
+                    <TextField
+                        key={selector}
+                        type={showPasswords[selector] ? 'text' : defaultType}
+                        label={`Credential for ${selector}`}
+                        value={credentials[selector]?.value || ''}
+                        onChange={(e) => handleCredentialChange(selector, e.target.value)}
+                        style={{ marginBottom: '20px' }}
+                        InputProps={{
+                            // Only show visibility toggle for password fields
+                            endAdornment: defaultType === 'password' ? (
+                                <InputAdornment position="end">
+                                    <IconButton
+                                        aria-label="toggle password visibility"
+                                        onClick={() => handleClickShowPassword(selector)}
+                                        edge="end"
+                                    >
+                                        {showPasswords[selector] ? <Visibility /> : <VisibilityOff />}
+                                    </IconButton>
+                                </InputAdornment>
+                            ) : undefined,
+                        }}
+                    />
+                ))}
+            </>
+        );
+    };
+
     const handleSave = async () => {
         if (!robot) return;
 
         try {
+            const credentialsForPayload = Object.entries(credentials).reduce((acc, [selector, info]) => {
+                const enforceType = info.type === 'password' ? 'password' : 'text';
+
+                acc[selector] = {
+                    value: info.value,
+                    type: enforceType
+                };
+                return acc;
+            }, {} as Record<string, CredentialInfo>);
+
             const payload = {
                 name: robot.recording_meta.name,
                 limit: robot.recording.workflow[0]?.what[0]?.args?.[0]?.limit,
-                credentials: credentials,
+                credentials: credentialsForPayload,
             };
 
             const success = await updateRecording(robot.recording_meta.id, payload);
 
             if (success) {
                 notify('success', t('robot_edit.notifications.update_success'));
-                handleStart(robot); // Inform parent about the updated robot
+                handleStart(robot);
                 handleClose();
 
                 setTimeout(() => {
@@ -233,87 +398,60 @@ export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettin
                     {t('robot_edit.title')}
                 </Typography>
                 <Box style={{ display: 'flex', flexDirection: 'column' }}>
-                    {
+                    {robot && (
-                        robot && (
+                        <>
-                            <>
+                            <TextField
+                                label={t('robot_edit.change_name')}
+                                key="Robot Name"
+                                type='text'
+                                value={robot.recording_meta.name}
+                                onChange={(e) => handleRobotNameChange(e.target.value)}
+                                style={{ marginBottom: '20px' }}
+                            />
+                            
+                            {robot.recording.workflow?.[0]?.what?.[0]?.args?.[0]?.limit !== undefined && (
                                 <TextField
-                                    label={t('robot_edit.change_name')}
+                                    label={t('robot_edit.robot_limit')}
-                                    key="Robot Name"
+                                    type="number"
-                                    type='text'
+                                    value={robot.recording.workflow[0].what[0].args[0].limit || ''}
-                                    value={robot.recording_meta.name}
+                                    onChange={(e) => {
-                                    onChange={(e) => handleRobotNameChange(e.target.value)}
+                                        const value = parseInt(e.target.value, 10);
+                                        if (value >= 1) {
+                                            handleLimitChange(value);
+                                        }
+                                    }}
+                                    inputProps={{ min: 1 }}
                                     style={{ marginBottom: '20px' }}
                                 />
-                                {robot.recording.workflow?.[0]?.what?.[0]?.args?.[0]?.limit !== undefined && (
+                            )}
-                                    <TextField
-                                        label={t('robot_edit.robot_limit')}
-                                        type="number"
-                                        value={robot.recording.workflow[0].what[0].args[0].limit || ''}
-                                        onChange={(e) => {
-                                            const value = parseInt(e.target.value, 10);
-                                            if (value >= 1) {
-                                                handleLimitChange(value);
-                                            }
-                                        }}
-                                        inputProps={{ min: 1 }}
-                                        style={{ marginBottom: '20px' }}
-                                    />
-                                )}
 
-                                {(robot.isLogin || credentialSelectors.length > 0) && (
+                            {(robot.isLogin || Object.keys(credentials).length > 0) && (
-                                    <>
+                                <>
-                                        <Typography variant="h6" style={{ marginBottom: '20px' }}>
+                                    {renderAllCredentialFields()}
-                                            {t('Login Credentials')}
+                                </>
-                                        </Typography>
+                            )}
-                                        
-                                        {credentialSelectors.map((selector) => (
-                                            <TextField
-                                                key={selector}
-                                                type={showPasswords[selector] ? 'text' : 'password'}
-                                                label={`Credential for ${selector}`}
-                                                value={credentials[selector] || ''}
-                                                onChange={(e) => handleCredentialChange(selector, e.target.value)}
-                                                style={{ marginBottom: '20px' }}
-                                                InputProps={{
-                                                    endAdornment: (
-                                                        <InputAdornment position="end">
-                                                            <IconButton
-                                                                aria-label="toggle password visibility"
-                                                                onClick={() => handleClickShowPassword(selector)}
-                                                                edge="end"
-                                                            >
-                                                                {showPasswords[selector] ? <Visibility /> : <VisibilityOff />}
-                                                            </IconButton>
-                                                        </InputAdornment>
-                                                    ),
-                                                }}
-                                            />
-                                        ))}
-                                    </>
-                                )}
 
-                                <Box mt={2} display="flex" justifyContent="flex-end">
+                            <Box mt={2} display="flex" justifyContent="flex-end">
-                                    <Button variant="contained" color="primary" onClick={handleSave}>
+                                <Button variant="contained" color="primary" onClick={handleSave}>
-                                        {t('robot_edit.save')}
+                                    {t('robot_edit.save')}
-                                    </Button>
+                                </Button>
-                                    <Button
+                                <Button
-                                        onClick={handleClose}
+                                    onClick={handleClose}
-                                        color="primary"
+                                    color="primary"
-                                        variant="outlined"
+                                    variant="outlined"
-                                        style={{ marginLeft: '10px' }}
+                                    style={{ marginLeft: '10px' }}
-                                        sx={{
+                                    sx={{
-                                            color: '#ff00c3 !important',
+                                        color: '#ff00c3 !important',
-                                            borderColor: '#ff00c3 !important',
+                                        borderColor: '#ff00c3 !important',
-                                            backgroundColor: 'whitesmoke !important',
+                                        backgroundColor: 'whitesmoke !important',
-                                        }}>
+                                    }}>
-                                        {t('robot_edit.cancel')}
+                                    {t('robot_edit.cancel')}
-                                    </Button>
+                                </Button>
-                                </Box>
+                            </Box>
-                            </>
+                        </>
-                        )
+                    )}
-                    }
                 </Box>
             </>
         </GenericModal>
     );
-};
+};