diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 399920df..5a46dedd 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -171,54 +171,82 @@ interface Credentials { [key: string]: CredentialInfo; } -function updateTypeActionsInWorkflow(workflow: any[], credentials: Credentials) { +function handleWorkflowActions(workflow: any[], credentials: Credentials) { return workflow.map(step => { if (!step.what) return step; - const indicesToRemove = new Set(); - step.what.forEach((action: any, index: number) => { - if (!action.action || !action.args?.[0]) return; + const newWhat: any[] = []; + const processedSelectors = new Set(); + + for (let i = 0; i < step.what.length; i++) { + const action = step.what[i]; - if ((action.action === 'type' || action.action === 'press') && credentials[action.args[0]]) { - indicesToRemove.add(index); - - if (step.what[index + 1]?.action === 'waitForLoadState') { - indicesToRemove.add(index + 1); - } + if (!action?.action || !action?.args?.[0]) { + newWhat.push(action); + continue; } - }); - const filteredWhat = step.what.filter((_: any, index: number) => !indicesToRemove.has(index)); + const selector = action.args[0]; + const credential = credentials[selector]; - Object.entries(credentials).forEach(([selector, credentialInfo]) => { - const clickIndex = filteredWhat.findIndex((action: any) => - action.action === 'click' && action.args?.[0] === selector - ); + if (!credential) { + newWhat.push(action); + continue; + } - if (clickIndex !== -1) { - const chars = credentialInfo.value.split(''); + if (action.action === 'click') { + newWhat.push(action); - chars.forEach((char, i) => { - filteredWhat.splice(clickIndex + 1 + (i * 2), 0, { + if (!processedSelectors.has(selector) && + i + 1 < step.what.length && + (step.what[i + 1].action === 'type' || step.what[i + 1].action === 'press')) { + + newWhat.push({ action: 'type', - args: [ - selector, - encrypt(char), - credentialInfo.type - ] + args: [selector, encrypt(credential.value), credential.type] }); - filteredWhat.splice(clickIndex + 2 + (i * 2), 0, { + newWhat.push({ action: 'waitForLoadState', args: ['networkidle'] }); + + processedSelectors.add(selector); + + while (i + 1 < step.what.length && + (step.what[i + 1].action === 'type' || + step.what[i + 1].action === 'press' || + step.what[i + 1].action === 'waitForLoadState')) { + i++; + } + } + } else if ((action.action === 'type' || action.action === 'press') && + !processedSelectors.has(selector)) { + newWhat.push({ + action: 'type', + args: [selector, encrypt(credential.value), credential.type] }); + + newWhat.push({ + action: 'waitForLoadState', + args: ['networkidle'] + }); + + processedSelectors.add(selector); + + // Skip subsequent type/press/waitForLoadState actions for this selector + while (i + 1 < step.what.length && + (step.what[i + 1].action === 'type' || + step.what[i + 1].action === 'press' || + step.what[i + 1].action === 'waitForLoadState')) { + i++; + } } - }); + } return { ...step, - what: filteredWhat + what: newWhat }; }); } @@ -251,7 +279,7 @@ router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, r let workflow = [...robot.recording.workflow]; // Create a copy of the workflow if (credentials) { - workflow = updateTypeActionsInWorkflow(workflow, credentials); + workflow = handleWorkflowActions(workflow, credentials); } // Update the limit @@ -289,9 +317,23 @@ router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, r } } - robot.set('recording', { ...robot.recording, workflow }); + const updates: any = { + recording: { + ...robot.recording, + workflow + } + }; - await robot.save(); + if (name) { + updates.recording_meta = { + ...robot.recording_meta, + name + }; + } + + await Robot.update(updates, { + where: { 'recording_meta.id': id } + }); const updatedRobot = await Robot.findOne({ where: { 'recording_meta.id': id } }); diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 509c0177..2eb5f130 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -22,7 +22,7 @@ import { getBestSelectorForAction } from "../utils"; import { browserPool } from "../../server"; import { uuid } from "uuidv4"; import { capture } from "../../utils/analytics" -import { encrypt } from "../../utils/auth"; +import { decrypt, encrypt } from "../../utils/auth"; interface PersistedGeneratedData { lastUsedSelector: string; @@ -42,6 +42,13 @@ interface MetaData { isLogin?: boolean; } +interface InputState { + selector: string; + value: string; + type: string; + cursorPosition: number; +} + /** * Workflow generator is used to transform the user's interactions into an automatically * generated correct workflows, using the ability of internal state persistence and @@ -428,26 +435,86 @@ export class WorkflowGenerator { } if ((elementInfo?.tagName === 'INPUT' || elementInfo?.tagName === 'TEXTAREA') && selector) { - // Calculate the exact position within the element - const elementPos = await page.evaluate((selector) => { - const element = document.querySelector(selector); - if (!element) return null; - const rect = element.getBoundingClientRect(); - return { - x: rect.left, - y: rect.top - }; - }, selector); + const positionAndCursor = await page.evaluate( + ({ selector, coords }) => { + const getCursorPosition = (element: any, clickX: any) => { + const text = element.value; + + const mirror = document.createElement('div'); + + const style = window.getComputedStyle(element); + mirror.style.cssText = ` + font: ${style.font}; + line-height: ${style.lineHeight}; + padding: ${style.padding}; + border: ${style.border}; + box-sizing: ${style.boxSizing}; + white-space: ${style.whiteSpace}; + overflow-wrap: ${style.overflowWrap}; + position: absolute; + top: -9999px; + left: -9999px; + width: ${element.offsetWidth}px; + `; + + document.body.appendChild(mirror); + + const paddingLeft = parseFloat(style.paddingLeft); + const borderLeft = parseFloat(style.borderLeftWidth); + + const adjustedClickX = clickX - (paddingLeft + borderLeft); + + let bestIndex = 0; + let bestDiff = Infinity; + + for (let i = 0; i <= text.length; i++) { + const textBeforeCursor = text.substring(0, i); + const span = document.createElement('span'); + span.textContent = textBeforeCursor; + mirror.innerHTML = ''; + mirror.appendChild(span); + + const textWidth = span.getBoundingClientRect().width; + + const diff = Math.abs(adjustedClickX - textWidth); + + if (diff < bestDiff) { + bestIndex = i; + bestDiff = diff; + } + } + + document.body.removeChild(mirror); + + return bestIndex; + }; - if (elementPos) { - const relativeX = coordinates.x - elementPos.x; - const relativeY = coordinates.y - elementPos.y; + const element = document.querySelector(selector) as HTMLInputElement | HTMLTextAreaElement; + if (!element) return null; + + const rect = element.getBoundingClientRect(); + const relativeX = coords.x - rect.left; + + return { + rect: { + x: rect.left, + y: rect.top + }, + cursorIndex: getCursorPosition(element, relativeX) + }; + }, + { selector, coords: coordinates } + ); + + if (positionAndCursor) { + const relativeX = coordinates.x - positionAndCursor.rect.x; + const relativeY = coordinates.y - positionAndCursor.rect.y; const pair: WhereWhatPair = { where, what: [{ action: 'click', - args: [selector, { position: { x: relativeX, y: relativeY } }] + args: [selector, { position: { x: relativeX, y: relativeY } }, { cursorIndex: positionAndCursor.cursorIndex }], }] }; @@ -1026,77 +1093,107 @@ export class WorkflowGenerator { * @param workflow The workflow to be optimized. */ private optimizeWorkflow = (workflow: WorkflowFile) => { - - // replace a sequence of press actions by a single fill action - let input = { - selector: '', - value: '', - type: '', - actionCounter: 0, - }; - - const pushTheOptimizedAction = (pair: WhereWhatPair, index: number) => { - if (input.value.length === 1) { - // when only one press action is present, keep it and add a waitForLoadState action - pair.what.splice(index + 1, 0, { - action: 'waitForLoadState', - args: ['networkidle'], - }) - } else { - // when more than one press action is present, add a type action - pair.what.splice(index - input.actionCounter, input.actionCounter, { - action: 'type', - args: [input.selector, encrypt(input.value), input.type], - }, { - action: 'waitForLoadState', - args: ['networkidle'], - }); + const inputStates = new Map(); + + for (const pair of workflow.workflow) { + let currentIndex = 0; + + while (currentIndex < pair.what.length) { + const condition = pair.what[currentIndex]; + + if (condition.action === 'click' && condition.args?.[2]?.cursorIndex !== undefined) { + const selector = condition.args[0]; + const cursorIndex = condition.args[2].cursorIndex; + + let state = inputStates.get(selector) || { + selector, + value: '', + type: 'text', + cursorPosition: -1 + }; + + state.cursorPosition = cursorIndex; + inputStates.set(selector, state); + + pair.what.splice(currentIndex, 1); + continue; + } + + if (condition.action === 'press' && condition.args?.[1]) { + const [selector, encryptedKey, type] = condition.args; + const key = decrypt(encryptedKey); + + let state = inputStates.get(selector); + if (!state) { + state = { + selector, + value: '', + type: type || 'text', + cursorPosition: -1 + }; + } else { + state.type = type || state.type; + } + + if (key.length === 1) { + if (state.cursorPosition === -1) { + state.value += key; + } else { + state.value = + state.value.slice(0, state.cursorPosition) + + key + + state.value.slice(state.cursorPosition); + state.cursorPosition++; + } + } else if (key === 'Backspace') { + if (state.cursorPosition > 0) { + state.value = + state.value.slice(0, state.cursorPosition - 1) + + state.value.slice(state.cursorPosition); + state.cursorPosition--; + } else if (state.cursorPosition === -1 && state.value.length > 0) { + state.value = state.value.slice(0, -1); + } + } else if (key === 'Delete') { + if (state.cursorPosition >= 0 && state.cursorPosition < state.value.length) { + state.value = + state.value.slice(0, state.cursorPosition) + + state.value.slice(state.cursorPosition + 1); + } else if (state.cursorPosition === -1 && state.value.length > 0) { + state.value = state.value.slice(0, -1); + } + } + + inputStates.set(selector, state); + + pair.what.splice(currentIndex, 1); + continue; + } + + currentIndex++; } } - - - for (const pair of workflow.workflow) { - pair.what.forEach((condition, index) => { - if (condition.action === 'press') { - if (condition.args && condition.args[1]) { - if (!input.selector) { - input.selector = condition.args[0]; - } - if (input.selector === condition.args[0]) { - input.actionCounter++; - if (condition.args[1].length === 1) { - input.value = input.value + condition.args[1]; - } else if (condition.args[1] === 'Backspace') { - input.value = input.value.slice(0, -1); - } else if (condition.args[1] !== 'Shift') { - pushTheOptimizedAction(pair, index); - pair.what.splice(index + 1, 0, { - action: 'waitForLoadState', - args: ['networkidle'], - }) - input = { selector: '', value: '', type: '', actionCounter: 0 }; - } - } else { - pushTheOptimizedAction(pair, index); - input = { - selector: condition.args[0], - value: condition.args[1], - type: condition.args[2], - actionCounter: 1, - }; - } - } - } else { - if (input.value.length !== 0) { - pushTheOptimizedAction(pair, index); - // clear the input - input = { selector: '', value: '', type: '', actionCounter: 0 }; - } + + for (const [selector, state] of inputStates.entries()) { + if (state.value) { + for (let i = workflow.workflow.length - 1; i >= 0; i--) { + const pair = workflow.workflow[i]; + + pair.what.push({ + action: 'type', + args: [selector, encrypt(state.value), state.type] + }, { + action: 'waitForLoadState', + args: ['networkidle'] + }); + + break; } - }); + } } + return workflow; - } + }; /** * Returns workflow params from the stored metadata. diff --git a/src/components/robot/RobotEdit.tsx b/src/components/robot/RobotEdit.tsx index 25edaa3f..9c1a0cf6 100644 --- a/src/components/robot/RobotEdit.tsx +++ b/src/components/robot/RobotEdit.tsx @@ -124,81 +124,113 @@ export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettin } }, [robot]); - const extractInitialCredentials = (workflow: any[]): Credentials => { + function extractInitialCredentials(workflow: any[]): Credentials { const credentials: Credentials = {}; - - // Helper function to check if a character is printable + const isPrintableCharacter = (char: string): boolean => { return char.length === 1 && !!char.match(/^[\x20-\x7E]$/); }; - - // Process each step in the workflow + workflow.forEach(step => { if (!step.what) return; - - // Keep track of the current input field being processed + let currentSelector = ''; let currentValue = ''; let currentType = ''; - - // Process actions in sequence to maintain correct text state - step.what.forEach((action: any) => { - if ( - (action.action === 'type' || action.action === 'press') && - action.args?.length >= 2 && - typeof action.args[1] === 'string' - ) { - const selector: string = action.args[0]; - const character: string = action.args[1]; - const inputType: string = action.args[2] || ''; - - // Detect `input[type="password"]` - if (!currentType && inputType.toLowerCase() === 'password') { - currentType = 'password'; + let i = 0; + + while (i < step.what.length) { + const action = step.what[i]; + + if (!action.action || !action.args?.[0]) { + i++; + continue; + } + + const selector = action.args[0]; + + // Handle full word type actions first + if (action.action === 'type' && + action.args?.length >= 2 && + typeof action.args[1] === 'string' && + action.args[1].length > 1) { + + if (!credentials[selector]) { + credentials[selector] = { + value: action.args[1], + type: action.args[2] || 'text' + }; } - - // If we're dealing with a new selector, store the previous one - if (currentSelector && selector !== currentSelector) { - if (!credentials[currentSelector]) { + i++; + continue; + } + + // Handle character-by-character sequences (both type and press) + if ((action.action === 'type' || action.action === 'press') && + action.args?.length >= 2 && + typeof action.args[1] === 'string') { + + if (selector !== currentSelector) { + if (currentSelector && currentValue) { credentials[currentSelector] = { value: currentValue, - type: currentType + type: currentType || 'text' }; - } else { - credentials[currentSelector].value = currentValue; } - } - - // Update current tracking variables - if (selector !== currentSelector) { currentSelector = selector; currentValue = credentials[selector]?.value || ''; - currentType = inputType || credentials[selector]?.type || ''; + currentType = action.args[2] || credentials[selector]?.type || 'text'; } - - // Handle different types of key actions - if (character === 'Backspace') { - // Remove the last character when backspace is pressed - currentValue = currentValue.slice(0, -1); - } else if (isPrintableCharacter(character)) { - // Add the character to the current value + + const character = action.args[1]; + + if (isPrintableCharacter(character)) { currentValue += character; + } else if (character === 'Backspace') { + currentValue = currentValue.slice(0, -1); } - // Note: We ignore other special keys like 'Shift', 'Enter', etc. + + if (!currentType && action.args[2]?.toLowerCase() === 'password') { + currentType = 'password'; + } + + let j = i + 1; + while (j < step.what.length) { + const nextAction = step.what[j]; + if (!nextAction.action || !nextAction.args?.[0] || + nextAction.args[0] !== selector || + (nextAction.action !== 'type' && nextAction.action !== 'press')) { + break; + } + if (nextAction.args[1] === 'Backspace') { + currentValue = currentValue.slice(0, -1); + } else if (isPrintableCharacter(nextAction.args[1])) { + currentValue += nextAction.args[1]; + } + j++; + } + + credentials[currentSelector] = { + value: currentValue, + type: currentType + }; + + i = j; + } else { + i++; } - }); - - // Store the final state of the last processed selector - if (currentSelector) { + } + + if (currentSelector && currentValue) { credentials[currentSelector] = { value: currentValue, - type: currentType + type: currentType || 'text' }; } }); - + return credentials; - }; + } const groupCredentialsByType = (credentials: Credentials): GroupedCredentials => { return Object.entries(credentials).reduce((acc: GroupedCredentials, [selector, info]) => {