Merge pull request #435 from getmaxun/key-fix
fix: handle displayed input texts
This commit is contained in:
Karishma Shukla
@@ -171,54 +171,82 @@ interface Credentials {
|
||||
[key: string]: CredentialInfo;
|
||||
}
|
||||
|
||||
function updateTypeActionsInWorkflow(workflow: any[], credentials: Credentials) {
|
||||
function handleWorkflowActions(workflow: any[], credentials: Credentials) {
|
||||
return workflow.map(step => {
|
||||
if (!step.what) return step;
|
||||
|
||||
const indicesToRemove = new Set<number>();
|
||||
step.what.forEach((action: any, index: number) => {
|
||||
if (!action.action || !action.args?.[0]) return;
|
||||
const newWhat: any[] = [];
|
||||
const processedSelectors = new Set<string>();
|
||||
|
||||
for (let i = 0; i < step.what.length; i++) {
|
||||
const action = step.what[i];
|
||||
|
||||
if ((action.action === 'type' || action.action === 'press') && credentials[action.args[0]]) {
|
||||
indicesToRemove.add(index);
|
||||
|
||||
if (step.what[index + 1]?.action === 'waitForLoadState') {
|
||||
indicesToRemove.add(index + 1);
|
||||
}
|
||||
if (!action?.action || !action?.args?.[0]) {
|
||||
newWhat.push(action);
|
||||
continue;
|
||||
}
|
||||
});
|
||||
|
||||
const filteredWhat = step.what.filter((_: any, index: number) => !indicesToRemove.has(index));
|
||||
const selector = action.args[0];
|
||||
const credential = credentials[selector];
|
||||
|
||||
Object.entries(credentials).forEach(([selector, credentialInfo]) => {
|
||||
const clickIndex = filteredWhat.findIndex((action: any) =>
|
||||
action.action === 'click' && action.args?.[0] === selector
|
||||
);
|
||||
if (!credential) {
|
||||
newWhat.push(action);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (clickIndex !== -1) {
|
||||
const chars = credentialInfo.value.split('');
|
||||
if (action.action === 'click') {
|
||||
newWhat.push(action);
|
||||
|
||||
chars.forEach((char, i) => {
|
||||
filteredWhat.splice(clickIndex + 1 + (i * 2), 0, {
|
||||
if (!processedSelectors.has(selector) &&
|
||||
i + 1 < step.what.length &&
|
||||
(step.what[i + 1].action === 'type' || step.what[i + 1].action === 'press')) {
|
||||
|
||||
newWhat.push({
|
||||
action: 'type',
|
||||
args: [
|
||||
selector,
|
||||
encrypt(char),
|
||||
credentialInfo.type
|
||||
]
|
||||
args: [selector, encrypt(credential.value), credential.type]
|
||||
});
|
||||
|
||||
filteredWhat.splice(clickIndex + 2 + (i * 2), 0, {
|
||||
newWhat.push({
|
||||
action: 'waitForLoadState',
|
||||
args: ['networkidle']
|
||||
});
|
||||
|
||||
processedSelectors.add(selector);
|
||||
|
||||
while (i + 1 < step.what.length &&
|
||||
(step.what[i + 1].action === 'type' ||
|
||||
step.what[i + 1].action === 'press' ||
|
||||
step.what[i + 1].action === 'waitForLoadState')) {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
} else if ((action.action === 'type' || action.action === 'press') &&
|
||||
!processedSelectors.has(selector)) {
|
||||
newWhat.push({
|
||||
action: 'type',
|
||||
args: [selector, encrypt(credential.value), credential.type]
|
||||
});
|
||||
|
||||
newWhat.push({
|
||||
action: 'waitForLoadState',
|
||||
args: ['networkidle']
|
||||
});
|
||||
|
||||
processedSelectors.add(selector);
|
||||
|
||||
// Skip subsequent type/press/waitForLoadState actions for this selector
|
||||
while (i + 1 < step.what.length &&
|
||||
(step.what[i + 1].action === 'type' ||
|
||||
step.what[i + 1].action === 'press' ||
|
||||
step.what[i + 1].action === 'waitForLoadState')) {
|
||||
i++;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
...step,
|
||||
what: filteredWhat
|
||||
what: newWhat
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -251,7 +279,7 @@ router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, r
|
||||
let workflow = [...robot.recording.workflow]; // Create a copy of the workflow
|
||||
|
||||
if (credentials) {
|
||||
workflow = updateTypeActionsInWorkflow(workflow, credentials);
|
||||
workflow = handleWorkflowActions(workflow, credentials);
|
||||
}
|
||||
|
||||
// Update the limit
|
||||
@@ -289,9 +317,23 @@ router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, r
|
||||
}
|
||||
}
|
||||
|
||||
robot.set('recording', { ...robot.recording, workflow });
|
||||
const updates: any = {
|
||||
recording: {
|
||||
...robot.recording,
|
||||
workflow
|
||||
}
|
||||
};
|
||||
|
||||
await robot.save();
|
||||
if (name) {
|
||||
updates.recording_meta = {
|
||||
...robot.recording_meta,
|
||||
name
|
||||
};
|
||||
}
|
||||
|
||||
await Robot.update(updates, {
|
||||
where: { 'recording_meta.id': id }
|
||||
});
|
||||
|
||||
const updatedRobot = await Robot.findOne({ where: { 'recording_meta.id': id } });
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ import { getBestSelectorForAction } from "../utils";
|
||||
import { browserPool } from "../../server";
|
||||
import { uuid } from "uuidv4";
|
||||
import { capture } from "../../utils/analytics"
|
||||
import { encrypt } from "../../utils/auth";
|
||||
import { decrypt, encrypt } from "../../utils/auth";
|
||||
|
||||
interface PersistedGeneratedData {
|
||||
lastUsedSelector: string;
|
||||
@@ -42,6 +42,13 @@ interface MetaData {
|
||||
isLogin?: boolean;
|
||||
}
|
||||
|
||||
interface InputState {
|
||||
selector: string;
|
||||
value: string;
|
||||
type: string;
|
||||
cursorPosition: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Workflow generator is used to transform the user's interactions into an automatically
|
||||
* generated correct workflows, using the ability of internal state persistence and
|
||||
@@ -428,26 +435,86 @@ export class WorkflowGenerator {
|
||||
}
|
||||
|
||||
if ((elementInfo?.tagName === 'INPUT' || elementInfo?.tagName === 'TEXTAREA') && selector) {
|
||||
// Calculate the exact position within the element
|
||||
const elementPos = await page.evaluate((selector) => {
|
||||
const element = document.querySelector(selector);
|
||||
if (!element) return null;
|
||||
const rect = element.getBoundingClientRect();
|
||||
return {
|
||||
x: rect.left,
|
||||
y: rect.top
|
||||
};
|
||||
}, selector);
|
||||
const positionAndCursor = await page.evaluate(
|
||||
({ selector, coords }) => {
|
||||
const getCursorPosition = (element: any, clickX: any) => {
|
||||
const text = element.value;
|
||||
|
||||
const mirror = document.createElement('div');
|
||||
|
||||
const style = window.getComputedStyle(element);
|
||||
mirror.style.cssText = `
|
||||
font: ${style.font};
|
||||
line-height: ${style.lineHeight};
|
||||
padding: ${style.padding};
|
||||
border: ${style.border};
|
||||
box-sizing: ${style.boxSizing};
|
||||
white-space: ${style.whiteSpace};
|
||||
overflow-wrap: ${style.overflowWrap};
|
||||
position: absolute;
|
||||
top: -9999px;
|
||||
left: -9999px;
|
||||
width: ${element.offsetWidth}px;
|
||||
`;
|
||||
|
||||
document.body.appendChild(mirror);
|
||||
|
||||
const paddingLeft = parseFloat(style.paddingLeft);
|
||||
const borderLeft = parseFloat(style.borderLeftWidth);
|
||||
|
||||
const adjustedClickX = clickX - (paddingLeft + borderLeft);
|
||||
|
||||
let bestIndex = 0;
|
||||
let bestDiff = Infinity;
|
||||
|
||||
for (let i = 0; i <= text.length; i++) {
|
||||
const textBeforeCursor = text.substring(0, i);
|
||||
const span = document.createElement('span');
|
||||
span.textContent = textBeforeCursor;
|
||||
mirror.innerHTML = '';
|
||||
mirror.appendChild(span);
|
||||
|
||||
const textWidth = span.getBoundingClientRect().width;
|
||||
|
||||
const diff = Math.abs(adjustedClickX - textWidth);
|
||||
|
||||
if (diff < bestDiff) {
|
||||
bestIndex = i;
|
||||
bestDiff = diff;
|
||||
}
|
||||
}
|
||||
|
||||
document.body.removeChild(mirror);
|
||||
|
||||
return bestIndex;
|
||||
};
|
||||
|
||||
if (elementPos) {
|
||||
const relativeX = coordinates.x - elementPos.x;
|
||||
const relativeY = coordinates.y - elementPos.y;
|
||||
const element = document.querySelector(selector) as HTMLInputElement | HTMLTextAreaElement;
|
||||
if (!element) return null;
|
||||
|
||||
const rect = element.getBoundingClientRect();
|
||||
const relativeX = coords.x - rect.left;
|
||||
|
||||
return {
|
||||
rect: {
|
||||
x: rect.left,
|
||||
y: rect.top
|
||||
},
|
||||
cursorIndex: getCursorPosition(element, relativeX)
|
||||
};
|
||||
},
|
||||
{ selector, coords: coordinates }
|
||||
);
|
||||
|
||||
if (positionAndCursor) {
|
||||
const relativeX = coordinates.x - positionAndCursor.rect.x;
|
||||
const relativeY = coordinates.y - positionAndCursor.rect.y;
|
||||
|
||||
const pair: WhereWhatPair = {
|
||||
where,
|
||||
what: [{
|
||||
action: 'click',
|
||||
args: [selector, { position: { x: relativeX, y: relativeY } }]
|
||||
args: [selector, { position: { x: relativeX, y: relativeY } }, { cursorIndex: positionAndCursor.cursorIndex }],
|
||||
}]
|
||||
};
|
||||
|
||||
@@ -1026,77 +1093,107 @@ export class WorkflowGenerator {
|
||||
* @param workflow The workflow to be optimized.
|
||||
*/
|
||||
private optimizeWorkflow = (workflow: WorkflowFile) => {
|
||||
|
||||
// replace a sequence of press actions by a single fill action
|
||||
let input = {
|
||||
selector: '',
|
||||
value: '',
|
||||
type: '',
|
||||
actionCounter: 0,
|
||||
};
|
||||
|
||||
const pushTheOptimizedAction = (pair: WhereWhatPair, index: number) => {
|
||||
if (input.value.length === 1) {
|
||||
// when only one press action is present, keep it and add a waitForLoadState action
|
||||
pair.what.splice(index + 1, 0, {
|
||||
action: 'waitForLoadState',
|
||||
args: ['networkidle'],
|
||||
})
|
||||
} else {
|
||||
// when more than one press action is present, add a type action
|
||||
pair.what.splice(index - input.actionCounter, input.actionCounter, {
|
||||
action: 'type',
|
||||
args: [input.selector, encrypt(input.value), input.type],
|
||||
}, {
|
||||
action: 'waitForLoadState',
|
||||
args: ['networkidle'],
|
||||
});
|
||||
const inputStates = new Map<string, InputState>();
|
||||
|
||||
for (const pair of workflow.workflow) {
|
||||
let currentIndex = 0;
|
||||
|
||||
while (currentIndex < pair.what.length) {
|
||||
const condition = pair.what[currentIndex];
|
||||
|
||||
if (condition.action === 'click' && condition.args?.[2]?.cursorIndex !== undefined) {
|
||||
const selector = condition.args[0];
|
||||
const cursorIndex = condition.args[2].cursorIndex;
|
||||
|
||||
let state = inputStates.get(selector) || {
|
||||
selector,
|
||||
value: '',
|
||||
type: 'text',
|
||||
cursorPosition: -1
|
||||
};
|
||||
|
||||
state.cursorPosition = cursorIndex;
|
||||
inputStates.set(selector, state);
|
||||
|
||||
pair.what.splice(currentIndex, 1);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (condition.action === 'press' && condition.args?.[1]) {
|
||||
const [selector, encryptedKey, type] = condition.args;
|
||||
const key = decrypt(encryptedKey);
|
||||
|
||||
let state = inputStates.get(selector);
|
||||
if (!state) {
|
||||
state = {
|
||||
selector,
|
||||
value: '',
|
||||
type: type || 'text',
|
||||
cursorPosition: -1
|
||||
};
|
||||
} else {
|
||||
state.type = type || state.type;
|
||||
}
|
||||
|
||||
if (key.length === 1) {
|
||||
if (state.cursorPosition === -1) {
|
||||
state.value += key;
|
||||
} else {
|
||||
state.value =
|
||||
state.value.slice(0, state.cursorPosition) +
|
||||
key +
|
||||
state.value.slice(state.cursorPosition);
|
||||
state.cursorPosition++;
|
||||
}
|
||||
} else if (key === 'Backspace') {
|
||||
if (state.cursorPosition > 0) {
|
||||
state.value =
|
||||
state.value.slice(0, state.cursorPosition - 1) +
|
||||
state.value.slice(state.cursorPosition);
|
||||
state.cursorPosition--;
|
||||
} else if (state.cursorPosition === -1 && state.value.length > 0) {
|
||||
state.value = state.value.slice(0, -1);
|
||||
}
|
||||
} else if (key === 'Delete') {
|
||||
if (state.cursorPosition >= 0 && state.cursorPosition < state.value.length) {
|
||||
state.value =
|
||||
state.value.slice(0, state.cursorPosition) +
|
||||
state.value.slice(state.cursorPosition + 1);
|
||||
} else if (state.cursorPosition === -1 && state.value.length > 0) {
|
||||
state.value = state.value.slice(0, -1);
|
||||
}
|
||||
}
|
||||
|
||||
inputStates.set(selector, state);
|
||||
|
||||
pair.what.splice(currentIndex, 1);
|
||||
continue;
|
||||
}
|
||||
|
||||
currentIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
for (const pair of workflow.workflow) {
|
||||
pair.what.forEach((condition, index) => {
|
||||
if (condition.action === 'press') {
|
||||
if (condition.args && condition.args[1]) {
|
||||
if (!input.selector) {
|
||||
input.selector = condition.args[0];
|
||||
}
|
||||
if (input.selector === condition.args[0]) {
|
||||
input.actionCounter++;
|
||||
if (condition.args[1].length === 1) {
|
||||
input.value = input.value + condition.args[1];
|
||||
} else if (condition.args[1] === 'Backspace') {
|
||||
input.value = input.value.slice(0, -1);
|
||||
} else if (condition.args[1] !== 'Shift') {
|
||||
pushTheOptimizedAction(pair, index);
|
||||
pair.what.splice(index + 1, 0, {
|
||||
action: 'waitForLoadState',
|
||||
args: ['networkidle'],
|
||||
})
|
||||
input = { selector: '', value: '', type: '', actionCounter: 0 };
|
||||
}
|
||||
} else {
|
||||
pushTheOptimizedAction(pair, index);
|
||||
input = {
|
||||
selector: condition.args[0],
|
||||
value: condition.args[1],
|
||||
type: condition.args[2],
|
||||
actionCounter: 1,
|
||||
};
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (input.value.length !== 0) {
|
||||
pushTheOptimizedAction(pair, index);
|
||||
// clear the input
|
||||
input = { selector: '', value: '', type: '', actionCounter: 0 };
|
||||
}
|
||||
|
||||
for (const [selector, state] of inputStates.entries()) {
|
||||
if (state.value) {
|
||||
for (let i = workflow.workflow.length - 1; i >= 0; i--) {
|
||||
const pair = workflow.workflow[i];
|
||||
|
||||
pair.what.push({
|
||||
action: 'type',
|
||||
args: [selector, encrypt(state.value), state.type]
|
||||
}, {
|
||||
action: 'waitForLoadState',
|
||||
args: ['networkidle']
|
||||
});
|
||||
|
||||
break;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return workflow;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns workflow params from the stored metadata.
|
||||
|
||||
Reference in New Issue
Block a user