vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #435 from getmaxun/key-fix

Browse Source 
fix: handle displayed input texts
This commit is contained in:
Karishma Shukla
2025-02-22 19:57:42 +05:30
committed by GitHub
parent f719db4c29 99696aaba2
commit 19f6ac5c50
3 changed files with 332 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
server/src/routes/storage.ts
View File

@@ -171,54 +171,82 @@ interface Credentials {
[key: string]: CredentialInfo;
}
function updateTypeActionsInWorkflow(workflow: any[], credentials: Credentials) {
function handleWorkflowActions(workflow: any[], credentials: Credentials) {
return workflow.map(step => {
if (!step.what) return step;
const indicesToRemove = new Set<number>();
step.what.forEach((action: any, index: number) => {
if (!action.action || !action.args?.[0]) return;
const newWhat: any[] = [];
const processedSelectors = new Set<string>();
for (let i = 0; i < step.what.length; i++) {
const action = step.what[i];
if ((action.action === 'type' || action.action === 'press') && credentials[action.args[0]]) {
indicesToRemove.add(index);
if (step.what[index + 1]?.action === 'waitForLoadState') {
indicesToRemove.add(index + 1);
}
if (!action?.action || !action?.args?.[0]) {
newWhat.push(action);
continue;
}
});
const filteredWhat = step.what.filter((_: any, index: number) => !indicesToRemove.has(index));
const selector = action.args[0];
const credential = credentials[selector];
Object.entries(credentials).forEach(([selector, credentialInfo]) => {
const clickIndex = filteredWhat.findIndex((action: any) =>
action.action === 'click' && action.args?.[0] === selector
);
if (!credential) {
newWhat.push(action);
continue;
}
if (clickIndex !== -1) {
const chars = credentialInfo.value.split('');
if (action.action === 'click') {
newWhat.push(action);
chars.forEach((char, i) => {
filteredWhat.splice(clickIndex + 1 + (i * 2), 0, {
if (!processedSelectors.has(selector) &&
i + 1 < step.what.length &&
(step.what[i + 1].action === 'type' || step.what[i + 1].action === 'press')) {
newWhat.push({
action: 'type',
args: [
selector,
encrypt(char),
credentialInfo.type
]
args: [selector, encrypt(credential.value), credential.type]
});
filteredWhat.splice(clickIndex + 2 + (i * 2), 0, {
newWhat.push({
action: 'waitForLoadState',
args: ['networkidle']
});
processedSelectors.add(selector);
while (i + 1 < step.what.length &&
(step.what[i + 1].action === 'type' ||
step.what[i + 1].action === 'press' ||
step.what[i + 1].action === 'waitForLoadState')) {
i++;
}
}
} else if ((action.action === 'type' || action.action === 'press') &&
!processedSelectors.has(selector)) {
newWhat.push({
action: 'type',
args: [selector, encrypt(credential.value), credential.type]
});
newWhat.push({
action: 'waitForLoadState',
args: ['networkidle']
});
processedSelectors.add(selector);
// Skip subsequent type/press/waitForLoadState actions for this selector
while (i + 1 < step.what.length &&
(step.what[i + 1].action === 'type' ||
step.what[i + 1].action === 'press' ||
step.what[i + 1].action === 'waitForLoadState')) {
i++;
}
}
});
}
return {
...step,
what: filteredWhat
what: newWhat
};
});
}
@@ -251,7 +279,7 @@ router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, r
let workflow = [...robot.recording.workflow]; // Create a copy of the workflow
if (credentials) {
workflow = updateTypeActionsInWorkflow(workflow, credentials);
workflow = handleWorkflowActions(workflow, credentials);
}
// Update the limit
@@ -289,9 +317,23 @@ router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, r
}
}
robot.set('recording', { ...robot.recording, workflow });
const updates: any = {
recording: {
...robot.recording,
workflow
}
};
await robot.save();
if (name) {
updates.recording_meta = {
...robot.recording_meta,
name
};
}
await Robot.update(updates, {
where: { 'recording_meta.id': id }
});
const updatedRobot = await Robot.findOne({ where: { 'recording_meta.id': id } });

259
server/src/workflow-management/classes/Generator.ts
View File

@@ -22,7 +22,7 @@ import { getBestSelectorForAction } from "../utils";
import { browserPool } from "../../server";
import { uuid } from "uuidv4";
import { capture } from "../../utils/analytics"
import { encrypt } from "../../utils/auth";
import { decrypt, encrypt } from "../../utils/auth";
interface PersistedGeneratedData {
lastUsedSelector: string;
@@ -42,6 +42,13 @@ interface MetaData {
isLogin?: boolean;
}
interface InputState {
selector: string;
value: string;
type: string;
cursorPosition: number;
}
/**
* Workflow generator is used to transform the user's interactions into an automatically
* generated correct workflows, using the ability of internal state persistence and
@@ -428,26 +435,86 @@ export class WorkflowGenerator {
}
if ((elementInfo?.tagName === 'INPUT' || elementInfo?.tagName === 'TEXTAREA') && selector) {
// Calculate the exact position within the element
const elementPos = await page.evaluate((selector) => {
const element = document.querySelector(selector);
if (!element) return null;
const rect = element.getBoundingClientRect();
return {
x: rect.left,
y: rect.top
};
}, selector);
const positionAndCursor = await page.evaluate(
({ selector, coords }) => {
const getCursorPosition = (element: any, clickX: any) => {
const text = element.value;
const mirror = document.createElement('div');
const style = window.getComputedStyle(element);
mirror.style.cssText = `
font: ${style.font};
line-height: ${style.lineHeight};
padding: ${style.padding};
border: ${style.border};
box-sizing: ${style.boxSizing};
white-space: ${style.whiteSpace};
overflow-wrap: ${style.overflowWrap};
position: absolute;
top: -9999px;
left: -9999px;
width: ${element.offsetWidth}px;
`;
document.body.appendChild(mirror);
const paddingLeft = parseFloat(style.paddingLeft);
const borderLeft = parseFloat(style.borderLeftWidth);
const adjustedClickX = clickX - (paddingLeft + borderLeft);
let bestIndex = 0;
let bestDiff = Infinity;
for (let i = 0; i <= text.length; i++) {
const textBeforeCursor = text.substring(0, i);
const span = document.createElement('span');
span.textContent = textBeforeCursor;
mirror.innerHTML = '';
mirror.appendChild(span);
const textWidth = span.getBoundingClientRect().width;
const diff = Math.abs(adjustedClickX - textWidth);
if (diff < bestDiff) {
bestIndex = i;
bestDiff = diff;
}
}
document.body.removeChild(mirror);
return bestIndex;
};
if (elementPos) {
const relativeX = coordinates.x - elementPos.x;
const relativeY = coordinates.y - elementPos.y;
const element = document.querySelector(selector) as HTMLInputElement | HTMLTextAreaElement;
if (!element) return null;
const rect = element.getBoundingClientRect();
const relativeX = coords.x - rect.left;
return {
rect: {
x: rect.left,
y: rect.top
},
cursorIndex: getCursorPosition(element, relativeX)
};
},
{ selector, coords: coordinates }
);
if (positionAndCursor) {
const relativeX = coordinates.x - positionAndCursor.rect.x;
const relativeY = coordinates.y - positionAndCursor.rect.y;
const pair: WhereWhatPair = {
where,
what: [{
action: 'click',
args: [selector, { position: { x: relativeX, y: relativeY } }]
args: [selector, { position: { x: relativeX, y: relativeY } }, { cursorIndex: positionAndCursor.cursorIndex }],
}]
};
@@ -1026,77 +1093,107 @@ export class WorkflowGenerator {
* @param workflow The workflow to be optimized.
*/
private optimizeWorkflow = (workflow: WorkflowFile) => {
// replace a sequence of press actions by a single fill action
let input = {
selector: '',
value: '',
type: '',
actionCounter: 0,
};
const pushTheOptimizedAction = (pair: WhereWhatPair, index: number) => {
if (input.value.length === 1) {
// when only one press action is present, keep it and add a waitForLoadState action
pair.what.splice(index + 1, 0, {
action: 'waitForLoadState',
args: ['networkidle'],
})
} else {
// when more than one press action is present, add a type action
pair.what.splice(index - input.actionCounter, input.actionCounter, {
action: 'type',
args: [input.selector, encrypt(input.value), input.type],
}, {
action: 'waitForLoadState',
args: ['networkidle'],
});
const inputStates = new Map<string, InputState>();
for (const pair of workflow.workflow) {
let currentIndex = 0;
while (currentIndex < pair.what.length) {
const condition = pair.what[currentIndex];
if (condition.action === 'click' && condition.args?.[2]?.cursorIndex !== undefined) {
const selector = condition.args[0];
const cursorIndex = condition.args[2].cursorIndex;
let state = inputStates.get(selector) || {
selector,
value: '',
type: 'text',
cursorPosition: -1
};
state.cursorPosition = cursorIndex;
inputStates.set(selector, state);
pair.what.splice(currentIndex, 1);
continue;
}
if (condition.action === 'press' && condition.args?.[1]) {
const [selector, encryptedKey, type] = condition.args;
const key = decrypt(encryptedKey);
let state = inputStates.get(selector);
if (!state) {
state = {
selector,
value: '',
type: type || 'text',
cursorPosition: -1
};
} else {
state.type = type || state.type;
}
if (key.length === 1) {
if (state.cursorPosition === -1) {
state.value += key;
} else {
state.value =
state.value.slice(0, state.cursorPosition) +
key +
state.value.slice(state.cursorPosition);
state.cursorPosition++;
}
} else if (key === 'Backspace') {
if (state.cursorPosition > 0) {
state.value =
state.value.slice(0, state.cursorPosition - 1) +
state.value.slice(state.cursorPosition);
state.cursorPosition--;
} else if (state.cursorPosition === -1 && state.value.length > 0) {
state.value = state.value.slice(0, -1);
}
} else if (key === 'Delete') {
if (state.cursorPosition >= 0 && state.cursorPosition < state.value.length) {
state.value =
state.value.slice(0, state.cursorPosition) +
state.value.slice(state.cursorPosition + 1);
} else if (state.cursorPosition === -1 && state.value.length > 0) {
state.value = state.value.slice(0, -1);
}
}
inputStates.set(selector, state);
pair.what.splice(currentIndex, 1);
continue;
}
currentIndex++;
}
}
for (const pair of workflow.workflow) {
pair.what.forEach((condition, index) => {
if (condition.action === 'press') {
if (condition.args && condition.args[1]) {
if (!input.selector) {
input.selector = condition.args[0];
}
if (input.selector === condition.args[0]) {
input.actionCounter++;
if (condition.args[1].length === 1) {
input.value = input.value + condition.args[1];
} else if (condition.args[1] === 'Backspace') {
input.value = input.value.slice(0, -1);
} else if (condition.args[1] !== 'Shift') {
pushTheOptimizedAction(pair, index);
pair.what.splice(index + 1, 0, {
action: 'waitForLoadState',
args: ['networkidle'],
})
input = { selector: '', value: '', type: '', actionCounter: 0 };
}
} else {
pushTheOptimizedAction(pair, index);
input = {
selector: condition.args[0],
value: condition.args[1],
type: condition.args[2],
actionCounter: 1,
};
}
}
} else {
if (input.value.length !== 0) {
pushTheOptimizedAction(pair, index);
// clear the input
input = { selector: '', value: '', type: '', actionCounter: 0 };
}
for (const [selector, state] of inputStates.entries()) {
if (state.value) {
for (let i = workflow.workflow.length - 1; i >= 0; i--) {
const pair = workflow.workflow[i];
pair.what.push({
action: 'type',
args: [selector, encrypt(state.value), state.type]
}, {
action: 'waitForLoadState',
args: ['networkidle']
});
break;
}
});
}
}
return workflow;
}
};
/**
* Returns workflow params from the stored metadata.

130
src/components/robot/RobotEdit.tsx
View File

@@ -124,81 +124,113 @@ export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettin
}
}, [robot]);
const extractInitialCredentials = (workflow: any[]): Credentials => {
function extractInitialCredentials(workflow: any[]): Credentials {
const credentials: Credentials = {};
// Helper function to check if a character is printable
const isPrintableCharacter = (char: string): boolean => {
return char.length === 1 && !!char.match(/^[\x20-\x7E]$/);
};
// Process each step in the workflow
workflow.forEach(step => {
if (!step.what) return;
// Keep track of the current input field being processed
let currentSelector = '';
let currentValue = '';
let currentType = '';
// Process actions in sequence to maintain correct text state
step.what.forEach((action: any) => {
if (
(action.action === 'type' || action.action === 'press') &&
action.args?.length >= 2 &&
typeof action.args[1] === 'string'
) {
const selector: string = action.args[0];
const character: string = action.args[1];
const inputType: string = action.args[2] || '';
// Detect `input[type="password"]`
if (!currentType && inputType.toLowerCase() === 'password') {
currentType = 'password';
let i = 0;
while (i < step.what.length) {
const action = step.what[i];
if (!action.action || !action.args?.[0]) {
i++;
continue;
}
const selector = action.args[0];
// Handle full word type actions first
if (action.action === 'type' &&
action.args?.length >= 2 &&
typeof action.args[1] === 'string' &&
action.args[1].length > 1) {
if (!credentials[selector]) {
credentials[selector] = {
value: action.args[1],
type: action.args[2] || 'text'
};
}
// If we're dealing with a new selector, store the previous one
if (currentSelector && selector !== currentSelector) {
if (!credentials[currentSelector]) {
i++;
continue;
}
// Handle character-by-character sequences (both type and press)
if ((action.action === 'type' || action.action === 'press') &&
action.args?.length >= 2 &&
typeof action.args[1] === 'string') {
if (selector !== currentSelector) {
if (currentSelector && currentValue) {
credentials[currentSelector] = {
value: currentValue,
type: currentType
type: currentType || 'text'
};
} else {
credentials[currentSelector].value = currentValue;
}
}
// Update current tracking variables
if (selector !== currentSelector) {
currentSelector = selector;
currentValue = credentials[selector]?.value || '';
currentType = inputType || credentials[selector]?.type || '';
currentType = action.args[2] || credentials[selector]?.type || 'text';
}
// Handle different types of key actions
if (character === 'Backspace') {
// Remove the last character when backspace is pressed
currentValue = currentValue.slice(0, -1);
} else if (isPrintableCharacter(character)) {
// Add the character to the current value
const character = action.args[1];
if (isPrintableCharacter(character)) {
currentValue += character;
} else if (character === 'Backspace') {
currentValue = currentValue.slice(0, -1);
}
// Note: We ignore other special keys like 'Shift', 'Enter', etc.
if (!currentType && action.args[2]?.toLowerCase() === 'password') {
currentType = 'password';
}
let j = i + 1;
while (j < step.what.length) {
const nextAction = step.what[j];
if (!nextAction.action || !nextAction.args?.[0] ||
nextAction.args[0] !== selector ||
(nextAction.action !== 'type' && nextAction.action !== 'press')) {
break;
}
if (nextAction.args[1] === 'Backspace') {
currentValue = currentValue.slice(0, -1);
} else if (isPrintableCharacter(nextAction.args[1])) {
currentValue += nextAction.args[1];
}
j++;
}
credentials[currentSelector] = {
value: currentValue,
type: currentType
};
i = j;
} else {
i++;
}
});
// Store the final state of the last processed selector
if (currentSelector) {
}
if (currentSelector && currentValue) {
credentials[currentSelector] = {
value: currentValue,
type: currentType
type: currentType || 'text'
};
}
});
return credentials;
};
}
const groupCredentialsByType = (credentials: Credentials): GroupedCredentials => {
return Object.entries(credentials).reduce((acc: GroupedCredentials, [selector, info]) => {