vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #871 from getmaxun/dataout-fix

Browse Source 
fix(maxun-core): broken backward compatibility for action names
This commit is contained in:
Karishma Shukla
2025-11-12 21:52:27 +05:30
committed by GitHub
parent 1e07aa5dbb 25fb8290eb
commit 791507fe21
4 changed files with 119 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
maxun-core/src/preprocessor.ts
View File

@@ -55,7 +55,7 @@ export default class Preprocessor {
*/
static getParams(workflow: WorkflowFile): string[] {
const getParamsRecurse = (object: any): string[] => {
if (typeof object === 'object') {
if (typeof object === 'object' && object !== null) {
// Recursion base case
if (object.$param) {
return [object.$param];
@@ -141,14 +141,24 @@ export default class Preprocessor {
}
const out = object;
// for every key (child) of the object
Object.keys(object!).forEach((key) => {
// if the field has only one key, which is `k`
if (Object.keys((<any>object)[key]).length === 1 && (<any>object)[key][k]) {
// process the current special tag (init param, hydrate regex...)
(<any>out)[key] = f((<any>object)[key][k]);
} else {
initSpecialRecurse((<any>object)[key], k, f);
const childValue = (<any>object)[key];
if (!childValue || typeof childValue !== 'object') {
return;
}
try {
const childKeys = Object.keys(childValue);
if (childKeys.length === 1 && childValue[k]) {
(<any>out)[key] = f(childValue[k]);
} else {
initSpecialRecurse(childValue, k, f);
}
} catch (error) {
console.warn(`Error processing key "${key}" in initSpecialRecurse:`, error);
}
});
return out;

70
server/src/workflow-management/classes/Interpreter.ts
View File

@@ -116,6 +116,16 @@ export class WorkflowInterpreter {
*/
private currentScrapeListIndex: number = 0;
/**
* Track action counts to generate unique names
*/
private actionCounts: Record<string, number> = {};
/**
* Track used action names to prevent duplicates
*/
private usedActionNames: Set<string> = new Set();
/**
* Current run ID for real-time persistence
*/
@@ -379,6 +389,8 @@ export class WorkflowInterpreter {
};
this.binaryData = [];
this.currentScrapeListIndex = 0;
this.actionCounts = {};
this.usedActionNames = new Set();
this.currentRunId = null;
this.persistenceBuffer = [];
this.persistenceInProgress = false;
@@ -394,6 +406,43 @@ export class WorkflowInterpreter {
logger.log('debug', `Set run ID for real-time persistence: ${runId}`);
};
/**
* Generates a unique action name for data storage
* @param actionType The type of action (scrapeList, scrapeSchema, etc.)
* @param providedName Optional name provided by the action
* @returns A unique action name
*/
private getUniqueActionName = (actionType: string, providedName?: string | null): string => {
if (providedName && providedName.trim() !== '' && !this.usedActionNames.has(providedName)) {
this.usedActionNames.add(providedName);
return providedName;
}
if (!this.actionCounts[actionType]) {
this.actionCounts[actionType] = 0;
}
let uniqueName: string;
let counter = this.actionCounts[actionType];
do {
counter++;
if (actionType === 'scrapeList') {
uniqueName = `List ${counter}`;
} else if (actionType === 'scrapeSchema') {
uniqueName = `Text ${counter}`;
} else if (actionType === 'screenshot') {
uniqueName = `Screenshot ${counter}`;
} else {
uniqueName = `${actionType} ${counter}`;
}
} while (this.usedActionNames.has(uniqueName));
this.actionCounts[actionType] = counter;
this.usedActionNames.add(uniqueName);
return uniqueName;
};
/**
* Persists extracted data to database with intelligent batching for performance
* Falls back to immediate persistence for critical operations
@@ -525,20 +574,8 @@ export class WorkflowInterpreter {
}
let actionName = this.currentActionName || "";
if (!actionName) {
if (!Array.isArray(data) && Object.keys(data).length === 1) {
const soleKey = Object.keys(data)[0];
const soleValue = data[soleKey];
if (Array.isArray(soleValue) || typeof soleValue === "object") {
actionName = soleKey;
data = soleValue;
}
}
}
if (!actionName) {
actionName = "Unnamed Action";
if (typeKey === "scrapeList") {
actionName = this.getUniqueActionName(typeKey, this.currentActionName);
}
const flattened = Array.isArray(data)
@@ -570,9 +607,10 @@ export class WorkflowInterpreter {
const { name, data, mimeType } = payload;
const base64Data = data.toString("base64");
const uniqueName = this.getUniqueActionName('screenshot', name);
const binaryItem = {
name,
name: uniqueName,
mimeType,
data: base64Data
};
@@ -582,7 +620,7 @@ export class WorkflowInterpreter {
await this.persistBinaryDataToDatabase(binaryItem);
this.socket.emit("binaryCallback", {
name,
name: uniqueName,
data: base64Data,
mimeType
});

16
src/components/robot/pages/RobotEditPage.tsx
View File

@@ -552,16 +552,12 @@ export const RobotEditPage = ({ handleStart }: RobotSettingsProps) => {
pair.what.forEach((action, actionIndex) => {
if (!editableActions.has(String(action.action))) return;
let currentName =
action.name ||
(action.args && action.args[0] && typeof action.args[0] === 'object') ||
'';
let currentName = action.name || '';
if (!currentName) {
switch (action.action) {
case 'scrapeSchema':
textCount++;
currentName = `Text ${textCount}`;
currentName = 'Texts';
break;
case 'screenshot':
screenshotCount++;
@@ -574,9 +570,6 @@ export const RobotEditPage = ({ handleStart }: RobotSettingsProps) => {
}
} else {
switch (action.action) {
case 'scrapeSchema':
textCount++;
break;
case 'screenshot':
screenshotCount++;
break;
@@ -599,10 +592,7 @@ export const RobotEditPage = ({ handleStart }: RobotSettingsProps) => {
switch (action.action) {
case 'scrapeSchema': {
const existingName =
currentName ||
(action.args && action.args[0] && typeof action.args[0] === "object") ||
"Texts";
const existingName = currentName || "Texts";
if (!textInputs.length) {
textInputs.push(

63
src/components/run/RunContent.tsx
View File

@@ -116,26 +116,28 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
const isLegacyPattern = rawKeys.every(key => /^item-\d+-\d+$/.test(key));
let normalizedScreenshotKeys: string[];
if (isLegacyPattern) {
const renamedKeys = rawKeys.map((_, index) => `Screenshot ${index + 1}`);
const keyMap: Record<string, string> = {};
renamedKeys.forEach((displayName, index) => {
keyMap[displayName] = rawKeys[index];
});
setScreenshotKeys(renamedKeys);
setScreenshotKeyMap(keyMap);
// Legacy unnamed screenshots → Screenshot 1, Screenshot 2...
normalizedScreenshotKeys = rawKeys.map((_, index) => `Screenshot ${index + 1}`);
} else {
const keyMap: Record<string, string> = {};
rawKeys.forEach(key => {
keyMap[key] = key;
// Same rule as captured lists: if name missing or generic, auto-label
normalizedScreenshotKeys = rawKeys.map((key, index) => {
if (!key || key.toLowerCase().includes("screenshot")) {
return `Screenshot ${index + 1}`;
}
return key;
});
setScreenshotKeys(rawKeys);
setScreenshotKeyMap(keyMap);
}
const keyMap: Record<string, string> = {};
normalizedScreenshotKeys.forEach((displayName, index) => {
keyMap[displayName] = rawKeys[index];
});
setScreenshotKeys(normalizedScreenshotKeys);
setScreenshotKeyMap(keyMap);
setCurrentScreenshotIndex(0);
} else {
setScreenshotKeys([]);
@@ -202,7 +204,14 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
const processSchemaData = (schemaOutput: any) => {
const keys = Object.keys(schemaOutput);
setSchemaKeys(keys);
const normalizedKeys = keys.map((key, index) => {
if (!key || key.toLowerCase().includes("scrapeschema")) {
return keys.length === 1 ? "Texts" : `Text ${index + 1}`;
}
return key;
});
setSchemaKeys(normalizedKeys);
const dataByKey: Record<string, any[]> = {};
const columnsByKey: Record<string, string[]> = {};
@@ -248,8 +257,17 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
}
});
setSchemaDataByKey(dataByKey);
setSchemaColumnsByKey(columnsByKey);
const remappedDataByKey: Record<string, any[]> = {};
const remappedColumnsByKey: Record<string, string[]> = {};
normalizedKeys.forEach((newKey, idx) => {
const oldKey = keys[idx];
remappedDataByKey[newKey] = dataByKey[oldKey];
remappedColumnsByKey[newKey] = columnsByKey[oldKey];
});
setSchemaDataByKey(remappedDataByKey);
setSchemaColumnsByKey(remappedColumnsByKey);
if (allData.length > 0) {
const allColumns = new Set<string>();
@@ -290,7 +308,14 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
setListData(tablesList);
setListColumns(columnsList);
setListKeys(keys);
const normalizedListKeys = keys.map((key, index) => {
if (!key || key.toLowerCase().includes("scrapelist")) {
return `List ${index + 1}`;
}
return key;
});
setListKeys(normalizedListKeys);
setCurrentListIndex(0);
};