vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: scrapeList pagination persistence and action data separation

Browse Source
This commit is contained in:
Rohit Rajan
2025-11-18 14:25:09 +05:30
parent 334fbbc391
commit 801ae5a365
2 changed files with 81 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
server/src/workflow-management/classes/Interpreter.ts
View File

@@ -567,20 +567,39 @@ export class WorkflowInterpreter {
typeKey = "scrapeSchema";
}
if (this.currentActionType === "scrapeList" && data.scrapeList) {
if (typeKey === "scrapeList" && data.scrapeList) {
data = data.scrapeList;
} else if (this.currentActionType === "scrapeSchema" && data.scrapeSchema) {
} else if (typeKey === "scrapeSchema" && data.scrapeSchema) {
data = data.scrapeSchema;
}
let actionName = this.currentActionName || "";
if (typeKey === "scrapeList") {
actionName = this.getUniqueActionName(typeKey, this.currentActionName);
let actionName = "";
if (typeKey === "scrapeList" && data && typeof data === "object" && !Array.isArray(data)) {
const keys = Object.keys(data);
if (keys.length === 1) {
actionName = keys[0];
data = data[actionName];
} else if (keys.length > 1) {
actionName = keys[keys.length - 1];
data = data[actionName];
}
}
if (!actionName) {
actionName = this.currentActionName || "";
if (typeKey === "scrapeList" && !actionName) {
actionName = this.getUniqueActionName(typeKey, "");
}
}
const flattened = Array.isArray(data)
? data
: (data?.List ?? (data && typeof data === 'object' ? Object.values(data).flat?.() ?? data : []));
: (
data?.List ??
(data && typeof data === "object"
? Object.values(data).flat?.() ?? data
: [])
);
if (!this.serializableDataByType[typeKey]) {
this.serializableDataByType[typeKey] = {};
@@ -588,16 +607,15 @@ export class WorkflowInterpreter {
this.serializableDataByType[typeKey][actionName] = flattened;
await this.persistDataToDatabase(typeKey, { [actionName]: flattened });
await this.persistDataToDatabase(typeKey, {
[actionName]: flattened,
});
this.socket.emit("serializableCallback", {
type: typeKey,
name: actionName,
data: flattened,
});
this.currentActionType = null;
this.currentActionName = null;
} catch (err: any) {
logger.log('error', `serializableCallback handler failed: ${err.message}`);
}