vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: scrapeList pagination persistence and action data separation

Browse Source
This commit is contained in:
Rohit Rajan
2025-11-18 14:25:09 +05:30
parent 334fbbc391
commit 801ae5a365
2 changed files with 81 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
maxun-core/src/interpret.ts
View File

@@ -82,6 +82,8 @@ export default class Interpreter extends EventEmitter {
scrapeSchema: {} scrapeSchema: {}
}; };
private scrapeListCounter: number = 0;
constructor(workflow: WorkflowFile, options?: Partial<InterpreterOptions>) { constructor(workflow: WorkflowFile, options?: Partial<InterpreterOptions>) {
super(); super();
this.workflow = workflow.workflow; this.workflow = workflow.workflow;
@@ -575,12 +577,13 @@ export default class Interpreter extends EventEmitter {
try { try {
await this.ensureScriptsLoaded(page); await this.ensureScriptsLoaded(page);
if (this.options.debugChannel?.incrementScrapeListIndex) { if (this.options.debugChannel?.incrementScrapeListIndex) {
this.options.debugChannel.incrementScrapeListIndex(); this.options.debugChannel.incrementScrapeListIndex();
} }
let scrapeResults = []; let scrapeResults = [];
let paginationUsed = false;
if (!config.pagination) { if (!config.pagination) {
scrapeResults = await page.evaluate((cfg) => { scrapeResults = await page.evaluate((cfg) => {
@@ -592,6 +595,7 @@ export default class Interpreter extends EventEmitter {
} }
}, config); }, config);
} else { } else {
paginationUsed = true;
scrapeResults = await this.handlePagination(page, config); scrapeResults = await this.handlePagination(page, config);
} }
@@ -599,25 +603,39 @@ export default class Interpreter extends EventEmitter {
scrapeResults = []; scrapeResults = [];
} }
const actionType = "scrapeList"; console.log(`ScrapeList completed with ${scrapeResults.length} results`);
const actionName = (config as any).__name || "List";
if (!this.serializableDataByType[actionType]) this.serializableDataByType[actionType] = {}; if (!paginationUsed) {
if (!this.serializableDataByType[actionType][actionName]) { const actionType = "scrapeList";
this.serializableDataByType[actionType][actionName] = []; let actionName = (config as any).__name || "";
if (!actionName || actionName.trim() === "") {
this.scrapeListCounter++;
actionName = `List ${this.scrapeListCounter}`;
}
if (!this.serializableDataByType[actionType]) this.serializableDataByType[actionType] = {};
if (!this.serializableDataByType[actionType][actionName]) {
this.serializableDataByType[actionType][actionName] = [];
}
this.serializableDataByType[actionType][actionName].push(...scrapeResults);
await this.options.serializableCallback({
scrapeList: this.serializableDataByType.scrapeList,
scrapeSchema: this.serializableDataByType.scrapeSchema
});
} }
this.serializableDataByType[actionType][actionName].push(...scrapeResults);
await this.options.serializableCallback({
scrapeList: this.serializableDataByType.scrapeList,
scrapeSchema: this.serializableDataByType.scrapeSchema
});
} catch (error) { } catch (error) {
console.error('ScrapeList action failed completely:', error.message); console.error('ScrapeList action failed completely:', error.message);
const actionType = "scrapeList"; const actionType = "scrapeList";
const actionName = (config as any).__name || "List"; let actionName = (config as any).__name || "";
if (!actionName || actionName.trim() === "") {
this.scrapeListCounter++;
actionName = `List ${this.scrapeListCounter}`;
}
if (!this.namedResults[actionType]) this.namedResults[actionType] = {}; if (!this.namedResults[actionType]) this.namedResults[actionType] = {};
this.namedResults[actionType][actionName] = []; this.namedResults[actionType][actionName] = [];
@@ -818,12 +836,26 @@ export default class Interpreter extends EventEmitter {
return []; return [];
} }
const actionType = "scrapeList";
let actionName = (config as any).__name || "";
if (!actionName || actionName.trim() === "") {
this.scrapeListCounter++;
actionName = `List ${this.scrapeListCounter}`;
}
if (!this.serializableDataByType[actionType]) {
this.serializableDataByType[actionType] = {};
}
if (!this.serializableDataByType[actionType][actionName]) {
this.serializableDataByType[actionType][actionName] = [];
}
let allResults: Record<string, any>[] = []; let allResults: Record<string, any>[] = [];
let previousHeight = 0; let previousHeight = 0;
let scrapedItems: Set<string> = new Set<string>(); let scrapedItems: Set<string> = new Set<string>();
let visitedUrls: Set<string> = new Set<string>(); let visitedUrls: Set<string> = new Set<string>();
const MAX_RETRIES = 3; const MAX_RETRIES = 3;
const RETRY_DELAY = 1000; // 1 second delay between retries const RETRY_DELAY = 1000;
const MAX_UNCHANGED_RESULTS = 5; const MAX_UNCHANGED_RESULTS = 5;
const debugLog = (message: string, ...args: any[]) => { const debugLog = (message: string, ...args: any[]) => {
@@ -831,7 +863,6 @@ export default class Interpreter extends EventEmitter {
}; };
const scrapeCurrentPage = async () => { const scrapeCurrentPage = async () => {
// Check abort flag before scraping current page
if (this.isAborted) { if (this.isAborted) {
debugLog("Workflow aborted, stopping scrapeCurrentPage"); debugLog("Workflow aborted, stopping scrapeCurrentPage");
return; return;
@@ -849,7 +880,6 @@ export default class Interpreter extends EventEmitter {
debugLog(`Page evaluation failed: ${error.message}`); debugLog(`Page evaluation failed: ${error.message}`);
return; return;
} }
const newResults = results.filter(item => { const newResults = results.filter(item => {
const uniqueKey = JSON.stringify(item); const uniqueKey = JSON.stringify(item);
if (scrapedItems.has(uniqueKey)) return false; if (scrapedItems.has(uniqueKey)) return false;
@@ -859,7 +889,11 @@ export default class Interpreter extends EventEmitter {
allResults = allResults.concat(newResults); allResults = allResults.concat(newResults);
debugLog("Results collected:", allResults.length); debugLog("Results collected:", allResults.length);
await this.options.serializableCallback(allResults); this.serializableDataByType[actionType][actionName] = [...allResults];
await this.options.serializableCallback({
scrapeList: this.serializableDataByType.scrapeList,
scrapeSchema: this.serializableDataByType.scrapeSchema
});
}; };
const checkLimit = () => { const checkLimit = () => {

38
server/src/workflow-management/classes/Interpreter.ts
View File

@@ -567,20 +567,39 @@ export class WorkflowInterpreter {
typeKey = "scrapeSchema"; typeKey = "scrapeSchema";
} }
if (this.currentActionType === "scrapeList" && data.scrapeList) { if (typeKey === "scrapeList" && data.scrapeList) {
data = data.scrapeList; data = data.scrapeList;
} else if (this.currentActionType === "scrapeSchema" && data.scrapeSchema) { } else if (typeKey === "scrapeSchema" && data.scrapeSchema) {
data = data.scrapeSchema; data = data.scrapeSchema;
} }
let actionName = this.currentActionName || ""; let actionName = "";
if (typeKey === "scrapeList") { if (typeKey === "scrapeList" && data && typeof data === "object" && !Array.isArray(data)) {
actionName = this.getUniqueActionName(typeKey, this.currentActionName); const keys = Object.keys(data);
if (keys.length === 1) {
actionName = keys[0];
data = data[actionName];
} else if (keys.length > 1) {
actionName = keys[keys.length - 1];
data = data[actionName];
}
}
if (!actionName) {
actionName = this.currentActionName || "";
if (typeKey === "scrapeList" && !actionName) {
actionName = this.getUniqueActionName(typeKey, "");
}
} }
const flattened = Array.isArray(data) const flattened = Array.isArray(data)
? data ? data
: (data?.List ?? (data && typeof data === 'object' ? Object.values(data).flat?.() ?? data : [])); : (
data?.List ??
(data && typeof data === "object"
? Object.values(data).flat?.() ?? data
: [])
);
if (!this.serializableDataByType[typeKey]) { if (!this.serializableDataByType[typeKey]) {
this.serializableDataByType[typeKey] = {}; this.serializableDataByType[typeKey] = {};
@@ -588,16 +607,15 @@ export class WorkflowInterpreter {
this.serializableDataByType[typeKey][actionName] = flattened; this.serializableDataByType[typeKey][actionName] = flattened;
await this.persistDataToDatabase(typeKey, { [actionName]: flattened }); await this.persistDataToDatabase(typeKey, {
[actionName]: flattened,
});
this.socket.emit("serializableCallback", { this.socket.emit("serializableCallback", {
type: typeKey, type: typeKey,
name: actionName, name: actionName,
data: flattened, data: flattened,
}); });
this.currentActionType = null;
this.currentActionName = null;
} catch (err: any) { } catch (err: any) {
logger.log('error', `serializableCallback handler failed: ${err.message}`); logger.log('error', `serializableCallback handler failed: ${err.message}`);
} }