vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add scrapeSchema data accumulation logic

Browse Source
This commit is contained in:
RohitR311
2024-12-03 22:21:26 +05:30
parent 7cc535adc1
commit 40e0c7e839
1 changed files with 13 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
maxun-core/src/interpret.ts
View File

@@ -329,12 +329,20 @@ export default class Interpreter extends EventEmitter {
scrapeSchema: async (schema: Record<string, { selector: string; tag: string, attribute: string; }>) => {
await this.ensureScriptsLoaded(page);
// Scrape data using the schema
const scrapeResult = await page.evaluate((schemaObj) => window.scrapeSchema(schemaObj), schema);
// Log result and accumulate it
console.log("Scrape result:", scrapeResult);
this.cumulativeResults.push(...(Array.isArray(scrapeResult) ? scrapeResult : [scrapeResult]));
const newResults = Array.isArray(scrapeResult) ? scrapeResult : [scrapeResult];
newResults.forEach((result) => {
Object.entries(result).forEach(([key, value]) => {
const keyExists = this.cumulativeResults.some(
(item) => key in item && item[key] !== undefined
);
if (!keyExists) {
this.cumulativeResults.push({ [key]: value });
}
});
});
const mergedResult: Record<string, string>[] = [
Object.fromEntries(