vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add scrapeSchema data accumulation logic

Browse Source
This commit is contained in:
RohitR311
2024-12-03 22:21:26 +05:30
parent 7cc535adc1
commit 40e0c7e839
1 changed files with 13 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
maxun-core/src/interpret.ts
View File

@@ -329,15 +329,23 @@ export default class Interpreter extends EventEmitter {
scrapeSchema: async (schema: Record<string, { selector: string; tag: string, attribute: string; }>) => { scrapeSchema: async (schema: Record<string, { selector: string; tag: string, attribute: string; }>) => {
await this.ensureScriptsLoaded(page); await this.ensureScriptsLoaded(page);
// Scrape data using the schema
const scrapeResult = await page.evaluate((schemaObj) => window.scrapeSchema(schemaObj), schema); const scrapeResult = await page.evaluate((schemaObj) => window.scrapeSchema(schemaObj), schema);
// Log result and accumulate it const newResults = Array.isArray(scrapeResult) ? scrapeResult : [scrapeResult];
console.log("Scrape result:", scrapeResult); newResults.forEach((result) => {
this.cumulativeResults.push(...(Array.isArray(scrapeResult) ? scrapeResult : [scrapeResult])); Object.entries(result).forEach(([key, value]) => {
const keyExists = this.cumulativeResults.some(
(item) => key in item && item[key] !== undefined
);
if (!keyExists) {
this.cumulativeResults.push({ [key]: value });
}
});
});
const mergedResult: Record<string, string>[] = [ const mergedResult: Record<string, string>[] = [
Object.fromEntries( Object.fromEntries(
Object.entries( Object.entries(
this.cumulativeResults.reduce((acc, curr) => { this.cumulativeResults.reduce((acc, curr) => {
Object.entries(curr).forEach(([key, value]) => { Object.entries(curr).forEach(([key, value]) => {