vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into iframe

Browse Source
This commit is contained in:
Rohit
2025-01-05 00:05:47 +05:30
committed by GitHub
parent 5e775fa83f 10bf769add
commit d1e82390f4
8 changed files with 1151 additions and 673 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
maxun-core/src/interpret.ts
View File

@@ -403,7 +403,7 @@ export default class Interpreter extends EventEmitter {
await this.options.serializableCallback(scrapeResults);
},
scrapeSchema: async (schema: Record<string, { selector: string; tag: string, attribute: string; }>) => {
scrapeSchema: async (schema: Record<string, { selector: string; tag: string, attribute: string; shadow: string}>) => {
await this.ensureScriptsLoaded(page);
const scrapeResult = await page.evaluate((schemaObj) => window.scrapeSchema(schemaObj), schema);
@@ -663,27 +663,42 @@ export default class Interpreter extends EventEmitter {
if (isApplicable) {
return actionId;
}
}
}
}
private removeIframeSelectors(workflow: Workflow) {
private removeShadowSelectors(workflow: Workflow) {
for (let actionId = workflow.length - 1; actionId >= 0; actionId--) {
const step = workflow[actionId];
// Check if step has where and selectors
if (step.where && Array.isArray(step.where.selectors)) {
// Filter out selectors that contain ">>"
step.where.selectors = step.where.selectors.filter(selector => !selector.includes(':>>'));
step.where.selectors = step.where.selectors.filter(selector => !selector.includes('>>'));
}
}
return workflow;
}
private removeSpecialSelectors(workflow: Workflow) {
for (let actionId = workflow.length - 1; actionId >= 0; actionId--) {
const step = workflow[actionId];
if (step.where && Array.isArray(step.where.selectors)) {
// Filter out if selector has EITHER ":>>" OR ">>"
step.where.selectors = step.where.selectors.filter(selector =>
!(selector.includes(':>>') || selector.includes('>>'))
);
}
}
return workflow;
}
private async runLoop(p: Page, workflow: Workflow) {
let workflowCopy: Workflow = JSON.parse(JSON.stringify(workflow));
workflowCopy = this.removeIframeSelectors(workflowCopy);
workflowCopy = this.removeSpecialSelectors(workflowCopy);
// apply ad-blocker to the current page
try {