vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(core): script injection for scrape | scrapeSchema

Browse Source
This commit is contained in:
karishmas6
2024-08-01 19:16:13 +05:30
parent 7d57e5adc0
commit 92bb30d8b7
1 changed files with 31 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
maxun-core/src/interpret.ts
View File

@@ -277,25 +277,39 @@ export default class Interpreter extends EventEmitter {
await page.close();
},
scrape: async (selector?: string) => {
const scrapeResults: Record<string, string>[] = <any>await page
// eslint-disable-next-line
// @ts-ignore
.evaluate((s) => scrape(s ?? null), selector);
// Check if 'scrape' function is available in the page context
const isScrapeAvailable = await page.evaluate(() => typeof window.scrape === 'function');
if (!isScrapeAvailable) {
// Inject the script that defines the 'scrape' function
await page.addScriptTag({ path: path.join(__dirname, 'browserSide', 'scraper.js') });
}
const scrapeResults: Record<string, string>[] = await page.evaluate((s) => window.scrape(s ?? null), selector);
await this.options.serializableCallback(scrapeResults);
},
scrapeSchema: async (schema: Record<string, string>) => {
},
scrapeSchema: async (schema: Record<string, string>) => {
// Check if 'scrapeSchema' function is available in the page context
const isScrapeSchemaAvailable = await page.evaluate(() => typeof window.scrapeSchema === 'function');
if (!isScrapeSchemaAvailable) {
// Inject the script that defines the 'scrapeSchema' function
await page.addScriptTag({ path: path.join(__dirname, 'browserSide', 'scraper.js') });
}
const handleLists = await Promise.all(
Object.values(schema).map((selector) => page.$$(selector)),
Object.values(schema).map((selector) => page.$$(selector)),
);
const namedHandleLists = Object.fromEntries(
Object.keys(schema).map((key, i) => [key, handleLists[i]]),
Object.keys(schema).map((key, i) => [key, handleLists[i]]),
);
const scrapeResult = await page.evaluate((n) => scrapeSchema(n), namedHandleLists);
this.options.serializableCallback(scrapeResult);
},
const scrapeResult = await page.evaluate((n) => window.scrapeSchema(n), namedHandleLists);
await this.options.serializableCallback(scrapeResult);
},
scroll: async (pages?: number) => {
await page.evaluate(async (pagesInternal) => {
for (let i = 1; i <= (pagesInternal ?? 1); i += 1) {
@@ -433,9 +447,9 @@ export default class Interpreter extends EventEmitter {
this.initializedWorkflow = Preprocessor.initWorkflow(this.workflow, params);
// @ts-ignore
if (await page.evaluate(() => !<any>window.scrape)) {
page.context().addInitScript({ path: path.join(__dirname, 'browserSide', 'scraper.js') });
}
// if (await page.evaluate(() => !<any>window.scrape)) {
// page.context().addInitScript({ path: path.join(__dirname, 'browserSide', 'scraper.js') });
// }
this.stopper = () => {
this.stopper = null;