vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: use scrapeListAuto

Browse Source
This commit is contained in:
karishmas6
2024-08-12 06:28:08 +05:30
parent 196cfd4cca
commit 2bd4bc9ff1
1 changed files with 12 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
maxun-core/src/interpret.ts
View File

@@ -291,12 +291,22 @@ export default class Interpreter extends EventEmitter {
await this.options.serializableCallback(scrapeResult); await this.options.serializableCallback(scrapeResult);
}, },
scrapeList: async (config: { listSelector: string, fields: any, limit?: number, flexible?: boolean }) => { scrapeList: async (config: { listSelector: string, fields: any, limit?: number, flexible?: boolean, pagination:any }) => {
await this.ensureScriptsLoaded(page); await this.ensureScriptsLoaded(page);
const scrapeResults: Record<string, any>[] = await page.evaluate((cfg) => window.scrapeList(cfg), config); const scrapeResults: Record<string, any>[] = await page.evaluate((cfg) => window.scrapeList(cfg), config);
await this.options.serializableCallback(scrapeResults); await this.options.serializableCallback(scrapeResults);
}, },
scrapeListAuto: async (config: { listSelector: string }) => {
await this.ensureScriptsLoaded(page);
const scrapeResults: { selector: string, innerText: string }[] = await page.evaluate((listSelector) => {
return window.scrapeListAuto(listSelector);
}, config.listSelector);
await this.options.serializableCallback(scrapeResults);
},
scroll: async (pages?: number) => { scroll: async (pages?: number) => {
await page.evaluate(async (pagesInternal) => { await page.evaluate(async (pagesInternal) => {
for (let i = 1; i <= (pagesInternal ?? 1); i += 1) { for (let i = 1; i <= (pagesInternal ?? 1); i += 1) {
@@ -417,7 +427,7 @@ export default class Interpreter extends EventEmitter {
} }
private async ensureScriptsLoaded(page: Page) { private async ensureScriptsLoaded(page: Page) {
const isScriptLoaded = await page.evaluate(() => typeof window.scrape === 'function' && typeof window.scrapeSchema === 'function' && typeof window.scrapeList === 'function'); const isScriptLoaded = await page.evaluate(() => typeof window.scrape === 'function' && typeof window.scrapeSchema === 'function' && typeof window.scrapeList === 'function' && typeof window.scrapeListAuto === 'function');
if (!isScriptLoaded) { if (!isScriptLoaded) {
await page.addInitScript({ path: path.join(__dirname, 'browserSide', 'scraper.js') }); await page.addInitScript({ path: path.join(__dirname, 'browserSide', 'scraper.js') });
} }