vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: remove timeout

Browse Source
This commit is contained in:
karishmas6
2024-04-20 00:51:10 +05:30
parent 41ebb3b2aa
commit 497a248163
1 changed files with 1 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
scraper/src/scraper.js
View File

@@ -1,13 +1,10 @@
import { PlaywrightCrawler, Configuration } from 'crawlee'; import { PlaywrightCrawler, Configuration } from 'crawlee';
async function scrapeData(url, selectors, waitForSeconds = 2) { async function scrapeData(url, selectors) {
const scrapedData = []; const scrapedData = [];
const crawler = new PlaywrightCrawler({ const crawler = new PlaywrightCrawler({
requestHandler: async ({ page, request }) => { requestHandler: async ({ page, request }) => {
await page.goto(url); await page.goto(url);
await page.waitForTimeout(waitForSeconds * 1000);
console.log('Received selectors:', selectors); console.log('Received selectors:', selectors);
for (const selector of selectors) { for (const selector of selectors) {