vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(engine): stale scrape

Browse Source
This commit is contained in:
karishmas6
2024-04-18 21:38:41 +05:30
parent a5e09f9098
commit 1eeb218f1d
1 changed files with 5 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
scraper/src/main.js
View File

@@ -1,7 +1,7 @@
import Fastify from 'fastify' import Fastify from 'fastify'
import cors from '@fastify/cors' import cors from '@fastify/cors'
import fetch from 'node-fetch'; import fetch from 'node-fetch';
import scraper from './scraper.js'; import playwright from 'playwright'
const fastify = Fastify(); const fastify = Fastify();
@@ -12,6 +12,10 @@ const corsOptions = {
await fastify.register(cors, corsOptions) await fastify.register(cors, corsOptions)
fastify.get('/', async (request, reply) => {
reply.send('Welcome to the Playwright Scraper API');
});
fastify.get('/proxy', async (request, reply) => { fastify.get('/proxy', async (request, reply) => {
const { url } = request.query; const { url } = request.query;
@@ -24,20 +28,6 @@ fastify.get('/proxy', async (request, reply) => {
} }
}); });
fastify.post('/scrape', async (request, reply) => {
const { url, selectors } = request.body;
try {
const data = await scraper(url, selectors);
console.log('Scraped data:', data);
reply.send(data);
} catch (error) {
console.error('Error scraping:', error);
reply.status(500).send({ error: 'Failed to scrape data' });
}
});
await fastify.listen(3000, (err, address) => { await fastify.listen(3000, (err, address) => {
if (err) throw err; if (err) throw err;
console.log(`Server listening on ${fastify.server.address().port}`) console.log(`Server listening on ${fastify.server.address().port}`)