fix: remove button menu
This commit is contained in:
31
api/src/main.py
Normal file
31
api/src/main.py
Normal file
@@ -0,0 +1,31 @@
|
||||
from fastapi import FastAPI
|
||||
from scrapy import signals
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.signalmanager import dispatcher
|
||||
from sample_spider import SampleSpider
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
|
||||
dispatcher.connect(start_crawler, signal=signals.engine_started)
|
||||
process = CrawlerProcess(settings={
|
||||
'USffER_AGENT': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
||||
})
|
||||
process.crawl(SampleSpider)
|
||||
process.start()
|
||||
|
||||
def start_crawler():
|
||||
|
||||
spider = SampleSpider()
|
||||
crawler = CrawlerProcess()
|
||||
crawler.crawl(spider)
|
||||
crawler.start()
|
||||
|
||||
@app.get("/scrape")
|
||||
async def scrape_endpoint():
|
||||
# Add your API endpoint logic here
|
||||
# Retrieve and return the scraped data
|
||||
return {"message": "Scraping in progress. Check logs for details."}
|
||||
8
api/src/sample_spider.py
Normal file
8
api/src/sample_spider.py
Normal file
@@ -0,0 +1,8 @@
|
||||
import scrapy
|
||||
|
||||
class SampleSpider(scrapy.Spider):
|
||||
name = 'SampleSpider'
|
||||
start_urls = ['https://www.google.com/']
|
||||
|
||||
def parse(self, response):
|
||||
print(response.body)
|
||||
Reference in New Issue
Block a user