diff --git a/scraper/src/load.js b/scraper/src/load.js new file mode 100644 index 00000000..5b824ba3 --- /dev/null +++ b/scraper/src/load.js @@ -0,0 +1,19 @@ +import { PlaywrightCrawler, Configuration } from "crawlee"; + +async function loadWebsite(url) { + let htmlContent = ''; + const crawler = new PlaywrightCrawler({ + requestHandler: async ({ page }) => { + await page.goto(url); + htmlContent = await page.content(); + } + }, + new Configuration({ + persistStorage: false, + })); + + await crawler.run([url]); + return htmlContent; +} + +export default loadWebsite; \ No newline at end of file