From dfdcafa5b835f86e1c6180e89bfa51daf50ba1a9 Mon Sep 17 00:00:00 2001 From: Rohit Rajan Date: Fri, 5 Dec 2025 21:58:23 +0530 Subject: [PATCH] feat: add scrape screenshot support --- server/src/api/record.ts | 49 +++++++++- server/src/markdownify/scrape.ts | 25 ++++++ server/src/models/Robot.ts | 2 +- server/src/pgboss-worker.ts | 46 +++++++++- server/src/routes/storage.ts | 2 +- .../workflow-management/scheduler/index.ts | 45 +++++++++- src/components/robot/pages/RobotCreate.tsx | 90 +++++++++++-------- .../robot/pages/RobotDuplicatePage.tsx | 2 +- src/components/robot/pages/RobotEditPage.tsx | 2 +- .../robot/pages/RobotSettingsPage.tsx | 2 +- src/components/run/RunContent.tsx | 70 ++++++++++++++- src/context/globalInfo.tsx | 2 +- 12 files changed, 281 insertions(+), 56 deletions(-) diff --git a/server/src/api/record.ts b/server/src/api/record.ts index b2c2422a..25c6f95a 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -16,7 +16,7 @@ import { WorkflowFile } from "maxun-core"; import { addGoogleSheetUpdateTask, googleSheetUpdateTasks, processGoogleSheetUpdates } from "../workflow-management/integrations/gsheet"; import { addAirtableUpdateTask, airtableUpdateTasks, processAirtableUpdates } from "../workflow-management/integrations/airtable"; import { sendWebhook } from "../routes/webhook"; -import { convertPageToHTML, convertPageToMarkdown } from '../markdownify/scrape'; +import { convertPageToHTML, convertPageToMarkdown, convertPageToScreenshot } from '../markdownify/scrape'; const router = Router(); @@ -689,7 +689,9 @@ async function executeRun(id: string, userId: string, requestedFormats?: string[ // Override if API request defines formats if (requestedFormats && Array.isArray(requestedFormats) && requestedFormats.length > 0) { - formats = requestedFormats.filter((f): f is 'markdown' | 'html' => ['markdown', 'html'].includes(f)); + formats = requestedFormats.filter((f): f is 'markdown' | 'html' | 'screenshot-visible' | 'screenshot-fullpage' => + ['markdown', 'html', 'screenshot-visible', 'screenshot-fullpage'].includes(f) + ); } await run.update({ @@ -707,6 +709,7 @@ async function executeRun(id: string, userId: string, requestedFormats?: string[ let markdown = ''; let html = ''; const serializableOutput: any = {}; + const binaryOutput: any = {}; const SCRAPE_TIMEOUT = 120000; @@ -728,14 +731,52 @@ async function executeRun(id: string, userId: string, requestedFormats?: string[ serializableOutput.html = [{ content: html }]; } + if (formats.includes("screenshot-visible")) { + const screenshotPromise = convertPageToScreenshot(url, currentPage, false); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error(`Screenshot conversion timed out after ${SCRAPE_TIMEOUT/1000}s`)), SCRAPE_TIMEOUT); + }); + const screenshotBuffer = await Promise.race([screenshotPromise, timeoutPromise]); + + if (!binaryOutput['screenshot-visible']) { + binaryOutput['screenshot-visible'] = { + data: screenshotBuffer.toString('base64'), + mimeType: 'image/png' + }; + } + } + + if (formats.includes("screenshot-fullpage")) { + const screenshotPromise = convertPageToScreenshot(url, currentPage, true); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error(`Screenshot conversion timed out after ${SCRAPE_TIMEOUT/1000}s`)), SCRAPE_TIMEOUT); + }); + const screenshotBuffer = await Promise.race([screenshotPromise, timeoutPromise]); + + if (!binaryOutput['screenshot-fullpage']) { + binaryOutput['screenshot-fullpage'] = { + data: screenshotBuffer.toString('base64'), + mimeType: 'image/png' + }; + } + } + await run.update({ status: 'success', finishedAt: new Date().toLocaleString(), log: `${formats.join(', ')} conversion completed successfully`, serializableOutput, - binaryOutput: {}, + binaryOutput, }); + // Upload binary output (screenshots) to MinIO if present + let uploadedBinaryOutput: Record = {}; + if (Object.keys(binaryOutput).length > 0) { + const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); + uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, binaryOutput); + await run.update({ binaryOutput: uploadedBinaryOutput }); + } + logger.log('info', `Markdown robot execution completed for API run ${id}`); // Push success socket event @@ -775,6 +816,8 @@ async function executeRun(id: string, userId: string, requestedFormats?: string[ if (formats.includes('markdown')) webhookPayload.markdown = markdown; if (formats.includes('html')) webhookPayload.html = html; + if (uploadedBinaryOutput['screenshot-visible']) webhookPayload.screenshot_visible = uploadedBinaryOutput['screenshot-visible']; + if (uploadedBinaryOutput['screenshot-fullpage']) webhookPayload.screenshot_fullpage = uploadedBinaryOutput['screenshot-fullpage']; try { await sendWebhook(plainRun.robotMetaId, 'run_completed', webhookPayload); diff --git a/server/src/markdownify/scrape.ts b/server/src/markdownify/scrape.ts index 09df4276..70c06084 100644 --- a/server/src/markdownify/scrape.ts +++ b/server/src/markdownify/scrape.ts @@ -123,3 +123,28 @@ export async function convertPageToHTML(url: string, page: Page): Promise { + try { + const screenshotType = fullPage ? 'full page' : 'visible viewport'; + logger.log('info', `[Scrape] Taking ${screenshotType} screenshot of ${url}`); + + await gotoWithFallback(page, url); + + const screenshot = await page.screenshot({ + type: 'png', + fullPage + }); + + return screenshot; + } catch (error: any) { + logger.error(`[Scrape] Error during screenshot: ${error.message}`); + throw error; + } +} diff --git a/server/src/models/Robot.ts b/server/src/models/Robot.ts index 39218de2..1ce269b0 100644 --- a/server/src/models/Robot.ts +++ b/server/src/models/Robot.ts @@ -11,7 +11,7 @@ interface RobotMeta { params: any[]; type?: 'extract' | 'scrape'; url?: string; - formats?: ('markdown' | 'html')[]; + formats?: ('markdown' | 'html' | 'screenshot-visible' | 'screenshot-fullpage')[]; } interface RobotWorkflow { diff --git a/server/src/pgboss-worker.ts b/server/src/pgboss-worker.ts index e7d4e115..1f7125f4 100644 --- a/server/src/pgboss-worker.ts +++ b/server/src/pgboss-worker.ts @@ -20,7 +20,7 @@ import { addAirtableUpdateTask, airtableUpdateTasks, processAirtableUpdates } fr import { io as serverIo } from "./server"; import { sendWebhook } from './routes/webhook'; import { BinaryOutputService } from './storage/mino'; -import { convertPageToMarkdown, convertPageToHTML } from './markdownify/scrape'; +import { convertPageToMarkdown, convertPageToHTML, convertPageToScreenshot } from './markdownify/scrape'; if (!process.env.DB_USER || !process.env.DB_PASSWORD || !process.env.DB_HOST || !process.env.DB_PORT || !process.env.DB_NAME) { throw new Error('Failed to start pgboss worker: one or more required environment variables are missing.'); @@ -244,6 +244,7 @@ async function processRunExecution(job: Job) { let markdown = ''; let html = ''; const serializableOutput: any = {}; + const binaryOutput: any = {}; const SCRAPE_TIMEOUT = 120000; @@ -265,15 +266,52 @@ async function processRunExecution(job: Job) { serializableOutput.html = [{ content: html }]; } + if (formats.includes("screenshot-visible")) { + const screenshotPromise = convertPageToScreenshot(url, currentPage, false); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error(`Screenshot conversion timed out after ${SCRAPE_TIMEOUT/1000}s`)), SCRAPE_TIMEOUT); + }); + const screenshotBuffer = await Promise.race([screenshotPromise, timeoutPromise]); + + if (!binaryOutput['screenshot-visible']) { + binaryOutput['screenshot-visible'] = { + data: screenshotBuffer.toString('base64'), + mimeType: 'image/png' + }; + } + } + + if (formats.includes("screenshot-fullpage")) { + const screenshotPromise = convertPageToScreenshot(url, currentPage, true); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error(`Screenshot conversion timed out after ${SCRAPE_TIMEOUT/1000}s`)), SCRAPE_TIMEOUT); + }); + const screenshotBuffer = await Promise.race([screenshotPromise, timeoutPromise]); + + if (!binaryOutput['screenshot-fullpage']) { + binaryOutput['screenshot-fullpage'] = { + data: screenshotBuffer.toString('base64'), + mimeType: 'image/png' + }; + } + } + // Success update await run.update({ status: 'success', finishedAt: new Date().toLocaleString(), log: `${formats.join(', ').toUpperCase()} conversion completed successfully`, serializableOutput, - binaryOutput: {}, + binaryOutput, }); + let uploadedBinaryOutput: Record = {}; + if (Object.keys(binaryOutput).length > 0) { + const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); + uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, binaryOutput); + await run.update({ binaryOutput: uploadedBinaryOutput }); + } + logger.log('info', `Markdown robot execution completed for run ${data.runId}`); // Notify sockets @@ -304,6 +342,8 @@ async function processRunExecution(job: Job) { if (formats.includes('markdown')) webhookPayload.markdown = markdown; if (formats.includes('html')) webhookPayload.html = html; + if (uploadedBinaryOutput['screenshot-visible']) webhookPayload.screenshot_visible = uploadedBinaryOutput['screenshot-visible']; + if (uploadedBinaryOutput['screenshot-fullpage']) webhookPayload.screenshot_fullpage = uploadedBinaryOutput['screenshot-fullpage']; await sendWebhook(plainRun.robotMetaId, 'run_completed', webhookPayload); logger.log('info', `Webhooks sent successfully for markdown robot run ${data.runId}`); @@ -427,7 +467,7 @@ async function processRunExecution(job: Job) { logger.log('info', `Workflow execution completed for run ${data.runId}`); - const binaryOutputService = new BinaryOutputService('maxuncloud-run-screenshots'); + const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput( run, interpretationInfo.binaryOutput diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 45d4bc53..3941b01f 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -456,7 +456,7 @@ router.post('/recordings/scrape', requireSignIn, async (req: AuthenticatedReques } // Validate format - const validFormats = ['markdown', 'html']; + const validFormats = ['markdown', 'html', 'screenshot-visible', 'screenshot-fullpage']; if (!Array.isArray(formats) || formats.length === 0) { return res.status(400).json({ error: 'At least one output format must be selected.' }); diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 49237522..652e72d8 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -13,7 +13,7 @@ import { WorkflowFile } from "maxun-core"; import { Page } from "playwright-core"; import { sendWebhook } from "../../routes/webhook"; import { addAirtableUpdateTask, airtableUpdateTasks, processAirtableUpdates } from "../integrations/airtable"; -import { convertPageToMarkdown, convertPageToHTML } from "../../markdownify/scrape"; +import { convertPageToMarkdown, convertPageToHTML, convertPageToScreenshot } from "../../markdownify/scrape"; async function createWorkflowAndStoreMetadata(id: string, userId: string) { try { @@ -268,6 +268,7 @@ async function executeRun(id: string, userId: string) { let markdown = ''; let html = ''; const serializableOutput: any = {}; + const binaryOutput: any = {}; const SCRAPE_TIMEOUT = 120000; @@ -290,13 +291,51 @@ async function executeRun(id: string, userId: string) { serializableOutput.html = [{ content: html }]; } + if (formats.includes("screenshot-visible")) { + const screenshotPromise = convertPageToScreenshot(url, currentPage, false); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error(`Screenshot conversion timed out after ${SCRAPE_TIMEOUT/1000}s`)), SCRAPE_TIMEOUT); + }); + const screenshotBuffer = await Promise.race([screenshotPromise, timeoutPromise]); + + if (!binaryOutput['screenshot-visible']) { + binaryOutput['screenshot-visible'] = { + data: screenshotBuffer.toString('base64'), + mimeType: 'image/png' + }; + } + } + + // Screenshot - full page + if (formats.includes("screenshot-fullpage")) { + const screenshotPromise = convertPageToScreenshot(url, currentPage, true); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error(`Screenshot conversion timed out after ${SCRAPE_TIMEOUT/1000}s`)), SCRAPE_TIMEOUT); + }); + const screenshotBuffer = await Promise.race([screenshotPromise, timeoutPromise]); + + if (!binaryOutput['screenshot-fullpage']) { + binaryOutput['screenshot-fullpage'] = { + data: screenshotBuffer.toString('base64'), + mimeType: 'image/png' + }; + } + } + await run.update({ status: 'success', finishedAt: new Date().toLocaleString(), log: `${formats.join(', ')} conversion completed successfully`, serializableOutput, - binaryOutput: {}, + binaryOutput, }); + + let uploadedBinaryOutput: Record = {}; + if (Object.keys(binaryOutput).length > 0) { + const binaryOutputService = new BinaryOutputService('maxun-run-screenshots'); + uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, binaryOutput); + await run.update({ binaryOutput: uploadedBinaryOutput }); + } logger.log('info', `Markdown robot execution completed for scheduled run ${id}`); @@ -335,6 +374,8 @@ async function executeRun(id: string, userId: string) { if (formats.includes('markdown')) webhookPayload.markdown = markdown; if (formats.includes('html')) webhookPayload.html = html; + if (uploadedBinaryOutput['screenshot-visible']) webhookPayload.screenshot_visible = uploadedBinaryOutput['screenshot-visible']; + if (uploadedBinaryOutput['screenshot-fullpage']) webhookPayload.screenshot_fullpage = uploadedBinaryOutput['screenshot-fullpage']; try { await sendWebhook(plainRun.robotMetaId, 'run_completed', webhookPayload); diff --git a/src/components/robot/pages/RobotCreate.tsx b/src/components/robot/pages/RobotCreate.tsx index 486f3bff..8552435e 100644 --- a/src/components/robot/pages/RobotCreate.tsx +++ b/src/components/robot/pages/RobotCreate.tsx @@ -16,10 +16,10 @@ import { CardContent, Tabs, Tab, - RadioGroup, - Radio, FormControl, - FormLabel + Select, + MenuItem, + InputLabel } from '@mui/material'; import { ArrowBack, PlayCircleOutline, Article, Code, Description } from '@mui/icons-material'; import { useGlobalInfoStore } from '../../../context/globalInfo'; @@ -376,7 +376,7 @@ const RobotCreate: React.FC = () => { /> - Turn websites into LLM-ready Markdown & clean HTML for AI apps. + Turn websites into LLM-ready Markdown, clean HTML, or screenshots for AI apps. @@ -399,40 +399,52 @@ const RobotCreate: React.FC = () => { sx={{ mb: 2 }} /> - -

Output Format (Select at least one)

- { - if (e.target.checked) { - setOutputFormats([...outputFormats, 'markdown']); - } else { - setOutputFormats(outputFormats.filter(f => f !== 'markdown')); - } - }} - /> - } - label="Markdown" - /> - - { - if (e.target.checked) { - setOutputFormats([...outputFormats, 'html']); - } else { - setOutputFormats(outputFormats.filter(f => f !== 'html')); - } - }} - /> - } - label="HTML" - /> - + + + Output Formats * + + +