vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add scrape screenshot support

Browse Source
This commit is contained in:
Rohit Rajan
2025-12-05 21:58:23 +05:30
parent 6986ba0700
commit dfdcafa5b8
12 changed files with 281 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
server/src/api/record.ts
View File

@@ -16,7 +16,7 @@ import { WorkflowFile } from "maxun-core";
import { addGoogleSheetUpdateTask, googleSheetUpdateTasks, processGoogleSheetUpdates } from "../workflow-management/integrations/gsheet";
import { addAirtableUpdateTask, airtableUpdateTasks, processAirtableUpdates } from "../workflow-management/integrations/airtable";
import { sendWebhook } from "../routes/webhook";
import { convertPageToHTML, convertPageToMarkdown } from '../markdownify/scrape';
import { convertPageToHTML, convertPageToMarkdown, convertPageToScreenshot } from '../markdownify/scrape';
const router = Router();
@@ -689,7 +689,9 @@ async function executeRun(id: string, userId: string, requestedFormats?: string[
// Override if API request defines formats
if (requestedFormats && Array.isArray(requestedFormats) && requestedFormats.length > 0) {
formats = requestedFormats.filter((f): f is 'markdown' | 'html' => ['markdown', 'html'].includes(f));
formats = requestedFormats.filter((f): f is 'markdown' | 'html' | 'screenshot-visible' | 'screenshot-fullpage' =>
['markdown', 'html', 'screenshot-visible', 'screenshot-fullpage'].includes(f)
);
}
await run.update({
@@ -707,6 +709,7 @@ async function executeRun(id: string, userId: string, requestedFormats?: string[
let markdown = '';
let html = '';
const serializableOutput: any = {};
const binaryOutput: any = {};
const SCRAPE_TIMEOUT = 120000;
@@ -728,14 +731,52 @@ async function executeRun(id: string, userId: string, requestedFormats?: string[
serializableOutput.html = [{ content: html }];
}
if (formats.includes("screenshot-visible")) {
const screenshotPromise = convertPageToScreenshot(url, currentPage, false);
const timeoutPromise = new Promise<never>((_, reject) => {
setTimeout(() => reject(new Error(`Screenshot conversion timed out after ${SCRAPE_TIMEOUT/1000}s`)), SCRAPE_TIMEOUT);
});
const screenshotBuffer = await Promise.race([screenshotPromise, timeoutPromise]);
if (!binaryOutput['screenshot-visible']) {
binaryOutput['screenshot-visible'] = {
data: screenshotBuffer.toString('base64'),
mimeType: 'image/png'
};
}
}
if (formats.includes("screenshot-fullpage")) {
const screenshotPromise = convertPageToScreenshot(url, currentPage, true);
const timeoutPromise = new Promise<never>((_, reject) => {
setTimeout(() => reject(new Error(`Screenshot conversion timed out after ${SCRAPE_TIMEOUT/1000}s`)), SCRAPE_TIMEOUT);
});
const screenshotBuffer = await Promise.race([screenshotPromise, timeoutPromise]);
if (!binaryOutput['screenshot-fullpage']) {
binaryOutput['screenshot-fullpage'] = {
data: screenshotBuffer.toString('base64'),
mimeType: 'image/png'
};
}
}
await run.update({
status: 'success',
finishedAt: new Date().toLocaleString(),
log: `${formats.join(', ')} conversion completed successfully`,
serializableOutput,
binaryOutput: {},
binaryOutput,
});
// Upload binary output (screenshots) to MinIO if present
let uploadedBinaryOutput: Record<string, string> = {};
if (Object.keys(binaryOutput).length > 0) {
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, binaryOutput);
await run.update({ binaryOutput: uploadedBinaryOutput });
}
logger.log('info', `Markdown robot execution completed for API run ${id}`);
// Push success socket event
@@ -775,6 +816,8 @@ async function executeRun(id: string, userId: string, requestedFormats?: string[
if (formats.includes('markdown')) webhookPayload.markdown = markdown;
if (formats.includes('html')) webhookPayload.html = html;
if (uploadedBinaryOutput['screenshot-visible']) webhookPayload.screenshot_visible = uploadedBinaryOutput['screenshot-visible'];
if (uploadedBinaryOutput['screenshot-fullpage']) webhookPayload.screenshot_fullpage = uploadedBinaryOutput['screenshot-fullpage'];
try {
await sendWebhook(plainRun.robotMetaId, 'run_completed', webhookPayload);

25
server/src/markdownify/scrape.ts
View File

@@ -123,3 +123,28 @@ export async function convertPageToHTML(url: string, page: Page): Promise<string
throw error;
}
}
/**
* Takes a screenshot of the page
* @param url - The URL to screenshot
* @param page - Existing Playwright page instance to use
* @param fullPage - Whether to capture the full scrollable page (true) or just visible viewport (false)
*/
export async function convertPageToScreenshot(url: string, page: Page, fullPage: boolean = false): Promise<Buffer> {
try {
const screenshotType = fullPage ? 'full page' : 'visible viewport';
logger.log('info', `[Scrape] Taking ${screenshotType} screenshot of ${url}`);
await gotoWithFallback(page, url);
const screenshot = await page.screenshot({
type: 'png',
fullPage
});
return screenshot;
} catch (error: any) {
logger.error(`[Scrape] Error during screenshot: ${error.message}`);
throw error;
}
}

2
server/src/models/Robot.ts
View File

@@ -11,7 +11,7 @@ interface RobotMeta {
params: any[];
type?: 'extract' | 'scrape';
url?: string;
formats?: ('markdown' | 'html')[];
formats?: ('markdown' | 'html' | 'screenshot-visible' | 'screenshot-fullpage')[];
}
interface RobotWorkflow {

46
server/src/pgboss-worker.ts
View File

@@ -20,7 +20,7 @@ import { addAirtableUpdateTask, airtableUpdateTasks, processAirtableUpdates } fr
import { io as serverIo } from "./server";
import { sendWebhook } from './routes/webhook';
import { BinaryOutputService } from './storage/mino';
import { convertPageToMarkdown, convertPageToHTML } from './markdownify/scrape';
import { convertPageToMarkdown, convertPageToHTML, convertPageToScreenshot } from './markdownify/scrape';
if (!process.env.DB_USER || !process.env.DB_PASSWORD || !process.env.DB_HOST || !process.env.DB_PORT || !process.env.DB_NAME) {
throw new Error('Failed to start pgboss worker: one or more required environment variables are missing.');
@@ -244,6 +244,7 @@ async function processRunExecution(job: Job<ExecuteRunData>) {
let markdown = '';
let html = '';
const serializableOutput: any = {};
const binaryOutput: any = {};
const SCRAPE_TIMEOUT = 120000;
@@ -265,15 +266,52 @@ async function processRunExecution(job: Job<ExecuteRunData>) {
serializableOutput.html = [{ content: html }];
}
if (formats.includes("screenshot-visible")) {
const screenshotPromise = convertPageToScreenshot(url, currentPage, false);
const timeoutPromise = new Promise<never>((_, reject) => {
setTimeout(() => reject(new Error(`Screenshot conversion timed out after ${SCRAPE_TIMEOUT/1000}s`)), SCRAPE_TIMEOUT);
});
const screenshotBuffer = await Promise.race([screenshotPromise, timeoutPromise]);
if (!binaryOutput['screenshot-visible']) {
binaryOutput['screenshot-visible'] = {
data: screenshotBuffer.toString('base64'),
mimeType: 'image/png'
};
}
}
if (formats.includes("screenshot-fullpage")) {
const screenshotPromise = convertPageToScreenshot(url, currentPage, true);
const timeoutPromise = new Promise<never>((_, reject) => {
setTimeout(() => reject(new Error(`Screenshot conversion timed out after ${SCRAPE_TIMEOUT/1000}s`)), SCRAPE_TIMEOUT);
});
const screenshotBuffer = await Promise.race([screenshotPromise, timeoutPromise]);
if (!binaryOutput['screenshot-fullpage']) {
binaryOutput['screenshot-fullpage'] = {
data: screenshotBuffer.toString('base64'),
mimeType: 'image/png'
};
}
}
// Success update
await run.update({
status: 'success',
finishedAt: new Date().toLocaleString(),
log: `${formats.join(', ').toUpperCase()} conversion completed successfully`,
serializableOutput,
binaryOutput: {},
binaryOutput,
});
let uploadedBinaryOutput: Record<string, string> = {};
if (Object.keys(binaryOutput).length > 0) {
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, binaryOutput);
await run.update({ binaryOutput: uploadedBinaryOutput });
}
logger.log('info', `Markdown robot execution completed for run ${data.runId}`);
// Notify sockets
@@ -304,6 +342,8 @@ async function processRunExecution(job: Job<ExecuteRunData>) {
if (formats.includes('markdown')) webhookPayload.markdown = markdown;
if (formats.includes('html')) webhookPayload.html = html;
if (uploadedBinaryOutput['screenshot-visible']) webhookPayload.screenshot_visible = uploadedBinaryOutput['screenshot-visible'];
if (uploadedBinaryOutput['screenshot-fullpage']) webhookPayload.screenshot_fullpage = uploadedBinaryOutput['screenshot-fullpage'];
await sendWebhook(plainRun.robotMetaId, 'run_completed', webhookPayload);
logger.log('info', `Webhooks sent successfully for markdown robot run ${data.runId}`);
@@ -427,7 +467,7 @@ async function processRunExecution(job: Job<ExecuteRunData>) {
logger.log('info', `Workflow execution completed for run ${data.runId}`);
const binaryOutputService = new BinaryOutputService('maxuncloud-run-screenshots');
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(
run,
interpretationInfo.binaryOutput

2
server/src/routes/storage.ts
View File

@@ -456,7 +456,7 @@ router.post('/recordings/scrape', requireSignIn, async (req: AuthenticatedReques
}
// Validate format
const validFormats = ['markdown', 'html'];
const validFormats = ['markdown', 'html', 'screenshot-visible', 'screenshot-fullpage'];
if (!Array.isArray(formats) || formats.length === 0) {
return res.status(400).json({ error: 'At least one output format must be selected.' });

45
server/src/workflow-management/scheduler/index.ts
View File

@@ -13,7 +13,7 @@ import { WorkflowFile } from "maxun-core";
import { Page } from "playwright-core";
import { sendWebhook } from "../../routes/webhook";
import { addAirtableUpdateTask, airtableUpdateTasks, processAirtableUpdates } from "../integrations/airtable";
import { convertPageToMarkdown, convertPageToHTML } from "../../markdownify/scrape";
import { convertPageToMarkdown, convertPageToHTML, convertPageToScreenshot } from "../../markdownify/scrape";
async function createWorkflowAndStoreMetadata(id: string, userId: string) {
try {
@@ -268,6 +268,7 @@ async function executeRun(id: string, userId: string) {
let markdown = '';
let html = '';
const serializableOutput: any = {};
const binaryOutput: any = {};
const SCRAPE_TIMEOUT = 120000;
@@ -290,13 +291,51 @@ async function executeRun(id: string, userId: string) {
serializableOutput.html = [{ content: html }];
}
if (formats.includes("screenshot-visible")) {
const screenshotPromise = convertPageToScreenshot(url, currentPage, false);
const timeoutPromise = new Promise<never>((_, reject) => {
setTimeout(() => reject(new Error(`Screenshot conversion timed out after ${SCRAPE_TIMEOUT/1000}s`)), SCRAPE_TIMEOUT);
});
const screenshotBuffer = await Promise.race([screenshotPromise, timeoutPromise]);
if (!binaryOutput['screenshot-visible']) {
binaryOutput['screenshot-visible'] = {
data: screenshotBuffer.toString('base64'),
mimeType: 'image/png'
};
}
}
// Screenshot - full page
if (formats.includes("screenshot-fullpage")) {
const screenshotPromise = convertPageToScreenshot(url, currentPage, true);
const timeoutPromise = new Promise<never>((_, reject) => {
setTimeout(() => reject(new Error(`Screenshot conversion timed out after ${SCRAPE_TIMEOUT/1000}s`)), SCRAPE_TIMEOUT);
});
const screenshotBuffer = await Promise.race([screenshotPromise, timeoutPromise]);
if (!binaryOutput['screenshot-fullpage']) {
binaryOutput['screenshot-fullpage'] = {
data: screenshotBuffer.toString('base64'),
mimeType: 'image/png'
};
}
}
await run.update({
status: 'success',
finishedAt: new Date().toLocaleString(),
log: `${formats.join(', ')} conversion completed successfully`,
serializableOutput,
binaryOutput: {},
binaryOutput,
});
let uploadedBinaryOutput: Record<string, string> = {};
if (Object.keys(binaryOutput).length > 0) {
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, binaryOutput);
await run.update({ binaryOutput: uploadedBinaryOutput });
}
logger.log('info', `Markdown robot execution completed for scheduled run ${id}`);
@@ -335,6 +374,8 @@ async function executeRun(id: string, userId: string) {
if (formats.includes('markdown')) webhookPayload.markdown = markdown;
if (formats.includes('html')) webhookPayload.html = html;
if (uploadedBinaryOutput['screenshot-visible']) webhookPayload.screenshot_visible = uploadedBinaryOutput['screenshot-visible'];
if (uploadedBinaryOutput['screenshot-fullpage']) webhookPayload.screenshot_fullpage = uploadedBinaryOutput['screenshot-fullpage'];
try {
await sendWebhook(plainRun.robotMetaId, 'run_completed', webhookPayload);