feat: add execution timeouts api, scheduled, manual run

server/src/api/record.ts

@@ -15,8 +15,8 @@ import { AuthenticatedRequest } from "../routes/record"
 import {capture} from "../utils/analytics";
 import { Page } from "playwright";
 import { WorkflowFile } from "maxun-core";
-import { googleSheetUpdateTasks, processGoogleSheetUpdates } from "../workflow-management/integrations/gsheet";
-import { airtableUpdateTasks, processAirtableUpdates } from "../workflow-management/integrations/airtable";
+import { addGoogleSheetUpdateTask, googleSheetUpdateTasks, processGoogleSheetUpdates } from "../workflow-management/integrations/gsheet";
+import { addAirtableUpdateTask, airtableUpdateTasks, processAirtableUpdates } from "../workflow-management/integrations/airtable";
 import { sendWebhook } from "../routes/webhook";
 import { convertPageToHTML, convertPageToMarkdown } from '../markdownify/scrape';
 
@@ -557,32 +557,44 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) {
     }
 }
 
+function withTimeout<T>(promise: Promise<T>, timeoutMs: number, operation: string): Promise<T> {
+  return Promise.race([
+    promise,
+    new Promise<T>((_, reject) =>
+      setTimeout(() => reject(new Error(`${operation} timed out after ${timeoutMs}ms`)), timeoutMs)
+    )
+  ]);
+}
+
 async function triggerIntegrationUpdates(runId: string, robotMetaId: string): Promise<void> {
   try {
-    googleSheetUpdateTasks[runId] = {
+    addGoogleSheetUpdateTask(runId, {
       robotId: robotMetaId,
       runId: runId,
       status: 'pending',
       retries: 5,
-    };
+    });
 
-    airtableUpdateTasks[runId] = {
+    addAirtableUpdateTask(runId, {
       robotId: robotMetaId,
       runId: runId,
       status: 'pending',
       retries: 5,
-    };
+    });
 
-    processAirtableUpdates().catch(err => logger.log('error', `Airtable update error: ${err.message}`));
-    processGoogleSheetUpdates().catch(err => logger.log('error', `Google Sheets update error: ${err.message}`));
+    withTimeout(processAirtableUpdates(), 65000, 'Airtable update')
+      .catch(err => logger.log('error', `Airtable update error: ${err.message}`));
+
+    withTimeout(processGoogleSheetUpdates(), 65000, 'Google Sheets update')
+      .catch(err => logger.log('error', `Google Sheets update error: ${err.message}`));
   } catch (err: any) {
     logger.log('error', `Failed to update integrations for run: ${runId}: ${err.message}`);
   }
 }
 
-async function readyForRunHandler(browserId: string, id: string, userId: string, requestedFormats?: string[]){
+async function readyForRunHandler(browserId: string, id: string, userId: string, socket: Socket){
     try {
-        const result = await executeRun(id, userId, requestedFormats);
+        const result = await executeRun(id, userId);
 
         if (result && result.success) {
             logger.log('info', `Interpretation of ${id} succeeded`);
@@ -599,6 +611,8 @@ async function readyForRunHandler(browserId: string, id: string, userId: string,
         logger.error(`Error during readyForRunHandler: ${error.message}`);
         await destroyRemoteBrowser(browserId, userId);
         return null;
+    } finally {
+        cleanupSocketConnection(socket, browserId, id);
     }
 }
 
@@ -688,15 +702,23 @@ async function executeRun(id: string, userId: string, requestedFormats?: string[
                 let html = '';
                 const serializableOutput: any = {};
 
-                // Markdown conversion
+                const SCRAPE_TIMEOUT = 120000;
+
                 if (formats.includes('markdown')) {
-                    markdown = await convertPageToMarkdown(url);
+                    const markdownPromise = convertPageToMarkdown(url);
+                    const timeoutPromise = new Promise<never>((_, reject) => {
+                        setTimeout(() => reject(new Error(`Markdown conversion timed out after ${SCRAPE_TIMEOUT/1000}s`)), SCRAPE_TIMEOUT);
+                    });
+                    markdown = await Promise.race([markdownPromise, timeoutPromise]);
                     serializableOutput.markdown = [{ content: markdown }];
                 }
 
-                // HTML conversion
                 if (formats.includes('html')) {
-                    html = await convertPageToHTML(url);
+                    const htmlPromise = convertPageToHTML(url);
+                    const timeoutPromise = new Promise<never>((_, reject) => {
+                        setTimeout(() => reject(new Error(`HTML conversion timed out after ${SCRAPE_TIMEOUT/1000}s`)), SCRAPE_TIMEOUT);
+                    });
+                    html = await Promise.race([htmlPromise, timeoutPromise]);
                     serializableOutput.html = [{ content: html }];
                 }
 
@@ -808,6 +830,22 @@ async function executeRun(id: string, userId: string, requestedFormats?: string[
                     );
                 }
 
+                try {
+                    await sendWebhook(plainRun.robotMetaId, 'run_failed', {
+                        robot_id: plainRun.robotMetaId,
+                        run_id: plainRun.runId,
+                        robot_name: recording.recording_meta.name,
+                        status: 'failed',
+                        finished_at: new Date().toLocaleString(),
+                        error: {
+                            message: error.message,
+                            type: 'ConversionError'
+                        }
+                    });
+                } catch (webhookError: any) {
+                    logger.log('warn', `Failed to send webhook for failed API scrape run ${plainRun.runId}: ${webhookError.message}`);
+                }
+
                 capture("maxun-oss-run-created-api", {
                     runId: plainRun.runId,
                     user_id: userId,
@@ -838,13 +876,24 @@ async function executeRun(id: string, userId: string, requestedFormats?: string[
 
         browser.interpreter.setRunId(plainRun.runId);
         
-        const interpretationInfo = await browser.interpreter.InterpretRecording(
+        const INTERPRETATION_TIMEOUT = 600000;
+
+        const interpretationPromise = browser.interpreter.InterpretRecording(
             workflow, currentPage, (newPage: Page) => currentPage = newPage, plainRun.interpreterSettings
         );
 
+        const timeoutPromise = new Promise<never>((_, reject) => {
+            setTimeout(() => reject(new Error(`Workflow interpretation timed out after ${INTERPRETATION_TIMEOUT/1000}s`)), INTERPRETATION_TIMEOUT);
+        });
+
+        const interpretationInfo = await Promise.race([interpretationPromise, timeoutPromise]);
+
         const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
         const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput);
 
+        if (browser && browser.interpreter) {
+            await browser.interpreter.clearState();
+        }
         await destroyRemoteBrowser(plainRun.browserId, userId);
 
         const updatedRun = await run.update({
@@ -854,6 +903,25 @@ async function executeRun(id: string, userId: string, requestedFormats?: string[
             binaryOutput: uploadedBinaryOutput,
         });
 
+        try {
+            const completionData = {
+                runId: plainRun.runId,
+                robotMetaId: plainRun.robotMetaId,
+                robotName: recording.recording_meta.name,
+                status: 'success',
+                finishedAt: new Date().toLocaleString(),
+                runByUserId: plainRun.runByUserId,
+                runByScheduleId: plainRun.runByScheduleId,
+                runByAPI: plainRun.runByAPI || false,
+                browserId: plainRun.browserId
+            };
+
+            serverIo.of('/queued-run').to(`user-${userId}`).emit('run-completed', completionData);
+            logger.log('info', `API run completed notification sent for run: ${plainRun.runId} to user-${userId}`);
+        } catch (socketError: any) {
+            logger.log('warn', `Failed to send run-completed notification for API run ${plainRun.runId}: ${socketError.message}`);
+        }
+
         let totalSchemaItemsExtracted = 0;
         let totalListItemsExtracted = 0;
         let extractedScreenshotsCount = 0;
@@ -950,6 +1018,17 @@ async function executeRun(id: string, userId: string, requestedFormats?: string[
         logger.log('info', `Error while running a robot with id: ${id} - ${error.message}`);
         const run = await Run.findOne({ where: { runId: id } });
         if (run) {
+            if (browser) {
+                try {
+                    if (browser.interpreter) {
+                        await browser.interpreter.clearState();
+                    }
+                    await destroyRemoteBrowser(run.browserId, userId);
+                } catch (cleanupError: any) {
+                    logger.error(`Failed to cleanup browser in error handler: ${cleanupError.message}`);
+                }
+            }
+
             await run.update({
                 status: 'failed',
                 finishedAt: new Date().toLocaleString(),
@@ -1020,6 +1099,8 @@ async function executeRun(id: string, userId: string, requestedFormats?: string[
 }
 
 export async function handleRunRecording(id: string, userId: string, requestedFormats?: string[]) {
+    let socket: Socket | null = null;
+
     try {
         const result = await createWorkflowAndStoreMetadata(id, userId);
         const { browserId, runId: newRunId } = result;
@@ -1028,41 +1109,79 @@ export async function handleRunRecording(id: string, userId: string, requestedFo
             throw new Error('browserId or runId or userId is undefined');
         }
 
-        const socket = io(`${process.env.BACKEND_URL ? process.env.BACKEND_URL : 'http://localhost:8080'}/${browserId}`, {
+        const CONNECTION_TIMEOUT = 30000;
+
+        socket = io(`${process.env.BACKEND_URL ? process.env.BACKEND_URL : 'http://localhost:8080'}/${browserId}`, {
             transports: ['websocket'],
-            rejectUnauthorized: false
+            rejectUnauthorized: false,
+            timeout: CONNECTION_TIMEOUT,
         });
 
-        socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId, userId, requestedFormats));
+        const readyHandler = () => readyForRunHandler(browserId, newRunId, userId, socket!);
+
+        socket.on('ready-for-run', readyHandler);
+
+        socket.on('connect_error', (error: Error) => {
+            logger.error(`Socket connection error for API run ${newRunId}: ${error.message}`);
+            cleanupSocketConnection(socket!, browserId, newRunId);
+        });
+
+        socket.on('disconnect', () => {
+            cleanupSocketConnection(socket!, browserId, newRunId);
+        });
 
         logger.log('info', `Running Robot: ${id}`);
 
-        socket.on('disconnect', () => {
-            cleanupSocketListeners(socket, browserId, newRunId, userId);
-        });
-
-        // Return the runId immediately, so the client knows the run is started
         return newRunId;
 
     } catch (error: any) {
         logger.error('Error running robot:', error);
+        if (socket) {
+            cleanupSocketConnection(socket, '', '');
+        }
     }
 }
 
-function cleanupSocketListeners(socket: Socket, browserId: string, id: string, userId: string) {
-    socket.off('ready-for-run', () => readyForRunHandler(browserId, id, userId));
-    logger.log('info', `Cleaned up listeners for browserId: ${browserId}, runId: ${id}`);
+function cleanupSocketConnection(socket: Socket, browserId: string, id: string) {
+    try {
+        socket.removeAllListeners();
+        socket.disconnect();
+
+        if (browserId) {
+            const namespace = serverIo.of(browserId);
+            namespace.removeAllListeners();
+            namespace.disconnectSockets(true);
+            const nsps = (serverIo as any)._nsps;
+            if (nsps && nsps.has(`/${browserId}`)) {
+                nsps.delete(`/${browserId}`);
+                logger.log('debug', `Deleted namespace /${browserId} from io._nsps Map`);
+            }
+        }
+
+        logger.log('info', `Cleaned up socket connection for browserId: ${browserId}, runId: ${id}`);
+    } catch (error: any) {
+        logger.error(`Error cleaning up socket connection: ${error.message}`);
+    }
 }
 
 async function waitForRunCompletion(runId: string, interval: number = 2000) {
+    const MAX_WAIT_TIME = 180 * 60 * 1000;
+    const startTime = Date.now();
+
     while (true) {
-        const run = await Run.findOne({ where: { runId }, raw: true });
+        if (Date.now() - startTime > MAX_WAIT_TIME) {
+            throw new Error('Run completion timeout after 3 hours');
+        }
+
+        const run = await Run.findOne({ where: { runId } });
         if (!run) throw new Error('Run not found');
 
         if (run.status === 'success') {
-            return run;
+            return run.toJSON();
         } else if (run.status === 'failed') {
             throw new Error('Run failed');
+        } else if (run.status === 'aborted' || run.status === 'aborting') {
+            throw new Error('Run was aborted');
         }
 
         await new Promise(resolve => setTimeout(resolve, interval));