From 35c3278933736bfd90ccc4a18c88c6140961cbc3 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 20 May 2025 17:37:44 +0530 Subject: [PATCH 001/350] feat: add action id param for steps --- src/context/browserSteps.tsx | 48 ++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/src/context/browserSteps.tsx b/src/context/browserSteps.tsx index db0eb239..6d8b8f07 100644 --- a/src/context/browserSteps.tsx +++ b/src/context/browserSteps.tsx @@ -6,12 +6,14 @@ export interface TextStep { label: string; data: string; selectorObj: SelectorObject; + actionId?: string; // Add actionId to track which action created this step } interface ScreenshotStep { id: number; type: 'screenshot'; fullPage: boolean; + actionId?: string; // Add actionId to track which action created this step } export interface ListStep { @@ -24,6 +26,7 @@ export interface ListStep { selector: string; }; limit?: number; + actionId?: string; // Add actionId to track which action created this step } export type BrowserStep = TextStep | ScreenshotStep | ListStep; @@ -38,15 +41,16 @@ export interface SelectorObject { interface BrowserStepsContextType { browserSteps: BrowserStep[]; - addTextStep: (label: string, data: string, selectorObj: SelectorObject) => void; - addListStep: (listSelector: string, fields: { [key: string]: TextStep }, listId: number, pagination?: { type: string; selector: string }, limit?: number) => void - addScreenshotStep: (fullPage: boolean) => void; + addTextStep: (label: string, data: string, selectorObj: SelectorObject, actionId: string) => void; + addListStep: (listSelector: string, fields: { [key: string]: TextStep }, listId: number, actionId: string, pagination?: { type: string; selector: string }, limit?: number) => void + addScreenshotStep: (fullPage: boolean, actionId: string) => void; deleteBrowserStep: (id: number) => void; updateBrowserTextStepLabel: (id: number, newLabel: string) => void; updateListTextFieldLabel: (listId: number, fieldKey: string, newLabel: string) => void; updateListStepLimit: (listId: number, limit: number) => void; updateListStepData: (listId: number, extractedData: any[]) => void; removeListTextField: (listId: number, fieldKey: string) => void; + deleteStepsByActionId: (actionId: string) => void; // New function to delete steps by actionId } const BrowserStepsContext = createContext(undefined); @@ -55,14 +59,14 @@ export const BrowserStepsProvider: React.FC<{ children: React.ReactNode }> = ({ const [browserSteps, setBrowserSteps] = useState([]); const [discardedFields, setDiscardedFields] = useState>(new Set()); - const addTextStep = (label: string, data: string, selectorObj: SelectorObject) => { + const addTextStep = (label: string, data: string, selectorObj: SelectorObject, actionId: string) => { setBrowserSteps(prevSteps => [ ...prevSteps, - { id: Date.now(), type: 'text', label, data, selectorObj } + { id: Date.now(), type: 'text', label, data, selectorObj, actionId } ]); }; - const addListStep = (listSelector: string, newFields: { [key: string]: TextStep }, listId: number, pagination?: { type: string; selector: string }, limit?: number) => { + const addListStep = (listSelector: string, newFields: { [key: string]: TextStep }, listId: number, actionId: string, pagination?: { type: string; selector: string }, limit?: number) => { setBrowserSteps(prevSteps => { const existingListStepIndex = prevSteps.findIndex(step => step.type === 'list' && step.id === listId); @@ -77,10 +81,14 @@ export const BrowserStepsProvider: React.FC<{ children: React.ReactNode }> = ({ if (existingListStep.fields[key]) { acc[key] = { ...field, - label: existingListStep.fields[key].label + label: existingListStep.fields[key].label, + actionId }; } else { - acc[key] = field; + acc[key] = { + ...field, + actionId + }; } } return acc; @@ -90,22 +98,31 @@ export const BrowserStepsProvider: React.FC<{ children: React.ReactNode }> = ({ ...existingListStep, fields: mergedFields, pagination: pagination || existingListStep.pagination, - limit: limit + limit: limit, + actionId }; return updatedSteps; } else { + const fieldsWithActionId = Object.entries(newFields).reduce((acc, [key, field]) => { + acc[key] = { + ...field, + actionId + }; + return acc; + }, {} as { [key: string]: TextStep }); + return [ ...prevSteps, - { id: listId, type: 'list', listSelector, fields: newFields, pagination, limit } + { id: listId, type: 'list', listSelector, fields: fieldsWithActionId, pagination, limit, actionId } ]; } }); }; - const addScreenshotStep = (fullPage: boolean) => { + const addScreenshotStep = (fullPage: boolean, actionId: string) => { setBrowserSteps(prevSteps => [ ...prevSteps, - { id: Date.now(), type: 'screenshot', fullPage } + { id: Date.now(), type: 'screenshot', fullPage, actionId } ]); }; @@ -113,6 +130,10 @@ export const BrowserStepsProvider: React.FC<{ children: React.ReactNode }> = ({ setBrowserSteps(prevSteps => prevSteps.filter(step => step.id !== id)); }; + const deleteStepsByActionId = (actionId: string) => { + setBrowserSteps(prevSteps => prevSteps.filter(step => step.actionId !== actionId)); + }; + const updateBrowserTextStepLabel = (id: number, newLabel: string) => { setBrowserSteps(prevSteps => prevSteps.map(step => @@ -199,6 +220,7 @@ export const BrowserStepsProvider: React.FC<{ children: React.ReactNode }> = ({ updateListStepLimit, updateListStepData, removeListTextField, + deleteStepsByActionId, // Export the new function }}> {children} @@ -211,4 +233,4 @@ export const useBrowserSteps = () => { throw new Error('useBrowserSteps must be used within a BrowserStepsProvider'); } return context; -}; +}; \ No newline at end of file From 111bfec00e304d66b75f17138c9a7bf063c91767 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 20 May 2025 17:44:00 +0530 Subject: [PATCH 002/350] feat: pass action id to browser steps --- src/components/browser/BrowserWindow.tsx | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/components/browser/BrowserWindow.tsx b/src/components/browser/BrowserWindow.tsx index 83750f38..f078d61c 100644 --- a/src/components/browser/BrowserWindow.tsx +++ b/src/components/browser/BrowserWindow.tsx @@ -85,7 +85,7 @@ export const BrowserWindow = () => { const [paginationSelector, setPaginationSelector] = useState(''); const { socket } = useSocketStore(); - const { notify } = useGlobalInfoStore(); + const { notify, currentTextActionId, currentListActionId } = useGlobalInfoStore(); const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext(); const { addTextStep, addListStep, updateListStepData } = useBrowserSteps(); @@ -326,8 +326,8 @@ export const BrowserWindow = () => { selector: highlighterData.selector, tag: highlighterData.elementInfo?.tagName, shadow: highlighterData.elementInfo?.isShadowRoot, - attribute - }); + attribute, + }, currentTextActionId || `text-${Date.now()}`); } else { // Show the modal if there are multiple options setAttributeOptions(options); @@ -344,7 +344,7 @@ export const BrowserWindow = () => { if (paginationType !== '' && paginationType !== 'scrollDown' && paginationType !== 'scrollUp' && paginationType !== 'none') { setPaginationSelector(highlighterData.selector); notify(`info`, t('browser_window.attribute_modal.notifications.pagination_select_success')); - addListStep(listSelector!, fields, currentListId || 0, { type: paginationType, selector: highlighterData.selector }); + addListStep(listSelector!, fields, currentListId || 0, currentListActionId || `list-${Date.now()}`, { type: paginationType, selector: highlighterData.selector }); socket?.emit('setPaginationMode', { pagination: false }); } return; @@ -412,6 +412,7 @@ export const BrowserWindow = () => { listSelector, updatedFields, currentListId, + currentListActionId || `list-${Date.now()}`, { type: '', selector: paginationSelector } ); } @@ -449,7 +450,7 @@ export const BrowserWindow = () => { tag: selectedElement.info?.tagName, shadow: selectedElement.info?.isShadowRoot, attribute: attribute - }); + }, currentTextActionId || `text-${Date.now()}`); } if (getList === true && listSelector && currentListId) { const newField: TextStep = { @@ -484,6 +485,7 @@ export const BrowserWindow = () => { listSelector, updatedFields, currentListId, + currentListActionId || `list-${Date.now()}`, { type: '', selector: paginationSelector } ); } From 9da3f7f291e07177d417c7697ccd8d2907727ee4 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 20 May 2025 17:44:49 +0530 Subject: [PATCH 003/350] feat: add current action id funcs --- src/context/globalInfo.tsx | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/context/globalInfo.tsx b/src/context/globalInfo.tsx index 58ee9672..55f96b06 100644 --- a/src/context/globalInfo.tsx +++ b/src/context/globalInfo.tsx @@ -80,6 +80,12 @@ interface GlobalInfo { }) => void; shouldResetInterpretationLog: boolean; resetInterpretationLog: () => void; + currentTextActionId: string; + setCurrentTextActionId: (actionId: string) => void; + currentListActionId: string; + setCurrentListActionId: (actionId: string) => void; + currentScreenshotActionId: string; + setCurrentScreenshotActionId: (actionId: string) => void; }; class GlobalInfoStore implements Partial { @@ -106,6 +112,9 @@ class GlobalInfoStore implements Partial { hasScrapeSchemaAction: false, }; shouldResetInterpretationLog = false; + currentTextActionId = ''; + currentListActionId = ''; + currentScreenshotActionId = ''; }; const globalInfoStore = new GlobalInfoStore(); @@ -129,6 +138,9 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => { const [recordingUrl, setRecordingUrl] = useState(globalInfoStore.recordingUrl); const [currentWorkflowActionsState, setCurrentWorkflowActionsState] = useState(globalInfoStore.currentWorkflowActionsState); const [shouldResetInterpretationLog, setShouldResetInterpretationLog] = useState(globalInfoStore.shouldResetInterpretationLog); + const [currentTextActionId, setCurrentTextActionId] = useState(''); + const [currentListActionId, setCurrentListActionId] = useState(''); + const [currentScreenshotActionId, setCurrentScreenshotActionId] = useState(''); const notify = (severity: 'error' | 'warning' | 'info' | 'success', message: string) => { setNotification({ severity, message, isOpen: true }); @@ -187,6 +199,12 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => { setCurrentWorkflowActionsState, shouldResetInterpretationLog, resetInterpretationLog, + currentTextActionId, + setCurrentTextActionId, + currentListActionId, + setCurrentListActionId, + currentScreenshotActionId, + setCurrentScreenshotActionId, }} > {children} From 515ad5b7c9acda082a107122b9a6412f81dbdf0b Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 20 May 2025 17:58:07 +0530 Subject: [PATCH 004/350] feat: discard steps based on action id --- src/components/recorder/RightSidePanel.tsx | 84 +++++++++++++++++----- 1 file changed, 65 insertions(+), 19 deletions(-) diff --git a/src/components/recorder/RightSidePanel.tsx b/src/components/recorder/RightSidePanel.tsx index 3383c941..e29e9d2a 100644 --- a/src/components/recorder/RightSidePanel.tsx +++ b/src/components/recorder/RightSidePanel.tsx @@ -52,7 +52,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture const [isCaptureListConfirmed, setIsCaptureListConfirmed] = useState(false); const { panelHeight } = useBrowserDimensionsStore(); - const { lastAction, notify, currentWorkflowActionsState, setCurrentWorkflowActionsState, resetInterpretationLog } = useGlobalInfoStore(); + const { lastAction, notify, currentWorkflowActionsState, setCurrentWorkflowActionsState, resetInterpretationLog, currentListActionId, setCurrentListActionId, currentTextActionId, setCurrentTextActionId, currentScreenshotActionId, setCurrentScreenshotActionId } = useGlobalInfoStore(); const { getText, startGetText, stopGetText, getList, startGetList, stopGetList, @@ -69,7 +69,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture startAction, finishAction } = useActionContext(); - const { browserSteps, updateBrowserTextStepLabel, deleteBrowserStep, addScreenshotStep, updateListTextFieldLabel, removeListTextField, updateListStepLimit } = useBrowserSteps(); + const { browserSteps, updateBrowserTextStepLabel, deleteBrowserStep, addScreenshotStep, updateListTextFieldLabel, removeListTextField, updateListStepLimit, deleteStepsByActionId } = useBrowserSteps(); const { id, socket } = useSocketStore(); const { t } = useTranslation(); @@ -139,15 +139,21 @@ export const RightSidePanel: React.FC = ({ onFinishCapture const handleStartGetText = () => { setIsCaptureTextConfirmed(false); + const newActionId = `text-${Date.now()}`; + setCurrentTextActionId(newActionId); startGetText(); } const handleStartGetList = () => { setIsCaptureListConfirmed(false); + const newActionId = `list-${Date.now()}`; + setCurrentListActionId(newActionId); startGetList(); } const handleStartGetScreenshot = () => { + const newActionId = `screenshot-${Date.now()}`; + setCurrentScreenshotActionId(newActionId); startGetScreenshot(); }; @@ -277,6 +283,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture socket?.emit('action', { action: 'scrapeSchema', settings }); } setIsCaptureTextConfirmed(true); + setCurrentTextActionId(''); resetInterpretationLog(); finishAction('text'); onFinishCapture(); @@ -337,6 +344,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture notify('error', t('right_panel.errors.unable_create_settings')); } handleStopGetList(); + setCurrentListActionId(''); resetInterpretationLog(); finishAction('list'); onFinishCapture(); @@ -434,35 +442,73 @@ export const RightSidePanel: React.FC = ({ onFinishCapture const discardGetText = useCallback(() => { stopGetText(); - browserSteps.forEach(step => { - if (step.type === 'text') { - deleteBrowserStep(step.id); - } - }); - setTextLabels({}); - setErrors({}); - setConfirmedTextSteps({}); + + if (currentTextActionId) { + const stepsToDelete = browserSteps + .filter(step => step.type === 'text' && step.actionId === currentTextActionId) + .map(step => step.id); + + deleteStepsByActionId(currentTextActionId); + + setTextLabels(prevLabels => { + const newLabels = { ...prevLabels }; + stepsToDelete.forEach(id => { + delete newLabels[id]; + }); + return newLabels; + }); + + setErrors(prevErrors => { + const newErrors = { ...prevErrors }; + stepsToDelete.forEach(id => { + delete newErrors[id]; + }); + return newErrors; + }); + + setConfirmedTextSteps(prev => { + const newConfirmed = { ...prev }; + stepsToDelete.forEach(id => { + delete newConfirmed[id]; + }); + return newConfirmed; + }); + } + + setCurrentTextActionId(''); setIsCaptureTextConfirmed(false); notify('error', t('right_panel.errors.capture_text_discarded')); - }, [browserSteps, stopGetText, deleteBrowserStep, notify, t]); + }, [currentTextActionId, browserSteps, stopGetText, deleteStepsByActionId, notify, t]); const discardGetList = useCallback(() => { stopGetList(); - browserSteps.forEach(step => { - if (step.type === 'list') { - deleteBrowserStep(step.id); - } - }); + + if (currentListActionId) { + const listStepsToDelete = browserSteps + .filter(step => step.type === 'list' && step.actionId === currentListActionId) + .map(step => step.id); + + deleteStepsByActionId(currentListActionId); + + setConfirmedListTextFields(prev => { + const newConfirmed = { ...prev }; + listStepsToDelete.forEach(id => { + delete newConfirmed[id]; + }); + return newConfirmed; + }); + } + resetListState(); stopPaginationMode(); stopLimitMode(); setShowPaginationOptions(false); setShowLimitOptions(false); setCaptureStage('initial'); - setConfirmedListTextFields({}); + setCurrentListActionId(''); setIsCaptureListConfirmed(false); notify('error', t('right_panel.errors.capture_list_discarded')); - }, [browserSteps, stopGetList, deleteBrowserStep, resetListState, setShowPaginationOptions, setShowLimitOptions, setCaptureStage, notify, t]); + }, [currentListActionId, browserSteps, stopGetList, deleteStepsByActionId, resetListState, setShowPaginationOptions, setShowLimitOptions, setCaptureStage, notify, t]); const captureScreenshot = (fullPage: boolean) => { const screenshotSettings: ScreenshotSettings = { @@ -474,7 +520,7 @@ export const RightSidePanel: React.FC = ({ onFinishCapture scale: 'device', }; socket?.emit('action', { action: 'screenshot', settings: screenshotSettings }); - addScreenshotStep(fullPage); + addScreenshotStep(fullPage, currentScreenshotActionId); stopGetScreenshot(); resetInterpretationLog(); finishAction('screenshot'); From 372914530693669731d5226fdcace2d61e2dc8f0 Mon Sep 17 00:00:00 2001 From: Rohit Date: Mon, 26 May 2025 21:41:40 +0530 Subject: [PATCH 005/350] feat: generate truly unique action ids --- src/components/browser/BrowserWindow.tsx | 10 +++++----- src/components/recorder/RightSidePanel.tsx | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/components/browser/BrowserWindow.tsx b/src/components/browser/BrowserWindow.tsx index f078d61c..8ce6e0f9 100644 --- a/src/components/browser/BrowserWindow.tsx +++ b/src/components/browser/BrowserWindow.tsx @@ -327,7 +327,7 @@ export const BrowserWindow = () => { tag: highlighterData.elementInfo?.tagName, shadow: highlighterData.elementInfo?.isShadowRoot, attribute, - }, currentTextActionId || `text-${Date.now()}`); + }, currentTextActionId || `text-${crypto.randomUUID()}`); } else { // Show the modal if there are multiple options setAttributeOptions(options); @@ -344,7 +344,7 @@ export const BrowserWindow = () => { if (paginationType !== '' && paginationType !== 'scrollDown' && paginationType !== 'scrollUp' && paginationType !== 'none') { setPaginationSelector(highlighterData.selector); notify(`info`, t('browser_window.attribute_modal.notifications.pagination_select_success')); - addListStep(listSelector!, fields, currentListId || 0, currentListActionId || `list-${Date.now()}`, { type: paginationType, selector: highlighterData.selector }); + addListStep(listSelector!, fields, currentListId || 0, currentListActionId || `list-${crypto.randomUUID()}`, { type: paginationType, selector: highlighterData.selector }); socket?.emit('setPaginationMode', { pagination: false }); } return; @@ -412,7 +412,7 @@ export const BrowserWindow = () => { listSelector, updatedFields, currentListId, - currentListActionId || `list-${Date.now()}`, + currentListActionId || `list-${crypto.randomUUID()}`, { type: '', selector: paginationSelector } ); } @@ -450,7 +450,7 @@ export const BrowserWindow = () => { tag: selectedElement.info?.tagName, shadow: selectedElement.info?.isShadowRoot, attribute: attribute - }, currentTextActionId || `text-${Date.now()}`); + }, currentTextActionId || `text-${crypto.randomUUID()}`); } if (getList === true && listSelector && currentListId) { const newField: TextStep = { @@ -485,7 +485,7 @@ export const BrowserWindow = () => { listSelector, updatedFields, currentListId, - currentListActionId || `list-${Date.now()}`, + currentListActionId || `list-${crypto.randomUUID()}`, { type: '', selector: paginationSelector } ); } diff --git a/src/components/recorder/RightSidePanel.tsx b/src/components/recorder/RightSidePanel.tsx index e29e9d2a..8d8cac22 100644 --- a/src/components/recorder/RightSidePanel.tsx +++ b/src/components/recorder/RightSidePanel.tsx @@ -139,20 +139,20 @@ export const RightSidePanel: React.FC = ({ onFinishCapture const handleStartGetText = () => { setIsCaptureTextConfirmed(false); - const newActionId = `text-${Date.now()}`; + const newActionId = `text-${crypto.randomUUID()}`; setCurrentTextActionId(newActionId); startGetText(); } const handleStartGetList = () => { setIsCaptureListConfirmed(false); - const newActionId = `list-${Date.now()}`; + const newActionId = `list-${crypto.randomUUID()}`; setCurrentListActionId(newActionId); startGetList(); } const handleStartGetScreenshot = () => { - const newActionId = `screenshot-${Date.now()}`; + const newActionId = `screenshot-${crypto.randomUUID()}`; setCurrentScreenshotActionId(newActionId); startGetScreenshot(); }; From c3a58fc23862ab3a70df20f5cb185f6ba050b0c5 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 27 May 2025 16:09:29 +0530 Subject: [PATCH 006/350] feat: add webhook integration button --- .../integration/IntegrationSettings.tsx | 42 +++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/src/components/integration/IntegrationSettings.tsx b/src/components/integration/IntegrationSettings.tsx index 5ca71840..455fe72c 100644 --- a/src/components/integration/IntegrationSettings.tsx +++ b/src/components/integration/IntegrationSettings.tsx @@ -23,7 +23,16 @@ interface IntegrationProps { isOpen: boolean; handleStart: (data: IntegrationSettings) => void; handleClose: () => void; - preSelectedIntegrationType?: "googleSheets" | "airtable" | null; + preSelectedIntegrationType?: "googleSheets" | "airtable" | "webhook" | null; +} + +export interface WebhookConfig { + id: string; + name: string; + url: string; + headers: { [key: string]: string }; + events: string[]; + active: boolean; } export interface IntegrationSettings { @@ -33,8 +42,9 @@ export interface IntegrationSettings { airtableBaseName?: string; airtableTableName?: string, airtableTableId?: string, + webhooks?: WebhookConfig[]; data: string; - integrationType: "googleSheets" | "airtable"; + integrationType: "googleSheets" | "airtable" | "webhook"; } const getCookie = (name: string): string | null => { @@ -64,6 +74,7 @@ export const IntegrationSettingsModal = ({ airtableBaseName: "", airtableTableName: "", airtableTableId: "", + webhooks: [], data: "", integrationType: preSelectedIntegrationType || "googleSheets", }); @@ -74,6 +85,18 @@ export const IntegrationSettingsModal = ({ const [loading, setLoading] = useState(false); const [error, setError] = useState(null); + // Webhook-specific state + const [newWebhook, setNewWebhook] = useState({ + id: "", + name: "", + url: "", + headers: {}, + events: ["scrape_completed"], + active: true, + }); + const [newHeaderKey, setNewHeaderKey] = useState(""); + const [newHeaderValue, setNewHeaderValue] = useState(""); + const { recordingId, notify, @@ -84,7 +107,7 @@ export const IntegrationSettingsModal = ({ const navigate = useNavigate(); const [selectedIntegrationType, setSelectedIntegrationType] = useState< - "googleSheets" | "airtable" | null + "googleSheets" | "airtable" | "webhook" | null >(preSelectedIntegrationType); const authenticateWithGoogle = () => { @@ -410,6 +433,19 @@ export const IntegrationSettingsModal = ({ Airtable Airtable + + From cfd1bc5ecbffe47893d464a7c7e34b3b3db88012 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 27 May 2025 16:13:55 +0530 Subject: [PATCH 007/350] feat: add webhook svg --- public/svg/webhook.svg | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 public/svg/webhook.svg diff --git a/public/svg/webhook.svg b/public/svg/webhook.svg new file mode 100644 index 00000000..959bc307 --- /dev/null +++ b/public/svg/webhook.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file From 2b12726d1be0810debf4e7f63d5ff82b9be9cc01 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 27 May 2025 20:06:46 +0530 Subject: [PATCH 008/350] feat: add migration for webhooks column --- .../migrations/20250527105655-add-webhooks.js | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 server/src/db/migrations/20250527105655-add-webhooks.js diff --git a/server/src/db/migrations/20250527105655-add-webhooks.js b/server/src/db/migrations/20250527105655-add-webhooks.js new file mode 100644 index 00000000..60eefd19 --- /dev/null +++ b/server/src/db/migrations/20250527105655-add-webhooks.js @@ -0,0 +1,27 @@ +'use strict'; + +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.addColumn('robot', 'webhooks', { + type: Sequelize.JSONB, + allowNull: true, + defaultValue: null, + comment: 'Webhook configurations for the robot' + }); + + // Optional: Add an index for better query performance if you plan to search within webhook data + await queryInterface.addIndex('robot', { + fields: ['webhooks'], + using: 'gin', // GIN index for JSONB columns + name: 'robot_webhooks_gin_idx' + }); + }, + + async down(queryInterface, Sequelize) { + // Remove the index first + await queryInterface.removeIndex('robot', 'robot_webhooks_gin_idx'); + + // Then remove the column + await queryInterface.removeColumn('robot', 'webhooks'); + } +}; \ No newline at end of file From 1777a598c13fe512c2e9f020879e57d7e8cfe24f Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 27 May 2025 20:08:01 +0530 Subject: [PATCH 009/350] feat: add webhook column model --- server/src/models/Robot.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/server/src/models/Robot.ts b/server/src/models/Robot.ts index 1681eaac..eae9438e 100644 --- a/server/src/models/Robot.ts +++ b/server/src/models/Robot.ts @@ -15,6 +15,19 @@ interface RobotWorkflow { workflow: WhereWhatPair[]; } +interface WebhookConfig { + id: string; + url: string; + events: string[]; + active: boolean; + createdAt: string; + updatedAt: string; + lastCalledAt?: string | null; + retryAttempts?: number; + retryDelay?: number; + timeout?: number; +} + interface RobotAttributes { id: string; userId?: number; @@ -32,6 +45,7 @@ interface RobotAttributes { airtable_refresh_token?: string | null; schedule?: ScheduleConfig | null; airtable_table_id?: string | null; + webhooks?: WebhookConfig[] | null; } interface ScheduleConfig { @@ -66,6 +80,7 @@ class Robot extends Model implements R public airtable_refresh_token!: string | null; public airtable_table_id!: string | null; public schedule!: ScheduleConfig | null; + public webhooks!: WebhookConfig[] | null; } Robot.init( @@ -135,6 +150,11 @@ Robot.init( type: DataTypes.JSONB, allowNull: true, }, + webhooks: { + type: DataTypes.JSONB, + allowNull: true, + defaultValue: null, + }, }, { sequelize, From d38a5cd39c71e1f2382dbe98c7eb31e3106cd995 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 27 May 2025 20:09:06 +0530 Subject: [PATCH 010/350] feat: add webhook navigation endpoint --- src/components/robot/Recordings.tsx | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/components/robot/Recordings.tsx b/src/components/robot/Recordings.tsx index 780d01cf..1b3fc7b2 100644 --- a/src/components/robot/Recordings.tsx +++ b/src/components/robot/Recordings.tsx @@ -114,6 +114,15 @@ export const Recordings = ({ preSelectedIntegrationType="airtable" /> ); + } else if (currentPath.endsWith("/integrate/webhook")) { + return ( + {}} + preSelectedIntegrationType="webhook" + /> + ); } else if (currentPath.endsWith("/integrate")) { return ( Date: Tue, 27 May 2025 20:09:52 +0530 Subject: [PATCH 011/350] feat: register webhook endpoint --- server/src/routes/index.ts | 4 +++- server/src/server.ts | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/server/src/routes/index.ts b/server/src/routes/index.ts index bc616273..3d8a3644 100644 --- a/server/src/routes/index.ts +++ b/server/src/routes/index.ts @@ -4,6 +4,7 @@ import { router as storage } from './storage'; import { router as auth } from './auth'; import { router as integration } from './integration'; import { router as proxy } from './proxy'; +import { router as webhook } from './webhook'; export { record, @@ -11,5 +12,6 @@ export { storage, auth, integration, - proxy + proxy, + webhook }; diff --git a/server/src/server.ts b/server/src/server.ts index 7f2d04d3..bd4a1697 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -4,7 +4,7 @@ import http from 'http'; import cors from 'cors'; import dotenv from 'dotenv'; dotenv.config(); -import { record, workflow, storage, auth, integration, proxy } from './routes'; +import { record, workflow, storage, auth, integration, proxy, webhook } from './routes'; import { BrowserPool } from "./browser-management/classes/BrowserPool"; import logger from './logger'; import { connectDB, syncDB } from './storage/db' @@ -88,6 +88,7 @@ export const browserPool = new BrowserPool(); // parse cookies - "cookie" is true in csrfProtection app.use(cookieParser()) +app.use('/webhook', webhook); app.use('/record', record); app.use('/workflow', workflow); app.use('/storage', storage); From a2e96a1779419a0e78f3785e33025090c1ebc865 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 27 May 2025 20:30:19 +0530 Subject: [PATCH 012/350] feat: add webhooks config modal --- .../integration/IntegrationSettings.tsx | 499 +++++++++++++++++- 1 file changed, 489 insertions(+), 10 deletions(-) diff --git a/src/components/integration/IntegrationSettings.tsx b/src/components/integration/IntegrationSettings.tsx index 455fe72c..09cabbd8 100644 --- a/src/components/integration/IntegrationSettings.tsx +++ b/src/components/integration/IntegrationSettings.tsx @@ -8,7 +8,23 @@ import { AlertTitle, Button, TextField, + IconButton, + Box, + Chip, + Card, + CardContent, + CardActions, + Switch, + FormControlLabel, + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + TableRow, + Paper, } from "@mui/material"; +import { Add as AddIcon, Delete as DeleteIcon, Edit as EditIcon, Science as ScienceIcon } from "@mui/icons-material"; import axios from "axios"; import { useGlobalInfoStore } from "../../context/globalInfo"; import { getStoredRecording } from "../../api/storage"; @@ -28,11 +44,10 @@ interface IntegrationProps { export interface WebhookConfig { id: string; - name: string; url: string; - headers: { [key: string]: string }; events: string[]; active: boolean; + lastCalledAt?: string | null; } export interface IntegrationSettings { @@ -85,17 +100,15 @@ export const IntegrationSettingsModal = ({ const [loading, setLoading] = useState(false); const [error, setError] = useState(null); - // Webhook-specific state + const [showWebhookForm, setShowWebhookForm] = useState(false); + const [editingWebhook, setEditingWebhook] = useState(null); const [newWebhook, setNewWebhook] = useState({ id: "", - name: "", url: "", - headers: {}, - events: ["scrape_completed"], + events: ["run_completed"], active: true, }); - const [newHeaderKey, setNewHeaderKey] = useState(""); - const [newHeaderValue, setNewHeaderValue] = useState(""); + const [urlError, setUrlError] = useState(null); const { recordingId, @@ -119,6 +132,207 @@ export const IntegrationSettingsModal = ({ window.location.href = `${apiUrl}/auth/airtable?robotId=${recordingId}`; }; + const validateWebhookData = (url: string, events: string[], excludeId?: string) => { + if (!url) { + setUrlError("Please provide webhook URL"); + return false; + } + + const existingWebhook = settings.webhooks?.find( + (webhook) => webhook.url === url && webhook.id !== excludeId + ); + + if (existingWebhook) { + setUrlError("This webhook URL is already in use"); + return false; + } + + if (!events || events.length === 0) { + setUrlError("Please select at least one event"); + return false; + } + + setUrlError(null); + return true; + }; + + const addWebhook = async () => { + if (!validateWebhookData(newWebhook.url, newWebhook.events)) { + if (!newWebhook.url) { + notify("error", "Please provide webhook URL"); + } else if (!newWebhook.events || newWebhook.events.length === 0) { + notify("error", "Please select at least one event"); + } + return; + } + + try { + setLoading(true); + const webhookWithId = { + ...newWebhook, + id: Date.now().toString(), + }; + + const response = await axios.post( + `${apiUrl}/webhook/add`, + { + webhook: webhookWithId, + robotId: recordingId, + }, + { withCredentials: true } + ); + + const updatedWebhooks = [...(settings.webhooks || []), webhookWithId]; + setSettings({ ...settings, webhooks: updatedWebhooks }); + + resetWebhookForm(); + await refreshRecordingData(); + notify("success", "Webhook added successfully"); + setLoading(false); + } catch (error: any) { + setLoading(false); + console.log("Error adding webhook:", error); + notify("error", `Error adding webhook: ${error.response?.data?.message || error.message}`); + } + }; + + const updateWebhook = async () => { + if (!editingWebhook) return; + + if (!validateWebhookData(newWebhook.url, newWebhook.events, editingWebhook)) { + if (!newWebhook.url) { + notify("error", "Please provide webhook URL"); + } else if (!newWebhook.events || newWebhook.events.length === 0) { + notify("error", "Please select at least one event"); + } + return; + } + + try { + setLoading(true); + await axios.post( + `${apiUrl}/webhook/update`, + { + webhook: newWebhook, + robotId: recordingId, + }, + { withCredentials: true } + ); + + const updatedWebhooks = (settings.webhooks || []).map(w => + w.id === editingWebhook ? newWebhook : w + ); + setSettings({ ...settings, webhooks: updatedWebhooks }); + + resetWebhookForm(); + await refreshRecordingData(); + notify("success", "Webhook updated successfully"); + setLoading(false); + } catch (error: any) { + setLoading(false); + console.error("Error updating webhook:", error); + notify("error", `Error updating webhook: ${error.response?.data?.message || error.message}`); + } + }; + + const removeWebhook = async (webhookId: string) => { + try { + setLoading(true); + await axios.post( + `${apiUrl}/webhook/remove`, + { + webhookId, + robotId: recordingId, + }, + { withCredentials: true } + ); + + const updatedWebhooks = (settings.webhooks || []).filter(w => w.id !== webhookId); + setSettings({ ...settings, webhooks: updatedWebhooks }); + + await refreshRecordingData(); + notify("success", "Webhook removed successfully"); + setLoading(false); + } catch (error: any) { + setLoading(false); + console.error("Error removing webhook:", error); + notify("error", `Error removing webhook: ${error.response?.data?.message || error.message}`); + } + }; + + const toggleWebhookStatus = async (webhookId: string) => { + try { + const webhook = settings.webhooks?.find(w => w.id === webhookId); + if (!webhook) return; + + const updatedWebhook = { ...webhook, active: !webhook.active }; + + await axios.post( + `${apiUrl}/webhook/update`, + { + webhook: updatedWebhook, + robotId: recordingId, + }, + { withCredentials: true } + ); + + const updatedWebhooks = (settings.webhooks || []).map(w => + w.id === webhookId ? updatedWebhook : w + ); + setSettings({ ...settings, webhooks: updatedWebhooks }); + + await refreshRecordingData(); + notify("success", `Webhook ${updatedWebhook.active ? "enabled" : "disabled"}`); + } catch (error: any) { + console.error("Error toggling webhook status:", error); + notify("error", `Error updating webhook: ${error.response?.data?.message || error.message}`); + } + }; + + const testWebhook = async (webhook: WebhookConfig) => { + try { + setLoading(true); + await axios.post( + `${apiUrl}/webhook/test`, + { + webhook, + robotId: recordingId, + }, + { withCredentials: true } + ); + + const updatedWebhooks = (settings.webhooks || []).map(w => + w.id === webhook.id ? { ...w, lastCalledAt: new Date().toISOString() } : w + ); + setSettings({ ...settings, webhooks: updatedWebhooks }); + + notify("success", "Test webhook sent successfully"); + setLoading(false); + } catch (error: any) { + setLoading(false); + console.error("Error testing webhook:", error); + notify("error", `Error testing webhook: ${error.response?.data?.message || error.message}`); + } + }; + + const editWebhook = (webhook: WebhookConfig) => { + setNewWebhook(webhook); + setEditingWebhook(webhook.id); + setShowWebhookForm(true); + }; + + const resetWebhookForm = () => { + setNewWebhook({ + id: "", + url: "", + events: ["run_completed"], + active: true, + }); + setShowWebhookForm(false); + setEditingWebhook(null); + setUrlError(null); + }; + // Fetch Google Sheets files const fetchSpreadsheetFiles = async () => { try { @@ -366,6 +580,12 @@ export const IntegrationSettingsModal = ({ airtableTableId: recording.airtable_table_id || "", integrationType: recording.airtable_base_id ? "airtable" : "googleSheets" })); + } else if (recording.webhooks && recording.webhooks.length > 0) { + setSettings(prev => ({ + ...prev, + webhooks: recording.webhooks, + integrationType: "webhook" + })); } } @@ -393,7 +613,50 @@ export const IntegrationSettingsModal = ({ } }, []); - // Add this UI at the top of the modal return statement + const formatEventName = (event: string) => { + switch (event) { + case "run_completed": + return "Run finished"; + case "run_completed_success": + return "Run finished successfully"; + case "run_failed": + return "Run failed"; + default: + return event; + } + }; + + const formatLastCalled = (lastCalledAt?: string | null) => { + if (!lastCalledAt) { + return "Not called yet"; + } + + const date = new Date(lastCalledAt); + const now = new Date(); + const diffMs = now.getTime() - date.getTime(); + const diffDays = Math.floor(diffMs / (1000 * 60 * 60 * 24)); + const diffHours = Math.floor(diffMs / (1000 * 60 * 60)); + const diffMinutes = Math.floor(diffMs / (1000 * 60)); + + if (diffMinutes < 1) { + return "Just now"; + } else if (diffMinutes < 60) { + return `${diffMinutes} minute${diffMinutes === 1 ? '' : 's'} ago`; + } else if (diffHours < 24) { + return `${diffHours} hour${diffHours === 1 ? '' : 's'} ago`; + } else if (diffDays < 7) { + return `${diffDays} day${diffDays === 1 ? '' : 's'} ago`; + } else { + return date.toLocaleDateString('en-US', { + year: 'numeric', + month: 'short', + day: 'numeric', + hour: '2-digit', + minute: '2-digit' + }); + } + }; + if (!selectedIntegrationType) { return ( {settings.integrationType === "googleSheets" && ( @@ -676,6 +940,219 @@ export const IntegrationSettingsModal = ({ )} )} + + {settings.integrationType === "webhook" && ( + <> + + Integrate using Webhooks + + + {settings.webhooks && settings.webhooks.length > 0 && ( + + + + + Webhook URL + Call when + Last called + Status + Actions + + + + {settings.webhooks.map((webhook) => ( + + {webhook.url} + {formatEventName(webhook.events[0])} + {formatLastCalled(webhook.lastCalledAt)} + + toggleWebhookStatus(webhook.id)} + size="small" + /> + + + + testWebhook(webhook)} + disabled={loading || !webhook.active} + title="Test" + > + + + editWebhook(webhook)} + disabled={loading} + title="Edit" + > + + + removeWebhook(webhook.id)} + disabled={loading} + title="Delete" + > + + + + + + ))} + +
+ + )} + + {!showWebhookForm && ( + + + { + setNewWebhook({ ...newWebhook, url: e.target.value }); + if (urlError) setUrlError(null); + }} + error={!!urlError} + helperText={urlError} + /> + setNewWebhook({ + ...newWebhook, + events: [e.target.value] + })} + sx={{ minWidth: "200px" }} + > + Run finished + Run finished successfully + Run failed + + + + + + Refer to the API documentation for examples and details. + + + + + )} + + {showWebhookForm && ( + + + + {editingWebhook ? "Edit Webhook" : "Add New Webhook"} + + + { + setNewWebhook({ ...newWebhook, url: e.target.value }); + if (urlError) setUrlError(null); + }} + sx={{ marginBottom: "15px" }} + placeholder="https://your-api.com/webhook/endpoint" + required + error={!!urlError} + helperText={urlError} + /> + + setNewWebhook({ + ...newWebhook, + events: typeof e.target.value === 'string' ? [e.target.value] : e.target.value + })} + SelectProps={{ + multiple: true, + renderValue: (selected) => ( + + {(selected as string[]).map((value) => ( + + ))} + + ), + }} + sx={{ marginBottom: "20px" }} + required + > + Run finished + Run finished successfully + Run failed + + + setNewWebhook({ ...newWebhook, active: e.target.checked })} + /> + } + label="Active" + sx={{ marginBottom: "10px" }} + /> + + + + + + + + )} + + )} ); @@ -685,10 +1162,12 @@ export const modalStyle = { top: "40%", left: "50%", transform: "translate(-50%, -50%)", - width: "50%", + width: "60%", backgroundColor: "background.paper", p: 4, height: "fit-content", display: "block", padding: "20px", + maxHeight: "90vh", + overflow: "auto", }; \ No newline at end of file From 6f11b69f12950979e36c26c218b56bdae26a9cf4 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 27 May 2025 22:12:00 +0530 Subject: [PATCH 013/350] feat: add webhook routes --- server/src/routes/webhook.ts | 445 +++++++++++++++++++++++++++++++++++ 1 file changed, 445 insertions(+) create mode 100644 server/src/routes/webhook.ts diff --git a/server/src/routes/webhook.ts b/server/src/routes/webhook.ts new file mode 100644 index 00000000..ddc9c1bf --- /dev/null +++ b/server/src/routes/webhook.ts @@ -0,0 +1,445 @@ +import { Router, Request, Response } from 'express'; +import Robot from '../models/Robot'; +import { requireSignIn } from '../middlewares/auth'; +import axios from 'axios'; + +export const router = Router(); + +interface AuthenticatedRequest extends Request { + user?: { id: string }; +} + +interface WebhookConfig { + id: string; + url: string; + events: string[]; + active: boolean; + createdAt: string; + updatedAt: string; + lastCalledAt?: string | null; + retryAttempts?: number; + retryDelay?: number; + timeout?: number; +} + +const updateWebhookLastCalled = async (robotId: string, webhookId: string): Promise => { + try { + const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + if (!robot || !robot.webhooks) { + return; + } + + const updatedWebhooks = robot.webhooks.map((w: WebhookConfig) => { + if (w.id === webhookId) { + return { + ...w, + lastCalledAt: new Date().toISOString() + }; + } + return w; + }); + + await robot.update({ webhooks: updatedWebhooks }); + } catch (error) { + console.error('Error updating webhook lastCalledAt:', error); + } +}; + +// Add new webhook +router.post('/add', requireSignIn, async (req: Request, res: Response) => { + const { webhook, robotId } = req.body; + const authenticatedReq = req as AuthenticatedRequest; + + try { + if (!authenticatedReq.user) { + return res.status(401).json({ ok: false, error: 'Unauthorized' }); + } + + if (!webhook || !robotId) { + return res.status(400).json({ ok: false, error: 'Webhook configuration and robot ID are required' }); + } + + if (!webhook.url) { + return res.status(400).json({ ok: false, error: 'Webhook URL is required' }); + } + + // Validate URL format + try { + new URL(webhook.url); + } catch (error) { + return res.status(400).json({ ok: false, error: 'Invalid webhook URL format' }); + } + + const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + + if (!robot) { + return res.status(404).json({ ok: false, error: 'Robot not found' }); + } + + const currentWebhooks = robot.webhooks || []; + + const existingWebhook = currentWebhooks.find((w: WebhookConfig) => w.url === webhook.url); + if (existingWebhook) { + return res.status(400).json({ ok: false, error: 'Webhook with this url already exists' }); + } + + const newWebhook: WebhookConfig = { + ...webhook, + id: webhook.id || Date.now().toString(), + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + lastCalledAt: null, + retryAttempts: webhook.retryAttempts || 3, + retryDelay: webhook.retryDelay || 5, + timeout: webhook.timeout || 30, + }; + + const updatedWebhooks = [...currentWebhooks, newWebhook]; + + await robot.update({ webhooks: updatedWebhooks }); + + res.status(200).json({ + ok: true, + message: 'Webhook added successfully', + webhook: newWebhook + }); + } catch (error: any) { + console.log(`Could not add webhook - ${error}`); + res.status(500).json({ ok: false, error: 'Could not add webhook configuration' }); + } +}); + +// Update existing webhook +router.post('/update', requireSignIn, async (req: Request, res: Response) => { + const { webhook, robotId } = req.body; + const authenticatedReq = req as AuthenticatedRequest; + + try { + if (!authenticatedReq.user) { + return res.status(401).json({ ok: false, error: 'Unauthorized' }); + } + + if (!webhook || !robotId || !webhook.id) { + return res.status(400).json({ ok: false, error: 'Webhook configuration, webhook ID, and robot ID are required' }); + } + + // Validate URL format if provided + if (webhook.url) { + try { + new URL(webhook.url); + } catch (error) { + return res.status(400).json({ ok: false, error: 'Invalid webhook URL format' }); + } + } + + const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + + if (!robot) { + return res.status(404).json({ ok: false, error: 'Robot not found' }); + } + + const currentWebhooks = robot.webhooks || []; + const webhookIndex = currentWebhooks.findIndex((w: WebhookConfig) => w.id === webhook.id); + + if (webhookIndex === -1) { + return res.status(404).json({ ok: false, error: 'Webhook not found' }); + } + + // Check for duplicate URLs (excluding current webhook) + const duplicateUrl = currentWebhooks.find((w: WebhookConfig, index: number) => + w.url === webhook.url && index !== webhookIndex + ); + if (duplicateUrl) { + return res.status(400).json({ ok: false, error: 'Webhook with this URL already exists' }); + } + + const updatedWebhook: WebhookConfig = { + ...currentWebhooks[webhookIndex], + ...webhook, + updatedAt: new Date().toISOString(), + lastCalledAt: currentWebhooks[webhookIndex].lastCalledAt + }; + + const updatedWebhooks = [...currentWebhooks]; + updatedWebhooks[webhookIndex] = updatedWebhook; + + await robot.update({ webhooks: updatedWebhooks }); + + res.status(200).json({ + ok: true, + message: 'Webhook updated successfully', + webhook: updatedWebhook + }); + } catch (error: any) { + console.log(`Could not update webhook - ${error}`); + res.status(500).json({ ok: false, error: 'Could not update webhook configuration' }); + } +}); + +// Remove webhook +router.post('/remove', requireSignIn, async (req: Request, res: Response) => { + const { webhookId, robotId } = req.body; + const authenticatedReq = req as AuthenticatedRequest; + + try { + if (!authenticatedReq.user) { + return res.status(401).json({ ok: false, error: 'Unauthorized' }); + } + + if (!webhookId || !robotId) { + return res.status(400).json({ ok: false, error: 'Webhook ID and robot ID are required' }); + } + + const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + + if (!robot) { + return res.status(404).json({ ok: false, error: 'Robot not found' }); + } + + const currentWebhooks = robot.webhooks || []; + const webhookExists = currentWebhooks.find((w: WebhookConfig) => w.id === webhookId); + + if (!webhookExists) { + return res.status(404).json({ ok: false, error: 'Webhook not found' }); + } + + const updatedWebhooks = currentWebhooks.filter((w: WebhookConfig) => w.id !== webhookId); + + await robot.update({ webhooks: updatedWebhooks }); + + res.status(200).json({ + ok: true, + message: 'Webhook removed successfully' + }); + } catch (error: any) { + console.log(`Could not remove webhook - ${error}`); + res.status(500).json({ ok: false, error: 'Could not remove webhook configuration' }); + } +}); + +// Get all webhooks for a robot +router.get('/list/:robotId', requireSignIn, async (req: Request, res: Response) => { + const { robotId } = req.params; + const authenticatedReq = req as AuthenticatedRequest; + + try { + if (!authenticatedReq.user) { + return res.status(401).json({ ok: false, error: 'Unauthorized' }); + } + + const robot = await Robot.findOne({ + where: { 'recording_meta.id': robotId }, + attributes: ['webhooks'] + }); + + if (!robot) { + return res.status(404).json({ ok: false, error: 'Robot not found' }); + } + + const webhooks = robot.webhooks || []; + + res.status(200).json({ + ok: true, + webhooks: webhooks + }); + } catch (error: any) { + console.log(`Could not retrieve webhooks - ${error}`); + res.status(500).json({ ok: false, error: 'Could not retrieve webhook configurations' }); + } +}); + +// Test webhook endpoint +router.post('/test', requireSignIn, async (req: Request, res: Response) => { + const { webhook, robotId } = req.body; + const authenticatedReq = req as AuthenticatedRequest; + + try { + if (!authenticatedReq.user) { + return res.status(401).json({ ok: false, error: 'Unauthorized' }); + } + + if (!webhook || !robotId) { + return res.status(400).json({ ok: false, error: 'Webhook configuration and robot ID are required' }); + } + + const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + + if (!robot) { + return res.status(404).json({ ok: false, error: 'Robot not found' }); + } + + // Create test payload + const testPayload = { + event_type: "webhook_test", + timestamp: new Date().toISOString(), + webhook_id: webhook.id, + data: { + workflow_id: robotId, + run_id: "110c4dae-c39b-4b30-a932-eff1022e4bb0", + robot_name: robot.recording_meta?.name || "Unknown Robot", + status: "test", + started_at: new Date().toISOString(), + finished_at: new Date().toISOString(), + execution_time_ms: 5000, + extracted_data: { + schema_items: { + "item-0": { title: "Test Item 1", value: "Sample Value 1" }, + "item-1": { title: "Test Item 2", value: "Sample Value 2" } + }, + list_items: { + "item-0": [ + { name: "List Item 1", price: "$10.99" }, + { name: "List Item 2", price: "$15.99" } + ] + }, + total_rows: 4, + schema_count: 2, + list_count: 2, + screenshots_count: 3 + }, + metadata: { + test_mode: true, + browser_id: "d27ace57-75cb-441c-8589-8ba34e52f7d1", + user_id: "108" + } + } + }; + + await updateWebhookLastCalled(robotId, webhook.id); + + const response = await axios.post(webhook.url, testPayload, { + timeout: (webhook.timeout || 30) * 1000, + validateStatus: (status) => status < 500 + }); + + const success = response.status >= 200 && response.status < 300; + + res.status(200).json({ + ok: true, + message: success ? 'Test webhook sent successfully' : 'Webhook endpoint responded with non-success status', + details: { + status: response.status, + statusText: response.statusText, + success: success + } + }); + } catch (error: any) { + console.log(`Could not test webhook - ${error}`); + + try { + await updateWebhookLastCalled(robotId, webhook.id); + } catch (updateError) { + console.error('Failed to update lastCalledAt after webhook error:', updateError); + } + + let errorMessage = 'Could not send test webhook'; + if (error.code === 'ECONNREFUSED') { + errorMessage = 'Connection refused - webhook URL is not accessible'; + } else if (error.code === 'ETIMEDOUT') { + errorMessage = 'Request timeout - webhook endpoint did not respond in time'; + } else if (error.response) { + errorMessage = `Webhook endpoint responded with error: ${error.response.status} ${error.response.statusText}`; + } + + res.status(500).json({ + ok: false, + error: errorMessage, + details: { + code: error.code, + message: error.message + } + }); + } +}); + +// Send webhook +export const sendWebhook = async (robotId: string, eventType: string, data: any): Promise => { + try { + const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + if (!robot || !robot.webhooks) { + return; + } + + const activeWebhooks = robot.webhooks.filter((w: WebhookConfig) => + w.active && w.events.includes(eventType) + ); + + if (activeWebhooks.length === 0) { + return; + } + + const webhookPromises = activeWebhooks.map(async (webhook: WebhookConfig) => { + const payload = { + event_type: eventType, + timestamp: new Date().toISOString(), + webhook_id: webhook.id, + data: data + }; + + return sendWebhookWithRetry(robotId, webhook, payload); + }); + + await Promise.allSettled(webhookPromises); + } catch (error) { + console.error('Error sending webhooks:', error); + } +}; + +// Helper function to send webhook with retry logic +const sendWebhookWithRetry = async (robotId: string, webhook: WebhookConfig, payload: any, attempt: number = 1): Promise => { + const maxRetries = webhook.retryAttempts || 3; + const retryDelay = webhook.retryDelay || 5; + const timeout = webhook.timeout || 30; + + try { + await updateWebhookLastCalled(robotId, webhook.id); + + const response = await axios.post(webhook.url, payload, { + timeout: timeout * 1000, + validateStatus: (status) => status >= 200 && status < 300 + }); + + console.log(`Webhook sent successfully to ${webhook.url}: ${response.status}`); + } catch (error: any) { + console.error(`Webhook failed for ${webhook.url} (attempt ${attempt}):`, error.message); + + if (attempt < maxRetries) { + const delay = retryDelay * Math.pow(2, attempt - 1); + console.log(`Retrying webhook ${webhook.url} in ${delay} seconds...`); + + setTimeout(async () => { + await sendWebhookWithRetry(robotId, webhook, payload, attempt + 1); + }, delay * 1000); + } else { + console.error(`Webhook ${webhook.url} failed after ${maxRetries} attempts`); + } + } +}; + +// Clear all webhooks for a robot +router.delete('/clear/:robotId', requireSignIn, async (req: Request, res: Response) => { + const { robotId } = req.params; + const authenticatedReq = req as AuthenticatedRequest; + + try { + if (!authenticatedReq.user) { + return res.status(401).json({ ok: false, error: 'Unauthorized' }); + } + + const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + + if (!robot) { + return res.status(404).json({ ok: false, error: 'Robot not found' }); + } + + await robot.update({ webhooks: [] }); + + res.status(200).json({ + ok: true, + message: 'All webhooks cleared successfully' + }); + } catch (error: any) { + console.log(`Could not clear webhooks - ${error}`); + res.status(500).json({ ok: false, error: 'Could not clear webhook configurations' }); + } +}); \ No newline at end of file From 96df256f3f193d0cb8fde041fe4a081b26308e0c Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 27 May 2025 22:12:37 +0530 Subject: [PATCH 014/350] feat: add webhook api functions --- src/api/webhook.ts | 149 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 src/api/webhook.ts diff --git a/src/api/webhook.ts b/src/api/webhook.ts new file mode 100644 index 00000000..ad5c75dc --- /dev/null +++ b/src/api/webhook.ts @@ -0,0 +1,149 @@ +import { default as axios } from "axios"; +import { apiUrl } from "../apiConfig"; + +export interface WebhookConfig { + id: string; + url: string; + events: string[]; + active: boolean; + createdAt?: string; + updatedAt?: string; + lastCalledAt?: string | null; + retryAttempts?: number; + retryDelay?: number; + timeout?: number; +} + +export interface WebhookResponse { + ok: boolean; + message?: string; + webhook?: WebhookConfig; + webhooks?: WebhookConfig[]; + error?: string; + details?: any; +} + +export const addWebhook = async (webhook: WebhookConfig, robotId: string): Promise => { + try { + const response = await axios.post(`${apiUrl}/webhook/add`, { + webhook, + robotId + }, { withCredentials: true }); + + if (response.status === 200) { + return response.data; + } else { + throw new Error(`Failed to add webhook. Status code: ${response.status}`); + } + } catch (error: any) { + console.error('Error adding webhook:', error.message || error); + return { + ok: false, + error: error.response?.data?.message || error.message || 'Failed to add webhook' + }; + } +}; + +export const updateWebhook = async (webhook: WebhookConfig, robotId: string): Promise => { + try { + const response = await axios.post(`${apiUrl}/webhook/update`, { + webhook, + robotId + }, { withCredentials: true }); + + if (response.status === 200) { + return response.data; + } else { + throw new Error(`Failed to update webhook. Status code: ${response.status}`); + } + } catch (error: any) { + console.error('Error updating webhook:', error.message || error); + return { + ok: false, + error: error.response?.data?.message || error.message || 'Failed to update webhook' + }; + } +}; + +export const removeWebhook = async (webhookId: string, robotId: string): Promise => { + try { + const response = await axios.post(`${apiUrl}/webhook/remove`, { + webhookId, + robotId + }, { withCredentials: true }); + + if (response.status === 200) { + return response.data; + } else { + throw new Error(`Failed to remove webhook. Status code: ${response.status}`); + } + } catch (error: any) { + console.error('Error removing webhook:', error.message || error); + return { + ok: false, + error: error.response?.data?.message || error.message || 'Failed to remove webhook' + }; + } +}; + +export const getWebhooks = async (robotId: string): Promise => { + try { + const response = await axios.get(`${apiUrl}/webhook/list/${robotId}`, { + withCredentials: true + }); + + if (response.status === 200) { + return response.data; + } else { + throw new Error(`Failed to fetch webhooks. Status code: ${response.status}`); + } + } catch (error: any) { + console.error('Error fetching webhooks:', error.message || error); + return { + ok: false, + error: error.response?.data?.message || error.message || 'Failed to fetch webhooks', + webhooks: [] + }; + } +}; + +export const testWebhook = async (webhook: WebhookConfig, robotId: string): Promise => { + try { + const response = await axios.post(`${apiUrl}/webhook/test`, { + webhook, + robotId + }, { withCredentials: true }); + + if (response.status === 200) { + return response.data; + } else { + throw new Error(`Failed to test webhook. Status code: ${response.status}`); + } + } catch (error: any) { + console.error('Error testing webhook:', error.message || error); + return { + ok: false, + error: error.response?.data?.message || error.message || 'Failed to test webhook' + }; + } +}; + +export const clearAllWebhooks = async (robotId: string): Promise => { + try { + const response = await axios.delete(`${apiUrl}/webhook/clear/${robotId}`, { + withCredentials: true + }); + + if (response.status === 200) { + return response.data; + } else { + throw new Error(`Failed to clear webhooks. Status code: ${response.status}`); + } + } catch (error: any) { + console.error('Error clearing webhooks:', error.message || error); + return { + ok: false, + error: error.response?.data?.message || error.message || 'Failed to clear webhooks' + }; + } +}; \ No newline at end of file From 13f8e95c8f987f1f9d2eb206546cd49c99ede942 Mon Sep 17 00:00:00 2001 From: Rohit Date: Tue, 27 May 2025 22:13:21 +0530 Subject: [PATCH 015/350] feat: use webhook api functions --- .../integration/IntegrationSettings.tsx | 231 +++++++++--------- 1 file changed, 121 insertions(+), 110 deletions(-) diff --git a/src/components/integration/IntegrationSettings.tsx b/src/components/integration/IntegrationSettings.tsx index 09cabbd8..607ff6e5 100644 --- a/src/components/integration/IntegrationSettings.tsx +++ b/src/components/integration/IntegrationSettings.tsx @@ -35,6 +35,8 @@ import Cookies from "js-cookie"; import { useTranslation } from "react-i18next"; import { useNavigate } from "react-router-dom"; +import { addWebhook, updateWebhook, removeWebhook, getWebhooks, testWebhook,WebhookConfig } from "../../api/webhook"; + interface IntegrationProps { isOpen: boolean; handleStart: (data: IntegrationSettings) => void; @@ -42,14 +44,6 @@ interface IntegrationProps { preSelectedIntegrationType?: "googleSheets" | "airtable" | "webhook" | null; } -export interface WebhookConfig { - id: string; - url: string; - events: string[]; - active: boolean; - lastCalledAt?: string | null; -} - export interface IntegrationSettings { spreadsheetId?: string; spreadsheetName?: string; @@ -156,7 +150,30 @@ export const IntegrationSettingsModal = ({ return true; }; - const addWebhook = async () => { + const fetchWebhooks = async () => { + try { + setLoading(true); + if (!recordingId) return; + + const response = await getWebhooks(recordingId); + + if (response.ok && response.webhooks) { + setSettings(prev => ({ + ...prev, + webhooks: response.webhooks + })); + } else { + notify("error", response.error || "Failed to fetch webhooks"); + } + setLoading(false); + } catch (error: any) { + setLoading(false); + console.error("Error fetching webhooks:", error); + notify("error", "Failed to fetch webhooks"); + } + }; + + const addWebhookSetting = async () => { if (!validateWebhookData(newWebhook.url, newWebhook.events)) { if (!newWebhook.url) { notify("error", "Please provide webhook URL"); @@ -166,6 +183,8 @@ export const IntegrationSettingsModal = ({ return; } + if (!recordingId) return; + try { setLoading(true); const webhookWithId = { @@ -173,31 +192,28 @@ export const IntegrationSettingsModal = ({ id: Date.now().toString(), }; - const response = await axios.post( - `${apiUrl}/webhook/add`, - { - webhook: webhookWithId, - robotId: recordingId, - }, - { withCredentials: true } - ); + const response = await addWebhook(webhookWithId, recordingId); - const updatedWebhooks = [...(settings.webhooks || []), webhookWithId]; - setSettings({ ...settings, webhooks: updatedWebhooks }); - - resetWebhookForm(); - await refreshRecordingData(); - notify("success", "Webhook added successfully"); + if (response.ok) { + const updatedWebhooks = [...(settings.webhooks || []), webhookWithId]; + setSettings({ ...settings, webhooks: updatedWebhooks }); + + resetWebhookForm(); + await refreshRecordingData(); + notify("success", "Webhook added successfully"); + } else { + notify("error", response.error || "Failed to add webhook"); + } setLoading(false); } catch (error: any) { setLoading(false); console.log("Error adding webhook:", error); - notify("error", `Error adding webhook: ${error.response?.data?.message || error.message}`); + notify("error", "Failed to add webhook"); } }; - const updateWebhook = async () => { - if (!editingWebhook) return; + const updateWebhookSetting = async () => { + if (!editingWebhook || !recordingId) return; if (!validateWebhookData(newWebhook.url, newWebhook.events, editingWebhook)) { if (!newWebhook.url) { @@ -210,112 +226,106 @@ export const IntegrationSettingsModal = ({ try { setLoading(true); - await axios.post( - `${apiUrl}/webhook/update`, - { - webhook: newWebhook, - robotId: recordingId, - }, - { withCredentials: true } - ); + const response = await updateWebhook(newWebhook, recordingId); - const updatedWebhooks = (settings.webhooks || []).map(w => - w.id === editingWebhook ? newWebhook : w - ); - setSettings({ ...settings, webhooks: updatedWebhooks }); + if (response.ok) { + const updatedWebhooks = (settings.webhooks || []).map(w => + w.id === editingWebhook ? newWebhook : w + ); + setSettings({ ...settings, webhooks: updatedWebhooks }); - resetWebhookForm(); - await refreshRecordingData(); - notify("success", "Webhook updated successfully"); + resetWebhookForm(); + await refreshRecordingData(); + notify("success", "Webhook updated successfully"); + } else { + notify("error", response.error || "Failed to update webhook"); + } setLoading(false); } catch (error: any) { setLoading(false); console.error("Error updating webhook:", error); - notify("error", `Error updating webhook: ${error.response?.data?.message || error.message}`); + notify("error", "Failed to update webhook"); } }; - const removeWebhook = async (webhookId: string) => { + const removeWebhookSetting = async (webhookId: string) => { + if (!recordingId) return; + try { setLoading(true); - await axios.post( - `${apiUrl}/webhook/remove`, - { - webhookId, - robotId: recordingId, - }, - { withCredentials: true } - ); + const response = await removeWebhook(webhookId, recordingId); - const updatedWebhooks = (settings.webhooks || []).filter(w => w.id !== webhookId); - setSettings({ ...settings, webhooks: updatedWebhooks }); + if (response.ok) { + const updatedWebhooks = (settings.webhooks || []).filter(w => w.id !== webhookId); + setSettings({ ...settings, webhooks: updatedWebhooks }); - await refreshRecordingData(); - notify("success", "Webhook removed successfully"); + await refreshRecordingData(); + notify("success", "Webhook removed successfully"); + } else { + notify("error", response.error || "Failed to remove webhook"); + } setLoading(false); } catch (error: any) { setLoading(false); console.error("Error removing webhook:", error); - notify("error", `Error removing webhook: ${error.response?.data?.message || error.message}`); + notify("error", "Failed to remove webhook"); } }; - const toggleWebhookStatus = async (webhookId: string) => { + const toggleWebhookStatusSetting = async (webhookId: string) => { + if (!recordingId) return; + try { const webhook = settings.webhooks?.find(w => w.id === webhookId); if (!webhook) return; const updatedWebhook = { ...webhook, active: !webhook.active }; - await axios.post( - `${apiUrl}/webhook/update`, - { - webhook: updatedWebhook, - robotId: recordingId, - }, - { withCredentials: true } - ); + const response = await updateWebhook(updatedWebhook, recordingId); - const updatedWebhooks = (settings.webhooks || []).map(w => - w.id === webhookId ? updatedWebhook : w - ); - setSettings({ ...settings, webhooks: updatedWebhooks }); + if (response.ok) { + const updatedWebhooks = (settings.webhooks || []).map(w => + w.id === webhookId ? updatedWebhook : w + ); + setSettings({ ...settings, webhooks: updatedWebhooks }); - await refreshRecordingData(); - notify("success", `Webhook ${updatedWebhook.active ? "enabled" : "disabled"}`); + await refreshRecordingData(); + notify("success", `Webhook ${updatedWebhook.active ? "enabled" : "disabled"}`); + } else { + notify("error", response.error || "Failed to update webhook"); + } } catch (error: any) { console.error("Error toggling webhook status:", error); - notify("error", `Error updating webhook: ${error.response?.data?.message || error.message}`); + notify("error", "Failed to update webhook"); } }; - const testWebhook = async (webhook: WebhookConfig) => { + const testWebhookSetting = async (webhook: WebhookConfig) => { + if (!recordingId) return; + try { setLoading(true); - await axios.post( - `${apiUrl}/webhook/test`, - { - webhook, - robotId: recordingId, - }, - { withCredentials: true } - ); + const response = await testWebhook(webhook, recordingId); - const updatedWebhooks = (settings.webhooks || []).map(w => - w.id === webhook.id ? { ...w, lastCalledAt: new Date().toISOString() } : w - ); - setSettings({ ...settings, webhooks: updatedWebhooks }); + if (response.ok) { + const updatedWebhooks = (settings.webhooks || []).map(w => + w.id === webhook.id ? { ...w, lastCalledAt: new Date().toISOString() } : w + ); + setSettings({ ...settings, webhooks: updatedWebhooks }); - notify("success", "Test webhook sent successfully"); + notify("success", "Test webhook sent successfully"); + } else { + notify("error", response.error || "Failed to test webhook"); + } setLoading(false); } catch (error: any) { setLoading(false); console.error("Error testing webhook:", error); - notify("error", `Error testing webhook: ${error.response?.data?.message || error.message}`); + notify("error", "Failed to test webhook"); } }; - const editWebhook = (webhook: WebhookConfig) => { + const editWebhookSetting = (webhook: WebhookConfig) => { setNewWebhook(webhook); setEditingWebhook(webhook.id); setShowWebhookForm(true); @@ -430,6 +440,9 @@ export const IntegrationSettingsModal = ({ if (!recordingId) return null; const updatedRecording = await getStoredRecording(recordingId); setRecording(updatedRecording); + + await fetchWebhooks(); + setRerenderRobots(true); return updatedRecording; }; @@ -568,8 +581,7 @@ export const IntegrationSettingsModal = ({ if (preSelectedIntegrationType) { setSettings(prev => ({ ...prev, integrationType: preSelectedIntegrationType })); - } - else if (recording.google_sheet_id) { + } else if (recording.google_sheet_id) { setSettings(prev => ({ ...prev, integrationType: "googleSheets" })); } else if (recording.airtable_base_id) { setSettings(prev => ({ @@ -578,15 +590,18 @@ export const IntegrationSettingsModal = ({ airtableBaseName: recording.airtable_base_name || "", airtableTableName: recording.airtable_table_name || "", airtableTableId: recording.airtable_table_id || "", - integrationType: recording.airtable_base_id ? "airtable" : "googleSheets" - })); - } else if (recording.webhooks && recording.webhooks.length > 0) { - setSettings(prev => ({ - ...prev, - webhooks: recording.webhooks, - integrationType: "webhook" + integrationType: "airtable" })); } + + await fetchWebhooks(); + + if (!preSelectedIntegrationType && !recording.google_sheet_id && !recording.airtable_base_id) { + const webhookResponse = await getWebhooks(recordingId); + if (webhookResponse.ok && webhookResponse.webhooks && webhookResponse.webhooks.length > 0) { + setSettings(prev => ({ ...prev, integrationType: "webhook" })); + } + } } setLoading(false); @@ -617,8 +632,6 @@ export const IntegrationSettingsModal = ({ switch (event) { case "run_completed": return "Run finished"; - case "run_completed_success": - return "Run finished successfully"; case "run_failed": return "Run failed"; default: @@ -680,7 +693,7 @@ export const IntegrationSettingsModal = ({ }} style={{ display: "flex", flexDirection: "column", alignItems: "center", background: 'white', color: '#ff00c3' }} > - Google Sheets + Google Sheets Google Sheets @@ -693,7 +706,7 @@ export const IntegrationSettingsModal = ({ }} style={{ display: "flex", flexDirection: "column", alignItems: "center", background: 'white', color: '#ff00c3' }} > - Airtable + Airtable Airtable @@ -706,7 +719,7 @@ export const IntegrationSettingsModal = ({ }} style={{ display: "flex", flexDirection: "column", alignItems: "center", background: 'white', color: '#ff00c3' }} > - Webhook + Webhook Webhooks @@ -968,7 +981,7 @@ export const IntegrationSettingsModal = ({ toggleWebhookStatus(webhook.id)} + onChange={() => toggleWebhookStatusSetting(webhook.id)} size="small" /> @@ -976,7 +989,7 @@ export const IntegrationSettingsModal = ({ testWebhook(webhook)} + onClick={() => testWebhookSetting(webhook)} disabled={loading || !webhook.active} title="Test" > @@ -984,7 +997,7 @@ export const IntegrationSettingsModal = ({ editWebhook(webhook)} + onClick={() => editWebhookSetting(webhook)} disabled={loading} title="Edit" > @@ -992,7 +1005,7 @@ export const IntegrationSettingsModal = ({ removeWebhook(webhook.id)} + onClick={() => removeWebhookSetting(webhook.id)} disabled={loading} title="Delete" > @@ -1033,7 +1046,6 @@ export const IntegrationSettingsModal = ({ sx={{ minWidth: "200px" }} > Run finished - Run finished successfully Run failed @@ -1057,7 +1069,7 @@ export const IntegrationSettingsModal = ({ if (!validateWebhookData(newWebhook.url, newWebhook.events)) { return; } - addWebhook(); + addWebhookSetting(); }} disabled={!newWebhook.url || !newWebhook.events || newWebhook.events.length === 0 || loading || !!urlError} > @@ -1112,7 +1124,6 @@ export const IntegrationSettingsModal = ({ required > Run finished - Run finished successfully Run failed @@ -1132,7 +1143,7 @@ export const IntegrationSettingsModal = ({ + */} From 9519414a10554e3a5250e20bce3dbb6668384de5 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 28 May 2025 13:31:30 +0530 Subject: [PATCH 025/350] feat: use uuid for webhook id --- server/src/routes/webhook.ts | 3 ++- src/components/integration/IntegrationSettings.tsx | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/routes/webhook.ts b/server/src/routes/webhook.ts index 87f0808a..2eb85bb6 100644 --- a/server/src/routes/webhook.ts +++ b/server/src/routes/webhook.ts @@ -2,6 +2,7 @@ import { Router, Request, Response } from 'express'; import Robot from '../models/Robot'; import { requireSignIn } from '../middlewares/auth'; import axios from 'axios'; +import { uuid } from "uuidv4"; export const router = Router(); @@ -85,7 +86,7 @@ router.post('/add', requireSignIn, async (req: Request, res: Response) => { const newWebhook: WebhookConfig = { ...webhook, - id: webhook.id || Date.now().toString(), + id: webhook.id || uuid(), createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), lastCalledAt: null, diff --git a/src/components/integration/IntegrationSettings.tsx b/src/components/integration/IntegrationSettings.tsx index 607ff6e5..b2a7dc26 100644 --- a/src/components/integration/IntegrationSettings.tsx +++ b/src/components/integration/IntegrationSettings.tsx @@ -29,6 +29,7 @@ import axios from "axios"; import { useGlobalInfoStore } from "../../context/globalInfo"; import { getStoredRecording } from "../../api/storage"; import { apiUrl } from "../../apiConfig.js"; +import { uuid } from "uuidv4"; import Cookies from "js-cookie"; @@ -189,7 +190,7 @@ export const IntegrationSettingsModal = ({ setLoading(true); const webhookWithId = { ...newWebhook, - id: Date.now().toString(), + id: uuid(), }; const response = await addWebhook(webhookWithId, recordingId); From 1d3a6674cdf0a2781e1f25657041b37965484433 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 28 May 2025 14:22:43 +0530 Subject: [PATCH 026/350] feat: replace uuidv4 package with uuid --- server/src/api/record.ts | 2 +- server/src/browser-management/controller.ts | 2 +- server/src/routes/storage.ts | 2 +- server/src/routes/webhook.ts | 2 +- server/src/workflow-management/classes/Generator.ts | 2 +- server/src/workflow-management/scheduler/index.ts | 2 +- src/components/integration/IntegrationSettings.tsx | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/api/record.ts b/server/src/api/record.ts index 6b7c5942..02e6e09b 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -7,7 +7,7 @@ import Robot from "../models/Robot"; import Run from "../models/Run"; const router = Router(); import { getDecryptedProxyConfig } from "../routes/proxy"; -import { uuid } from "uuidv4"; +import { v4 as uuid } from "uuid"; import { createRemoteBrowserForRun, destroyRemoteBrowser } from "../browser-management/controller"; import logger from "../logger"; import { browserPool } from "../server"; diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index 7c5271bf..3da388a0 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -3,7 +3,7 @@ * Holds the singleton instances of browser pool and socket.io server. */ import { Socket } from "socket.io"; -import { uuid } from 'uuidv4'; +import { v4 as uuid } from "uuid"; import { createSocketConnection, createSocketConnectionForRun } from "../socket-connection/connection"; import { io, browserPool } from "../server"; diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 0942600c..b4e8cdfd 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -4,7 +4,7 @@ import { createRemoteBrowserForRun, getActiveBrowserIdByState } from "../browser import { chromium } from 'playwright-extra'; import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import { browserPool } from "../server"; -import { uuid } from "uuidv4"; +import { v4 as uuid } from "uuid"; import moment from 'moment-timezone'; import cron from 'node-cron'; import { getDecryptedProxyConfig } from './proxy'; diff --git a/server/src/routes/webhook.ts b/server/src/routes/webhook.ts index 2eb85bb6..bba8ec8f 100644 --- a/server/src/routes/webhook.ts +++ b/server/src/routes/webhook.ts @@ -2,7 +2,7 @@ import { Router, Request, Response } from 'express'; import Robot from '../models/Robot'; import { requireSignIn } from '../middlewares/auth'; import axios from 'axios'; -import { uuid } from "uuidv4"; +import { v4 as uuid } from "uuid"; export const router = Router(); diff --git a/server/src/workflow-management/classes/Generator.ts b/server/src/workflow-management/classes/Generator.ts index 1be328aa..27123e22 100644 --- a/server/src/workflow-management/classes/Generator.ts +++ b/server/src/workflow-management/classes/Generator.ts @@ -15,7 +15,7 @@ import { import { CustomActions } from "../../../../src/shared/types"; import Robot from "../../models/Robot"; import { getBestSelectorForAction } from "../utils"; -import { uuid } from "uuidv4"; +import { v4 as uuid } from "uuid"; import { capture } from "../../utils/analytics" import { decrypt, encrypt } from "../../utils/auth"; diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 464b3984..69b38f73 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -1,4 +1,4 @@ -import { uuid } from "uuidv4"; +import { v4 as uuid } from "uuid"; import { chromium } from 'playwright-extra'; import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import { io, Socket } from "socket.io-client"; diff --git a/src/components/integration/IntegrationSettings.tsx b/src/components/integration/IntegrationSettings.tsx index b2a7dc26..bf647938 100644 --- a/src/components/integration/IntegrationSettings.tsx +++ b/src/components/integration/IntegrationSettings.tsx @@ -29,7 +29,7 @@ import axios from "axios"; import { useGlobalInfoStore } from "../../context/globalInfo"; import { getStoredRecording } from "../../api/storage"; import { apiUrl } from "../../apiConfig.js"; -import { uuid } from "uuidv4"; +import { v4 as uuid } from "uuid"; import Cookies from "js-cookie"; From b493e6b8facdc4d84879cf274d2130c310cefc6d Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 28 May 2025 14:35:57 +0530 Subject: [PATCH 027/350] feat: map multiple webhook events --- src/components/integration/IntegrationSettings.tsx | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/components/integration/IntegrationSettings.tsx b/src/components/integration/IntegrationSettings.tsx index bf647938..53e54540 100644 --- a/src/components/integration/IntegrationSettings.tsx +++ b/src/components/integration/IntegrationSettings.tsx @@ -977,7 +977,18 @@ export const IntegrationSettingsModal = ({ {settings.webhooks.map((webhook) => ( {webhook.url} - {formatEventName(webhook.events[0])} + + + {webhook.events.map((event) => ( + + ))} + + {formatLastCalled(webhook.lastCalledAt)} Date: Wed, 28 May 2025 14:48:26 +0530 Subject: [PATCH 028/350] feat: improve access and validation feedback --- src/components/integration/IntegrationSettings.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/components/integration/IntegrationSettings.tsx b/src/components/integration/IntegrationSettings.tsx index 53e54540..e385a4d8 100644 --- a/src/components/integration/IntegrationSettings.tsx +++ b/src/components/integration/IntegrationSettings.tsx @@ -1046,6 +1046,8 @@ export const IntegrationSettingsModal = ({ }} error={!!urlError} helperText={urlError} + required + aria-describedby="webhook-url-help" /> Run finished Run failed From e82d700edba62b00c9d43fe4676a7a2053053800 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 28 May 2025 14:51:09 +0530 Subject: [PATCH 029/350] feat: validate url format --- src/components/integration/IntegrationSettings.tsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/components/integration/IntegrationSettings.tsx b/src/components/integration/IntegrationSettings.tsx index e385a4d8..c964d6b7 100644 --- a/src/components/integration/IntegrationSettings.tsx +++ b/src/components/integration/IntegrationSettings.tsx @@ -133,6 +133,13 @@ export const IntegrationSettingsModal = ({ return false; } + try { + new URL(url); + } catch { + setUrlError("Please provide a valid URL"); + return false; + } + const existingWebhook = settings.webhooks?.find( (webhook) => webhook.url === url && webhook.id !== excludeId ); From 6834b00be67168270f6a4894a5a2d8c59d907543 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 28 May 2025 14:53:38 +0530 Subject: [PATCH 030/350] feat: improve captured text payload structure --- server/src/api/record.ts | 2 +- server/src/pgboss-worker.ts | 2 +- server/src/workflow-management/scheduler/index.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/api/record.ts b/server/src/api/record.ts index 02e6e09b..e05aa8ce 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -677,7 +677,7 @@ async function executeRun(id: string, userId: string) { started_at: plainRun.startedAt, finished_at: new Date().toLocaleString(), extracted_data: { - captured_texts: categorizedOutput.scrapeSchema["schema_merged"] || [], + captured_texts: Object.values(categorizedOutput.scrapeSchema).flat() || [], captured_lists: categorizedOutput.scrapeList, total_rows: totalRowsExtracted, captured_texts_count: totalSchemaItemsExtracted, diff --git a/server/src/pgboss-worker.ts b/server/src/pgboss-worker.ts index 2bcbd21e..1a32f79b 100644 --- a/server/src/pgboss-worker.ts +++ b/server/src/pgboss-worker.ts @@ -365,7 +365,7 @@ async function processRunExecution(job: Job) { started_at: plainRun.startedAt, finished_at: new Date().toLocaleString(), extracted_data: { - captured_texts: categorizedOutput.scrapeSchema["schema_merged"] || [], + captured_texts: Object.values(categorizedOutput.scrapeSchema).flat() || [], captured_lists: categorizedOutput.scrapeList, total_rows: totalRowsExtracted, captured_texts_count: totalSchemaItemsExtracted, diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 69b38f73..b40e55f2 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -204,7 +204,7 @@ async function executeRun(id: string, userId: string) { started_at: plainRun.startedAt, finished_at: new Date().toLocaleString(), extracted_data: { - captured_texts: categorizedOutput.scrapeSchema["schema_merged"] || [], + captured_texts: Object.values(categorizedOutput.scrapeSchema).flat() || [], captured_lists: categorizedOutput.scrapeList, total_rows: totalRowsExtracted, captured_texts_count: totalSchemaItemsExtracted, From 3f78a1323e83f0b11073cf150ef47889335a8a18 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 28 May 2025 15:00:30 +0530 Subject: [PATCH 031/350] feat: rm spread op to reduce time complexity --- .../classes/Interpreter.ts | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/server/src/workflow-management/classes/Interpreter.ts b/server/src/workflow-management/classes/Interpreter.ts index fd7d627a..ca853489 100644 --- a/server/src/workflow-management/classes/Interpreter.ts +++ b/server/src/workflow-management/classes/Interpreter.ts @@ -374,23 +374,17 @@ export class WorkflowInterpreter { scrapeSchemaOutput: Object.keys(mergedScrapeSchema).length > 0 ? { "schema_merged": [mergedScrapeSchema] } : this.serializableDataByType.scrapeSchema.reduce((reducedObject, item, index) => { - return { - [`schema_${index}`]: item, - ...reducedObject, - } - }, {}), + reducedObject[`schema_${index}`] = item; + return reducedObject; + }, {} as Record), scrapeListOutput: this.serializableDataByType.scrapeList.reduce((reducedObject, item, index) => { - return { - [`list_${index}`]: item, - ...reducedObject, - } - }, {}), + reducedObject[`list_${index}`] = item; + return reducedObject; + }, {} as Record), binaryOutput: this.binaryData.reduce((reducedObject, item, index) => { - return { - [`item_${index}`]: item, - ...reducedObject, - } - }, {}) + reducedObject[`item_${index}`] = item; + return reducedObject; + }, {} as Record) } logger.log('debug', `Interpretation finished`); From 76dffd260d7278ef54c291b1e5f741ef5154f774 Mon Sep 17 00:00:00 2001 From: Rohit Date: Wed, 28 May 2025 15:06:09 +0530 Subject: [PATCH 032/350] feat: rm redundant validation --- .../integration/IntegrationSettings.tsx | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/components/integration/IntegrationSettings.tsx b/src/components/integration/IntegrationSettings.tsx index c964d6b7..cf907106 100644 --- a/src/components/integration/IntegrationSettings.tsx +++ b/src/components/integration/IntegrationSettings.tsx @@ -1079,20 +1079,7 @@ export const IntegrationSettingsModal = ({ )} + + {t('mainmenu.apidocs')} + ); }; From 4c474093cb167b121147f93377a10312f54be0d5 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 28 May 2025 23:43:45 +0530 Subject: [PATCH 034/350] feat: wrap inside typographu --- src/components/api/ApiKey.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/components/api/ApiKey.tsx b/src/components/api/ApiKey.tsx index 4286fd3c..00b331a8 100644 --- a/src/components/api/ApiKey.tsx +++ b/src/components/api/ApiKey.tsx @@ -158,9 +158,12 @@ const ApiKeyManager = () => { )} + + View and test your API endpoints: - {t('mainmenu.apidocs')} + here + ); }; From 5f65097c31085deb69b30d4544bf497d60930ffe Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 28 May 2025 23:44:56 +0530 Subject: [PATCH 035/350] feat: link style --- src/components/api/ApiKey.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/api/ApiKey.tsx b/src/components/api/ApiKey.tsx index 00b331a8..e6d58c51 100644 --- a/src/components/api/ApiKey.tsx +++ b/src/components/api/ApiKey.tsx @@ -159,8 +159,8 @@ const ApiKeyManager = () => { )} - View and test your API endpoints: - + View and test your API endpoints + here From eef2326964174b1984ad0feba7bbf7fc86441c9b Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 28 May 2025 23:45:07 +0530 Subject: [PATCH 036/350] chore: lint --- src/components/api/ApiKey.tsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/components/api/ApiKey.tsx b/src/components/api/ApiKey.tsx index e6d58c51..d6cc83a5 100644 --- a/src/components/api/ApiKey.tsx +++ b/src/components/api/ApiKey.tsx @@ -159,11 +159,11 @@ const ApiKeyManager = () => { )} - View and test your API endpoints - - here - - + View and test your API endpoints + + here + + ); }; From a0827f4e099db90afe14a9dcc12f960e042abff3 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 28 May 2025 23:48:54 +0530 Subject: [PATCH 037/350] feat: move instructions above --- src/components/api/ApiKey.tsx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/components/api/ApiKey.tsx b/src/components/api/ApiKey.tsx index d6cc83a5..ea6c21ac 100644 --- a/src/components/api/ApiKey.tsx +++ b/src/components/api/ApiKey.tsx @@ -110,6 +110,12 @@ const ApiKeyManager = () => { {t('apikey.title')} + + + View and test your API endpoints + + here + {apiKey ? ( @@ -158,12 +164,6 @@ const ApiKeyManager = () => { )} - - View and test your API endpoints - - here - - ); }; From 82d361be7640a12ccb766b6385453892907afe36 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 28 May 2025 23:49:06 +0530 Subject: [PATCH 038/350] chore: lint --- src/components/api/ApiKey.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/api/ApiKey.tsx b/src/components/api/ApiKey.tsx index ea6c21ac..c0248907 100644 --- a/src/components/api/ApiKey.tsx +++ b/src/components/api/ApiKey.tsx @@ -111,7 +111,7 @@ const ApiKeyManager = () => { {t('apikey.title')} - + View and test your API endpoints here From 2155b1a694cc7ce0c4a38864d5c5309b0385db0e Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 28 May 2025 23:50:04 +0530 Subject: [PATCH 039/350] feat: move instructions above --- src/components/api/ApiKey.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/components/api/ApiKey.tsx b/src/components/api/ApiKey.tsx index c0248907..b02a136f 100644 --- a/src/components/api/ApiKey.tsx +++ b/src/components/api/ApiKey.tsx @@ -108,15 +108,15 @@ const ApiKeyManager = () => { return ( - - {t('apikey.title')} - - + View and test your API endpoints here + + {t('apikey.title')} + {apiKey ? ( From 1c151aedbd9debc563c0ee826219108bf60d8270 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 28 May 2025 23:51:40 +0530 Subject: [PATCH 040/350] feat: add docs --- src/components/api/ApiKey.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/components/api/ApiKey.tsx b/src/components/api/ApiKey.tsx index b02a136f..ac8c3ea1 100644 --- a/src/components/api/ApiKey.tsx +++ b/src/components/api/ApiKey.tsx @@ -109,10 +109,13 @@ const ApiKeyManager = () => { return ( - View and test your API endpoints + Start by creating an API key below. Then, - here + test your API + or read the + API documentation + for setup instructions. {t('apikey.title')} From 033dd05a03ddc7779e344552ad20d46897cac21d Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 28 May 2025 23:51:54 +0530 Subject: [PATCH 041/350] chore: lint --- src/components/api/ApiKey.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/api/ApiKey.tsx b/src/components/api/ApiKey.tsx index ac8c3ea1..ff8375cd 100644 --- a/src/components/api/ApiKey.tsx +++ b/src/components/api/ApiKey.tsx @@ -111,10 +111,10 @@ const ApiKeyManager = () => { Start by creating an API key below. Then, - test your API + test your API or read the - API documentation + API documentation for setup instructions. From 0ed1bac2dca5729d9119143c75c8177cf1efb0b9 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 28 May 2025 23:52:55 +0530 Subject: [PATCH 042/350] fix: url margins --- src/components/api/ApiKey.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/api/ApiKey.tsx b/src/components/api/ApiKey.tsx index ff8375cd..9a6c8bb7 100644 --- a/src/components/api/ApiKey.tsx +++ b/src/components/api/ApiKey.tsx @@ -110,10 +110,10 @@ const ApiKeyManager = () => { Start by creating an API key below. Then, - + test your API - or read the + or read the API documentation for setup instructions. From 0e67f8807edeb4c4e33dc5a38cb4f5d606171876 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 28 May 2025 23:53:43 +0530 Subject: [PATCH 043/350] feat: use body1 --- src/components/api/ApiKey.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/api/ApiKey.tsx b/src/components/api/ApiKey.tsx index 9a6c8bb7..db927227 100644 --- a/src/components/api/ApiKey.tsx +++ b/src/components/api/ApiKey.tsx @@ -108,7 +108,7 @@ const ApiKeyManager = () => { return ( - + Start by creating an API key below. Then, test your API From 71f98e0be72ab6de3de42ccbdf070b270fe16bcd Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 28 May 2025 23:54:34 +0530 Subject: [PATCH 044/350] feat: set margin bottom --- src/components/api/ApiKey.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/api/ApiKey.tsx b/src/components/api/ApiKey.tsx index db927227..e8dc9a97 100644 --- a/src/components/api/ApiKey.tsx +++ b/src/components/api/ApiKey.tsx @@ -108,7 +108,7 @@ const ApiKeyManager = () => { return ( - + Start by creating an API key below. Then, test your API From c8b036b91e37647d51b19a75a97711cfc646d0c8 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 28 May 2025 23:56:08 +0530 Subject: [PATCH 045/350] feat: align left --- src/components/api/ApiKey.tsx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/components/api/ApiKey.tsx b/src/components/api/ApiKey.tsx index e8dc9a97..cac33daf 100644 --- a/src/components/api/ApiKey.tsx +++ b/src/components/api/ApiKey.tsx @@ -117,7 +117,12 @@ const ApiKeyManager = () => { API documentation for setup instructions. - + {t('apikey.title')} {apiKey ? ( From d767e89dfeef76b79bb76d38685557c1f186704e Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 28 May 2025 23:56:17 +0530 Subject: [PATCH 046/350] chore: lint --- src/components/api/ApiKey.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/api/ApiKey.tsx b/src/components/api/ApiKey.tsx index cac33daf..c5603446 100644 --- a/src/components/api/ApiKey.tsx +++ b/src/components/api/ApiKey.tsx @@ -110,7 +110,7 @@ const ApiKeyManager = () => { Start by creating an API key below. Then, - + test your API or read the From c128e187f3cba53e1e5630cf77adc9069c2ec601 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Wed, 28 May 2025 23:56:55 +0530 Subject: [PATCH 047/350] feat: increase margin bottom --- src/components/api/ApiKey.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/api/ApiKey.tsx b/src/components/api/ApiKey.tsx index c5603446..244eaa33 100644 --- a/src/components/api/ApiKey.tsx +++ b/src/components/api/ApiKey.tsx @@ -108,7 +108,7 @@ const ApiKeyManager = () => { return ( - + Start by creating an API key below. Then, test your API From eb00e398635b4831d83ff94a62a57662ef71db83 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 29 May 2025 00:00:10 +0530 Subject: [PATCH 048/350] fix: docs url --- src/components/api/ApiKey.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/api/ApiKey.tsx b/src/components/api/ApiKey.tsx index 244eaa33..9feb9551 100644 --- a/src/components/api/ApiKey.tsx +++ b/src/components/api/ApiKey.tsx @@ -113,7 +113,7 @@ const ApiKeyManager = () => { test your API - or read the + or read the API documentation for setup instructions. From c788c2a400977512b791fe710f481163eede278b Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 29 May 2025 01:40:43 +0530 Subject: [PATCH 049/350] chore: core 0.0.17 --- maxun-core/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maxun-core/package.json b/maxun-core/package.json index c9620bcb..06dc0734 100644 --- a/maxun-core/package.json +++ b/maxun-core/package.json @@ -1,6 +1,6 @@ { "name": "maxun-core", - "version": "0.0.16", + "version": "0.0.17", "description": "Core package for Maxun, responsible for data extraction", "main": "build/index.js", "typings": "build/index.d.ts", From 1727b74704dab0c65a3b6e9adf554f7952ce7c79 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 29 May 2025 01:41:26 +0530 Subject: [PATCH 050/350] chore: v0.0.15 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 795795bd..b75e8583 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "maxun", - "version": "0.0.14", + "version": "0.0.15", "author": "Maxun", "license": "AGPL-3.0-or-later", "dependencies": { From 1062d377f2e448d68d6fac0ebbe21e32d4437e31 Mon Sep 17 00:00:00 2001 From: Karishma Shukla Date: Thu, 29 May 2025 14:09:50 +0530 Subject: [PATCH 051/350] fix: increase timeout on url change --- server/src/browser-management/inputHandlers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/browser-management/inputHandlers.ts b/server/src/browser-management/inputHandlers.ts index 9b6551a5..6b5016f3 100644 --- a/server/src/browser-management/inputHandlers.ts +++ b/server/src/browser-management/inputHandlers.ts @@ -386,7 +386,7 @@ const handleChangeUrl = async (generator: WorkflowGenerator, page: Page, url: st if (url) { await generator.onChangeUrl(url, page); try { - await page.goto(url, { waitUntil: 'networkidle', timeout: 10000 }); + await page.goto(url, { waitUntil: 'networkidle', timeout: 100000 }); logger.log('debug', `Went to ${url}`); } catch (e) { const { message } = e as Error; From 37c9db62129897cd329f64e8fdd43a6334edab1d Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 29 May 2025 15:49:52 +0530 Subject: [PATCH 052/350] chore: use 0.0.17 core --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b75e8583..9652ed87 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "lodash": "^4.17.21", "loglevel": "^1.8.0", "loglevel-plugin-remote": "^0.6.8", - "maxun-core": "^0.0.16", + "maxun-core": "^0.0.17", "minio": "^8.0.1", "moment-timezone": "^0.5.45", "node-cron": "^3.0.3", From cc83e1ad28155c77e81830e51e6351486634f2ba Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 29 May 2025 16:31:16 +0530 Subject: [PATCH 053/350] fix: define module in commonjs scope --- server/src/db/config/database.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/db/config/database.js b/server/src/db/config/database.js index ae6972d1..4607c899 100644 --- a/server/src/db/config/database.js +++ b/server/src/db/config/database.js @@ -1,4 +1,4 @@ -import dotenv from 'dotenv'; +const dotenv = require('dotenv'); dotenv.config({ path: './.env' }); // Validate required environment variables From e9dab714ecd65a7e894143b09d61ab2ec2e67be4 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 29 May 2025 16:38:25 +0530 Subject: [PATCH 054/350] feat: run migrations before start script --- server/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/Dockerfile b/server/Dockerfile index af0860fc..00310418 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -43,8 +43,9 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* \ && mkdir -p /tmp/.X11-unix && chmod 1777 /tmp/.X11-unix -# Expose the backend port +# Expose backend port EXPOSE ${BACKEND_PORT:-8080} -# Start the backend using the start script -CMD ["npm", "run", "server"] +# Run migrations & start backend using start script +#CMD ["npm", "run", "server"] +CMD ["sh", "-c", "npm run migrate && npm run server"] \ No newline at end of file From 8d1ba511d55b5d88642a86ba20e828d6f0ad07ca Mon Sep 17 00:00:00 2001 From: amhsirak Date: Thu, 29 May 2025 16:40:39 +0530 Subject: [PATCH 055/350] feat: link webhooks docs --- src/components/integration/IntegrationSettings.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/integration/IntegrationSettings.tsx b/src/components/integration/IntegrationSettings.tsx index cf907106..17f0ec72 100644 --- a/src/components/integration/IntegrationSettings.tsx +++ b/src/components/integration/IntegrationSettings.tsx @@ -1074,7 +1074,7 @@ export const IntegrationSettingsModal = ({ - Refer to the API documentation for examples and details. + Refer to the API documentation for examples and details. + From 50285955adccbac848036b51f8f31ff0ecd8817d Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 13 Jun 2025 18:55:27 +0530 Subject: [PATCH 123/350] fix: format --- src/components/dashboard/MainMenu.tsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/components/dashboard/MainMenu.tsx b/src/components/dashboard/MainMenu.tsx index 2d1e4124..214cc9b9 100644 --- a/src/components/dashboard/MainMenu.tsx +++ b/src/components/dashboard/MainMenu.tsx @@ -115,11 +115,11 @@ export const MainMenu = ({ value = 'robots', handleChangeContent }: MainMenuProp - + From dd60265b4f8b5845b2a50ce6a78207b4cbc4672f Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 13 Jun 2025 18:56:07 +0530 Subject: [PATCH 124/350] fix: import description icon --- src/components/dashboard/MainMenu.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/dashboard/MainMenu.tsx b/src/components/dashboard/MainMenu.tsx index 214cc9b9..12d0a1ec 100644 --- a/src/components/dashboard/MainMenu.tsx +++ b/src/components/dashboard/MainMenu.tsx @@ -4,7 +4,7 @@ import Tab from '@mui/material/Tab'; import Box from '@mui/material/Box'; import { useNavigate } from 'react-router-dom'; import { Paper, Button, useTheme } from "@mui/material"; -import { AutoAwesome, FormatListBulleted, VpnKey, Usb, CloudQueue, Code, } from "@mui/icons-material"; +import { AutoAwesome, FormatListBulleted, VpnKey, Usb, CloudQueue, Code, Description} from "@mui/icons-material"; import { apiUrl } from "../../apiConfig"; import { useTranslation } from 'react-i18next'; import i18n from '../../i18n'; From 971075e108bbed6d15c4f0267ed73b9f1f9cad70 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 13 Jun 2025 18:57:20 +0530 Subject: [PATCH 125/350] feat: replace onClick with href --- src/components/dashboard/MainMenu.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/components/dashboard/MainMenu.tsx b/src/components/dashboard/MainMenu.tsx index 12d0a1ec..76210793 100644 --- a/src/components/dashboard/MainMenu.tsx +++ b/src/components/dashboard/MainMenu.tsx @@ -115,9 +115,7 @@ export const MainMenu = ({ value = 'robots', handleChangeContent }: MainMenuProp - From a4f40a3b94cad3ac573ea9d61bf48464f6d0d368 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 13 Jun 2025 18:58:21 +0530 Subject: [PATCH 126/350] feat: use Description as startIcon --- src/components/dashboard/MainMenu.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/dashboard/MainMenu.tsx b/src/components/dashboard/MainMenu.tsx index 76210793..24d68003 100644 --- a/src/components/dashboard/MainMenu.tsx +++ b/src/components/dashboard/MainMenu.tsx @@ -115,8 +115,8 @@ export const MainMenu = ({ value = 'robots', handleChangeContent }: MainMenuProp - From 722a021e086a68efc474b053c786788884f9abcd Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 13 Jun 2025 18:59:02 +0530 Subject: [PATCH 127/350] chore: remove unused imports --- src/components/dashboard/MainMenu.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/components/dashboard/MainMenu.tsx b/src/components/dashboard/MainMenu.tsx index 24d68003..b9761e62 100644 --- a/src/components/dashboard/MainMenu.tsx +++ b/src/components/dashboard/MainMenu.tsx @@ -4,8 +4,7 @@ import Tab from '@mui/material/Tab'; import Box from '@mui/material/Box'; import { useNavigate } from 'react-router-dom'; import { Paper, Button, useTheme } from "@mui/material"; -import { AutoAwesome, FormatListBulleted, VpnKey, Usb, CloudQueue, Code, Description} from "@mui/icons-material"; -import { apiUrl } from "../../apiConfig"; +import { AutoAwesome, FormatListBulleted, VpnKey, Usb, CloudQueue, Description} from "@mui/icons-material"; import { useTranslation } from 'react-i18next'; import i18n from '../../i18n'; From 7b5faf54ee3adb5a94fa6a47f5e0af19ea6b888e Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 13 Jun 2025 18:59:49 +0530 Subject: [PATCH 128/350] chore: rename to Documentation --- src/components/dashboard/MainMenu.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/dashboard/MainMenu.tsx b/src/components/dashboard/MainMenu.tsx index b9761e62..4f761394 100644 --- a/src/components/dashboard/MainMenu.tsx +++ b/src/components/dashboard/MainMenu.tsx @@ -115,7 +115,7 @@ export const MainMenu = ({ value = 'robots', handleChangeContent }: MainMenuProp {t('mainmenu.feedback')} From eba5e25e42d5ccae6d017aa18a77740f922f841c Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 13 Jun 2025 19:00:28 +0530 Subject: [PATCH 129/350] feat: rearrange order of menu buttons --- src/components/dashboard/MainMenu.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/components/dashboard/MainMenu.tsx b/src/components/dashboard/MainMenu.tsx index 4f761394..cfd6b5b3 100644 --- a/src/components/dashboard/MainMenu.tsx +++ b/src/components/dashboard/MainMenu.tsx @@ -111,12 +111,12 @@ export const MainMenu = ({ value = 'robots', handleChangeContent }: MainMenuProp {/* */} - + From 8c2eb514421be810f13cdde45f28d3151500abb9 Mon Sep 17 00:00:00 2001 From: amhsirak Date: Fri, 13 Jun 2025 19:00:56 +0530 Subject: [PATCH 130/350] chore: lint --- src/components/dashboard/MainMenu.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/dashboard/MainMenu.tsx b/src/components/dashboard/MainMenu.tsx index cfd6b5b3..993e78a8 100644 --- a/src/components/dashboard/MainMenu.tsx +++ b/src/components/dashboard/MainMenu.tsx @@ -4,7 +4,7 @@ import Tab from '@mui/material/Tab'; import Box from '@mui/material/Box'; import { useNavigate } from 'react-router-dom'; import { Paper, Button, useTheme } from "@mui/material"; -import { AutoAwesome, FormatListBulleted, VpnKey, Usb, CloudQueue, Description} from "@mui/icons-material"; +import { AutoAwesome, FormatListBulleted, VpnKey, Usb, CloudQueue, Description } from "@mui/icons-material"; import { useTranslation } from 'react-i18next'; import i18n from '../../i18n'; @@ -112,7 +112,7 @@ export const MainMenu = ({ value = 'robots', handleChangeContent }: MainMenuProp {t('mainmenu.apidocs')} */} +
+ Technical details + 
${error.toString()}
+
+ + + + `); + iframeDoc.close(); + + window.addEventListener("message", (event) => { + if (event.data === "retry-dom-mode") { + if (socket) { + socket.emit("enable-dom-streaming"); + } + } + }); + } catch (e) { + console.error("Failed to write error message to iframe:", e); + } + } + }; + + useEffect(() => { + return () => { + if (iframeRef.current) { + const iframeDoc = iframeRef.current.contentDocument; + if (iframeDoc) { + const handlers = (iframeDoc as any)._domRendererHandlers; + if (handlers) { + Object.entries(handlers).forEach(([event, handler]) => { + iframeDoc.removeEventListener( + event, + handler as EventListener, + true + ); + }); + } + } + } + }; + }, []); + + return ( +
+