diff --git a/.gitignore b/.gitignore index 74279fc9..a2a80766 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ /build package-lock.json + +docker-compose.yml \ No newline at end of file diff --git a/ENVEXAMPLE b/ENVEXAMPLE index 650c7027..3c999de4 100644 --- a/ENVEXAMPLE +++ b/ENVEXAMPLE @@ -30,3 +30,8 @@ GOOGLE_REDIRECT_URI=your_google_redirect_uri # Telemetry Settings - Please keep it enabled. Keeping it enabled helps us understand how the product is used and assess the impact of any new changes. MAXUN_TELEMETRY=true + +#airtable integration + +AIRTABLE_CLIENT_ID=your_airtable_client_id +AIRTABLE_REDIRECT_URI=http://localhost:8080/auth/airtable/callback diff --git a/package.json b/package.json index 0580d2de..0f1e977d 100644 --- a/package.json +++ b/package.json @@ -16,16 +16,19 @@ "@types/bcrypt": "^5.0.2", "@types/body-parser": "^1.19.5", "@types/csurf": "^1.11.5", + "@types/express-session": "^1.18.1", "@types/jsonwebtoken": "^9.0.7", "@types/node": "22.7.9", "@types/react": "^18.0.5", "@types/react-dom": "^18.0.1", "@types/uuid": "^8.3.4", + "airtable": "^0.12.2", "axios": "^0.26.0", "bcrypt": "^5.1.1", "body-parser": "^1.20.3", "buffer": "^6.0.3", "bullmq": "^5.12.15", + "connect-redis": "^8.0.1", "cookie-parser": "^1.4.6", "cors": "^2.8.5", "cron-parser": "^4.9.0", @@ -33,6 +36,7 @@ "csurf": "^1.11.0", "dotenv": "^16.0.0", "express": "^4.17.2", + "express-session": "^1.18.1", "fortawesome": "^0.0.1-security", "google-auth-library": "^9.14.1", "googleapis": "^144.0.0", @@ -52,6 +56,7 @@ "moment-timezone": "^0.5.45", "node-cron": "^3.0.3", "pg": "^8.13.0", + "pkce-challenge": "^4.1.0", "playwright": "^1.48.2", "playwright-extra": "^4.3.6", "posthog-node": "^4.2.1", @@ -65,6 +70,7 @@ "react-router-dom": "^6.26.1", "react-simple-code-editor": "^0.11.2", "react-transition-group": "^4.4.2", + "redis": "^4.7.0", "sequelize": "^6.37.3", "sequelize-typescript": "^2.1.6", "sharp": "^0.33.5", @@ -106,6 +112,7 @@ "@types/prismjs": "^1.26.0", "@types/react-highlight": "^0.12.5", "@types/react-transition-group": "^4.4.4", + "@types/redis": "^4.0.11", "@types/styled-components": "^5.1.23", "@types/swagger-jsdoc": "^6.0.4", "@types/swagger-ui-express": "^4.1.6", @@ -115,6 +122,7 @@ "cross-env": "^7.0.3", "js-cookie": "^3.0.5", "nodemon": "^2.0.15", + "sequelize-cli": "^6.6.2", "ts-node": "^10.4.0", "vite": "^5.4.10" } diff --git a/public/locales/de.json b/public/locales/de.json index 7314b677..aade3c67 100644 --- a/public/locales/de.json +++ b/public/locales/de.json @@ -305,29 +305,90 @@ } }, "integration_settings": { - "title": "Mit Google Sheet integrieren", + "title": "Integrationseinstellungen", "descriptions": { - "sync_info": "Wenn Sie diese Option aktivieren, werden bei jeder erfolgreichen Ausführung dieses Roboters die erfassten Daten in Ihrem Google Sheet ergänzt.", "authenticated_as": "Authentifiziert als: {{email}}" }, - "alerts": { - "success": { - "title": "Google Sheet erfolgreich integriert.", - "content": "Jedes Mal, wenn dieser Roboter eine erfolgreiche Ausführung erstellt, werden die erfassten Daten Ihrem Google Sheet {{sheetName}} hinzugefügt. Sie können die Datenaktualisierungen", - "here": "hier", - "note": "Hinweis:", - "sync_limitation": "Die vor der Integration mit Google Sheets extrahierten Daten werden nicht im Google Sheet synchronisiert. Nur die nach der Integration extrahierten Daten werden synchronisiert." + "buttons": { + "submit": "Absenden", + "remove_integration": "Integration entfernen" + }, + "google": { + "title": "Mit Google Sheet integrieren", + "descriptions": { + "sync_info": "Wenn Sie diese Option aktivieren, werden bei jeder erfolgreichen Ausführung einer Aufgabe durch diesen Roboter die erfassten Daten an Ihr Google Sheet angehängt.", + "authenticated_as": "Authentifiziert als: {{email}}" + }, + "alerts": { + "success": { + "title": "Google Sheet erfolgreich integriert.", + "content": "Jedes Mal, wenn dieser Roboter einen erfolgreichen Lauf erstellt, werden seine erfassten Daten an Ihr Google Sheet {{sheetName}} angehängt. Sie können die Datenaktualisierungen überprüfen", + "here": "hier", + "note": "Hinweis:", + "sync_limitation": "Die vor der Integration mit Google Sheets extrahierten Daten werden nicht im Google Sheet synchronisiert. Nur die nach der Integration extrahierten Daten werden synchronisiert." + } + }, + "buttons": { + "authenticate": "Mit Google authentifizieren", + "fetch_sheets": "Google Tabellenkalkulationen abrufen", + "remove_integration": "Integration entfernen", + "submit": "Absenden" + }, + "fields": { + "select_sheet": "Google Sheet auswählen", + "selected_sheet": "Ausgewähltes Sheet: {{name}} (ID: {{id}})" + }, + "errors": { + "auth_error": "Fehler bei der Authentifizierung mit Google", + "fetch_error": "Fehler beim Abrufen der Tabellenkalkulationsdateien: {{message}}", + "update_error": "Fehler beim Aktualisieren der Google Sheet-ID: {{message}}", + "remove_error": "Fehler beim Entfernen der Google Sheets-Integration: {{message}}" + }, + "notifications": { + "sheet_selected": "Google Sheet erfolgreich ausgewählt", + "integration_removed": "Google Sheets-Integration erfolgreich entfernt" } }, - "buttons": { - "authenticate": "Mit Google authentifizieren", - "fetch_sheets": "Google Sheets abrufen", - "remove_integration": "Integration entfernen", - "submit": "Absenden" - }, - "fields": { - "select_sheet": "Google Sheet auswählen", - "selected_sheet": "Ausgewähltes Sheet: {{name}} (ID: {{id}})" + "airtable": { + "title": "Mit Airtable integrieren", + "descriptions": { + "sync_info": "Wenn Sie diese Option aktivieren, werden bei jeder erfolgreichen Ausführung einer Aufgabe durch diesen Roboter die erfassten Daten an Ihre Airtable angehängt.", + "authenticated_as": "Mit Airtable authentifiziert" + }, + "alerts": { + "success": { + "title": "Mit Airtable verbunden", + "content": "Ihr Roboter ist jetzt mit der Basis: {{baseName}} und Tabelle: {{tableName}} verbunden. Sie können darauf zugreifen ", + "here": "hier", + "note": "Hinweis:", + "sync_limitation": "Nur nach der Integration erfasste Daten werden mit Airtable synchronisiert." + } + }, + "buttons": { + "authenticate": "Mit Airtable verbinden", + "fetch_bases": "Airtable Basen abrufen", + "fetch_tables": "Airtable Tabellen abrufen", + "remove_integration": "Airtable-Integration entfernen", + "submit": "Basis und Tabelle auswählen" + }, + "fields": { + "select_base": "Airtable Basis auswählen", + "select_table": "Airtable Tabelle auswählen", + "selected_base": "Ausgewählte Basis: {{name}}", + "selected_table": "Ausgewählte Tabelle: {{name}}" + }, + "errors": { + "auth_error": "Fehler bei der Authentifizierung mit Airtable", + "fetch_error": "Fehler beim Abrufen von Airtable-Basen: {{message}}", + "fetch_tables_error": "Fehler beim Abrufen von Airtable-Tabellen: {{message}}", + "update_error": "Fehler beim Aktualisieren der Airtable-Basis: {{message}}", + "remove_error": "Fehler beim Entfernen der Airtable-Integration: {{message}}" + }, + "notifications": { + "base_selected": "Airtable-Basis erfolgreich ausgewählt", + "table_selected": "Airtable-Tabelle erfolgreich ausgewählt", + "integration_removed": "Airtable-Integration erfolgreich entfernt" + } } }, "robot_duplication": { diff --git a/public/locales/en.json b/public/locales/en.json index 703ec9c0..4f7ebdc4 100644 --- a/public/locales/en.json +++ b/public/locales/en.json @@ -307,38 +307,90 @@ } }, "integration_settings": { - "title": "Integrate with Google Sheet", + "title": "Integration Settings", "descriptions": { - "sync_info": "If you enable this option, every time this robot runs a task successfully, its captured data will be appended to your Google Sheet.", "authenticated_as": "Authenticated as: {{email}}" }, - "alerts": { - "success": { - "title": "Google Sheet Integrated Successfully.", - "content": "Every time this robot creates a successful run, its captured data is appended to your {{sheetName}} Google Sheet. You can check the data updates", - "here": "here", - "note": "Note:", - "sync_limitation": "The data extracted before integrating with Google Sheets will not be synced in the Google Sheet. Only the data extracted after the integration will be synced." + "buttons": { + "submit": "Submit", + "remove_integration": "Remove Integration" + }, + "google": { + "title": "Integrate with Google Sheet", + "descriptions": { + "sync_info": "If you enable this option, every time this robot runs a task successfully, its captured data will be appended to your Google Sheet.", + "authenticated_as": "Authenticated as: {{email}}" + }, + "alerts": { + "success": { + "title": "Google Sheet Integrated Successfully.", + "content": "Every time this robot creates a successful run, its captured data is appended to your {{sheetName}} Google Sheet. You can check the data updates", + "here": "here", + "note": "Note:", + "sync_limitation": "The data extracted before integrating with Google Sheets will not be synced in the Google Sheet. Only the data extracted after the integration will be synced." + } + }, + "buttons": { + "authenticate": "Authenticate with Google", + "fetch_sheets": "Fetch Google Spreadsheets", + "remove_integration": "Remove Integration", + "submit": "Submit" + }, + "fields": { + "select_sheet": "Select Google Sheet", + "selected_sheet": "Selected Sheet: {{name}} (ID: {{id}})" + }, + "errors": { + "auth_error": "Error authenticating with Google", + "fetch_error": "Error fetching spreadsheet files: {{message}}", + "update_error": "Error updating Google Sheet ID: {{message}}", + "remove_error": "Error removing Google Sheets integration: {{message}}" + }, + "notifications": { + "sheet_selected": "Google Sheet selected successfully", + "integration_removed": "Google Sheets integration removed successfully" } }, - "buttons": { - "authenticate": "Authenticate with Google", - "fetch_sheets": "Fetch Google Spreadsheets", - "remove_integration": "Remove Integration", - "submit": "Submit" - }, - "fields": { - "select_sheet": "Select Google Sheet", - "selected_sheet": "Selected Sheet: {{name}} (ID: {{id}})" - }, - "errors": { - "auth_error": "Error authenticating with Google", - "fetch_error": "Error fetching spreadsheet files: {{message}}", - "update_error": "Error updating Google Sheet ID: {{message}}", - "remove_error": "Error removing Google Sheets integration: {{message}}" - }, - "notifications": { - "sheet_selected": "Google Sheet selected successfully" + "airtable": { + "title": "Integrate with Airtable", + "descriptions": { + "sync_info": "If you enable this option, every time this robot runs a task successfully, its captured data will be appended to your Airtable Base.", + "authenticated_as": "Authenticated with Airtable successfully. You can now select the base and table to integrate with." + }, + "alerts": { + "success": { + "title": "Airtable Base Integrated Successfully", + "content": "Every time this robot creates a successful run, its captured data is appended to your {{baseName}} Base and {{tableName}} Table. You can check updates", + "here": "here", + "note": "Note:", + "sync_limitation": "Only data captured after integration will be synced to Airtable." + } + }, + "buttons": { + "authenticate": "Connect with Airtable", + "fetch_bases": "Fetch Airtable Bases", + "fetch_tables": "Fetch Airtable Tables", + "remove_integration": "Remove Integration", + "submit": "Select Base and Table" + }, + "fields": { + "select_base": "Select Airtable Base", + "select_table": "Select Airtable Table", + "selected_base": "Selected Base: {{name}}", + "selected_table": "Selected Table: {{name}}" + }, + "errors": { + "auth_error": "Error authenticating with Airtable", + "fetch_error": "Error fetching Airtable bases: {{message}}", + "fetch_tables_error": "Error fetching Airtable tables: {{message}}", + "update_error": "Error updating Airtable base: {{message}}", + "remove_error": "Error removing Airtable integration: {{message}}" + }, + "notifications": { + "base_selected": "Airtable base selected successfully", + "table_selected": "Airtable table selected successfully", + "integration_removed": "Airtable integration removed successfully" + } } }, "robot_duplication": { diff --git a/public/locales/es.json b/public/locales/es.json index ca4edc5d..07234806 100644 --- a/public/locales/es.json +++ b/public/locales/es.json @@ -306,29 +306,90 @@ } }, "integration_settings": { - "title": "Integrar con Google Sheet", + "title": "Ajustes de Integración", "descriptions": { - "sync_info": "Si habilitas esta opción, cada vez que este robot ejecute una tarea exitosamente, sus datos capturados se añadirán a tu Google Sheet.", "authenticated_as": "Autenticado como: {{email}}" }, - "alerts": { - "success": { - "title": "Google Sheet integrado exitosamente.", - "content": "Cada vez que este robot crea una ejecución exitosa, sus datos capturados se añaden a tu Google Sheet {{sheetName}}. Puedes verificar las actualizaciones de datos", - "here": "aquí", - "note": "Nota:", - "sync_limitation": "Los datos extraídos antes de la integración con Google Sheets no se sincronizarán en el Google Sheet. Solo se sincronizarán los datos extraídos después de la integración." + "buttons": { + "submit": "Enviar", + "remove_integration": "Eliminar Integración" + }, + "google": { + "title": "Integrar con Google Sheet", + "descriptions": { + "sync_info": "Si habilita esta opción, cada vez que este robot ejecute una tarea correctamente, sus datos capturados se añadirán a su Google Sheet.", + "authenticated_as": "Autenticado como: {{email}}" + }, + "alerts": { + "success": { + "title": "Google Sheet Integrado Correctamente.", + "content": "Cada vez que este robot crea una ejecución exitosa, sus datos capturados se añaden a su Google Sheet {{sheetName}}. Puede comprobar las actualizaciones de datos", + "here": "aquí", + "note": "Nota:", + "sync_limitation": "Los datos extraídos antes de la integración con Google Sheets no se sincronizarán en la Google Sheet. Solo los datos extraídos después de la integración se sincronizarán." + } + }, + "buttons": { + "authenticate": "Autenticar con Google", + "fetch_sheets": "Obtener Hojas de Cálculo de Google", + "remove_integration": "Eliminar Integración", + "submit": "Enviar" + }, + "fields": { + "select_sheet": "Seleccionar Google Sheet", + "selected_sheet": "Hoja Seleccionada: {{name}} (ID: {{id}})" + }, + "errors": { + "auth_error": "Error al autenticar con Google", + "fetch_error": "Error al obtener archivos de hojas de cálculo: {{message}}", + "update_error": "Error al actualizar ID de Google Sheet: {{message}}", + "remove_error": "Error al eliminar la integración de Google Sheets: {{message}}" + }, + "notifications": { + "sheet_selected": "Google Sheet seleccionado correctamente", + "integration_removed": "Integración de Google Sheets eliminada correctamente" } }, - "buttons": { - "authenticate": "Autenticar con Google", - "fetch_sheets": "Obtener Google Sheets", - "remove_integration": "Eliminar integración", - "submit": "Enviar" - }, - "fields": { - "select_sheet": "Seleccionar Google Sheet", - "selected_sheet": "Hoja seleccionada: {{name}} (ID: {{id}})" + "airtable": { + "title": "Integrar con Airtable", + "descriptions": { + "sync_info": "Si habilita esta opción, cada vez que este robot ejecute una tarea correctamente, sus datos capturados se añadirán a su Airtable.", + "authenticated_as": "Autenticado con Airtable" + }, + "alerts": { + "success": { + "title": "Conectado a Airtable", + "content": "Su robot ahora está conectado a la base: {{baseName}} y tabla: {{tableName}}. Puede acceder a ella ", + "here": "aquí", + "note": "Nota:", + "sync_limitation": "Solo los datos capturados después de la integración se sincronizarán con Airtable." + } + }, + "buttons": { + "authenticate": "Conectar con Airtable", + "fetch_bases": "Obtener Bases de Airtable", + "fetch_tables": "Obtener Tablas de Airtable", + "remove_integration": "Eliminar Integración de Airtable", + "submit": "Seleccionar Base y Tabla" + }, + "fields": { + "select_base": "Seleccionar Base de Airtable", + "select_table": "Seleccionar Tabla de Airtable", + "selected_base": "Base Seleccionada: {{name}}", + "selected_table": "Tabla Seleccionada: {{name}}" + }, + "errors": { + "auth_error": "Error al autenticar con Airtable", + "fetch_error": "Error al obtener bases de Airtable: {{message}}", + "fetch_tables_error": "Error al obtener tablas de Airtable: {{message}}", + "update_error": "Error al actualizar base de Airtable: {{message}}", + "remove_error": "Error al eliminar la integración de Airtable: {{message}}" + }, + "notifications": { + "base_selected": "Base de Airtable seleccionada correctamente", + "table_selected": "Tabla de Airtable seleccionada correctamente", + "integration_removed": "Integración de Airtable eliminada correctamente" + } } }, "robot_duplication": { diff --git a/public/locales/ja.json b/public/locales/ja.json index 14e8e9e6..a57ff750 100644 --- a/public/locales/ja.json +++ b/public/locales/ja.json @@ -306,29 +306,90 @@ } }, "integration_settings": { - "title": "Google Sheetと連携", + "title": "連携設定", "descriptions": { - "sync_info": "このオプションを有効にすると、このロボットがタスクを正常に実行するたびに、取得したデータがGoogle Sheetに追加されます。", - "authenticated_as": "認証済みユーザー: {{email}}" - }, - "alerts": { - "success": { - "title": "Google Sheetの連携が完了しました。", - "content": "このロボットが正常に実行を完了するたびに、取得したデータはGoogle Sheet {{sheetName}}に追加されます。データの更新は", - "here": "こちら", - "note": "注意：", - "sync_limitation": "Google Sheetsとの連携前に抽出されたデータは同期されません。連携後に抽出されたデータのみが同期されます。" - } + "authenticated_as": "認証済み: {{email}}" }, "buttons": { - "authenticate": "Googleで認証", - "fetch_sheets": "Google Sheetsを取得", - "remove_integration": "連携を解除", - "submit": "送信" + "submit": "送信", + "remove_integration": "連携を解除" }, - "fields": { - "select_sheet": "Google Sheetを選択", - "selected_sheet": "選択したシート: {{name}} (ID: {{id}})" + "google": { + "title": "Google シートと連携", + "descriptions": { + "sync_info": "このオプションを有効にすると、このロボットがタスクを正常に実行するたびに、取得したデータがGoogle シートに追加されます。", + "authenticated_as": "認証済み: {{email}}" + }, + "alerts": { + "success": { + "title": "Google シートの連携に成功しました。", + "content": "このロボットが正常に実行されるたびに、取得したデータが{{sheetName}}という名前のGoogle シートに追加されます。データの更新を確認できます", + "here": "こちら", + "note": "注意:", + "sync_limitation": "Google シートとの連携前に抽出されたデータはGoogle シートに同期されません。連携後に抽出されたデータのみが同期されます。" + } + }, + "buttons": { + "authenticate": "Googleで認証", + "fetch_sheets": "Google スプレッドシートを取得", + "remove_integration": "連携を解除", + "submit": "送信" + }, + "fields": { + "select_sheet": "Google シートを選択", + "selected_sheet": "選択したシート: {{name}} (ID: {{id}})" + }, + "errors": { + "auth_error": "Googleでの認証エラー", + "fetch_error": "スプレッドシートファイルの取得エラー: {{message}}", + "update_error": "Google シートIDの更新エラー: {{message}}", + "remove_error": "Google シート連携の解除エラー: {{message}}" + }, + "notifications": { + "sheet_selected": "Google シートが正常に選択されました", + "integration_removed": "Google シートの連携が正常に解除されました" + } + }, + "airtable": { + "title": "Airtableと連携", + "descriptions": { + "sync_info": "このオプションを有効にすると、このロボットがタスクを正常に実行するたびに、取得したデータがAirtableに追加されます。", + "authenticated_as": "Airtableで認証済み" + }, + "alerts": { + "success": { + "title": "Airtableに接続しました", + "content": "ロボットはベース: {{baseName}} とテーブル: {{tableName}} に接続されました。アクセスできます ", + "here": "こちら", + "note": "注意:", + "sync_limitation": "連携後に取得されたデータのみがAirtableに同期されます。" + } + }, + "buttons": { + "authenticate": "Airtableと接続", + "fetch_bases": "Airtableベースを取得", + "fetch_tables": "Airtableテーブルを取得", + "remove_integration": "Airtable連携を解除", + "submit": "ベースとテーブルを選択" + }, + "fields": { + "select_base": "Airtableベースを選択", + "select_table": "Airtableテーブルを選択", + "selected_base": "選択したベース: {{name}}", + "selected_table": "選択したテーブル: {{name}}" + }, + "errors": { + "auth_error": "Airtableでの認証エラー", + "fetch_error": "Airtableベースの取得エラー: {{message}}", + "fetch_tables_error": "Airtableテーブルの取得エラー: {{message}}", + "update_error": "Airtableベースの更新エラー: {{message}}", + "remove_error": "Airtable連携の解除エラー: {{message}}" + }, + "notifications": { + "base_selected": "Airtableベースが正常に選択されました", + "table_selected": "Airtableテーブルが正常に選択されました", + "integration_removed": "Airtable連携が正常に解除されました" + } } }, "robot_duplication": { diff --git a/public/locales/zh.json b/public/locales/zh.json index 5f7a367d..f0204da7 100644 --- a/public/locales/zh.json +++ b/public/locales/zh.json @@ -306,29 +306,90 @@ } }, "integration_settings": { - "title": "与Google Sheet集成", + "title": "集成设置", "descriptions": { - "sync_info": "如果启用此选项，每次机器人成功运行任务时，捕获的数据都会追加到您的Google Sheet中。", - "authenticated_as": "已验证身份: {{email}}" - }, - "alerts": { - "success": { - "title": "Google Sheet集成成功。", - "content": "每次此机器人创建成功运行时，捕获的数据都会追加到您的Google Sheet {{sheetName}}中。您可以查看数据更新", - "here": "在此处", - "note": "注意：", - "sync_limitation": "与Google Sheets集成之前提取的数据将不会同步到Google Sheet中。只有集成后提取的数据才会同步。" - } + "authenticated_as": "已认证为：{{email}}" }, "buttons": { - "authenticate": "使用Google验证", - "fetch_sheets": "获取Google Sheets", - "remove_integration": "移除集成", - "submit": "提交" + "submit": "提交", + "remove_integration": "移除集成" }, - "fields": { - "select_sheet": "选择Google Sheet", - "selected_sheet": "已选择表格: {{name}} (ID: {{id}})" + "google": { + "title": "与Google表格集成", + "descriptions": { + "sync_info": "如果启用此选项，每次机器人成功运行任务时，其捕获的数据将附加到您的Google表格中。", + "authenticated_as": "已认证为：{{email}}" + }, + "alerts": { + "success": { + "title": "Google表格集成成功。", + "content": "每次此机器人创建成功运行时，其捕获的数据将附加到您的{{sheetName}} Google表格中。您可以检查数据更新", + "here": "在这里", + "note": "注意：", + "sync_limitation": "在与Google表格集成之前提取的数据将不会在Google表格中同步。只有集成后提取的数据才会同步。" + } + }, + "buttons": { + "authenticate": "使用Google认证", + "fetch_sheets": "获取Google电子表格", + "remove_integration": "移除集成", + "submit": "提交" + }, + "fields": { + "select_sheet": "选择Google表格", + "selected_sheet": "已选择表格：{{name}}（ID：{{id}}）" + }, + "errors": { + "auth_error": "使用Google认证时出错", + "fetch_error": "获取电子表格文件时出错：{{message}}", + "update_error": "更新Google表格ID时出错：{{message}}", + "remove_error": "移除Google表格集成时出错：{{message}}" + }, + "notifications": { + "sheet_selected": "Google表格选择成功", + "integration_removed": "Google表格集成已成功移除" + } + }, + "airtable": { + "title": "与Airtable集成", + "descriptions": { + "sync_info": "如果启用此选项，每次机器人成功运行任务时，其捕获的数据将附加到您的Airtable中。", + "authenticated_as": "已使用Airtable认证" + }, + "alerts": { + "success": { + "title": "已连接到Airtable", + "content": "您的机器人现已连接到基础：{{baseName}}和表格：{{tableName}}。您可以访问它", + "here": "在这里", + "note": "注意：", + "sync_limitation": "只有集成后捕获的数据才会同步到Airtable。" + } + }, + "buttons": { + "authenticate": "连接Airtable", + "fetch_bases": "获取Airtable基础", + "fetch_tables": "获取Airtable表格", + "remove_integration": "移除Airtable集成", + "submit": "选择基础和表格" + }, + "fields": { + "select_base": "选择Airtable基础", + "select_table": "选择Airtable表格", + "selected_base": "已选择基础：{{name}}", + "selected_table": "已选择表格：{{name}}" + }, + "errors": { + "auth_error": "使用Airtable认证时出错", + "fetch_error": "获取Airtable基础时出错：{{message}}", + "fetch_tables_error": "获取Airtable表格时出错：{{message}}", + "update_error": "更新Airtable基础时出错：{{message}}", + "remove_error": "移除Airtable集成时出错：{{message}}" + }, + "notifications": { + "base_selected": "Airtable基础选择成功", + "table_selected": "Airtable表格选择成功", + "integration_removed": "Airtable集成已成功移除" + } } }, "robot_duplication": { diff --git a/public/svg/airtable.svg b/public/svg/airtable.svg new file mode 100644 index 00000000..e3883c37 --- /dev/null +++ b/public/svg/airtable.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/svg/gsheet.svg b/public/svg/gsheet.svg new file mode 100644 index 00000000..84d26be4 --- /dev/null +++ b/public/svg/gsheet.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/server/config/config.json b/server/config/config.json new file mode 100644 index 00000000..395914f9 --- /dev/null +++ b/server/config/config.json @@ -0,0 +1,26 @@ +{ + "development": { + "username": "postgres", + "password": "postgres", + "database": "maxun", + "host": "localhost", + "port": 5432, + "dialect": "postgres" + }, + "test": { + "username": "postgres", + "password": "postgres", + "database": "maxun_test", + "host": "localhost", + "port": 5432, + "dialect": "postgres" + }, + "production": { + "username": "postgres", + "password": "postgres", + "database": "maxun_prod", + "host": "localhost", + "port": 5432, + "dialect": "postgres" + } + } \ No newline at end of file diff --git a/server/migrations/20250125120216-add-airtable-columns-to-robot.js b/server/migrations/20250125120216-add-airtable-columns-to-robot.js new file mode 100644 index 00000000..ac2c852b --- /dev/null +++ b/server/migrations/20250125120216-add-airtable-columns-to-robot.js @@ -0,0 +1,16 @@ +'use strict'; + +module.exports = { + up: async (queryInterface, Sequelize) => { + + await queryInterface.addColumn('robot', 'airtable_refresh_token', { + type: Sequelize.STRING, + allowNull: true, + }); + }, + + down: async (queryInterface, Sequelize) => { + + await queryInterface.removeColumn('robot', 'airtable_refresh_token'); + }, +}; \ No newline at end of file diff --git a/server/migrations/20250125182726-update-airtable-token-length.js b/server/migrations/20250125182726-update-airtable-token-length.js new file mode 100644 index 00000000..b75d788c --- /dev/null +++ b/server/migrations/20250125182726-update-airtable-token-length.js @@ -0,0 +1,22 @@ +// migrations/XXXXXXXXXXXXXX-update-airtable-token-length.js +module.exports = { + up: async (queryInterface, Sequelize) => { + await queryInterface.changeColumn('robot', 'airtable_access_token', { + type: Sequelize.TEXT + }); + + await queryInterface.changeColumn('robot', 'airtable_refresh_token', { + type: Sequelize.TEXT + }); + }, + + down: async (queryInterface, Sequelize) => { + await queryInterface.changeColumn('robot', 'airtable_access_token', { + type: Sequelize.STRING(255) + }); + + await queryInterface.changeColumn('robot', 'airtable_refresh_token', { + type: Sequelize.STRING(255) + }); + } +}; \ No newline at end of file diff --git a/server/migrations/20250207133740-added_table_id.js b/server/migrations/20250207133740-added_table_id.js new file mode 100644 index 00000000..0a40d0c7 --- /dev/null +++ b/server/migrations/20250207133740-added_table_id.js @@ -0,0 +1,16 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up (queryInterface, Sequelize) { + await queryInterface.addColumn('robot', 'airtable_table_id', { + type: Sequelize.TEXT + }); + }, + + async down (queryInterface, Sequelize) { + await queryInterface.removeColumn('robot', 'airtable_table_id', { + type: Sequelize.TEXT + }); + } +}; diff --git a/server/src/models/Robot.ts b/server/src/models/Robot.ts index 3b2717d6..1d03999e 100644 --- a/server/src/models/Robot.ts +++ b/server/src/models/Robot.ts @@ -25,7 +25,13 @@ interface RobotAttributes { google_sheet_id?: string | null; google_access_token?: string | null; google_refresh_token?: string | null; + airtable_base_id?: string | null; + airtable_base_name?: string | null; + airtable_table_name?: string | null; + airtable_access_token?: string | null; + airtable_refresh_token?: string | null; schedule?: ScheduleConfig | null; + airtable_table_id?: string | null; } interface ScheduleConfig { @@ -49,10 +55,16 @@ class Robot extends Model implements R public recording_meta!: RobotMeta; public recording!: RobotWorkflow; public google_sheet_email!: string | null; - public google_sheet_name?: string | null; - public google_sheet_id?: string | null; + public google_sheet_name!: string | null; + public google_sheet_id!: string | null; public google_access_token!: string | null; public google_refresh_token!: string | null; + public airtable_base_id!: string | null; + public airtable_base_name!: string | null; + public airtable_table_name!: string | null; + public airtable_access_token!: string | null; + public airtable_refresh_token!: string | null; + public airtable_table_id!: string | null; public schedule!: ScheduleConfig | null; } @@ -95,6 +107,30 @@ Robot.init( type: DataTypes.STRING, allowNull: true, }, + airtable_base_id: { + type: DataTypes.STRING, + allowNull: true, + }, + airtable_base_name: { + type: DataTypes.STRING, + allowNull: true, + }, + airtable_table_name: { + type: DataTypes.STRING, + allowNull: true, + }, + airtable_table_id: { + type: DataTypes.STRING, + allowNull: true, + }, + airtable_access_token: { + type: DataTypes.TEXT, + allowNull: true, + }, + airtable_refresh_token: { + type: DataTypes.TEXT, + allowNull: true, + }, schedule: { type: DataTypes.JSONB, allowNull: true, diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index 1afe45e6..4a435cd6 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -1,4 +1,5 @@ import { Router, Request, Response } from "express"; + import User from "../models/User"; import Robot from "../models/Robot"; import jwt from "jsonwebtoken"; @@ -7,6 +8,16 @@ import { requireSignIn } from "../middlewares/auth"; import { genAPIKey } from "../utils/api"; import { google } from "googleapis"; import { capture } from "../utils/analytics"; +import crypto from 'crypto'; + + +declare module "express-session" { + interface SessionData { + code_verifier: string; + robotId: string; + } +} + export const router = Router(); interface AuthenticatedRequest extends Request { @@ -632,4 +643,292 @@ router.post( }); } } -); \ No newline at end of file +); + + +// Airtable OAuth Routes +router.get("/airtable", requireSignIn, (req: Request, res) => { + const authenticatedReq = req as AuthenticatedRequest; + const { robotId } = authenticatedReq.query; + if (!robotId) { + return res.status(400).json({ message: "Robot ID is required" }); + } + + // Generate PKCE codes + const code_verifier = crypto.randomBytes(64).toString('base64url'); + const code_challenge = crypto.createHash('sha256') + .update(code_verifier) + .digest('base64url'); + + // Store in session + authenticatedReq.session.code_verifier = code_verifier; + authenticatedReq.session.robotId = robotId.toString(); + + const params = new URLSearchParams({ + client_id: process.env.AIRTABLE_CLIENT_ID!, + redirect_uri: process.env.AIRTABLE_REDIRECT_URI!, + response_type: 'code', + state: robotId.toString(), + scope: 'data.records:read data.records:write schema.bases:read schema.bases:write', + code_challenge: code_challenge, + code_challenge_method: 'S256' + }); + + res.redirect(`https://airtable.com/oauth2/v1/authorize?${params}`); +}); + +router.get("/airtable/callback", requireSignIn, async (req: Request, res) => { + const authenticatedReq = req as AuthenticatedRequest; + const baseUrl = process.env.PUBLIC_URL || "http://localhost:5173"; + + try { + const { code, state, error } = authenticatedReq.query; + + if (error) { + return res.redirect( + `${baseUrl}/robots/${state}/integrate?error=${encodeURIComponent(error.toString())}` + ); + } + + if (!code || !state) { + return res.status(400).json({ message: "Missing authorization code or state" }); + } + + // Verify session data + if (!authenticatedReq.session?.code_verifier || authenticatedReq.session.robotId !== state.toString()) { + return res.status(400).json({ + message: "Session expired - please restart the OAuth flow" + }); + } + + // Exchange code for tokens + const tokenResponse = await fetch("https://airtable.com/oauth2/v1/token", { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + body: new URLSearchParams({ + grant_type: "authorization_code", + code: code.toString(), + client_id: process.env.AIRTABLE_CLIENT_ID!, + redirect_uri: process.env.AIRTABLE_REDIRECT_URI!, + code_verifier: authenticatedReq.session.code_verifier + }), + }); + + if (!tokenResponse.ok) { + const errorData = await tokenResponse.json(); + console.error('Token exchange failed:', errorData); + return res.redirect( + `${baseUrl}/robots/${state}/integrate?error=${encodeURIComponent(errorData.error_description || 'Authentication failed')}` + ); + } + + const tokens = await tokenResponse.json(); + + // Update robot with credentials + const robot = await Robot.findOne({ + where: { "recording_meta.id": req.session.robotId } + }); + + if (!robot) { + return res.status(404).json({ message: "Robot not found" }); + } + + await robot.update({ + airtable_access_token: tokens.access_token, + airtable_refresh_token: tokens.refresh_token, + }); + + res.cookie("airtable_auth_status", "success", { + httpOnly: false, + maxAge: 60000, + }); // 1-minute expiration + // res.cookie("airtable_auth_message", "Robot successfully authenticated", { + // httpOnly: false, + // maxAge: 60000, + // }); + + res.cookie('robot_auth_robotId', req.session.robotId, { + httpOnly: false, + maxAge: 60000, + }); + + // Clear session data + authenticatedReq.session.destroy((err) => { + if (err) console.error('Session cleanup error:', err); + }); + + const redirectUrl = `${baseUrl}/robots/`; + + res.redirect(redirectUrl); + } catch (error: any) { + console.error('Airtable callback error:', error); + res.redirect( + `${baseUrl}/robots/${req.session.robotId}/integrate?error=${encodeURIComponent(error.message)}` + ); + } +}); + +// Get Airtable bases +router.get("/airtable/bases", requireSignIn, async (req: Request, res) => { + const authenticatedReq = req as AuthenticatedRequest; + try { + const { robotId } = authenticatedReq.query; + if (!robotId) { + return res.status(400).json({ message: "Robot ID is required" }); + } + + const robot = await Robot.findOne({ + where: { "recording_meta.id": robotId.toString() }, + raw: true, + }); + + if (!robot?.airtable_access_token) { + return res.status(400).json({ message: "Robot not authenticated with Airtable" }); + } + + const response = await fetch('https://api.airtable.com/v0/meta/bases', { + headers: { + 'Authorization': `Bearer ${robot.airtable_access_token}` + } + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.error.message || 'Failed to fetch bases'); + } + + const data = await response.json(); + res.json(data.bases.map((base: any) => ({ + id: base.id, + name: base.name + }))); + + } catch (error: any) { + res.status(500).json({ message: error.message }); + } +}); + +// Update robot with selected base +router.post("/airtable/update", requireSignIn, async (req: Request, res) => { + const authenticatedReq = req as AuthenticatedRequest; + const { baseId, robotId , baseName, tableName, tableId} = req.body; + + if (!baseId || !robotId) { + return res.status(400).json({ message: "Base ID and Robot ID are required" }); + } + + try { + const robot = await Robot.findOne({ + where: { "recording_meta.id": robotId } + }); + + if (!robot) { + return res.status(404).json({ message: "Robot not found" }); + } + + await robot.update({ + airtable_base_id: baseId, + airtable_table_name: tableName, + airtable_table_id: tableId, + airtable_base_name: baseName, + }); + + capture("maxun-oss-airtable-integration-created", { + user_id: authenticatedReq.user?.id, + robot_id: robotId, + created_at: new Date().toISOString(), + }); + + res.json({ message: "Airtable base updated successfully" }); + + } catch (error: any) { + res.status(500).json({ message: error.message }); + } +}); + +// Remove Airtable integration +router.post("/airtable/remove", requireSignIn, async (req: Request, res) => { + const authenticatedReq = req as AuthenticatedRequest; + const { robotId } = authenticatedReq.body; + if (!robotId) { + return res.status(400).json({ message: "Robot ID is required" }); + } + + try { + const robot = await Robot.findOne({ + where: { "recording_meta.id": robotId } + }); + + if (!robot) { + return res.status(404).json({ message: "Robot not found" }); + } + + await robot.update({ + airtable_access_token: null, + airtable_refresh_token: null, + airtable_base_id: null, + airtable_base_name: null, + airtable_table_name: null, + airtable_table_id: null, + }); + + capture("maxun-oss-airtable-integration-removed", { + user_id: authenticatedReq.user?.id, + robot_id: robotId, + deleted_at: new Date().toISOString(), + }); + + res.json({ message: "Airtable integration removed successfully" }); + + } catch (error: any) { + res.status(500).json({ message: error.message }); + } +}); + + + +// Fetch tables from an Airtable base +router.get("/airtable/tables", requireSignIn, async (req: Request, res) => { + const authenticatedReq = req as AuthenticatedRequest; + try { + const { baseId, robotId } = authenticatedReq.query; + + if (!baseId || !robotId) { + return res.status(400).json({ message: "Base ID and Robot ID are required" }); + } + + const robot = await Robot.findOne({ + where: { "recording_meta.id": robotId.toString() }, + raw: true, + }); + + if (!robot?.airtable_access_token) { + return res.status(400).json({ message: "Robot not authenticated with Airtable" }); + } + + const response = await fetch(`https://api.airtable.com/v0/meta/bases/${baseId}/tables`, { + headers: { + 'Authorization': `Bearer ${robot.airtable_access_token}` + } + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.error.message || 'Failed to fetch tables'); + } + + const data = await response.json(); + res.json(data.tables.map((table: any) => ({ + id: table.id, + name: table.name, + fields: table.fields + }))); + + } catch (error: any) { + res.status(500).json({ message: error.message }); + } +}); + + diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index bb3df751..5ab9ef7a 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -21,6 +21,7 @@ import { tryCatch } from 'bullmq'; import { encrypt, decrypt } from '../utils/auth'; import { WorkflowFile } from 'maxun-core'; import { Page } from 'playwright'; +import { airtableUpdateTasks, processAirtableUpdates } from '../workflow-management/integrations/airtable'; chromium.use(stealthPlugin()); export const router = Router(); @@ -667,6 +668,15 @@ router.post('/runs/run/:id', requireSignIn, async (req: AuthenticatedRequest, re status: 'pending', retries: 5, }; + + airtableUpdateTasks[plainRun.runId] = { + robotId: plainRun.robotMetaId, + runId: plainRun.runId, + status: 'pending', + retries: 5, + }; + + processAirtableUpdates(); processGoogleSheetUpdates(); } catch (err: any) { logger.log('error', `Failed to update Google Sheet for run: ${plainRun.runId}: ${err.message}`); diff --git a/server/src/server.ts b/server/src/server.ts index 581ecadc..9c0eff23 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -18,8 +18,12 @@ import { fork } from 'child_process'; import { capture } from "./utils/analytics"; import swaggerUi from 'swagger-ui-express'; import swaggerSpec from './swagger/config'; + +import session from 'express-session'; + import Run from './models/Run'; + const app = express(); app.use(cors({ origin: process.env.PUBLIC_URL ? process.env.PUBLIC_URL : 'http://localhost:5173', @@ -27,6 +31,16 @@ app.use(cors({ })); app.use(express.json()); + +app.use( + session({ + secret: 'your_secret_key', // Replace with a secure secret key + resave: false, // Do not resave the session if it hasn't changed + saveUninitialized: true, // Save new sessions + cookie: { secure: false }, // Set to true if using HTTPS + }) +); + const server = http.createServer(app); /** diff --git a/server/src/workflow-management/integrations/airtable.ts b/server/src/workflow-management/integrations/airtable.ts new file mode 100644 index 00000000..93474721 --- /dev/null +++ b/server/src/workflow-management/integrations/airtable.ts @@ -0,0 +1,324 @@ +import Airtable from "airtable"; +import axios from "axios"; +import logger from "../../logger"; +import Run from "../../models/Run"; +import Robot from "../../models/Robot"; + +interface AirtableUpdateTask { + robotId: string; + runId: string; + status: 'pending' | 'completed' | 'failed'; + retries: number; +} + +const MAX_RETRIES = 3; +const BASE_API_DELAY = 2000; + +export let airtableUpdateTasks: { [runId: string]: AirtableUpdateTask } = {}; + +async function refreshAirtableToken(refreshToken: string) { + try { + const response = await axios.post( + "https://airtable.com/oauth2/v1/token", + new URLSearchParams({ + grant_type: "refresh_token", + refresh_token: refreshToken, + client_id: process.env.AIRTABLE_CLIENT_ID!, + }), + { + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + } + ); + + return response.data; + } catch (error: any) { + logger.log("error", `Failed to refresh Airtable token: ${error.message}`); + throw new Error(`Token refresh failed: ${error.response?.data?.error_description || error.message}`); + } +} + +export async function updateAirtable(robotId: string, runId: string) { + try { + const run = await Run.findOne({ where: { runId } }); + if (!run) throw new Error(`Run not found for runId: ${runId}`); + + const plainRun = run.toJSON(); + if (plainRun.status !== 'success') { + console.log('Run status is not success'); + return; + } + + let data: { [key: string]: any }[] = []; + if (plainRun.serializableOutput?.['item-0']) { + data = plainRun.serializableOutput['item-0'] as { [key: string]: any }[]; + } else if (plainRun.binaryOutput?.['item-0']) { + data = [{ "File URL": plainRun.binaryOutput['item-0'] }]; + } + + const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + if (!robot) throw new Error(`Robot not found for robotId: ${robotId}`); + + const plainRobot = robot.toJSON(); + if (plainRobot.airtable_base_id && plainRobot.airtable_table_name && plainRobot.airtable_table_id) { + console.log(`Writing to Airtable base ${plainRobot.airtable_base_id}`); + await writeDataToAirtable( + robotId, + plainRobot.airtable_base_id, + plainRobot.airtable_table_name, + plainRobot.airtable_table_id, + data + ); + console.log(`Data written to Airtable for ${robotId}`); + } + } catch (error: any) { + console.error(`Airtable update failed: ${error.message}`); + throw error; + } +} + +async function withTokenRefresh(robotId: string, apiCall: (accessToken: string) => Promise): Promise { + const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + if (!robot) throw new Error(`Robot not found for robotId: ${robotId}`); + + let accessToken = robot.get('airtable_access_token') as string; + let refreshToken = robot.get('airtable_refresh_token') as string; + + if (!accessToken || !refreshToken) { + throw new Error('Airtable credentials not configured'); + } + + try { + return await apiCall(accessToken); + } catch (error: any) { + if (error.response?.status === 401 || + (error.statusCode === 401) || + error.message.includes('unauthorized') || + error.message.includes('expired')) { + + logger.log("info", `Refreshing expired Airtable token for robot: ${robotId}`); + + try { + const tokens = await refreshAirtableToken(refreshToken); + + await robot.update({ + airtable_access_token: tokens.access_token, + airtable_refresh_token: tokens.refresh_token || refreshToken + }); + + return await apiCall(tokens.access_token); + } catch (refreshError: any) { + logger.log("error", `Failed to refresh token: ${refreshError.message}`); + throw new Error(`Token refresh failed: ${refreshError.message}`); + } + } + + throw error; + } +} + +export async function writeDataToAirtable( + robotId: string, + baseId: string, + tableName: string, + tableId: string, + data: any[] +) { + try { + return await withTokenRefresh(robotId, async (accessToken: string) => { + const airtable = new Airtable({ apiKey: accessToken }); + const base = airtable.base(baseId); + + const existingFields = await getExistingFields(base, tableName); + console.log(`Found ${existingFields.length} existing fields in Airtable`); + + const dataFields = [...new Set(data.flatMap(row => Object.keys(row)))]; + console.log(`Found ${dataFields.length} fields in data: ${dataFields.join(', ')}`); + + const missingFields = dataFields.filter(field => !existingFields.includes(field)); + console.log(`Found ${missingFields.length} missing fields: ${missingFields.join(', ')}`); + + for (const field of missingFields) { + const sampleRow = data.find(row => field in row); + if (sampleRow) { + const sampleValue = sampleRow[field]; + try { + await createAirtableField(baseId, tableName, field, sampleValue, accessToken, tableId); + console.log(`Successfully created field: ${field}`); + } catch (fieldError: any) { + console.warn(`Warning: Could not create field "${field}": ${fieldError.message}`); + } + } + } + + await deleteEmptyRecords(base, tableName); + + const BATCH_SIZE = 10; + for (let i = 0; i < data.length; i += BATCH_SIZE) { + const batch = data.slice(i, i + BATCH_SIZE); + await retryableAirtableWrite(base, tableName, batch); + } + + logger.log('info', `Successfully wrote ${data.length} records to Airtable`); + }); + } catch (error: any) { + logger.log('error', `Airtable write failed: ${error.message}`); + throw error; + } +} + +async function deleteEmptyRecords(base: Airtable.Base, tableName: string): Promise { + console.log('Checking for empty records to clear...'); + + try { + const existingRecords = await base(tableName).select().all(); + console.log(`Found ${existingRecords.length} total records`); + + const emptyRecords = existingRecords.filter(record => { + const fields = record.fields; + return !fields || Object.keys(fields).length === 0 || + Object.values(fields).every(value => + value === null || value === undefined || value === ''); + }); + + if (emptyRecords.length > 0) { + const BATCH_SIZE = 10; + for (let i = 0; i < emptyRecords.length; i += BATCH_SIZE) { + const batch = emptyRecords.slice(i, i + BATCH_SIZE); + const recordIds = batch.map(record => record.id); + await base(tableName).destroy(recordIds); + } + } + } catch (error: any) { + console.warn(`Warning: Could not clear empty records: ${error.message}`); + console.warn('Will continue without deleting empty records'); + } +} + +async function retryableAirtableWrite( + base: Airtable.Base, + tableName: string, + batch: any[], + retries = MAX_RETRIES +): Promise { + try { + await base(tableName).create(batch.map(row => ({ fields: row }))); + } catch (error) { + if (retries > 0) { + await new Promise(resolve => setTimeout(resolve, BASE_API_DELAY)); + return retryableAirtableWrite(base, tableName, batch, retries - 1); + } + throw error; + } +} + +// Helper functions +async function getExistingFields(base: Airtable.Base, tableName: string): Promise { + try { + const records = await base(tableName).select({ pageSize: 5 }).firstPage(); + if (records.length > 0) { + const fieldNames = new Set(); + records.forEach(record => { + Object.keys(record.fields).forEach(field => fieldNames.add(field)); + }); + + const headers = Array.from(fieldNames); + console.log(`Found ${headers.length} headers from records: ${headers.join(', ')}`); + return headers; + } + return []; + } catch (error) { + return []; + } +} + +async function createAirtableField( + baseId: string, + tableName: string, + fieldName: string, + sampleValue: any, + accessToken: string, + tableId: string, + retries = MAX_RETRIES +): Promise { + try { + const fieldType = inferFieldType(sampleValue); + + console.log(`Creating field ${fieldName} with type ${fieldType}`); + + const response = await axios.post( + `https://api.airtable.com/v0/meta/bases/${baseId}/tables/${tableId}/fields`, + { name: fieldName, type: fieldType }, + { headers: { Authorization: `Bearer ${accessToken}` } } + ); + + logger.log('info', `Created field: ${fieldName} (${fieldType})`); + return response.data; + } catch (error: any) { + if (retries > 0 && error.response?.status === 429) { + await new Promise(resolve => setTimeout(resolve, BASE_API_DELAY)); + return createAirtableField(baseId, tableName, fieldName, sampleValue, accessToken, tableId, retries - 1); + } + + if (error.response?.status === 422) { + console.log(`Field ${fieldName} may already exist or has validation issues`); + return; + } + + const errorMessage = error.response?.data?.error?.message || error.message; + const statusCode = error.response?.status || 'No Status Code'; + console.warn(`Field creation issue (${statusCode}): ${errorMessage}`); + } +} + +function inferFieldType(value: any): string { + if (value === null || value === undefined) return 'singleLineText'; + if (typeof value === 'number') return 'number'; + if (typeof value === 'boolean') return 'checkbox'; + if (value instanceof Date) return 'dateTime'; + if (Array.isArray(value)) { + return value.length > 0 && typeof value[0] === 'object' ? 'multipleRecordLinks' : 'multipleSelects'; + } + if (typeof value === 'string' && isValidUrl(value)) return 'url'; + return 'singleLineText'; +} + +function isValidUrl(str: string): boolean { + try { + new URL(str); + return true; + } catch (_) { + return false; + } +} + +export const processAirtableUpdates = async () => { + while (true) { + let hasPendingTasks = false; + + for (const runId in airtableUpdateTasks) { + const task = airtableUpdateTasks[runId]; + if (task.status !== 'pending') continue; + + hasPendingTasks = true; + try { + await updateAirtable(task.robotId, task.runId); + delete airtableUpdateTasks[runId]; + } catch (error: any) { + task.retries += 1; + if (task.retries >= MAX_RETRIES) { + task.status = 'failed'; + logger.log('error', `Permanent failure for run ${runId}: ${error.message}`); + } + } + } + + if (!hasPendingTasks) { + console.log('No pending Airtable update tasks, exiting processor'); + break; + } + + await new Promise(resolve => setTimeout(resolve, 5000)); + } +}; \ No newline at end of file diff --git a/src/components/integration/IntegrationSettings.tsx b/src/components/integration/IntegrationSettings.tsx index 31c93368..36a809d4 100644 --- a/src/components/integration/IntegrationSettings.tsx +++ b/src/components/integration/IntegrationSettings.tsx @@ -6,90 +6,150 @@ import { CircularProgress, Alert, AlertTitle, - Chip, + Button, + TextField, } from "@mui/material"; -import Button from "@mui/material/Button"; -import TextField from "@mui/material/TextField"; import axios from "axios"; import { useGlobalInfoStore } from "../../context/globalInfo"; import { getStoredRecording } from "../../api/storage"; import { apiUrl } from "../../apiConfig.js"; -import { useTranslation } from "react-i18next"; +import Cookies from "js-cookie"; + +import { useTranslation } from "react-i18next"; +import { useNavigate } from "react-router-dom"; interface IntegrationProps { isOpen: boolean; handleStart: (data: IntegrationSettings) => void; handleClose: () => void; + preSelectedIntegrationType?: "googleSheets" | "airtable" | null; } export interface IntegrationSettings { - spreadsheetId: string; - spreadsheetName: string; + spreadsheetId?: string; + spreadsheetName?: string; + airtableBaseId?: string; + airtableBaseName?: string; + airtableTableName?: string, + airtableTableId?: string, data: string; + integrationType: "googleSheets" | "airtable"; } +const getCookie = (name: string): string | null => { + const value = `; ${document.cookie}`; + const parts = value.split(`; ${name}=`); + if (parts.length === 2) { + return parts.pop()?.split(";").shift() || null; + } + return null; +}; + +const removeCookie = (name: string): void => { + document.cookie = `${name}=; expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/`; +}; + export const IntegrationSettingsModal = ({ isOpen, handleStart, handleClose, + preSelectedIntegrationType = null, }: IntegrationProps) => { const { t } = useTranslation(); const [settings, setSettings] = useState({ spreadsheetId: "", spreadsheetName: "", + airtableBaseId: "", + airtableBaseName: "", + airtableTableName: "", + airtableTableId: "", data: "", + integrationType: preSelectedIntegrationType || "googleSheets", }); - const [spreadsheets, setSpreadsheets] = useState< - { id: string; name: string }[] - >([]); + const [spreadsheets, setSpreadsheets] = useState<{ id: string; name: string }[]>([]); + const [airtableBases, setAirtableBases] = useState<{ id: string; name: string }[]>([]); + const [airtableTables, setAirtableTables] = useState<{ id: string; name: string }[]>([]); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); - const { recordingId, notify } = useGlobalInfoStore(); + const { + recordingId, + notify, + setRerenderRobots + } = useGlobalInfoStore(); + const [recording, setRecording] = useState(null); + const navigate = useNavigate(); + + const [selectedIntegrationType, setSelectedIntegrationType] = useState< + "googleSheets" | "airtable" | null + >(preSelectedIntegrationType); const authenticateWithGoogle = () => { window.location.href = `${apiUrl}/auth/google?robotId=${recordingId}`; }; - const handleOAuthCallback = async () => { - try { - const response = await axios.get(`${apiUrl}/auth/google/callback`); - const { google_sheet_email, files } = response.data; - } catch (error) { - setError("Error authenticating with Google"); - } + // Authenticate with Airtable + const authenticateWithAirtable = () => { + window.location.href = `${apiUrl}/auth/airtable?robotId=${recordingId}`; }; + // Fetch Google Sheets files const fetchSpreadsheetFiles = async () => { try { const response = await axios.get( `${apiUrl}/auth/gsheets/files?robotId=${recordingId}`, - { - withCredentials: true, - } + { withCredentials: true } ); setSpreadsheets(response.data); } catch (error: any) { - console.error( - "Error fetching spreadsheet files:", - error.response?.data?.message || error.message - ); - notify( - "error", - t('integration_settings.errors.fetch_error', { - message: error.response?.data?.message || error.message - }) - ); + setLoading(false); + console.error("Error fetching spreadsheet files:", error); + notify("error", t("integration_settings.google.errors.fetch_error", { + message: error.response?.data?.message || error.message, + })); } }; + // Fetch Airtable bases + const fetchAirtableBases = async () => { + try { + const response = await axios.get( + `${apiUrl}/auth/airtable/bases?robotId=${recordingId}`, + { withCredentials: true } + ); + setAirtableBases(response.data); + } catch (error: any) { + setLoading(false); + console.error("Error fetching Airtable bases:", error); + notify("error", t("integration_settings.airtable.errors.fetch_error", { + message: error.response?.data?.message || error.message, + })); + } + }; + + const fetchAirtableTables = async (baseId: string, recordingId: string) => { + try { + const response = await axios.get( + `${apiUrl}/auth/airtable/tables?robotId=${recordingId}&baseId=${baseId}`, + { withCredentials: true } + ); + setAirtableTables(response.data); + } + catch (error: any) { + setLoading(false); + console.error("Error fetching Airtable tables:", error); + notify("error", t("integration_settings.airtable.errors.fetch_tables_error", { + message: error.response?.data?.message || error.message, + })); + } + } + + // Handle Google Sheets selection const handleSpreadsheetSelect = (e: React.ChangeEvent) => { - const selectedSheet = spreadsheets.find( - (sheet) => sheet.id === e.target.value - ); + const selectedSheet = spreadsheets.find((sheet) => sheet.id === e.target.value); if (selectedSheet) { setSettings({ ...settings, @@ -99,9 +159,48 @@ export const IntegrationSettingsModal = ({ } }; + // Handle Airtable base selection + const handleAirtableBaseSelect = async (e: React.ChangeEvent) => { + const selectedBase = airtableBases.find((base) => base.id === e.target.value); + + if (selectedBase) { + setSettings((prevSettings) => ({ + ...prevSettings, + airtableBaseId: selectedBase.id, + airtableBaseName: selectedBase.name, + })); + + if (recordingId) { + await fetchAirtableTables(selectedBase.id, recordingId); + } else { + console.error("Recording ID is null"); + } + } + }; + + const handleAirtabletableSelect = (e: React.ChangeEvent) => { + const selectedTable = airtableTables.find((table) => table.id === e.target.value); + if (selectedTable) { + setSettings((prevSettings) => ({ + ...prevSettings, + airtableTableId: e.target.value, + airtableTableName: selectedTable?.name || "", + })); + } + }; + + const refreshRecordingData = async () => { + if (!recordingId) return null; + const updatedRecording = await getStoredRecording(recordingId); + setRecording(updatedRecording); + setRerenderRobots(true); + return updatedRecording; + }; + const updateGoogleSheetId = async () => { try { - const response = await axios.post( + setLoading(true); + await axios.post( `${apiUrl}/auth/gsheets/update`, { spreadsheetId: settings.spreadsheetId, @@ -110,53 +209,212 @@ export const IntegrationSettingsModal = ({ }, { withCredentials: true } ); - notify(`success`, t('integration_settings.notifications.sheet_selected')); - console.log("Google Sheet ID updated:", response.data); + + // Refresh recording data immediately + await refreshRecordingData(); + + notify("success", t("integration_settings.google.notifications.sheet_selected")); + setLoading(false); } catch (error: any) { - console.error( - "Error updating Google Sheet ID:", - error.response?.data?.message || error.message - ); + setLoading(false); + console.error("Error updating Google Sheet ID:", error); + notify("error", t("integration_settings.google.errors.update_error", { + message: error.response?.data?.message || error.message, + })); } }; - const removeIntegration = async () => { + // Update Airtable integration + const updateAirtableBase = async () => { try { + setLoading(true); + await axios.post( + `${apiUrl}/auth/airtable/update`, + { + baseId: settings.airtableBaseId, + baseName: settings.airtableBaseName, + robotId: recordingId, + tableName: settings.airtableTableName, + tableId: settings.airtableTableId, + }, + { withCredentials: true } + ); + + await refreshRecordingData(); + + notify("success", t("integration_settings.airtable.notifications.base_selected")); + setLoading(false); + } catch (error: any) { + setLoading(false); + console.error("Error updating Airtable base:", error); + notify("error", t("integration_settings.airtable.errors.update_error", { + message: error.response?.data?.message || error.message, + })); + } + }; + + // Remove Google Sheets integration + const removeGoogleSheetsIntegration = async () => { + try { + setLoading(true); await axios.post( `${apiUrl}/auth/gsheets/remove`, { robotId: recordingId }, { withCredentials: true } ); - setRecording(null); + // Clear UI state setSpreadsheets([]); - setSettings({ spreadsheetId: "", spreadsheetName: "", data: "" }); + setSettings({ ...settings, spreadsheetId: "", spreadsheetName: "" }); + + // Refresh recording data + await refreshRecordingData(); + + notify("success", t("integration_settings.google.notifications.integration_removed")); + setLoading(false); } catch (error: any) { - console.error( - "Error removing Google Sheets integration:", - error.response?.data?.message || error.message + setLoading(false); + console.error("Error removing Google Sheets integration:", error); + notify("error", t("integration_settings.google.errors.remove_error", { + message: error.response?.data?.message || error.message, + })); + } + }; + + // Remove Airtable integration + const removeAirtableIntegration = async () => { + try { + setLoading(true); + await axios.post( + `${apiUrl}/auth/airtable/remove`, + { robotId: recordingId }, + { withCredentials: true } ); + + setAirtableBases([]); + setAirtableTables([]); + setSettings({ ...settings, airtableBaseId: "", airtableBaseName: "", airtableTableName: "", airtableTableId: "" }); + + await refreshRecordingData(); + + notify("success", t("integration_settings.airtable.notifications.integration_removed")); + setLoading(false); + } catch (error: any) { + setLoading(false); + console.error("Error removing Airtable integration:", error); + notify("error", t("integration_settings.airtable.errors.remove_error", { + message: error.response?.data?.message || error.message, + })); + } + }; + + const handleAirtableOAuthCallback = async () => { + try { + const response = await axios.get(`${apiUrl}/auth/airtable/callback`); + if (response.data.success) { + await refreshRecordingData(); + } + } catch (error) { + setError(t("integration_settings.airtable.errors.auth_error")); } }; useEffect(() => { - // Check if we're on the callback URL - const urlParams = new URLSearchParams(window.location.search); - const code = urlParams.get("code"); - if (code) { - handleOAuthCallback(); - } - const fetchRecordingInfo = async () => { if (!recordingId) return; + + setLoading(true); + const recording = await getStoredRecording(recordingId); if (recording) { setRecording(recording); + + if (preSelectedIntegrationType) { + setSettings(prev => ({ ...prev, integrationType: preSelectedIntegrationType })); + } + else if (recording.google_sheet_id) { + setSettings(prev => ({ ...prev, integrationType: "googleSheets" })); + } else if (recording.airtable_base_id) { + setSettings(prev => ({ + ...prev, + airtableBaseId: recording.airtable_base_id || "", + airtableBaseName: recording.airtable_base_name || "", + airtableTableName: recording.airtable_table_name || "", + airtableTableId: recording.airtable_table_id || "", + integrationType: recording.airtable_base_id ? "airtable" : "googleSheets" + })); + } } + + setLoading(false); }; fetchRecordingInfo(); - }, [recordingId]); + }, [recordingId, preSelectedIntegrationType]); + + useEffect(() => { + const status = getCookie("airtable_auth_status"); + const message = getCookie("airtable_auth_message"); + + if (status === "success") { + notify("success", message || t("integration_settings.airtable.notifications.auth_success")); + removeCookie("airtable_auth_status"); + removeCookie("airtable_auth_message"); + refreshRecordingData(); + } + + const urlParams = new URLSearchParams(window.location.search); + const code = urlParams.get("code"); + if (code) { + handleAirtableOAuthCallback(); + } + }, []); + + // Add this UI at the top of the modal return statement + if (!selectedIntegrationType) { + return ( + +
+
+ + + +
+
+ + ); + } return ( @@ -166,118 +424,215 @@ export const IntegrationSettingsModal = ({ alignItems: "flex-start", marginLeft: "65px", }}> - - {t('integration_settings.title')} - - {recording && recording.google_sheet_id ? ( + {settings.integrationType === "googleSheets" && ( <> - - {t('integration_settings.alerts.success.title')} - {t('integration_settings.alerts.success.content', { sheetName: recording.google_sheet_name })} - - {t('integration_settings.alerts.success.here')} - . -
- {t('integration_settings.alerts.success.note')} {t('integration_settings.alerts.success.sync_limitation')} - - - - ) : ( - <> - {!recording?.google_sheet_email ? ( + + {t("integration_settings.google.title")} + + + {recording?.google_sheet_id ? ( <> -

{t('integration_settings.descriptions.sync_info')}

+ + {t("integration_settings.google.alerts.success.title")} + {t("integration_settings.google.alerts.success.content", { + sheetName: recording.google_sheet_name, + })} + + {t("integration_settings.google.alerts.success.here")} + + ) : ( <> - {recording.google_sheet_email && ( - - {t('integration_settings.descriptions.authenticated_as', { - email: recording.google_sheet_email - })} - - )} - - {loading ? ( - - ) : error ? ( - {error} - ) : spreadsheets.length === 0 ? ( + {!recording?.google_sheet_email ? ( <> -
+

{t("integration_settings.google.descriptions.sync_info")}

+ + + ) : ( + <> + + {t("integration_settings.google.descriptions.authenticated_as", { + email: recording.google_sheet_email, + })} + + {loading ? ( + + ) : error ? ( + {error} + ) : spreadsheets.length === 0 ? ( - -
- - ) : ( - <> - - {spreadsheets.map((sheet) => ( - - {sheet.name} - - ))} - - - {settings.spreadsheetId && ( - - {t('integration_settings.fields.selected_sheet', { - name: spreadsheets.find((s) => s.id === settings.spreadsheetId)?.name, - id: settings.spreadsheetId - })} - + ) : ( + <> + + {spreadsheets.map((sheet) => ( + + {sheet.name} + + ))} + + + )} + + )} + + )} + + )} + {settings.integrationType === "airtable" && ( + <> + + {t("integration_settings.airtable.title")} + + + {recording?.airtable_base_id ? ( + <> + + {t("integration_settings.airtable.alerts.success.title")} + {t("integration_settings.airtable.alerts.success.content", { + baseName: recording.airtable_base_name, + tableName: recording.airtable_table_name + })} + + {t("integration_settings.airtable.alerts.success.here")} + + + + + ) : ( + <> + {!recording?.airtable_access_token ? ( + <> +

{t("integration_settings.airtable.descriptions.sync_info")}

+ ) : ( + <> + + {t("integration_settings.airtable.descriptions.authenticated_as")} + + {loading ? ( + + ) : error ? ( + {error} + ) : airtableBases.length === 0 ? ( + + ) : ( + <> + + {airtableBases.map((base) => ( + + {base.name} + + ))} + + + {airtableTables.map((table) => ( + + {table.name} + + ))} + + + + )} + )} )} diff --git a/src/components/robot/Recordings.tsx b/src/components/robot/Recordings.tsx index e63f5e1c..780d01cf 100644 --- a/src/components/robot/Recordings.tsx +++ b/src/components/robot/Recordings.tsx @@ -59,12 +59,18 @@ export const Recordings = ({ }; const authStatus = getAndClearCookie('robot_auth_status'); + const airtableAuthStatus = getAndClearCookie('airtable_auth_status'); const robotId = getAndClearCookie('robot_auth_robotId'); - if (authStatus === 'success' && robotId) { + if (airtableAuthStatus === 'success' && robotId) { + console.log("Airtable Auth Status:", airtableAuthStatus); + notify(airtableAuthStatus, t("recordingtable.notifications.auth_success")); + handleNavigate(`/robots/${robotId}/integrate/airtable`, robotId, "", []); + } + else if (authStatus === 'success' && robotId) { + console.log("Google Auth Status:", authStatus); notify(authStatus, t("recordingtable.notifications.auth_success")); - - handleNavigate(`/robots/${robotId}/integrate`, robotId, "", []);'' + handleNavigate(`/robots/${robotId}/integrate/google`, robotId, "", []); } }, []); @@ -90,6 +96,24 @@ export const Recordings = ({ handleStart={handleScheduleRecording} /> ); + } else if (currentPath.endsWith("/integrate/google")) { + return ( + {}} + preSelectedIntegrationType="googleSheets" + /> + ); + } else if (currentPath.endsWith("/integrate/airtable")) { + return ( + {}} + preSelectedIntegrationType="airtable" + /> + ); } else if (currentPath.endsWith("/integrate")) { return (