diff --git a/server/migrations/20250207133740-added_table_id.js b/server/migrations/20250207133740-added_table_id.js new file mode 100644 index 00000000..0a40d0c7 --- /dev/null +++ b/server/migrations/20250207133740-added_table_id.js @@ -0,0 +1,16 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up (queryInterface, Sequelize) { + await queryInterface.addColumn('robot', 'airtable_table_id', { + type: Sequelize.TEXT + }); + }, + + async down (queryInterface, Sequelize) { + await queryInterface.removeColumn('robot', 'airtable_table_id', { + type: Sequelize.TEXT + }); + } +}; diff --git a/server/src/models/Robot.ts b/server/src/models/Robot.ts index ffd4746d..b8fa26e1 100644 --- a/server/src/models/Robot.ts +++ b/server/src/models/Robot.ts @@ -30,6 +30,7 @@ interface RobotAttributes { airtable_access_token?: string | null; // New field for Airtable access token airtable_refresh_token?: string | null; // New field for Airtable refresh token schedule?: ScheduleConfig | null; + airtable_table_id?: string | null; } interface ScheduleConfig { @@ -61,6 +62,7 @@ class Robot extends Model implements R public airtable_table_name!: string | null; // New field for Airtable table name public airtable_access_token!: string | null; // New field for Airtable access token public airtable_refresh_token!: string | null; // New field for Airtable refresh token + public airtable_table_id!: string | null; public schedule!: ScheduleConfig | null; } @@ -111,6 +113,10 @@ Robot.init( type: DataTypes.STRING, allowNull: true, }, + airtable_table_id: { + type: DataTypes.STRING, + allowNull: true, + }, airtable_access_token: { type: DataTypes.TEXT, allowNull: true, diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index 733f4039..6b186511 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -749,7 +749,7 @@ router.get("/airtable/bases", async (req: AuthenticatedRequest, res) => { // Update robot with selected base router.post("/airtable/update", async (req: AuthenticatedRequest, res) => { - const { baseId, robotId , tableName} = req.body; + const { baseId, robotId , tableName,tableId} = req.body; if (!baseId || !robotId) { return res.status(400).json({ message: "Base ID and Robot ID are required" }); @@ -767,6 +767,7 @@ router.post("/airtable/update", async (req: AuthenticatedRequest, res) => { await robot.update({ airtable_base_id: baseId, airtable_table_name: tableName, + airtable_table_id: tableId, }); diff --git a/server/src/workflow-management/integrations/airtable.ts b/server/src/workflow-management/integrations/airtable.ts index 9ace046c..fdcd1a3b 100644 --- a/server/src/workflow-management/integrations/airtable.ts +++ b/server/src/workflow-management/integrations/airtable.ts @@ -44,6 +44,7 @@ export async function updateAirtable(robotId: string, runId: string) { robotId, plainRobot.airtable_base_id, plainRobot.airtable_table_name, + plainRobot.airtable_table_id || '', data ); console.log(`Data written to Airtable for ${robotId}`); @@ -58,6 +59,7 @@ export async function writeDataToAirtable( robotId: string, baseId: string, tableName: string, + tableId: string, data: any[] ) { try { @@ -70,17 +72,20 @@ export async function writeDataToAirtable( const airtable = new Airtable({ apiKey: accessToken }); const base = airtable.base(baseId); + // Dynamic field creation logic const existingFields = await getExistingFields(base, tableName); const dataFields = [...new Set(data.flatMap(row => Object.keys(row)))]; const missingFields = dataFields.filter(field => !existingFields.includes(field)); for (const field of missingFields) { - const sampleValue = data.find(row => row[field])?.[field]; - if (sampleValue) { - await createAirtableField(baseId, tableName, field, sampleValue, accessToken); + const sampleRow = data.find(row => field in row); + if (sampleRow) { + const sampleValue = sampleRow[field]; + await createAirtableField(baseId, tableName, field, sampleValue, accessToken, tableId); } } + // Batch processing with retries const batchSize = 10; for (let i = 0; i < data.length; i += batchSize) { const batch = data.slice(i, i + batchSize); @@ -94,6 +99,7 @@ export async function writeDataToAirtable( } } +// Helper functions async function getExistingFields(base: Airtable.Base, tableName: string): Promise { try { const records = await base(tableName).select({ maxRecords: 1 }).firstPage(); @@ -109,48 +115,61 @@ async function createAirtableField( fieldName: string, sampleValue: any, accessToken: string, + tableId: string, + retries = MAX_RETRIES ): Promise { try { - let fieldType = inferFieldType(sampleValue); + const sanitizedFieldName = sanitizeFieldName(fieldName); + const fieldType = inferFieldType(sampleValue); - // Fallback if field type is unknown - if (!fieldType) { - fieldType = 'singleLineText'; - logger.log('warn', `Unknown field type for ${fieldName}, defaulting to singleLineText`); - } - - console.log(`Creating field: ${fieldName}, Type: ${fieldType}`); - await axios.post( - `https://api.airtable.com/v0/meta/bases/${baseId}/tables/${tableName}/fields`, - { name: fieldName, type: fieldType }, - { - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json' - } - } + `https://api.airtable.com/v0/meta/bases/${baseId}/tables/${tableId}/fields`, + { name: sanitizedFieldName, type: fieldType }, + { headers: { Authorization: `Bearer ${accessToken}` } } ); - - logger.log('info', `Created field: ${fieldName} (${fieldType})`); + logger.log('info', `Created field: ${sanitizedFieldName} (${fieldType})`); } catch (error: any) { if (retries > 0 && error.response?.status === 429) { - await delay(BASE_API_DELAY * (MAX_RETRIES - retries + 2)); - return createAirtableField(baseId, tableName, fieldName, sampleValue, accessToken, retries - 1); + await delay(BASE_API_DELAY * (MAX_RETRIES - retries + 1)); + return createAirtableField(baseId, tableName, fieldName, sampleValue, accessToken, tableId, retries - 1); } - throw new Error(`Field creation failed: ${error.response?.data?.error?.message || 'Unknown error'}`); + + const errorMessage = error.response?.data?.error?.message || error.message; + const statusCode = error.response?.status || 'No Status Code'; + throw new Error(`Field creation failed (${statusCode}): ${errorMessage}`); } } +function sanitizeFieldName(fieldName: string): string { + return fieldName + .trim() + .replace(/^[^a-zA-Z]+/, '') + .replace(/[^\w\s]/gi, ' ') + .substring(0, 50); +} + function inferFieldType(value: any): string { + if (value === null || value === undefined) return 'singleLineText'; if (typeof value === 'number') return 'number'; if (typeof value === 'boolean') return 'checkbox'; if (value instanceof Date) return 'dateTime'; - if (Array.isArray(value)) return 'multipleSelects'; + if (Array.isArray(value)) { + return value.length > 0 && typeof value[0] === 'object' ? 'multipleRecordLinks' : 'multipleSelects'; + } + if (typeof value === 'string' && isValidUrl(value)) return 'url'; return 'singleLineText'; } +function isValidUrl(str: string): boolean { + try { + new URL(str); + return true; + } catch (_) { + return false; + } +} + async function retryableAirtableWrite( base: Airtable.Base, tableName: string, @@ -188,7 +207,7 @@ export const processAirtableUpdates = async () => { task.retries += 1; if (task.retries >= MAX_RETRIES) { task.status = 'failed'; - logger.log('error', `Permanent failure for run ${runId}`); + logger.log('error', `Permanent failure for run ${runId}: ${error.message}`); } } } @@ -196,4 +215,4 @@ export const processAirtableUpdates = async () => { if (!hasPendingTasks) break; await delay(5000); } -}; +}; \ No newline at end of file diff --git a/src/components/integration/IntegrationSettings.tsx b/src/components/integration/IntegrationSettings.tsx index 2f251f64..26322435 100644 --- a/src/components/integration/IntegrationSettings.tsx +++ b/src/components/integration/IntegrationSettings.tsx @@ -18,6 +18,7 @@ import Cookies from "js-cookie"; import { useTranslation } from "react-i18next"; import { SignalCellularConnectedNoInternet0BarSharp } from "@mui/icons-material"; +import { table } from "console"; interface IntegrationProps { isOpen: boolean; @@ -257,6 +258,7 @@ export const IntegrationSettingsModal = ({ baseName: settings.airtableBaseName, robotId: recordingId, tableName: settings.airtableTableName, + tableId: settings.airtableTableId, }, { withCredentials: true } );