From 54a850b0f3f7fa7e0decb749a7740cc8310bb265 Mon Sep 17 00:00:00 2001
From: Rohit <rohit.rajan031101@gmail.com>
Date: Mon, 7 Jul 2025 22:50:34 +0530
Subject: [PATCH 1/6] feat: revamp cap text output preview ui

---
 src/components/run/InterpretationLog.tsx | 72 ++++++++++++++----------
 1 file changed, 43 insertions(+), 29 deletions(-)

diff --git a/src/components/run/InterpretationLog.tsx b/src/components/run/InterpretationLog.tsx
index 0f1f1185..faf965e9 100644
--- a/src/components/run/InterpretationLog.tsx
+++ b/src/components/run/InterpretationLog.tsx
@@ -379,49 +379,63 @@ export const InterpretationLog: React.FC<InterpretationLogProps> = ({ isOpen, se
                 )}
                 
                 {(activeTab === availableTabs.findIndex(tab => tab.id === 'captureText') || singleContentType === 'captureText') && captureTextData.length > 0 && (
-                  <TableContainer component={Paper} sx={{ boxShadow: 'none', borderRadius: 0 }}>
-                    <Table>
-                      <TableHead>
-                        <TableRow>
-                          {getCaptureTextColumns.map((column) => (
+                  <Box sx={{ p: 2 }}>
+                    <TableContainer component={Paper} sx={{ boxShadow: 'none', borderRadius: 0 }}>
+                      <Table>
+                        <TableHead>
+                          <TableRow>
                             <TableCell 
-                              key={column}
                               sx={{ 
                                 borderBottom: '1px solid',
                                 borderColor: darkMode ? '#3a4453' : '#dee2e6',
-                                backgroundColor: darkMode ? '#2a3441' : '#f8f9fa'
+                                backgroundColor: darkMode ? '#2a3441' : '#f8f9fa',
                               }}
                             >
-                              {column}
+                              Label
                             </TableCell>
-                          ))}
-                        </TableRow>
-                      </TableHead>
-                      <TableBody>
-                        {captureTextData.map((row, idx) => (
-                          <TableRow 
-                            key={idx}
-                            sx={{ 
-                              borderBottom: '1px solid',
-                              borderColor: darkMode ? '#3a4453' : '#dee2e6'
-                            }}
-                          >
-                            {getCaptureTextColumns.map((column) => (
+                            <TableCell 
+                              sx={{ 
+                                borderBottom: '1px solid',
+                                borderColor: darkMode ? '#3a4453' : '#dee2e6',
+                                backgroundColor: darkMode ? '#2a3441' : '#f8f9fa',
+                              }}
+                            >
+                              Value
+                            </TableCell>
+                          </TableRow>
+                        </TableHead>
+                        <TableBody>
+                          {getCaptureTextColumns.map((column, index) => (
+                            <TableRow 
+                              key={column}
+                              sx={{ 
+                                borderBottom: index < getCaptureTextColumns.length - 1 ? '1px solid' : 'none',
+                                borderColor: darkMode ? '#3a4453' : '#dee2e6'
+                              }}
+                            >
+                              <TableCell 
+                                sx={{ 
+                                  borderBottom: 'none',
+                                  py: 2,
+                                  fontWeight: 500
+                                }}
+                              >
+                                {column}
+                              </TableCell>
                               <TableCell 
-                                key={column}
                                 sx={{ 
                                   borderBottom: 'none',
                                   py: 2
                                 }}
                               >
-                                {row[column]}
+                                {captureTextData[0][column]}
                               </TableCell>
-                            ))}
-                          </TableRow>
-                        ))}
-                      </TableBody>
-                    </Table>
-                  </TableContainer>
+                            </TableRow>
+                          ))}
+                        </TableBody>
+                      </Table>
+                    </TableContainer>
+                  </Box>
                 )}
               </Box>
             </>

From c8b8262818d78e76c0dd93cb027da05523a22a67 Mon Sep 17 00:00:00 2001
From: Rohit <rohit.rajan031101@gmail.com>
Date: Mon, 7 Jul 2025 22:57:17 +0530
Subject: [PATCH 2/6] feat: revamp cap text run ui

---
 src/components/run/RunContent.tsx | 82 ++++++++++++++++++++++++-------
 1 file changed, 65 insertions(+), 17 deletions(-)

diff --git a/src/components/run/RunContent.tsx b/src/components/run/RunContent.tsx
index 25eb5063..2ad3be11 100644
--- a/src/components/run/RunContent.tsx
+++ b/src/components/run/RunContent.tsx
@@ -27,6 +27,7 @@ import TableHead from '@mui/material/TableHead';
 import TableRow from '@mui/material/TableRow';
 import 'highlight.js/styles/github.css';
 import { useTranslation } from "react-i18next";
+import { useThemeMode } from "../../context/theme-provider";
 
 interface RunContentProps {
   row: Data,
@@ -54,6 +55,8 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
   const [legacyColumns, setLegacyColumns] = useState<string[]>([]);
   const [isLegacyData, setIsLegacyData] = useState<boolean>(false);
 
+  const { darkMode } = useThemeMode();
+
   useEffect(() => {
     setTab(tab);
   }, [interpretationInProgress]);
@@ -262,8 +265,7 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
 
     if (!currentData || currentData.length === 0) return null;
 
-    const downloadData = isPaginatedList ? currentData : data;
-    const downloadColumns = isPaginatedList ? currentColumns : columns;
+    const isSchemaData = title.toLowerCase().includes('text') || title.toLowerCase().includes('schema');
 
     return (
       <Accordion defaultExpanded sx={{ mb: 2 }}>
@@ -283,7 +285,7 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
             <Box>
               <Button 
                 component="a"
-                onClick={() => downloadJSON(downloadData, jsonFilename)}
+                onClick={() => downloadJSON(currentData, jsonFilename)}
                 sx={{ 
                   color: '#FF00C3', 
                   textTransform: 'none',
@@ -297,12 +299,12 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
                   }
                 }}
               >
-                Download as JSON
+                {t('run_content.captured_data.download_json', 'Download as JSON')}
               </Button>
 
               <Button 
                 component="a"
-                onClick={() => downloadCSV(downloadData, downloadColumns, csvFilename)}
+                onClick={() => downloadCSV(currentData, currentColumns, csvFilename)}
                 sx={{ 
                   color: '#FF00C3', 
                   textTransform: 'none',
@@ -315,7 +317,7 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
                   }
                 }}
               >
-                Download as CSV
+                {t('run_content.captured_data.download_csv', 'Download as CSV')}
               </Button>
             </Box>
 
@@ -338,6 +340,7 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
                   onClick={() => navigateListTable('next')}
                   disabled={currentListIndex === listData.length - 1}
                   sx={{
+                    borderColor: '#FF00C3',
                     color: currentListIndex === listData.length - 1 ? 'gray' : '#FF00C3',
                     '&.Mui-disabled': {
                       borderColor: 'rgba(0, 0, 0, 0.12)'
@@ -353,21 +356,66 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
             <Table stickyHeader aria-label="sticky table">
               <TableHead>
                 <TableRow>
-                  {(isPaginatedList ? currentColumns : columns).map((column) => (
-                    <TableCell key={column}>{column}</TableCell>
-                  ))}
+                  {isSchemaData ? (
+                    <>
+                      <TableCell 
+                        sx={{ 
+                          borderBottom: '1px solid',
+                          borderColor: darkMode ? '#3a4453' : '#dee2e6',
+                          backgroundColor: darkMode ? '#2a3441' : '#f8f9fa'
+                        }}
+                      >
+                        Label
+                      </TableCell>
+                      <TableCell 
+                        sx={{ 
+                          borderBottom: '1px solid',
+                          borderColor: darkMode ? '#3a4453' : '#dee2e6',
+                          backgroundColor: darkMode ? '#2a3441' : '#f8f9fa'
+                        }}
+                      >
+                        Value
+                      </TableCell>
+                    </>
+                  ) : (
+                    (isPaginatedList ? currentColumns : columns).map((column) => (
+                      <TableCell 
+                        key={column}
+                        sx={{ 
+                          borderBottom: '1px solid',
+                          borderColor: darkMode ? '#3a4453' : '#dee2e6',
+                          backgroundColor: darkMode ? '#2a3441' : '#f8f9fa'
+                        }}
+                      >
+                        {column}
+                      </TableCell>
+                    ))
+                  )}
                 </TableRow>
               </TableHead>
               <TableBody>
-                {(isPaginatedList ? currentData : data).map((row, index) => (
-                  <TableRow key={index}>
-                    {(isPaginatedList ? currentColumns : columns).map((column) => (
-                      <TableCell key={column}>
-                        {row[column] === undefined || row[column] === "" ? "-" : row[column]}
+                {isSchemaData ? (
+                  currentColumns.map((column) => (
+                    <TableRow key={column}>
+                      <TableCell sx={{ fontWeight: 500 }}>
+                        {column}
                       </TableCell>
-                    ))}
-                  </TableRow>
-                ))}
+                      <TableCell>
+                        {currentData[0][column] === undefined || currentData[0][column] === "" ? "-" : currentData[0][column]}
+                      </TableCell>
+                    </TableRow>
+                  ))
+                ) : (
+                  currentData.map((row, index) => (
+                    <TableRow key={index}>
+                      {(isPaginatedList ? currentColumns : columns).map((column) => (
+                        <TableCell key={column}>
+                          {row[column] === undefined || row[column] === "" ? "-" : row[column]}
+                        </TableCell>
+                      ))}
+                    </TableRow>
+                  ))
+                )}
               </TableBody>
             </Table>
           </TableContainer>

From 708f846245cacb08f28b82fb2a46d08fcb0ad70e Mon Sep 17 00:00:00 2001
From: Rohit <rohit.rajan031101@gmail.com>
Date: Tue, 8 Jul 2025 13:40:55 +0530
Subject: [PATCH 3/6] feat: structure text data for csv format

---
 src/components/run/RunContent.tsx | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/src/components/run/RunContent.tsx b/src/components/run/RunContent.tsx
index 2ad3be11..38491854 100644
--- a/src/components/run/RunContent.tsx
+++ b/src/components/run/RunContent.tsx
@@ -194,17 +194,27 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
   };
 
   // Function to convert table data to CSV format
-  const convertToCSV = (data: any[], columns: string[]): string => {
-    const header = columns.join(',');
-    const rows = data.map(row =>
-      columns.map(col => JSON.stringify(row[col] || "", null, 2)).join(',')
-    );
-    return [header, ...rows].join('\n');
+  const convertToCSV = (data: any[], columns: string[], isSchemaData: boolean = false): string => {
+    if (isSchemaData) {
+      // For schema data, export as Label-Value pairs
+      const header = 'Label,Value';
+      const rows = columns.map(column => 
+        `"${column}","${data[0][column] || ""}"`
+      );
+      return [header, ...rows].join('\n');
+    } else {
+      // For regular table data, export as normal table
+      const header = columns.join(',');
+      const rows = data.map(row =>
+        columns.map(col => JSON.stringify(row[col] || "", null, 2)).join(',')
+      );
+      return [header, ...rows].join('\n');
+    }
   };
 
   // Function to download a specific dataset as CSV
-  const downloadCSV = (data: any[], columns: string[], filename: string) => {
-    const csvContent = convertToCSV(data, columns);
+  const downloadCSV = (data: any[], columns: string[], filename: string, isSchemaData: boolean = false) => {
+    const csvContent = convertToCSV(data, columns, isSchemaData);
     const blob = new Blob([csvContent], { type: 'text/csv;charset=utf-8;' });
     const url = URL.createObjectURL(blob);
 
@@ -304,7 +314,7 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
 
               <Button 
                 component="a"
-                onClick={() => downloadCSV(currentData, currentColumns, csvFilename)}
+                onClick={() => downloadCSV(currentData, currentColumns, csvFilename, isSchemaData)}
                 sx={{ 
                   color: '#FF00C3', 
                   textTransform: 'none',

From 782f38acd393043862c4116e3a56c49696c350d2 Mon Sep 17 00:00:00 2001
From: Rohit <rohit.rajan031101@gmail.com>
Date: Tue, 8 Jul 2025 14:08:26 +0530
Subject: [PATCH 4/6] feat: struct capture text data google sheets

---
 server/src/workflow-management/integrations/gsheet.ts | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/server/src/workflow-management/integrations/gsheet.ts b/server/src/workflow-management/integrations/gsheet.ts
index 402ace5d..2a29bdcc 100644
--- a/server/src/workflow-management/integrations/gsheet.ts
+++ b/server/src/workflow-management/integrations/gsheet.ts
@@ -114,7 +114,16 @@ async function processOutputType(
     
     await ensureSheetExists(spreadsheetId, sheetName, robotConfig);
     
-    await writeDataToSheet(robotId, spreadsheetId, data, sheetName, robotConfig);
+    let formattedData = data;
+    if (outputType === 'Text' && data.length > 0) {
+      const schemaItem = data[0];
+      formattedData = Object.entries(schemaItem).map(([key, value]) => ({
+        Label: key,
+        Value: value
+      }));
+    }
+    
+    await writeDataToSheet(robotId, spreadsheetId, formattedData, sheetName, robotConfig);
     console.log(`Data written to ${sheetName} sheet for ${outputType} data`);
   }
 }

From 3916ac91c4b3ad07449ac3d29a523aa10e2eac39 Mon Sep 17 00:00:00 2001
From: Rohit <rohit.rajan031101@gmail.com>
Date: Tue, 8 Jul 2025 15:21:59 +0530
Subject: [PATCH 5/6] feat: struct capture text data airtable

---
 .../integrations/airtable.ts                  | 298 ++++++++----------
 1 file changed, 132 insertions(+), 166 deletions(-)

diff --git a/server/src/workflow-management/integrations/airtable.ts b/server/src/workflow-management/integrations/airtable.ts
index 401bc11d..0c9dfe71 100644
--- a/server/src/workflow-management/integrations/airtable.ts
+++ b/server/src/workflow-management/integrations/airtable.ts
@@ -44,65 +44,118 @@ async function refreshAirtableToken(refreshToken: string) {
   }
 }
 
+
 function mergeRelatedData(serializableOutput: SerializableOutput, binaryOutput: Record<string, string>) {
-  const mergedRecords: Record<string, any>[] = [];
+  const allRecords: Record<string, any>[] = [];
   
-  const maxLength = Math.max(
-    ...[
-      ...(serializableOutput.scrapeSchema ?? []).map(arr => arr?.length ?? 0),
-      ...(serializableOutput.scrapeList ?? []).map(arr => arr?.length ?? 0),
-      0
-    ]
-  );
+  // Process all data types together, not separately
+  // This creates an interleaved/mixed result like in your image
   
-  for (let i = 0; i < maxLength; i++) {
-    mergedRecords.push({});
-  }
+  let schemaIndex = 0;
+  let listIndex = 0;
+  let screenshotIndex = 0;
   
+  // Flatten all data arrays for processing
+  const schemaData: Array<{key: string, value: any}> = [];
+  const listData: any[] = [];
+  const screenshotData: Array<{key: string, url: string}> = [];
+  
+  // Collect schema data
   if (serializableOutput.scrapeSchema) {
     for (const schemaArray of serializableOutput.scrapeSchema) {
       if (!Array.isArray(schemaArray)) continue;
-      
-      for (let i = 0; i < schemaArray.length; i++) {
-        if (i >= mergedRecords.length) break;
-        mergedRecords[i] = { ...mergedRecords[i], ...schemaArray[i] };
+      for (const schemaItem of schemaArray) {
+        Object.entries(schemaItem).forEach(([key, value]) => {
+          if (key && key.trim() !== '' && value !== null && value !== undefined && value !== '') {
+            schemaData.push({key, value});
+          }
+        });
       }
     }
   }
   
+  // Collect list data
   if (serializableOutput.scrapeList) {
     for (const listArray of serializableOutput.scrapeList) {
       if (!Array.isArray(listArray)) continue;
-      
-      for (let i = 0; i < listArray.length; i++) {
-        if (i >= mergedRecords.length) break;
-        mergedRecords[i] = { ...mergedRecords[i], ...listArray[i] };
-      }
-    }
-  }
-  
-  if (binaryOutput && Object.keys(binaryOutput).length > 0) {
-    for (let i = 0; i < mergedRecords.length; i++) {
-      const screenshotKey = `item-${i}`;
-      if (binaryOutput[screenshotKey]) {
-        mergedRecords[i].Screenshot = binaryOutput[screenshotKey];
-        mergedRecords[i].Key = screenshotKey;
-      }
-    }
-    
-    for (const [key, url] of Object.entries(binaryOutput)) {
-      if (mergedRecords.some(record => record.Key === key)) {
-        continue;
-      }
-      
-      mergedRecords.push({
-        "Key": key,
-        "Screenshot": url
+      listArray.forEach(listItem => {
+        const hasContent = Object.values(listItem).some(value => 
+          value !== null && value !== undefined && value !== ''
+        );
+        if (hasContent) {
+          listData.push(listItem);
+        }
       });
     }
   }
   
-  return mergedRecords;
+  // Collect screenshot data
+  if (binaryOutput && Object.keys(binaryOutput).length > 0) {
+    Object.entries(binaryOutput).forEach(([key, url]) => {
+      if (key && key.trim() !== '' && url && url.trim() !== '') {
+        screenshotData.push({key, url});
+      }
+    });
+  }
+  
+  // Mix all data types together to create consecutive records
+  const maxLength = Math.max(schemaData.length, listData.length, screenshotData.length);
+  
+  for (let i = 0; i < maxLength; i++) {
+    const record: Record<string, any> = {};
+    
+    // Add schema data if available
+    if (i < schemaData.length) {
+      record.Label = schemaData[i].key;
+      record.Value = schemaData[i].value;
+    }
+    
+    // Add list data if available
+    if (i < listData.length) {
+      Object.entries(listData[i]).forEach(([key, value]) => {
+        if (value !== null && value !== undefined && value !== '') {
+          record[key] = value;
+        }
+      });
+    }
+    
+    // Add screenshot data if available
+    if (i < screenshotData.length) {
+      record.Key = screenshotData[i].key;
+      record.Screenshot = screenshotData[i].url;
+    }
+    
+    // Only add record if it has at least one value
+    if (Object.keys(record).length > 0) {
+      allRecords.push(record);
+    }
+  }
+  
+  // Add any remaining data that didn't fit in the parallel processing
+  // This handles cases where one data type has significantly more records than others
+  
+  // Add remaining schema data
+  for (let i = maxLength; i < schemaData.length; i++) {
+    allRecords.push({
+      Label: schemaData[i].key,
+      Value: schemaData[i].value
+    });
+  }
+  
+  // Add remaining list data
+  for (let i = maxLength; i < listData.length; i++) {
+    allRecords.push(listData[i]);
+  }
+  
+  // Add remaining screenshot data
+  for (let i = maxLength; i < screenshotData.length; i++) {
+    allRecords.push({
+      Key: screenshotData[i].key,
+      Screenshot: screenshotData[i].url
+    });
+  }
+  
+  return allRecords;
 }
 
 export async function updateAirtable(robotId: string, runId: string) {
@@ -210,11 +263,13 @@ export async function writeDataToAirtable(
       const airtable = new Airtable({ apiKey: accessToken });
       const base = airtable.base(baseId);
 
+      await deleteEmptyRecords(base, tableName);
+
       const processedData = data.map(item => {
         const cleanedItem: Record<string, any> = {};
         
         for (const [key, value] of Object.entries(item)) {
-          if (value === null || value === undefined) {
+          if (value === null || value === undefined || value === '') {
             cleanedItem[key] = '';
           } else if (typeof value === 'object' && !Array.isArray(value)) {
             cleanedItem[key] = JSON.stringify(value);
@@ -224,113 +279,55 @@ export async function writeDataToAirtable(
         }
         
         return cleanedItem;
+      }).filter(record => {
+        return Object.values(record).some(value => value !== null && value !== undefined && value !== '');
       });
 
-      const existingFields = await getExistingFields(base, tableName);
-      console.log(`Found ${existingFields.length} existing fields in Airtable: ${existingFields.join(', ')}`); 
+      if (processedData.length === 0) {
+        console.log('No valid data to write after filtering. Skipping.');
+        return;
+      }
 
-      const dataFields = [...new Set(processedData.flatMap(row => Object.keys(row)))];  
+      const dataFields = [...new Set(processedData.flatMap(row => Object.keys(row)))];
       console.log(`Found ${dataFields.length} fields in data: ${dataFields.join(', ')}`);
 
+      const existingFields = await getExistingFields(base, tableName);
       const missingFields = dataFields.filter(field => !existingFields.includes(field));
-      const hasNewColumns = missingFields.length > 0;
-      console.log(`Found ${missingFields.length} new fields: ${missingFields.join(', ')}`);
-
-      for (const field of missingFields) {
-        const sampleRow = processedData.find(row => field in row);
-        if (sampleRow) {
-          const sampleValue = sampleRow[field];
-          try {
-            await createAirtableField(baseId, tableName, field, sampleValue, accessToken, tableId);
-            console.log(`Successfully created field: ${field}`);
-            
-            await new Promise(resolve => setTimeout(resolve, 200));
-          } catch (fieldError: any) {
-            console.warn(`Warning: Could not create field "${field}": ${fieldError.message}`);
-          }
-        }
-      }
       
-      let existingRecords: Array<{ id: string, fields: Record<string, any> }> = [];
-      
-      if (hasNewColumns) {
-        existingRecords = await fetchAllRecords(base, tableName);
-        console.log(`Found ${existingRecords.length} existing records in Airtable`);
-      }
-      
-      if (hasNewColumns && existingRecords.length > 0) {
-        const recordsToUpdate = [];
-        const recordsToCreate = [];
+      if (missingFields.length > 0) {
+        console.log(`Creating ${missingFields.length} new fields: ${missingFields.join(', ')}`);
         
-        const newColumnData = processedData.map(record => {
-          const newColumnsOnly: Record<string, any> = {};
-          missingFields.forEach(field => {
-            if (field in record) {
-              newColumnsOnly[field] = record[field];
-            }
-          });
-          return newColumnsOnly;
-        });
-        
-        for (let i = 0; i < Math.min(existingRecords.length, newColumnData.length); i++) {
-          if (Object.keys(newColumnData[i]).length > 0) {
-            recordsToUpdate.push({
-              id: existingRecords[i].id,
-              fields: newColumnData[i]
-            });
-          }
-        }
-        
-        const existingColumnsBeingUpdated = dataFields.filter(field => 
-          existingFields.includes(field) && !missingFields.includes(field)
-        );
-        
-        if (existingColumnsBeingUpdated.length > 0) {
-          recordsToCreate.push(...processedData.map(record => ({ fields: record })));
-          console.log(`Will append ${recordsToCreate.length} new records with all data`);
-        } else {
-          if (processedData.length > existingRecords.length) {
-            const additionalRecords = processedData.slice(existingRecords.length);
-            recordsToCreate.push(...additionalRecords.map(record => ({ fields: record })));
-            console.log(`Will append ${recordsToCreate.length} additional records`);
-          }
-        }
-        
-        if (recordsToUpdate.length > 0) {
-          console.log(`Updating ${recordsToUpdate.length} existing records with new columns`);
-          const BATCH_SIZE = 10;
-          for (let i = 0; i < recordsToUpdate.length; i += BATCH_SIZE) {
-            const batch = recordsToUpdate.slice(i, i + BATCH_SIZE);
-            console.log(`Updating batch ${Math.floor(i/BATCH_SIZE) + 1} of ${Math.ceil(recordsToUpdate.length/BATCH_SIZE)}`);
-            
+        for (const field of missingFields) {
+          const sampleRow = processedData.find(row => field in row && row[field] !== '');
+          if (sampleRow) {
+            const sampleValue = sampleRow[field];
             try {
-              await retryableAirtableUpdate(base, tableName, batch);
-            } catch (batchError: any) {
-              console.error(`Error updating batch: ${batchError.message}`);
-              throw batchError;
+              await createAirtableField(baseId, tableName, field, sampleValue, accessToken, tableId);
+              console.log(`Successfully created field: ${field}`);
+              await new Promise(resolve => setTimeout(resolve, 200));
+            } catch (fieldError: any) {
+              console.warn(`Warning: Could not create field "${field}": ${fieldError.message}`);
             }
-            
-            await new Promise(resolve => setTimeout(resolve, 500));
           }
         }
-      } else {
-        console.log(`Appending all ${processedData.length} records to Airtable`);
-        const recordsToCreate = processedData.map(record => ({ fields: record }));
+      }
+
+      console.log(`Appending all ${processedData.length} records to Airtable`);
+      const recordsToCreate = processedData.map(record => ({ fields: record }));
+      
+      const BATCH_SIZE = 10;
+      for (let i = 0; i < recordsToCreate.length; i += BATCH_SIZE) {
+        const batch = recordsToCreate.slice(i, i + BATCH_SIZE);
+        console.log(`Creating batch ${Math.floor(i/BATCH_SIZE) + 1} of ${Math.ceil(recordsToCreate.length/BATCH_SIZE)}`);
         
-        const BATCH_SIZE = 10;
-        for (let i = 0; i < recordsToCreate.length; i += BATCH_SIZE) {
-          const batch = recordsToCreate.slice(i, i + BATCH_SIZE);
-          console.log(`Creating batch ${Math.floor(i/BATCH_SIZE) + 1} of ${Math.ceil(recordsToCreate.length/BATCH_SIZE)}`);
-          
-          try {
-            await retryableAirtableCreate(base, tableName, batch);
-          } catch (batchError: any) {
-            console.error(`Error creating batch: ${batchError.message}`);
-            throw batchError;
-          }
-          
-          await new Promise(resolve => setTimeout(resolve, 500));
+        try {
+          await retryableAirtableCreate(base, tableName, batch);
+        } catch (batchError: any) {
+          console.error(`Error creating batch: ${batchError.message}`);
+          throw batchError;
         }
+        
+        await new Promise(resolve => setTimeout(resolve, 500));
       }
       
       await deleteEmptyRecords(base, tableName);
@@ -343,20 +340,6 @@ export async function writeDataToAirtable(
   }
 }
 
-async function fetchAllRecords(base: Airtable.Base, tableName: string): Promise<Array<{ id: string, fields: Record<string, any> }>> {
-  try {
-    console.log(`Fetching all records from ${tableName}...`);
-    const records = await base(tableName).select().all();
-    return records.map(record => ({
-      id: record.id,
-      fields: record.fields
-    }));
-  } catch (error: any) {
-    console.warn(`Warning: Could not fetch all records: ${error.message}`);
-    return [];
-  }
-}
-
 async function deleteEmptyRecords(base: Airtable.Base, tableName: string): Promise<void> {
   console.log('Checking for empty records to clear...');
   
@@ -407,23 +390,6 @@ async function retryableAirtableCreate(
   }
 }
 
-async function retryableAirtableUpdate(
-  base: Airtable.Base,
-  tableName: string,
-  batch: any[],
-  retries = MAX_RETRIES
-): Promise<void> {
-  try {
-    await base(tableName).update(batch);
-  } catch (error) {
-    if (retries > 0) {
-      await new Promise(resolve => setTimeout(resolve, BASE_API_DELAY));
-      return retryableAirtableUpdate(base, tableName, batch, retries - 1);
-    }
-    throw error;
-  }
-}
-
 // Helper functions
 async function getExistingFields(base: Airtable.Base, tableName: string): Promise<string[]> {
   try {

From bed04598cc5c6525253c08fe0752565af3ce2b45 Mon Sep 17 00:00:00 2001
From: Rohit <rohit.rajan031101@gmail.com>
Date: Tue, 8 Jul 2025 15:23:40 +0530
Subject: [PATCH 6/6] chore: stale code cleanup

---
 .../integrations/airtable.ts                   | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/server/src/workflow-management/integrations/airtable.ts b/server/src/workflow-management/integrations/airtable.ts
index 0c9dfe71..5f72c836 100644
--- a/server/src/workflow-management/integrations/airtable.ts
+++ b/server/src/workflow-management/integrations/airtable.ts
@@ -48,14 +48,6 @@ async function refreshAirtableToken(refreshToken: string) {
 function mergeRelatedData(serializableOutput: SerializableOutput, binaryOutput: Record<string, string>) {
   const allRecords: Record<string, any>[] = [];
   
-  // Process all data types together, not separately
-  // This creates an interleaved/mixed result like in your image
-  
-  let schemaIndex = 0;
-  let listIndex = 0;
-  let screenshotIndex = 0;
-  
-  // Flatten all data arrays for processing
   const schemaData: Array<{key: string, value: any}> = [];
   const listData: any[] = [];
   const screenshotData: Array<{key: string, url: string}> = [];
@@ -104,13 +96,11 @@ function mergeRelatedData(serializableOutput: SerializableOutput, binaryOutput:
   for (let i = 0; i < maxLength; i++) {
     const record: Record<string, any> = {};
     
-    // Add schema data if available
     if (i < schemaData.length) {
       record.Label = schemaData[i].key;
       record.Value = schemaData[i].value;
     }
     
-    // Add list data if available
     if (i < listData.length) {
       Object.entries(listData[i]).forEach(([key, value]) => {
         if (value !== null && value !== undefined && value !== '') {
@@ -119,22 +109,16 @@ function mergeRelatedData(serializableOutput: SerializableOutput, binaryOutput:
       });
     }
     
-    // Add screenshot data if available
     if (i < screenshotData.length) {
       record.Key = screenshotData[i].key;
       record.Screenshot = screenshotData[i].url;
     }
     
-    // Only add record if it has at least one value
     if (Object.keys(record).length > 0) {
       allRecords.push(record);
     }
   }
   
-  // Add any remaining data that didn't fit in the parallel processing
-  // This handles cases where one data type has significantly more records than others
-  
-  // Add remaining schema data
   for (let i = maxLength; i < schemaData.length; i++) {
     allRecords.push({
       Label: schemaData[i].key,
@@ -142,12 +126,10 @@ function mergeRelatedData(serializableOutput: SerializableOutput, binaryOutput:
     });
   }
   
-  // Add remaining list data
   for (let i = maxLength; i < listData.length; i++) {
     allRecords.push(listData[i]);
   }
   
-  // Add remaining screenshot data
   for (let i = maxLength; i < screenshotData.length; i++) {
     allRecords.push({
       Key: screenshotData[i].key,