vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: rm other actions logic

Browse Source
This commit is contained in:
Rohit
2025-04-30 19:24:38 +05:30
parent c7e3a66511
commit f1d0cbdaae
13 changed files with 9 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
public/locales/de.json
View File

@@ -547,8 +547,7 @@
"view_full": "Vollständige Daten anzeigen",
"items": "Elemente",
"schema_title": "Text erfassen",
"list_title": "Liste erfassen",
"other_title": "Andere Daten"
"list_title": "Liste erfassen"
},
"captured_screenshot": {
"title": "Erfasste Screenshots",

3
public/locales/en.json
View File

@@ -568,8 +568,7 @@
"view_full": "View Full Data",
"items": "items",
"schema_title": "Capture Text",
"list_title": "Capture List",
"other_title": "Other Data"
"list_title": "Capture List"
},
"captured_screenshot": {
"title": "Captured Screenshots",

3
public/locales/es.json
View File

@@ -548,8 +548,7 @@
"view_full": "Ver Datos Completos",
"items": "elementos",
"schema_title": "Capturar Texto",
"list_title": "Capturar Lista",
"other_title": "Otros Datos"
"list_title": "Capturar Lista"
},
"captured_screenshot": {
"title": "Capturas de Pantalla",

3
public/locales/ja.json
View File

@@ -548,8 +548,7 @@
"view_full": "すべてのデータを表示",
"items": "アイテム",
"schema_title": "テキストをキャプチャ",
"list_title": "リストをキャプチャ",
"other_title": "その他のデータ"
"list_title": "リストをキャプチャ"
},
"captured_screenshot": {
"title": "キャプチャされたスクリーンショット",

3
public/locales/zh.json
View File

@@ -548,8 +548,7 @@
"view_full": "查看完整数据",
"items": "项目",
"schema_title": "捕获文本",
"list_title": "捕获列表",
"other_title": "其他数据"
"list_title": "捕获列表"
},
"captured_screenshot": {
"title": "已捕获的截图",

2
server/src/api/record.ts
View File

@@ -589,7 +589,6 @@ async function executeRun(id: string, userId: string) {
const categorizedOutput = {
scrapeSchema: interpretationInfo.scrapeSchemaOutput || {},
scrapeList: interpretationInfo.scrapeListOutput || {},
other: interpretationInfo.otherOutput || {}
};
await destroyRemoteBrowser(plainRun.browserId, userId);
@@ -603,7 +602,6 @@ async function executeRun(id: string, userId: string) {
serializableOutput: {
scrapeSchema: Object.values(categorizedOutput.scrapeSchema),
scrapeList: Object.values(categorizedOutput.scrapeList),
other: Object.values(categorizedOutput.other),
},
binaryOutput: uploadedBinaryOutput,
});

7
server/src/pgboss-worker.ts
View File

@@ -265,8 +265,7 @@ async function processRunExecution(job: Job<ExecuteRunData>) {
const categorizedOutput = {
scrapeSchema: interpretationInfo.scrapeSchemaOutput || {},
scrapeList: interpretationInfo.scrapeListOutput || {},
other: interpretationInfo.otherOutput || {}
scrapeList: interpretationInfo.scrapeListOutput || {}
};
await run.update({
@@ -278,7 +277,6 @@ async function processRunExecution(job: Job<ExecuteRunData>) {
serializableOutput: {
scrapeSchema: Object.values(categorizedOutput.scrapeSchema),
scrapeList: Object.values(categorizedOutput.scrapeList),
other: Object.values(categorizedOutput.other),
},
binaryOutput: uploadedBinaryOutput,
});
@@ -482,7 +480,6 @@ async function abortRun(runId: string, userId: string): Promise<boolean> {
let categorizedOutput = {
scrapeSchema: {},
scrapeList: {},
other: {}
};
let binaryOutput: Record<string, any> = {};
@@ -496,7 +493,6 @@ async function abortRun(runId: string, userId: string): Promise<boolean> {
categorizedOutput = {
scrapeSchema: collectDataByType(browser.interpreter.serializableDataByType.scrapeSchema || []),
scrapeList: collectDataByType(browser.interpreter.serializableDataByType.scrapeList || []),
other: collectDataByType(browser.interpreter.serializableDataByType.other || [])
};
}
@@ -516,7 +512,6 @@ async function abortRun(runId: string, userId: string): Promise<boolean> {
serializableOutput: {
scrapeSchema: Object.values(categorizedOutput.scrapeSchema),
scrapeList: Object.values(categorizedOutput.scrapeList),
other: Object.values(categorizedOutput.other),
},
binaryOutput,
});

20
server/src/workflow-management/classes/Interpreter.ts
View File

@@ -92,11 +92,9 @@ export class WorkflowInterpreter {
public serializableDataByType: {
scrapeSchema: any[],
scrapeList: any[],
other: any[]
} = {
scrapeSchema: [],
scrapeList: [],
other: []
};
/**
@@ -216,12 +214,7 @@ export class WorkflowInterpreter {
type: 'captureList',
data
});
} else {
this.socket.emit('serializableCallback', {
type: 'other',
data
});
}
}
},
binaryCallback: (data: string, mimetype: string) => {
this.socket.emit('binaryCallback', {
@@ -292,7 +285,6 @@ export class WorkflowInterpreter {
this.serializableDataByType = {
scrapeSchema: [],
scrapeList: [],
other: []
};
this.binaryData = [];
}
@@ -342,9 +334,7 @@ export class WorkflowInterpreter {
}
} else if (this.currentActionType === 'scrapeList') {
this.serializableDataByType.scrapeList.push(data);
} else {
this.serializableDataByType.other.push(data);
}
}
this.socket.emit('serializableCallback', data);
},
@@ -394,12 +384,6 @@ export class WorkflowInterpreter {
...reducedObject,
}
}, {}),
otherOutput: this.serializableDataByType.other.reduce((reducedObject, item, index) => {
return {
[`item-${index}`]: item,
...reducedObject,
}
}, {}),
binaryOutput: this.binaryData.reduce((reducedObject, item, index) => {
return {
[`item-${index}`]: item,

13
server/src/workflow-management/integrations/airtable.ts
View File

@@ -14,7 +14,6 @@ interface AirtableUpdateTask {
interface SerializableOutput {
scrapeSchema?: any[];
scrapeList?: any[];
other?: any[];
}
const MAX_RETRIES = 3;
@@ -52,7 +51,6 @@ function mergeRelatedData(serializableOutput: SerializableOutput, binaryOutput:
...[
...(serializableOutput.scrapeSchema ?? []).map(arr => arr?.length ?? 0),
...(serializableOutput.scrapeList ?? []).map(arr => arr?.length ?? 0),
...(serializableOutput.other ?? []).map(arr => arr?.length ?? 0),
0
]
);
@@ -83,17 +81,6 @@ function mergeRelatedData(serializableOutput: SerializableOutput, binaryOutput:
}
}
if (serializableOutput.other) {
for (const otherArray of serializableOutput.other) {
if (!Array.isArray(otherArray)) continue;
for (let i = 0; i < otherArray.length; i++) {
if (i >= mergedRecords.length) break;
mergedRecords[i] = { ...mergedRecords[i], ...otherArray[i] };
}
}
}
if (binaryOutput && Object.keys(binaryOutput).length > 0) {
for (let i = 0; i < mergedRecords.length; i++) {
const screenshotKey = `item-${i}`;

11
server/src/workflow-management/integrations/gsheet.ts
View File

@@ -13,7 +13,6 @@ interface GoogleSheetUpdateTask {
interface SerializableOutput {
scrapeSchema?: any[];
scrapeList?: any[];
other?: any[];
}
const MAX_RETRIES = 5;
@@ -69,16 +68,6 @@ export async function updateGoogleSheet(robotId: string, runId: string) {
plainRobot
);
}
if (serializableOutput.other && serializableOutput.other.length > 0) {
await processOutputType(
robotId,
spreadsheetId,
'Other',
serializableOutput.other,
plainRobot
);
}
}
if (plainRun.binaryOutput && Object.keys(plainRun.binaryOutput).length > 0) {

2
server/src/workflow-management/scheduler/index.ts
View File

@@ -135,7 +135,6 @@ async function executeRun(id: string, userId: string) {
const categorizedOutput = {
scrapeSchema: interpretationInfo.scrapeSchemaOutput || {},
scrapeList: interpretationInfo.scrapeListOutput || {},
other: interpretationInfo.otherOutput || {}
};
await destroyRemoteBrowser(plainRun.browserId, userId);
@@ -149,7 +148,6 @@ async function executeRun(id: string, userId: string) {
serializableOutput: {
scrapeSchema: Object.values(categorizedOutput.scrapeSchema),
scrapeList: Object.values(categorizedOutput.scrapeList),
other: Object.values(categorizedOutput.other),
},
binaryOutput: uploadedBinaryOutput,
});

96
src/components/run/InterpretationLog.tsx
View File

@@ -96,17 +96,6 @@ export const InterpretationLog: React.FC<InterpretationLogProps> = ({ isOpen, se
const tabIndex = availableTabs.findIndex(tab => tab.id === 'captureText');
if (tabIndex !== -1) setActiveTab(tabIndex);
}
} else if (type === 'other') {
if (Array.isArray(data)) {
setOtherData(prev => [...prev, data]);
} else {
setOtherData(prev => [...prev, [data]]);
}
if (otherData.length === 0) {
const availableTabs = getAvailableTabs();
const tabIndex = availableTabs.findIndex(tab => tab.id === 'other');
if (tabIndex !== -1) setActiveTab(tabIndex);
}
}
scrollLogToBottom();
@@ -178,10 +167,6 @@ export const InterpretationLog: React.FC<InterpretationLogProps> = ({ isOpen, se
tabs.push({ id: 'captureScreenshot', label: 'Screenshots' });
}
if (otherData.length > 0) {
tabs.push({ id: 'other', label: 'Other' });
}
return tabs;
}, [captureListData.length, captureTextData.length, screenshotData.length, otherData.length]);
@@ -464,87 +449,6 @@ export const InterpretationLog: React.FC<InterpretationLogProps> = ({ isOpen, se
</Table>
</TableContainer>
)}
{activeTab === availableTabs.findIndex(tab => tab.id === 'other') && otherData.length > 0 && (
<Box>
{otherData.length > 1 && (
<Box sx={{
display: 'flex',
justifyContent: 'space-between',
alignItems: 'center',
mb: 2,
mt: 2
}}>
<Typography variant="body2">
{`Dataset ${otherPage + 1} of ${otherData.length}`}
</Typography>
<Box>
<Button
onClick={() => setOtherPage(prev => Math.max(0, prev - 1))}
disabled={otherPage === 0}
size="small"
>
Previous
</Button>
<Button
onClick={() => setOtherPage(prev => Math.min(otherData.length - 1, prev + 1))}
disabled={otherPage >= otherData.length - 1}
size="small"
sx={{ ml: 1 }}
>
Next
</Button>
</Box>
</Box>
)}
<TableContainer component={Paper} sx={{ boxShadow: 'none', borderRadius: 0 }}>
<Table>
<TableHead>
<TableRow>
{otherData[otherPage] && otherData[otherPage].length > 0 &&
Object.keys(otherData[otherPage][0]).map((column) => (
<TableCell
key={column}
sx={{
borderBottom: '1px solid',
borderColor: darkMode ? '#3a4453' : '#dee2e6',
backgroundColor: darkMode ? '#2a3441' : '#f8f9fa'
}}
>
{column}
</TableCell>
))
}
</TableRow>
</TableHead>
<TableBody>
{otherData[otherPage] &&
otherData[otherPage].map((row: any, idx: any) => (
<TableRow
key={idx}
sx={{
borderBottom: '1px solid',
borderColor: darkMode ? '#3a4453' : '#dee2e6'
}}
>
{Object.keys(row).map((column) => (
<TableCell
key={column}
sx={{
borderBottom: 'none',
py: 2
}}
>
{row[column]}
</TableCell>
))}
</TableRow>
))}
</TableBody>
</Table>
</TableContainer>
</Box>
)}
</Box>
</>
) : (

43
src/components/run/RunContent.tsx
View File

@@ -45,9 +45,6 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
const [listData, setListData] = useState<any[][]>([]);
const [listColumns, setListColumns] = useState<string[][]>([]);
const [currentListIndex, setCurrentListIndex] = useState<number>(0);
const [otherData, setOtherData] = useState<any[]>([]);
const [otherColumns, setOtherColumns] = useState<string[]>([]);
const [expandedView, setExpandedView] = useState<string | null>(null);
@@ -66,7 +63,6 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
if (!row.serializableOutput.scrapeSchema &&
!row.serializableOutput.scrapeList &&
!row.serializableOutput.other &&
Object.keys(row.serializableOutput).length > 0) {
setIsLegacyData(true);
@@ -83,10 +79,6 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
if (row.serializableOutput.scrapeList) {
processScrapeList(row.serializableOutput.scrapeList);
}
if (row.serializableOutput.other && Object.keys(row.serializableOutput.other).length > 0) {
processDataCategory(row.serializableOutput.other, setOtherData, setOtherColumns);
}
}, [row.serializableOutput]);
const processLegacyData = (legacyOutput: Record<string, any>) => {
@@ -240,7 +232,6 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
allData = {
schema: schemaData,
list: listData.flat(),
other: otherData
};
}
@@ -619,13 +610,6 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
csvFilename = 'list_data.csv';
jsonFilename = 'list_data.json';
break;
case 'other':
data = otherData;
columns = otherColumns;
title = t('run_content.captured_data.other_title');
csvFilename = 'other_data.csv';
jsonFilename = 'other_data.json';
break;
case 'legacy':
data = legacyData;
columns = legacyColumns;
@@ -714,7 +698,7 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
);
};
const hasData = schemaData.length > 0 || listData.length > 0 || otherData.length > 0 || legacyData.length > 0;
const hasData = schemaData.length > 0 || listData.length > 0 || legacyData.length > 0;
const hasScreenshots = row.binaryOutput && Object.keys(row.binaryOutput).length > 0;
return (
@@ -881,15 +865,6 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
'list_data.json',
true
)}
{renderDataTable(
otherData,
otherColumns,
t('run_content.captured_data.other_title'),
<MoreHorizIcon sx={{ color: '#FF00C3' }} />,
'other_data.csv',
'other_data.json'
)}
</>
) : (
<Grid container spacing={3}>
@@ -897,7 +872,6 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
const dataCategoriesCount = [
schemaData.length > 0,
listData.length > 0,
otherData.length > 0
].filter(Boolean).length;
const columnWidth = dataCategoriesCount === 1 ? 12 : dataCategoriesCount === 2 ? 6 : 4;
@@ -932,20 +906,6 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
)}
</Grid>
)}
{otherData.length > 0 && (
<Grid item xs={12} md={columnWidth} sx={{ display: 'flex' }}>
{renderDataCard(
otherData,
otherColumns,
t('run_content.captured_data.other_title'),
<MoreHorizIcon sx={{ color: '#FF00C3' }} />,
'other',
'other_data.csv',
'other_data.json'
)}
</Grid>
)}
</>
);
})()}
@@ -954,7 +914,6 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe
)}
{renderExpandedView('schema')}
{renderExpandedView('other')}
{renderExpandedView('legacy')}
{listData.map((_, index) => renderExpandedView(`list-${index}`))}