vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: mandatory user id for prompt llm

Browse Source
This commit is contained in:
Rohit Rajan
2025-12-11 13:13:26 +05:30
parent 0836a7efbe
commit d8e5ca7095
2 changed files with 25 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
server/src/api/sdk.ts
View File

@@ -651,12 +651,12 @@ router.post("/sdk/extract/llm", requireAPIKey, async (req: AuthenticatedRequest,
}); });
} }
const workflowResult = await WorkflowEnricher.generateWorkflowFromPrompt(url, prompt, { const workflowResult = await WorkflowEnricher.generateWorkflowFromPrompt(url, prompt, user.id, {
provider: llmProvider, provider: llmProvider,
model: llmModel, model: llmModel,
apiKey: llmApiKey, apiKey: llmApiKey,
baseUrl: llmBaseUrl baseUrl: llmBaseUrl
}, user.id); });
if (!workflowResult.success || !workflowResult.workflow) { if (!workflowResult.success || !workflowResult.workflow) {
return res.status(400).json({ return res.status(400).json({

42
server/src/sdk/workflowEnricher.ts
View File

@@ -183,7 +183,6 @@ export class WorkflowEnricher {
enrichedFields = autoDetectResult.fields; enrichedFields = autoDetectResult.fields;
listSelector = autoDetectResult.listSelector!; listSelector = autoDetectResult.listSelector!;
logger.info('Auto-detected', Object.keys(enrichedFields).length, 'fields');
} catch (error: any) { } catch (error: any) {
errors.push(`Field auto-detection failed: ${error.message}`); errors.push(`Field auto-detection failed: ${error.message}`);
continue; continue;
@@ -277,13 +276,13 @@ export class WorkflowEnricher {
static async generateWorkflowFromPrompt( static async generateWorkflowFromPrompt(
url: string, url: string,
prompt: string, prompt: string,
userId: string,
llmConfig?: { llmConfig?: {
provider?: 'anthropic' | 'openai' | 'ollama'; provider?: 'anthropic' | 'openai' | 'ollama';
model?: string; model?: string;
apiKey?: string; apiKey?: string;
baseUrl?: string; baseUrl?: string;
}, },
userId: string = 'sdk-validation-user',
): Promise<{ success: boolean; workflow?: any[]; url?: string; errors?: string[] }> { ): Promise<{ success: boolean; workflow?: any[]; url?: string; errors?: string[] }> {
let browserId: string | null = null; let browserId: string | null = null;
const validator = new SelectorValidator(); const validator = new SelectorValidator();
@@ -299,7 +298,13 @@ export class WorkflowEnricher {
await validator.initialize(page as any, url); await validator.initialize(page as any, url);
const validatorPage = (validator as any).page; const validatorPage = (validator as any).page;
const screenshotBuffer = await page.screenshot({ fullPage: true, type: 'png' }); // Use JPEG with quality 85 for faster processing and smaller file size
// Vision models handle this compression well while maintaining accuracy
const screenshotBuffer = await page.screenshot({
fullPage: true,
type: 'jpeg',
quality: 85
});
const screenshotBase64 = screenshotBuffer.toString('base64'); const screenshotBase64 = screenshotBuffer.toString('base64');
const elementGroups = await this.analyzePageGroups(validator); const elementGroups = await this.analyzePageGroups(validator);
@@ -994,14 +999,18 @@ Example - if extracting products:
throw new Error('Failed to auto-detect fields from selected group'); throw new Error('Failed to auto-detect fields from selected group');
} }
logger.info(`Auto-detected ${Object.keys(autoDetectResult.fields).length} fields`); logger.info('Extracting field samples and detecting pagination in parallel...');
const [fieldSamples, paginationResult] = await Promise.all([
logger.info('Extracting field samples for semantic labeling...'); this.extractFieldSamples(
const fieldSamples = await this.extractFieldSamples( autoDetectResult.fields,
autoDetectResult.fields, autoDetectResult.listSelector || '',
autoDetectResult.listSelector || '', validator
validator ),
); validator.autoDetectPagination(llmDecision.itemSelector).catch((error: any) => {
logger.warn('Pagination auto-detection failed:', error.message);
return { success: false, type: 'none', selector: '' };
})
]);
logger.info('Generating semantic field labels with LLM...'); logger.info('Generating semantic field labels with LLM...');
const fieldLabels = await this.generateFieldLabels( const fieldLabels = await this.generateFieldLabels(
@@ -1021,14 +1030,9 @@ Example - if extracting products:
let paginationType = 'none'; let paginationType = 'none';
let paginationSelector = ''; let paginationSelector = '';
try { if (paginationResult.success && paginationResult.type) {
const paginationResult = await validator.autoDetectPagination(llmDecision.itemSelector); paginationType = paginationResult.type;
if (paginationResult.success && paginationResult.type) { paginationSelector = paginationResult.selector || '';
paginationType = paginationResult.type;
paginationSelector = paginationResult.selector || '';
}
} catch (error: any) {
logger.warn('Pagination auto-detection failed:', error.message);
} }
const limit = llmDecision.limit || 100; const limit = llmDecision.limit || 100;