vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: mandatory user id for prompt llm

Browse Source
This commit is contained in:
Rohit Rajan
2025-12-11 13:13:26 +05:30
parent 0836a7efbe
commit d8e5ca7095
2 changed files with 25 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
server/src/api/sdk.ts
View File

@@ -651,12 +651,12 @@ router.post("/sdk/extract/llm", requireAPIKey, async (req: AuthenticatedRequest,
});
}
const workflowResult = await WorkflowEnricher.generateWorkflowFromPrompt(url, prompt, {
const workflowResult = await WorkflowEnricher.generateWorkflowFromPrompt(url, prompt, user.id, {
provider: llmProvider,
model: llmModel,
apiKey: llmApiKey,
baseUrl: llmBaseUrl
}, user.id);
});
if (!workflowResult.success || !workflowResult.workflow) {
return res.status(400).json({

42
server/src/sdk/workflowEnricher.ts
View File

@@ -183,7 +183,6 @@ export class WorkflowEnricher {
enrichedFields = autoDetectResult.fields;
listSelector = autoDetectResult.listSelector!;
logger.info('Auto-detected', Object.keys(enrichedFields).length, 'fields');
} catch (error: any) {
errors.push(`Field auto-detection failed: ${error.message}`);
continue;
@@ -277,13 +276,13 @@ export class WorkflowEnricher {
static async generateWorkflowFromPrompt(
url: string,
prompt: string,
userId: string,
llmConfig?: {
provider?: 'anthropic' | 'openai' | 'ollama';
model?: string;
apiKey?: string;
baseUrl?: string;
},
userId: string = 'sdk-validation-user',
): Promise<{ success: boolean; workflow?: any[]; url?: string; errors?: string[] }> {
let browserId: string | null = null;
const validator = new SelectorValidator();
@@ -299,7 +298,13 @@ export class WorkflowEnricher {
await validator.initialize(page as any, url);
const validatorPage = (validator as any).page;
const screenshotBuffer = await page.screenshot({ fullPage: true, type: 'png' });
// Use JPEG with quality 85 for faster processing and smaller file size
// Vision models handle this compression well while maintaining accuracy
const screenshotBuffer = await page.screenshot({
fullPage: true,
type: 'jpeg',
quality: 85
});
const screenshotBase64 = screenshotBuffer.toString('base64');
const elementGroups = await this.analyzePageGroups(validator);
@@ -994,14 +999,18 @@ Example - if extracting products:
throw new Error('Failed to auto-detect fields from selected group');
}
logger.info(`Auto-detected ${Object.keys(autoDetectResult.fields).length} fields`);
logger.info('Extracting field samples for semantic labeling...');
const fieldSamples = await this.extractFieldSamples(
autoDetectResult.fields,
autoDetectResult.listSelector || '',
validator
);
logger.info('Extracting field samples and detecting pagination in parallel...');
const [fieldSamples, paginationResult] = await Promise.all([
this.extractFieldSamples(
autoDetectResult.fields,
autoDetectResult.listSelector || '',
validator
),
validator.autoDetectPagination(llmDecision.itemSelector).catch((error: any) => {
logger.warn('Pagination auto-detection failed:', error.message);
return { success: false, type: 'none', selector: '' };
})
]);
logger.info('Generating semantic field labels with LLM...');
const fieldLabels = await this.generateFieldLabels(
@@ -1021,14 +1030,9 @@ Example - if extracting products:
let paginationType = 'none';
let paginationSelector = '';
try {
const paginationResult = await validator.autoDetectPagination(llmDecision.itemSelector);
if (paginationResult.success && paginationResult.type) {
paginationType = paginationResult.type;
paginationSelector = paginationResult.selector || '';
}
} catch (error: any) {
logger.warn('Pagination auto-detection failed:', error.message);
if (paginationResult.success && paginationResult.type) {
paginationType = paginationResult.type;
paginationSelector = paginationResult.selector || '';
}
const limit = llmDecision.limit || 100;