feat: add server side crawl and search logic

2026-01-02 15:46:10 +05:30
parent 3689eb96bd
commit 9898dc410d
15 changed files with 1577 additions and 124 deletions
--- a/server/src/models/Robot.ts
+++ b/server/src/models/Robot.ts
@@ -9,7 +9,7 @@ interface RobotMeta {
  pairs: number;
  updatedAt: string;
  params: any[];
-  type?: 'extract' | 'scrape';
+  type?: 'extract' | 'scrape' | 'crawl' | 'search';
  url?: string;
  formats?: ('markdown' | 'html' | 'screenshot-visible' | 'screenshot-fullpage')[];
  isLLM?: boolean;