From b14d84d83ab24f9c35cbca7da1d05165af8e9b53 Mon Sep 17 00:00:00 2001
From: amhsirak <karishmashuklaa@gmail.com>
Date: Thu, 20 Nov 2025 03:51:53 +0530
Subject: [PATCH] fix: -rm debug turndown

---
 server/src/markdownify/debug_turndown.ts | 132 -----------------------
 1 file changed, 132 deletions(-)
 delete mode 100644 server/src/markdownify/debug_turndown.ts

diff --git a/server/src/markdownify/debug_turndown.ts b/server/src/markdownify/debug_turndown.ts
deleted file mode 100644
index 1d62b109..00000000
--- a/server/src/markdownify/debug_turndown.ts
+++ /dev/null
@@ -1,132 +0,0 @@
-import { getPageSource } from './get_html';
-import { getProcessedText } from './get_llm_input_text';
-import * as cheerio from 'cheerio';
-import TurndownService from 'turndown';
-
-async function debugTurndown() {
-    const testUrls = [
-        "https://amazon.com/",
-    ];
-
-    for (const url of testUrls) {
-        console.log(`\n${'='.repeat(70)}`);
-        console.log(`🔍 Testing URL: ${url}`);
-        console.log(`${'='.repeat(70)}`);
-        
-        try {
-            const pageSource = await getPageSource(url, {
-                wait: 3.0, // Longer wait time
-                timeout: 15000 // 15 second timeout
-            });
-            
-            if (!pageSource || pageSource.length < 100) {
-                console.error("❌ No page source received or content too short");
-                continue;
-            }
-
-            // Save raw HTML for inspection
-            const fs = await import('fs/promises');
-            const domain = new URL(url).hostname;
-            await fs.writeFile(`debug_${domain}_raw.html`, pageSource);
-            console.log(`💾 Raw HTML saved to debug_${domain}_raw.html (${pageSource.length} chars)`);
-
-            // Parse with cheerio
-            const $ = cheerio.load(pageSource);
-            
-            // Check what's in the body
-            const bodyText = $('body').text();
-            console.log(`📄 Body text length: ${bodyText.length} chars`);
-            console.log(`📄 Body preview: ${bodyText.substring(0, 200)}...`);
-
-            // Test content extraction
-            const contentSelectors = [
-                'main', 'article', '[role="main"]', '.content', '.main-content',
-                '#content', '#main', '.post', '.article'
-            ];
-
-            let mainContent: cheerio.Cheerio<any> = $('body');
-            let foundSelector = 'body (fallback)';
-            
-            for (const selector of contentSelectors) {
-                const $content = $(selector).first();
-                if ($content.length > 0 && $content.text().trim().length > 10) {
-                    console.log(`✅ Found content with selector: ${selector}`);
-                    console.log(`📝 Content text length: ${$content.text().length}`);
-                    mainContent = $content;
-                    foundSelector = selector;
-                    break;
-                }
-            }
-
-            console.log(`🎯 Using content from: ${foundSelector}`);
-
-            // Test Turndown directly
-            console.log("\n🧪 Testing Turndown directly...");
-            const turndownService = new TurndownService();
-            
-            if (mainContent.length > 0) {
-                const contentHtml = mainContent.html() || '';
-                if (contentHtml && contentHtml.length > 10) {
-                    console.log(`📦 Content HTML length: ${contentHtml.length} chars`);
-                    
-                    try {
-                        const contentMarkdown = turndownService.turndown(contentHtml);
-                        console.log(`📝 Turndown result length: ${contentMarkdown.length} chars`);
-                        
-                        if (contentMarkdown.length > 0) {
-                            console.log(`📝 Markdown preview: ${contentMarkdown.substring(0, 300)}...`);
-                            await fs.writeFile(`debug_${domain}_turndown.md`, contentMarkdown);
-                            console.log(`💾 Turndown output saved to debug_${domain}_turndown.md`);
-                        } else {
-                            console.log("❌ Turndown produced empty markdown");
-                        }
-                    } catch (turndownError) {
-                        console.error("❌ Turndown conversion failed:", turndownError);
-                    }
-                } else {
-                    console.log("❌ No HTML content found for Turndown");
-                }
-            }
-
-            // Test our full function
-            console.log("\n🧪 Testing full getProcessedText function...");
-            const result = await getProcessedText(pageSource, url, {
-                keepImages: true,
-                keepWebpageLinks: true,
-                removeScriptTag: true,
-                removeStyleTag: true,
-                formatAsMarkdown: true
-            });
-
-            console.log("📊 Result metadata:");
-            console.log(`- Markdown length: ${result.metadata.markdownLength} chars`);
-            console.log(`- Plain text length: ${result.metadata.textLength} chars`);
-            console.log(`- Has content: ${result.metadata.hasContent}`);
-            console.log(`- Content score: ${result.metadata.contentScore}/10`);
-
-            if (result.markdown && result.markdown.length > 0) {
-                console.log(`📄 Markdown preview (300 chars):`);
-                console.log(result.markdown.substring(0, 300) + '...');
-                await fs.writeFile(`debug_${domain}_full.md`, result.markdown);
-                console.log(`💾 Full output saved to debug_${domain}_full.md`);
-            } else {
-                console.log("❌ Empty markdown from full function");
-                
-                // Debug why it's empty
-                if (result.plainText && result.plainText.length > 0) {
-                    console.log("ℹ️  But plain text has content, so markdown conversion failed");
-                    await fs.writeFile(`debug_${domain}_plain.txt`, result.plainText);
-                    console.log(`💾 Plain text saved to debug_${domain}_plain.txt`);
-                }
-            }
-
-        } catch (error) {
-            console.error(`💥 Error processing ${url}:`, error);
-        }
-        
-        // Small delay between requests
-        await new Promise(resolve => setTimeout(resolve, 1000));
-    }
-}
-
-debugTurndown().catch(console.error);
\ No newline at end of file