diff --git a/server/src/markdownify/get_llm_ready_text.ts b/server/src/markdownify/get_llm_ready_text.ts new file mode 100644 index 00000000..4d0515c6 --- /dev/null +++ b/server/src/markdownify/get_llm_ready_text.ts @@ -0,0 +1,29 @@ +import { getPageSource, GetPageSourceOptions } from './get_html'; +import { getProcessedText, ProcessTextOptions } from './get_llm_input_text'; + +export interface UrlToLlmTextOptions extends GetPageSourceOptions, ProcessTextOptions { + // Combined options from both interfaces +} + +export async function urlToLlmText( + url: string, + options: UrlToLlmTextOptions = {} +): Promise { + try { + const pageSource = await getPageSource(url, options); + + if (!pageSource) { + return ''; + } + + const llmText = await getProcessedText(pageSource, url, options); + return llmText; + + } catch (error) { + console.error('Error while scraping url: ', error); + return ''; + } +} + +// Export individual functions as well +export { getPageSource, getProcessedText }; \ No newline at end of file