fix t_llm_request is unbounded problem (#657)
This commit is contained in:
@@ -190,12 +190,12 @@ class LLMAPIHandlerFactory:
|
||||
}
|
||||
).encode("utf-8"),
|
||||
)
|
||||
t_llm_request = time.perf_counter()
|
||||
try:
|
||||
# TODO (kerem): add a timeout to this call
|
||||
# TODO (kerem): add a retry mechanism to this call (acompletion_with_retries)
|
||||
# TODO (kerem): use litellm fallbacks? https://litellm.vercel.app/docs/tutorials/fallbacks#how-does-completion_with_fallbacks-work
|
||||
LOG.info("Calling LLM API", llm_key=llm_key, model=llm_config.model_name)
|
||||
t_llm_request = time.perf_counter()
|
||||
response = await litellm.acompletion(
|
||||
model=llm_config.model_name,
|
||||
messages=messages,
|
||||
|
||||
Reference in New Issue
Block a user