2025-04-04 22:33:52 -04:00
|
|
|
import tiktoken
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def count_tokens(text: str) -> int:
|
2026-01-19 15:12:12 -08:00
|
|
|
"""
|
|
|
|
|
tiktoken sends a request to https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken to get the token
|
|
|
|
|
"""
|
|
|
|
|
try:
|
|
|
|
|
return len(tiktoken.encoding_for_model("gpt-4o").encode(text))
|
|
|
|
|
except Exception:
|
|
|
|
|
return 0
|
