make token counter request optional (#4489)
This commit is contained in:
@@ -2,4 +2,10 @@ import tiktoken
|
|||||||
|
|
||||||
|
|
||||||
def count_tokens(text: str) -> int:
|
def count_tokens(text: str) -> int:
|
||||||
return len(tiktoken.encoding_for_model("gpt-4o").encode(text))
|
"""
|
||||||
|
tiktoken sends a request to https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken to get the token
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return len(tiktoken.encoding_for_model("gpt-4o").encode(text))
|
||||||
|
except Exception:
|
||||||
|
return 0
|
||||||
|
|||||||
Reference in New Issue
Block a user