scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-25 01:02:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 80

fix(tokenizer): preserve caller-supplied Python tokenizer counts

Browse source
This commit is contained in:
Yujong Lee 2026-09-20 18:26:39 -07:00
parent 12ca5ac618
commit fd98ab921b
1 changed files with 4 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/litellm_core_utils/token_counter.py
View file

@ -626,7 +626,10 @@ def _get_exact_count_function(
tokenizer: Final[Tokenizer | NativeTokenizer] = tokenizer_json["tokenizer"]
def count_tokens(text: str) -> int:
return tokenizer.count(text)
count: Final = getattr(tokenizer, "count", None)
if callable(count):
return count(text)
return len(tokenizer.encode_batch_fast([text])[0])
return count_tokens
elif tokenizer_json["type"] == "openai_tokenizer":