scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-25 01:02:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 80

refactor(tokenizer): count custom tokenizers directly

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Yujong Lee 2026-09-21 01:05:46 +00:00
parent 2d452a01f8
commit 12ca5ac618
1 changed files with 1 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/litellm_core_utils/token_counter.py
View file

@ -626,10 +626,7 @@ def _get_exact_count_function(
tokenizer: Final[Tokenizer | NativeTokenizer] = tokenizer_json["tokenizer"]
def count_tokens(text: str) -> int:
count: Final = getattr(tokenizer, "count", None)
if callable(count):
return count(text)
return len(tokenizer.encode_batch_fast([text])[0])
return tokenizer.count(text)
return count_tokens
elif tokenizer_json["type"] == "openai_tokenizer":