From 12ca5ac618113f5c3fa54fe8d4e490fb34db3431 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Mon, 21 Sep 2026 01:05:46 +0000 Subject: [PATCH] refactor(tokenizer): count custom tokenizers directly Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 54b548a8ae0..1d475c6b396 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -626,10 +626,7 @@ def _get_exact_count_function( tokenizer: Final[Tokenizer | NativeTokenizer] = tokenizer_json["tokenizer"] def count_tokens(text: str) -> int: - count: Final = getattr(tokenizer, "count", None) - if callable(count): - return count(text) - return len(tokenizer.encode_batch_fast([text])[0]) + return tokenizer.count(text) return count_tokens elif tokenizer_json["type"] == "openai_tokenizer":