From fd98ab921b76fbe40b89d0d73b5648d00d5f4f11 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Sun, 20 Sep 2026 18:26:39 -0700 Subject: [PATCH] fix(tokenizer): preserve caller-supplied Python tokenizer counts --- litellm/litellm_core_utils/token_counter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 1d475c6b396..54b548a8ae0 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -626,7 +626,10 @@ def _get_exact_count_function( tokenizer: Final[Tokenizer | NativeTokenizer] = tokenizer_json["tokenizer"] def count_tokens(text: str) -> int: - return tokenizer.count(text) + count: Final = getattr(tokenizer, "count", None) + if callable(count): + return count(text) + return len(tokenizer.encode_batch_fast([text])[0]) return count_tokens elif tokenizer_json["type"] == "openai_tokenizer":