From 1a1ee402cb67db97bdf952d3f0b230273e140a2f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:49:08 -0700 Subject: [PATCH] fix(token_counter): count DeepSeek R1 and R1-0528 with DeepSeek's own tokenizer --- litellm/utils.py | 4 ++- .../litellm_core_utils/test_token_counter.py | 25 +++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 6a9beeec916..ac57f13b698 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2659,7 +2659,9 @@ def uses_anthropic_tokenizer(model: str) -> bool: HuggingFaceTokenizerKind = Literal["cohere", "anthropic", "deepseek", "llama2", "llama3"] _DEEPSEEK_TOKENIZER_MODEL: Final = re.compile( - r"deepseek[-_.](?:chat[-_.])?v[34]|(?:^|/)deepseek-(?:chat|reasoner|flash)(?::free)?$", re.IGNORECASE + r"deepseek[-_.](?:chat[-_.])?v[34]|(?:^|/)deepseek-(?:chat|reasoner|flash)(?::free)?$" + r"|deepseek[-_.]r1(?:-0528)?(?:-(?:turbo|basic|maas|671b|v1:0)|:(?:671b|free)|/community)?$", + re.IGNORECASE, ) _BUNDLED_TOKENIZER_LOAD_LOCKS: Final[Mapping[HuggingFaceTokenizerKind, threading.Lock]] = MappingProxyType( diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 9b0f6607708..2ed8a034efd 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1176,9 +1176,20 @@ def _deepseek_token_count(text: str) -> int: "deepseek/deepseek-reasoner", "openrouter/deepseek/deepseek-chat-v3-0324", "openrouter/deepseek/deepseek-chat:free", + "azure_ai/DeepSeek-R1-0528", + "azure_ai/deepseek-r1", + "deepseek/deepseek-r1", + "deepinfra/deepseek-ai/DeepSeek-R1-0528-Turbo", + "bedrock/us.deepseek.r1-v1:0", + "openrouter/deepseek/deepseek-r1-0528:free", + "vertex_ai/deepseek-ai/deepseek-r1-0528-maas", + "fireworks_ai/accounts/fireworks/models/deepseek-r1-basic", + "novita/deepseek/deepseek-r1/community", + "lambda_ai/deepseek-r1-671b", + "ollama/deepseek-r1:671b", ], ) -def test_token_counter_counts_deepseek_v3_and_v4_with_the_packaged_deepseek_tokenizer(model: str): +def test_token_counter_counts_deepseek_v3_v4_and_r1_with_the_packaged_deepseek_tokenizer(model: str): expected: Final = _deepseek_token_count(_CODE_SAMPLE) assert expected != len(tiktoken.get_encoding("cl100k_base").encode(_CODE_SAMPLE)) @@ -1187,7 +1198,17 @@ def test_token_counter_counts_deepseek_v3_and_v4_with_the_packaged_deepseek_toke @pytest.mark.parametrize( "model", - ["together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", "fireworks_ai/accounts/fireworks/models/deepseek-v2p5"], + [ + "together_ai/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", + "fireworks_ai/accounts/fireworks/models/deepseek-v2p5", + "novita/deepseek/deepseek-r1-0528-qwen3-8b", + "fireworks_ai/accounts/fireworks/models/deepseek-r1-0528-distill-qwen3-8b", + "deepinfra/deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + "llamagate/deepseek-r1-8b", + "llamagate/deepseek-r1-7b-qwen", + "ollama/deepseek-r1:8b", + "cloudflare/@cf/deepseek-ai/deepseek-r1-distill-qwen-32b", + ], ) def test_token_counter_skips_the_deepseek_tokenizer_for_deepseek_models_built_on_other_vocabularies(model: str): assert litellm.token_counter(model=model, text=_CODE_SAMPLE) != _deepseek_token_count(_CODE_SAMPLE)