scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

Merge fc844d624d into b1b236826a

Browse source
This commit is contained in:
quietjules 2026-11-06 06:11:31 +00:00 • committed by GitHub
commit 32f6d960b6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 30 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
litellm/utils.py
View file

@ -2859,7 +2859,9 @@ def uses_anthropic_tokenizer(model: str) -> bool:
HuggingFaceTokenizerKind = Literal["cohere", "anthropic", "deepseek", "llama2", "llama3"]
_DEEPSEEK_TOKENIZER_MODEL: Final = re.compile(
r"deepseek[-_.](?:chat[-_.])?v[34]|(?:^|/)deepseek-(?:chat|reasoner|flash)(?::free)?$", re.IGNORECASE
r"deepseek[-_.](?:chat[-_.])?v[34]|(?:^|/)deepseek-(?:chat|reasoner|flash)(?::free)?$"
r"|deepseek[-_.]r1(?:-0528)?(?:-(?:turbo|basic|maas|671b|v1:0)|:(?:671b|free)|/community)?$",
re.IGNORECASE,
)
_BUNDLED_TOKENIZER_LOAD_LOCKS: Final[Mapping[HuggingFaceTokenizerKind, threading.Lock]] = MappingProxyType(

6
tests/integration/providers/test_chat_template_tool_count_wire.py
View file

@ -38,6 +38,7 @@ pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
_API_KEY: Final = "scripted-provider-key"
_DEEPSEEK_V4: Final = "azure_ai/DeepSeek-V4-Flash-0731"
_DEEPSEEK_R1: Final = "azure_ai/DeepSeek-R1-0528"
_DEEPSEEK_R1_DISTILL: Final = "deepinfra/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
_QWEN3_5: Final = "databricks/databricks-qwen35-122b-a10b"
_GROK_3: Final = "azure_ai/grok-3"
_DEEPSEEK_V4_TOOL_PREAMBLE: Final = 222
@ -189,9 +190,10 @@ def _counted_with_cl100k(delta: int) -> None:
("backend", "tokenizer_type", "counted_with"),
(
(_DEEPSEEK_V4, "huggingface_tokenizer", _counted_with_deepseek),
(_DEEPSEEK_R1, "openai_tokenizer", _counted_with_cl100k),
(_DEEPSEEK_R1, "huggingface_tokenizer", _counted_with_deepseek),
(_DEEPSEEK_R1_DISTILL, "openai_tokenizer", _counted_with_cl100k),
),
ids=("deepseek-v4", "deepseek-r1"),
ids=("deepseek-v4", "deepseek-r1", "deepseek-r1-distill"),
)
def test_utils_token_counter_names_the_tokenizer_that_counted_deepseek_text(
gateway: Gateway, backend: str, tokenizer_type: str, counted_with: Callable[[int], None]

25
tests/unit/litellm_core_utils/test_token_counter.py
View file

@ -1235,9 +1235,20 @@ def _deepseek_token_count(text: str) -> int:
"deepseek/deepseek-reasoner",
"openrouter/deepseek/deepseek-chat-v3-0324",
"openrouter/deepseek/deepseek-chat:free",
"azure_ai/DeepSeek-R1-0528",
"azure_ai/deepseek-r1",
"deepseek/deepseek-r1",
"deepinfra/deepseek-ai/DeepSeek-R1-0528-Turbo",
"bedrock/us.deepseek.r1-v1:0",
"openrouter/deepseek/deepseek-r1-0528:free",
"vertex_ai/deepseek-ai/deepseek-r1-0528-maas",
"fireworks_ai/accounts/fireworks/models/deepseek-r1-basic",
"novita/deepseek/deepseek-r1/community",
"lambda_ai/deepseek-r1-671b",
"ollama/deepseek-r1:671b",
],
)
def test_token_counter_counts_deepseek_v3_and_v4_with_the_packaged_deepseek_tokenizer(model: str):
def test_token_counter_counts_deepseek_v3_v4_and_r1_with_the_packaged_deepseek_tokenizer(model: str):
expected: Final = _deepseek_token_count(_CODE_SAMPLE)
assert expected != len(tiktoken.get_encoding("cl100k_base").encode(_CODE_SAMPLE))
@ -1246,7 +1257,17 @@ def test_token_counter_counts_deepseek_v3_and_v4_with_the_packaged_deepseek_toke
@pytest.mark.parametrize(
"model",
["deepinfra/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", "fireworks_ai/accounts/fireworks/models/deepseek-v2p5"],
[
"deepinfra/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B",
"fireworks_ai/accounts/fireworks/models/deepseek-v2p5",
"novita/deepseek/deepseek-r1-0528-qwen3-8b",
"fireworks_ai/accounts/fireworks/models/deepseek-r1-0528-distill-qwen3-8b",
"deepinfra/deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
"llamagate/deepseek-r1-8b",
"llamagate/deepseek-r1-7b-qwen",
"ollama/deepseek-r1:8b",
"cloudflare/@cf/deepseek-ai/deepseek-r1-distill-qwen-32b",
],
)
def test_token_counter_skips_the_deepseek_tokenizer_for_deepseek_models_built_on_other_vocabularies(model: str):
assert litellm.token_counter(model=model, text=_CODE_SAMPLE) != _deepseek_token_count(_CODE_SAMPLE)