From 32d4d83111c3aac97d6b6b211fbb99bbd51840a0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 02:50:52 -0700 Subject: [PATCH] fix(count_tokens): count custom-named deployments with their base_model's tokenizer and tool template --- litellm/proxy/proxy_server.py | 18 +++- tests/unit/proxy/test_proxy_token_counter.py | 90 +++++++++++++++++++- 2 files changed, 103 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5e37ff1ba65..7971b94614d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15185,12 +15185,22 @@ def _deployment_aws_region(deployment: _TokenCountDeployment) -> str: ) +def _deployment_base_model(deployment: _TokenCountDeployment) -> str | None: + return _non_empty_str(deployment.get("model_info", {}).get("base_model")) or _non_empty_str( + deployment.get("litellm_params", {}).get("base_model") + ) + + +def _unprefixed_base_model(deployment: _TokenCountDeployment) -> str | None: + base_model: Final = _deployment_base_model(deployment) + return None if base_model is None else base_model.split("/", 1)[-1] + + def _token_count_settings_model(deployment: _TokenCountDeployment, request_model: str) -> tuple[str, str | None]: - model_info: Final = deployment.get("model_info", {}) - params: Final = deployment.get("litellm_params", {}) - base_model: Final = _non_empty_str(model_info.get("base_model")) or _non_empty_str(params.get("base_model")) + base_model: Final = _deployment_base_model(deployment) if base_model is not None: return base_model, None + params: Final = deployment.get("litellm_params", {}) model: Final = _non_empty_str(params.get("model")) if model is None: return request_model, None @@ -15546,7 +15556,7 @@ async def _count_request_tokens( count_source="local_estimate", ) estimate: Final = await _local_token_estimate( - model=deployment_openai_model(deployment_view) or model_to_use, + model=deployment_openai_model(deployment_view) or _unprefixed_base_model(deployment_view) or model_to_use, prompt=prompt if counted_messages is None else None, messages=counted_messages, tools=counted_tools, diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index b21730c64f6..9df7b77a495 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -3989,6 +3989,9 @@ async def test_responses_count_on_claude_counts_the_turns_previous_response_id_r spend_logs.assert_awaited_once_with("chatcmpl-stored", ("chatcmpl-stored", "resp_client_form")) +_MULTILINGUAL_TEXT: Final = "日本語のテキストとहिन्दी पाठ और русский текст. " * 20 + + @pytest.mark.asyncio async def test_token_counter_counts_a_custom_azure_deployment_name_with_its_base_models_tokenizer( monkeypatch: pytest.MonkeyPatch, @@ -4007,7 +4010,7 @@ async def test_token_counter_counts_a_custom_azure_deployment_name_with_its_base ] ), ) - messages: Final = [{"role": "user", "content": "日本語のテキストとहिन्दी पाठ और русский текст. " * 20}] + messages: Final = [{"role": "user", "content": _MULTILINGUAL_TEXT}] named: Final = await token_counter(request=TokenCountRequest(model="named", messages=messages)) served: Final = await token_counter(request=TokenCountRequest(model="served", messages=messages)) @@ -4015,6 +4018,91 @@ async def test_token_counter_counts_a_custom_azure_deployment_name_with_its_base assert named.total_tokens == served.total_tokens +_LOOKUP_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + "type": "function", + "function": { + "name": "lookup_order", + "description": "Look up an order by its id", + "parameters": {"type": "object", "properties": {"order_id": {"type": "string"}}, "required": ["order_id"]}, + }, + } +) + + +_FLASH_BASE_MODEL: Final = "azure_ai/deepseek-v4-flash" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("served_model", "named_params", "named_model_info", "tools"), + [ + pytest.param( + "azure_ai/DeepSeek-V4-Flash-0731", + {"model": "azure_ai/my-flash"}, + {"base_model": _FLASH_BASE_MODEL}, + [dict(_LOOKUP_TOOL)], + id="model_info base_model, tool template", + ), + pytest.param( + "azure_ai/DeepSeek-V4-Flash-0731", + {"model": "azure_ai/my-flash"}, + {"base_model": _FLASH_BASE_MODEL}, + None, + id="model_info base_model, tokenizer", + ), + pytest.param( + "azure_ai/DeepSeek-V4-Flash-0731", + {"model": "azure_ai/my-flash", "base_model": _FLASH_BASE_MODEL}, + {}, + [dict(_LOOKUP_TOOL)], + id="litellm_params base_model", + ), + pytest.param( + "azure_ai/Qwen3.5-397B-A17B", + {"model": "azure_ai/my-qwen"}, + {"base_model": "azure_ai/qwen3.5-397b-a17b"}, + [dict(_LOOKUP_TOOL)], + id="qwen3.5 base_model, tool template", + ), + pytest.param( + "anthropic/claude-opus-5-5", + {"model": "anthropic/my-opus"}, + {"base_model": "anthropic/claude-opus-5-5"}, + None, + id="provider-prefixed claude base_model, tokenizer", + ), + ], +) +async def test_token_counter_counts_a_custom_named_deployment_like_the_base_model_it_declares( + monkeypatch: pytest.MonkeyPatch, + served_model: str, + named_params: Mapping[str, str], + named_model_info: Mapping[str, str], + tools: list[dict[str, JsonValue]] | None, +) -> None: + credentials: Final = {"api_key": "sk-test", "api_base": "https://example.services.ai.azure.com"} + monkeypatch.setattr( + "litellm.proxy.proxy_server.llm_router", + Router( + model_list=[ + { + "model_name": "named", + "litellm_params": {**named_params, **credentials}, + "model_info": dict(named_model_info), + }, + {"model_name": "served", "litellm_params": {"model": served_model, **credentials}}, + ] + ), + ) + messages: Final = [{"role": "user", "content": _MULTILINGUAL_TEXT}] + + named: Final = await token_counter(request=TokenCountRequest(model="named", messages=messages, tools=tools)) + served: Final = await token_counter(request=TokenCountRequest(model="served", messages=messages, tools=tools)) + + assert named.total_tokens == served.total_tokens + + _COUNTED_HELLO: Final[Mapping[str, JsonValue]] = MappingProxyType( {"model": "count-log-gpt", "messages": [{"role": "user", "content": "Hello, how many tokens is this?"}]} )