From e0c9f1a7381744c7c40f8d5710a13da252777c61 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:27:39 -0700 Subject: [PATCH 1/7] fix(count_tokens): count grok-4-1, grok-4.3 and DeepSeek V4.1 tool requests like their chat templates Render grok-4-1, grok-4.3 and DeepSeek V4.1 tool schemas the way each chat template does, and add a per-deployment token_count_tool_framing next to token_count_framing for the hidden tool preamble a host bills. The Azure AI Foundry grok-4-1, xAI grok-4.3 and Fireworks DeepSeek V4.1 cost map rows carry the measured framing, overridable in model_info --- .../crates/model-catalog/src/model_info.rs | 1 + .../model-catalog/src/wire/attributes.rs | 1 + .../prompt_templates/chat_template_tools.py | 17 +- ...odel_prices_and_context_window_backup.json | 6 + .../common_utils/token_count_settings.py | 10 +- litellm/proxy/proxy_server.py | 4 +- litellm/types/utils.py | 5 +- litellm/utils.py | 1 + model_prices_and_context_window.json | 6 + ...ls_prompt_templates_chat_template_tools.py | 17 +- .../litellm_core_utils/test_token_counter.py | 22 ++- .../common_utils/test_token_count_settings.py | 23 +++ .../test_model_management_endpoints.py | 1 + tests/unit/proxy/test_proxy_token_counter.py | 145 ++++++++++++++++++ .../unit/test_router_model_cost_isolation.py | 3 +- tests/unit/test_utils.py | 1 + 16 files changed, 251 insertions(+), 12 deletions(-) diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs index 8f7549c1c6d..84decaf36d5 100644 --- a/litellm-rust/crates/model-catalog/src/model_info.rs +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -186,6 +186,7 @@ pub struct ModelInfoFields { pub supports_image_size: Option, pub supported_audio_formats: Option, pub token_count_framing: Option, + pub token_count_tool_framing: Option, pub token_count_model: Option, pub token_count_tokenizer: Option, pub vertex_ai_audio_api: Option, diff --git a/litellm-rust/crates/model-catalog/src/wire/attributes.rs b/litellm-rust/crates/model-catalog/src/wire/attributes.rs index 93c10a81524..3f1c6ae6df1 100644 --- a/litellm-rust/crates/model-catalog/src/wire/attributes.rs +++ b/litellm-rust/crates/model-catalog/src/wire/attributes.rs @@ -107,6 +107,7 @@ attributes! { token_count_framing: u64 as Count, token_count_model: TokenCountModel, token_count_tokenizer: TokenCountTokenizer, + token_count_tool_framing: u64 as Count, /// Provider default tokens-per-minute limit. tpm: u64 as Count, use_openai_responses_path: bool, diff --git a/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py b/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py index 60134456577..67ed6f82b41 100644 --- a/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py +++ b/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py @@ -12,10 +12,14 @@ class ChatTemplateToolSection: render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str] -def _deepseek_v4_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: +def _json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: return "\n".join(json.dumps(function, ensure_ascii=False) for function in functions) +def _compact_json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: + return "\n".join(json.dumps(function, ensure_ascii=False, separators=(",", ":")) for function in functions) + + def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: return "".join( "\n" + json.dumps({"type": "function", "function": function}, ensure_ascii=False) for function in functions @@ -23,13 +27,22 @@ def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: _TOOL_SECTIONS: Final = ( + ( + re.compile(r"deepseek-v4(?:[-._]|p)1(?!\d)", re.IGNORECASE), + ChatTemplateToolSection(211, 0, _json_line_tool_schemas), + ), ( re.compile( r"deepseek-v4(?![.\d]|p\d)|^(?:deepseek/)?deepseek-(?:chat|reasoner|flash)(?::free)?$", re.IGNORECASE ), - ChatTemplateToolSection(222, 0, _deepseek_v4_tool_schemas), + ChatTemplateToolSection(222, 0, _json_line_tool_schemas), ), (re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)), + ( + re.compile(r"grok-4[-.]1(?![.\d])", re.IGNORECASE), + ChatTemplateToolSection(0, 0, _compact_json_line_tool_schemas), + ), + (re.compile(r"grok-4[-.]3(?![.\d])", re.IGNORECASE), ChatTemplateToolSection(0, 0, _json_line_tool_schemas)), ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 50b3560f05b..61e45b648c7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13404,6 +13404,8 @@ "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", + "token_count_framing": 149, + "token_count_tool_framing": 151, "max_input_tokens": 128000, "max_output_tokens": 128000, "mode": "chat", @@ -53982,6 +53984,8 @@ "input_cost_per_token_above_200k_tokens_batches": 2e-06, "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", + "token_count_framing": 185, + "token_count_tool_framing": 29, "max_input_tokens": 1000000, "max_output_tokens": 1000000, "mode": "chat", @@ -63397,6 +63401,7 @@ "input_cost_per_token": 3e-07, "input_cost_per_token_priority": 3.75e-07, "litellm_provider": "fireworks_ai", + "token_count_framing": 23, "max_input_tokens": 1048576, "max_output_tokens": 393216, "mode": "chat", @@ -63495,6 +63500,7 @@ "input_cost_per_token": 3e-07, "input_cost_per_token_priority": 3.75e-07, "litellm_provider": "fireworks_ai", + "token_count_framing": 23, "max_input_tokens": 1048576, "max_output_tokens": 393216, "mode": "chat", diff --git a/litellm/proxy/common_utils/token_count_settings.py b/litellm/proxy/common_utils/token_count_settings.py index dd04e531f2d..d1ea99b02d3 100644 --- a/litellm/proxy/common_utils/token_count_settings.py +++ b/litellm/proxy/common_utils/token_count_settings.py @@ -56,6 +56,7 @@ class TokenCountSettings: framing: int | None count_model: TokenCountModel | None tokenizer: TokenCountTokenizer | None = None + tool_framing: int | None = None @dataclass(frozen=True, slots=True) @@ -105,6 +106,11 @@ def _framing(value: object) -> int | None: return None +def _configured_framing(configured: Mapping[str, object], default: Mapping[str, object], key: str) -> int | None: + framing: Final = _framing(configured.get(key)) + return framing if framing is not None else _framing(default.get(key)) + + def _non_blank(value: object) -> str | None: return value if isinstance(value, str) and value.strip() else None @@ -175,13 +181,13 @@ def _tokenizer(value: object) -> TokenCountTokenizer | None: def resolve_token_count_settings( configured: Mapping[str, object], default: Mapping[str, object], deployment_region: Callable[[], str] ) -> TokenCountSettings: - framing: Final = _framing(configured.get("token_count_framing")) return TokenCountSettings( - framing=framing if framing is not None else _framing(default.get("token_count_framing")), + framing=_configured_framing(configured, default, "token_count_framing"), count_model=_resolved_count_model( configured.get("token_count_model"), default.get("token_count_model"), deployment_region ), tokenizer=_tokenizer(default.get("token_count_tokenizer")), + tool_framing=_configured_framing(configured, default, "token_count_tool_framing"), ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5e37ff1ba65..ab4d56f0cef 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15530,7 +15530,9 @@ async def _count_request_tokens( counted_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats list[ChatCompletionToolParam] | None, tools if counted_messages is not None else None ) - framing: Final = settings.framing if counted_messages is not None and settings.framing is not None else 0 + message_framing: Final = 0 if counted_messages is None else settings.framing or 0 + tool_framing: Final = (settings.tool_framing or 0) if counted_tools else 0 + framing: Final = message_framing + tool_framing kimi_k3_messages: Final = counted_messages if settings.tokenizer == "kimi_k3" else None exact: Final = ( await _kimi_k3_count(request, messages_format, model_to_use, kimi_k3_messages, counted_tools) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 266a6a20131..d6482ea46b0 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -238,6 +238,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): bedrock_converse_supports_strict_tools: bool | None supports_regex_lookaround: ReadOnly[bool | None] token_count_framing: ReadOnly[int | None] + token_count_tool_framing: ReadOnly[int | None] token_count_model: ReadOnly[TokenCountModelInfo | None] token_count_tokenizer: ReadOnly[TokenCountTokenizer | None] @@ -4137,7 +4138,9 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( {"supports_regex_lookaround", "supports_assistant_prefill", "supports_bedrock_native_structured_output"} ) -DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset({"token_count_framing", "token_count_model"}) +DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset( + {"token_count_framing", "token_count_tool_framing", "token_count_model"} +) SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = ( frozenset(ModelInfoBase.__required_keys__ | ModelInfoBase.__optional_keys__) diff --git a/litellm/utils.py b/litellm/utils.py index 6a9beeec916..a2016ebeb04 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7056,6 +7056,7 @@ def get_model_info_helper( supports_image_size=_model_info.get("supports_image_size", None), supported_audio_formats=_model_info.get("supported_audio_formats", None), token_count_framing=_model_info.get("token_count_framing", None), + token_count_tool_framing=_model_info.get("token_count_tool_framing", None), token_count_model=_model_info.get("token_count_model", None), token_count_tokenizer=_model_info.get("token_count_tokenizer", None), vertex_ai_audio_api=_model_info.get("vertex_ai_audio_api", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 50b3560f05b..61e45b648c7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13404,6 +13404,8 @@ "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", + "token_count_framing": 149, + "token_count_tool_framing": 151, "max_input_tokens": 128000, "max_output_tokens": 128000, "mode": "chat", @@ -53982,6 +53984,8 @@ "input_cost_per_token_above_200k_tokens_batches": 2e-06, "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", + "token_count_framing": 185, + "token_count_tool_framing": 29, "max_input_tokens": 1000000, "max_output_tokens": 1000000, "mode": "chat", @@ -63397,6 +63401,7 @@ "input_cost_per_token": 3e-07, "input_cost_per_token_priority": 3.75e-07, "litellm_provider": "fireworks_ai", + "token_count_framing": 23, "max_input_tokens": 1048576, "max_output_tokens": 393216, "mode": "chat", @@ -63495,6 +63500,7 @@ "input_cost_per_token": 3e-07, "input_cost_per_token_priority": 3.75e-07, "litellm_provider": "fireworks_ai", + "token_count_framing": 23, "max_input_tokens": 1048576, "max_output_tokens": 393216, "mode": "chat", diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_chat_template_tools.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_chat_template_tools.py index 51249e01d4d..a8162860684 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_chat_template_tools.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_chat_template_tools.py @@ -9,7 +9,10 @@ _BARE_FUNCTION: Final = {"name": "g", "parameters": {}} # Tool schemas as rendered by render_tools in encoding/encoding_dsv4.py (deepseek-ai/DeepSeek-V4-Flash) and by -# chat_template.jinja (Qwen/Qwen3.5-122B-A10B), both read on Hugging Face on 2026-10-17 +# chat_template.jinja (Qwen/Qwen3.5-122B-A10B), both read on Hugging Face on 2026-10-17, and by chat_template.jinja +# (deepseek-ai/DeepSeek-V4.1-Flash) read on 2026-10-29. Grok's template is closed, so its two renderings are the ones +# whose token counts left a constant preamble across seven tool shapes billed by Azure AI Foundry +# grok-4-1-fast-reasoning and xAI grok-4.3 on 2026-10-29 @pytest.mark.parametrize( ("model", "rendered"), [ @@ -17,6 +20,18 @@ _BARE_FUNCTION: Final = {"name": "g", "parameters": {}} "DeepSeek-V4-Flash-0731", '{"name": "f", "description": "é", "parameters": {}}\n{"name": "g", "parameters": {}}', ), + ( + "DeepSeek-V4.1-Flash", + '{"name": "f", "description": "é", "parameters": {}}\n{"name": "g", "parameters": {}}', + ), + ( + "grok-4-1-fast-reasoning", + '{"name":"f","description":"é","parameters":{}}\n{"name":"g","parameters":{}}', + ), + ( + "grok-4.3", + '{"name": "f", "description": "é", "parameters": {}}\n{"name": "g", "parameters": {}}', + ), ( "databricks-qwen35-122b-a10b", '\n{"type": "function", "function": {"name": "f", "description": "é", "parameters": {}}}' diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 9b0f6607708..4eab59cb6b4 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -595,8 +595,14 @@ def test_token_counter_tools_match_billed_prompt_for_json_tool_chat_templates( ("deepseek/deepseek-chat:free", "DeepSeek-V4-Flash-0731"), ("deepseek-reasoner", "DeepSeek-V4-Flash-0731"), ("deepseek/deepseek-flash", "DeepSeek-V4-Flash-0731"), - ("DeepSeek-V4.1-Flash", "deepseek-v3.2"), - ("accounts/fireworks/models/deepseek-v4p1-flash", "deepseek-v3.2"), + ("DeepSeek-V4.1-Flash", "accounts/fireworks/models/deepseek-v4p1-flash"), + ("databricks-deepseek-v4-1-flash", "accounts/fireworks/models/deepseek-v4p1-flash"), + ("deepseek-ai/deepseek-v4.1-flash-maas", "accounts/fireworks/models/deepseek-v4p1-flash"), + ("deepseek-v4p1-flash-1017", "accounts/fireworks/models/deepseek-v4p1-flash"), + ("grok-4.1-fast-non-reasoning", "grok-4-1-fast-reasoning"), + ("xai.grok-4.1-fast", "grok-4-1-fast-reasoning"), + ("grok-4.3-latest", "grok-4.3"), + ("xai.grok-4-3", "grok-4.3"), ("Qwen/Qwen3.5-122B-A10B", "databricks-qwen35-122b-a10b"), ("qwen3-235b-a22b", "mistral-large-latest"), ], @@ -715,9 +721,17 @@ def test_token_counter_counts_anthropic_tools_as_sent_to_chat_template_families( # Qwen3.5's chat_template.jinja opens a system turn for the tool schemas when the request has none, 5 tokens that a # request with a system message already pays, while DeepSeek V4's encoding_dsv4.py appends them to the first turn -# either way (both read on Hugging Face on 2026-10-17) +# either way (both read on Hugging Face on 2026-10-17). A system message cost the same with and without a tool on +# Fireworks deepseek-v4p1-flash, Azure AI Foundry grok-4-1-fast-reasoning and xAI grok-4.3 on 2026-10-29 @pytest.mark.parametrize( - ("model", "implicit_system_turn_tokens"), [("DeepSeek-V4-Flash-0731", 0), ("databricks-qwen35-122b-a10b", 5)] + ("model", "implicit_system_turn_tokens"), + [ + ("DeepSeek-V4-Flash-0731", 0), + ("databricks-qwen35-122b-a10b", 5), + ("accounts/fireworks/models/deepseek-v4p1-flash", 0), + ("grok-4-1-fast-reasoning", 0), + ("grok-4.3", 0), + ], ) def test_token_counter_counts_the_tools_system_turn_only_without_a_system_message( model: str, implicit_system_turn_tokens: int diff --git a/tests/unit/proxy/common_utils/test_token_count_settings.py b/tests/unit/proxy/common_utils/test_token_count_settings.py index 43edc7e449f..fac1da5c6c4 100644 --- a/tests/unit/proxy/common_utils/test_token_count_settings.py +++ b/tests/unit/proxy/common_utils/test_token_count_settings.py @@ -131,6 +131,29 @@ def test_shipped_australian_count_region_stays_with_an_australian_deployment() - assert settings.count_model == TokenCountModel(model="au.anthropic.claude-opus-5-5", region="ap-southeast-4") +@pytest.mark.parametrize( + ("configured", "default", "expected"), + [ + ({}, {"token_count_tool_framing": 151}, (None, 151)), + ({"token_count_tool_framing": 40}, {"token_count_tool_framing": 151}, (None, 40)), + ({"token_count_tool_framing": 0}, {"token_count_tool_framing": 151}, (None, 0)), + ({"token_count_tool_framing": 40.0}, {}, (None, 40)), + ({"token_count_tool_framing": -1}, {"token_count_tool_framing": 151}, (None, 151)), + ({"token_count_tool_framing": True}, {"token_count_tool_framing": 151}, (None, 151)), + ({"token_count_tool_framing": "40"}, {}, (None, None)), + ({"token_count_tool_framing": 40}, {"token_count_framing": 97}, (97, 40)), + ({"token_count_framing": 40}, {"token_count_tool_framing": 151}, (40, 151)), + ], +) +def test_tool_framing_resolves_like_framing_and_apart_from_it( + configured: dict[str, object], default: dict[str, object], expected: tuple[int | None, int | None] +) -> None: + settings: Final = resolve_token_count_settings( + configured=configured, default=default, deployment_region=_in_us_west_2 + ) + assert (settings.framing, settings.tool_framing) == expected + + def test_no_setting_anywhere_means_the_model_counts_itself_without_framing() -> None: assert resolve_token_count_settings( configured={}, default={}, deployment_region=_in_us_west_2 diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index b892dc97e5c..afab406478d 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -4467,6 +4467,7 @@ class TestModelInfoCostMapEchoFilter: [ ("bedrock/us.anthropic.claude-sonnet-5-5", "token_count_model", {}), ("bedrock/us.moonshotai.kimi-k3", "token_count_framing", 0), + ("azure_ai/grok-4-1-fast-reasoning", "token_count_tool_framing", 0), ], ) def test_token_count_setting_is_stored_only_when_the_operator_changed_it( diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index b21730c64f6..04e1a812346 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -2652,6 +2652,151 @@ async def test_framing_comes_from_the_row_the_deployment_provider_or_base_model_ assert aliased.total_tokens == litellm.token_counter(model="gpt-5.4-mini", messages=messages) + framing +@pytest.mark.asyncio +async def test_tool_framing_is_added_once_and_only_when_the_request_carries_tools(monkeypatch: pytest.MonkeyPatch): + def deployment(name: str, model_info: dict[str, object]) -> dict[str, object]: + return { + "model_name": name, + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": f"sk-unused-{name}"}, + "model_info": model_info, + } + + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router( + model_list=[ + deployment("plain", {}), + deployment("tool-framed", {"token_count_tool_framing": 50}), + deployment("both", {"token_count_framing": 7, "token_count_tool_framing": 50}), + ] + ), + ) + messages: Final = [{"role": "user", "content": "What is the weather in Paris?"}] + tools: Final = [_KIMI_WEATHER] + + async def count(model: str, request_tools: list[dict[str, object]] | None) -> int: + response: Final = await token_counter( + request=TokenCountRequest(model=model, messages=messages, tools=request_tools) + ) + return response.total_tokens + + plain_with_tools: Final = await count("plain", tools) + plain_without_tools: Final = await count("plain", None) + + assert await count("tool-framed", tools) == plain_with_tools + 50 + assert await count("both", tools) == plain_with_tools + 57 + assert await count("tool-framed", None) == plain_without_tools + assert await count("tool-framed", []) == plain_without_tools + assert await count("both", None) == plain_without_tools + 7 + + +_GET_WEATHER: Final = { + "name": "get_weather", + "description": "Get the current weather for a city", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string", "description": "City name"}}, + "required": ["city"], + }, +} +_THREE_TOOLS: Final = ( + _GET_WEATHER, + { + "name": "get_time", + "description": "Get the local time in a timezone", + "input_schema": { + "type": "object", + "properties": {"timezone": {"type": "string", "description": "IANA timezone"}}, + "required": ["timezone"], + }, + }, + { + "name": "search_docs", + "description": "Search internal documents and return the top matches", + "input_schema": { + "type": "object", + "properties": {"query": {"type": "string"}, "limit": {"type": "integer", "description": "Max results"}}, + "required": ["query"], + }, + }, +) +_FORM_FIELDS: Final = tuple(f"field_{index}" for index in range(10)) +_FILL_FORM: Final = { + "name": "fill_form", + "description": "Fill a form", + "input_schema": { + "type": "object", + "properties": { + field: {"type": "string", "description": f"Value for field number {index}"} + for index, field in enumerate(_FORM_FIELDS) + }, + "required": list(_FORM_FIELDS), + }, +} +_WEATHER_SYSTEM: Final = "You are a helpful assistant that answers weather questions briefly." +_AZURE_GROK_4_1: Final = "azure_ai/grok-4-1-fast-reasoning" +_FIREWORKS_DEEPSEEK_V4_1: Final = "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash" +_XAI_GROK_4_3: Final = "xai/grok-4.3" + + +# usage.input_tokens plus cache_read_input_tokens that POST /v1/messages billed for these requests with the user turn +# "What is the weather in Paris?" through Azure AI Foundry grok-4-1-fast-reasoning, Fireworks deepseek-v4p1-flash and +# xAI grok-4.3 on 2026-10-29 +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "system", "tools", "billed_prompt_tokens"), + [ + (_AZURE_GROK_4_1, None, (), 163), + (_AZURE_GROK_4_1, None, (_GET_WEATHER,), 357), + (_AZURE_GROK_4_1, None, _THREE_TOOLS, 445), + (_AZURE_GROK_4_1, None, (_FILL_FORM,), 550), + (_AZURE_GROK_4_1, _WEATHER_SYSTEM, (_GET_WEATHER,), 368), + (_FIREWORKS_DEEPSEEK_V4_1, None, (), 37), + (_FIREWORKS_DEEPSEEK_V4_1, None, (_GET_WEATHER,), 304), + (_FIREWORKS_DEEPSEEK_V4_1, None, _THREE_TOOLS, 430), + (_FIREWORKS_DEEPSEEK_V4_1, None, (_FILL_FORM,), 550), + (_FIREWORKS_DEEPSEEK_V4_1, _WEATHER_SYSTEM, (_GET_WEATHER,), 315), + (_XAI_GROK_4_3, None, (), 199), + (_XAI_GROK_4_3, None, (_GET_WEATHER,), 283), + (_XAI_GROK_4_3, None, _THREE_TOOLS, 402), + (_XAI_GROK_4_3, None, (_FILL_FORM,), 530), + (_XAI_GROK_4_3, _WEATHER_SYSTEM, (_GET_WEATHER,), 297), + ], +) +async def test_local_estimate_matches_the_billed_prompt_with_the_shipped_framing( + monkeypatch: pytest.MonkeyPatch, + model: str, + system: str | None, + tools: tuple[dict[str, object], ...], + billed_prompt_tokens: int, +): + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "billed", + "litellm_params": {"model": model, "api_key": "unused", "api_base": "https://unused.invalid"}, + } + ] + ), + ) + + counted: Final = await token_counter( + request=TokenCountRequest( + model="billed", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=list(tools) or None, + system=system, + ) + ) + + assert counted.count_source == "local_estimate" + assert abs(counted.total_tokens - billed_prompt_tokens) <= billed_prompt_tokens * 0.05 + + _KIMI_WEATHER: Final = { "type": "function", "function": { diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 20b0055f729..181e09a4ffd 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -615,7 +615,7 @@ def test_capability_flag_stays_on_the_deployment_that_set_it(flag: str) -> None: def test_token_count_settings_stay_on_the_deployment_that_set_them() -> None: - """A deployment's ``token_count_model`` and ``token_count_framing`` must not land on + """A deployment's ``token_count_model`` and framing settings must not land on the shared ``{provider}/{model}`` key, or a sibling deployment of that model would count with them.""" from litellm.proxy.common_utils.token_count_settings import cost_map_entry @@ -624,6 +624,7 @@ def test_token_count_settings_stay_on_the_deployment_that_set_them() -> None: configured: Final = { "token_count_model": {"model": "anthropic.claude-opus-5-5", "region": "us-west-2"}, "token_count_framing": 7, + "token_count_tool_framing": 3, } builtin: Final = {field: cost_map_entry(backend_model).get(field) for field in configured} assert all(builtin[field] != value for field, value in configured.items()), builtin diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 665a540e462..371c06c6acb 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1172,6 +1172,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "supports_vision": {"type": "boolean"}, "supports_web_search": {"type": "boolean"}, "token_count_framing": {"type": "integer"}, + "token_count_tool_framing": {"type": "integer"}, "token_count_model": { "type": "object", "properties": {"model": {"type": "string"}, "region": {"type": "string"}}, From 3fbae7a2082a7669f690090173ccefc895a98ff0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:14:21 -0700 Subject: [PATCH 2/7] fix(count_tokens): count Azure grok-4-1 Responses requests with their own framing and use the tool framing in place of the base framing --- .../crates/model-catalog/src/model_info.rs | 2 + .../model-catalog/src/wire/attributes.rs | 2 + ...odel_prices_and_context_window_backup.json | 6 +- .../common_utils/token_count_settings.py | 9 ++ litellm/proxy/proxy_server.py | 10 ++- litellm/types/utils.py | 10 ++- litellm/utils.py | 2 + model_prices_and_context_window.json | 6 +- .../test_chat_template_tool_count_wire.py | 10 +-- .../common_utils/test_token_count_settings.py | 69 ++++++++++++--- .../test_model_management_endpoints.py | 2 + tests/unit/proxy/test_proxy_token_counter.py | 88 ++++++++++++++++--- .../unit/test_router_model_cost_isolation.py | 2 + tests/unit/test_utils.py | 2 + 14 files changed, 181 insertions(+), 39 deletions(-) diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs index 84decaf36d5..141702bf857 100644 --- a/litellm-rust/crates/model-catalog/src/model_info.rs +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -187,6 +187,8 @@ pub struct ModelInfoFields { pub supported_audio_formats: Option, pub token_count_framing: Option, pub token_count_tool_framing: Option, + pub token_count_responses_framing: Option, + pub token_count_responses_tool_framing: Option, pub token_count_model: Option, pub token_count_tokenizer: Option, pub vertex_ai_audio_api: Option, diff --git a/litellm-rust/crates/model-catalog/src/wire/attributes.rs b/litellm-rust/crates/model-catalog/src/wire/attributes.rs index 3f1c6ae6df1..9f3178cb480 100644 --- a/litellm-rust/crates/model-catalog/src/wire/attributes.rs +++ b/litellm-rust/crates/model-catalog/src/wire/attributes.rs @@ -106,6 +106,8 @@ attributes! { thinking_always_on: bool, token_count_framing: u64 as Count, token_count_model: TokenCountModel, + token_count_responses_framing: u64 as Count, + token_count_responses_tool_framing: u64 as Count, token_count_tokenizer: TokenCountTokenizer, token_count_tool_framing: u64 as Count, /// Provider default tokens-per-minute limit. diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 61e45b648c7..e0af2f61448 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13405,7 +13405,9 @@ "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", "token_count_framing": 149, - "token_count_tool_framing": 151, + "token_count_tool_framing": 300, + "token_count_responses_framing": 145, + "token_count_responses_tool_framing": 104, "max_input_tokens": 128000, "max_output_tokens": 128000, "mode": "chat", @@ -53985,7 +53987,7 @@ "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "token_count_framing": 185, - "token_count_tool_framing": 29, + "token_count_tool_framing": 214, "max_input_tokens": 1000000, "max_output_tokens": 1000000, "mode": "chat", diff --git a/litellm/proxy/common_utils/token_count_settings.py b/litellm/proxy/common_utils/token_count_settings.py index d1ea99b02d3..e9d8c93b0f1 100644 --- a/litellm/proxy/common_utils/token_count_settings.py +++ b/litellm/proxy/common_utils/token_count_settings.py @@ -57,6 +57,13 @@ class TokenCountSettings: count_model: TokenCountModel | None tokenizer: TokenCountTokenizer | None = None tool_framing: int | None = None + responses_framing: int | None = None + responses_tool_framing: int | None = None + + def hidden_prompt_tokens(self, carries_tools: bool, responses_api: bool) -> int: + chat: Final = self.tool_framing if carries_tools and self.tool_framing is not None else self.framing + responses: Final = self.responses_tool_framing if carries_tools else self.responses_framing + return (responses if responses_api and responses is not None else chat) or 0 @dataclass(frozen=True, slots=True) @@ -188,6 +195,8 @@ def resolve_token_count_settings( ), tokenizer=_tokenizer(default.get("token_count_tokenizer")), tool_framing=_configured_framing(configured, default, "token_count_tool_framing"), + responses_framing=_configured_framing(configured, default, "token_count_responses_framing"), + responses_tool_framing=_configured_framing(configured, default, "token_count_responses_tool_framing"), ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ab4d56f0cef..12aca6af8fe 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15530,9 +15530,13 @@ async def _count_request_tokens( counted_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats list[ChatCompletionToolParam] | None, tools if counted_messages is not None else None ) - message_framing: Final = 0 if counted_messages is None else settings.framing or 0 - tool_framing: Final = (settings.tool_framing or 0) if counted_tools else 0 - framing: Final = message_framing + tool_framing + framing: Final = ( + 0 + if counted_messages is None + else settings.hidden_prompt_tokens( + carries_tools=bool(counted_tools), responses_api=responses_request is not None + ) + ) kimi_k3_messages: Final = counted_messages if settings.tokenizer == "kimi_k3" else None exact: Final = ( await _kimi_k3_count(request, messages_format, model_to_use, kimi_k3_messages, counted_tools) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index d6482ea46b0..f80801f867c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -239,6 +239,8 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_regex_lookaround: ReadOnly[bool | None] token_count_framing: ReadOnly[int | None] token_count_tool_framing: ReadOnly[int | None] + token_count_responses_framing: ReadOnly[int | None] + token_count_responses_tool_framing: ReadOnly[int | None] token_count_model: ReadOnly[TokenCountModelInfo | None] token_count_tokenizer: ReadOnly[TokenCountTokenizer | None] @@ -4139,7 +4141,13 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( ) DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset( - {"token_count_framing", "token_count_tool_framing", "token_count_model"} + { + "token_count_framing", + "token_count_tool_framing", + "token_count_responses_framing", + "token_count_responses_tool_framing", + "token_count_model", + } ) SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = ( diff --git a/litellm/utils.py b/litellm/utils.py index a2016ebeb04..a68f783ca41 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7057,6 +7057,8 @@ def get_model_info_helper( supported_audio_formats=_model_info.get("supported_audio_formats", None), token_count_framing=_model_info.get("token_count_framing", None), token_count_tool_framing=_model_info.get("token_count_tool_framing", None), + token_count_responses_framing=_model_info.get("token_count_responses_framing", None), + token_count_responses_tool_framing=_model_info.get("token_count_responses_tool_framing", None), token_count_model=_model_info.get("token_count_model", None), token_count_tokenizer=_model_info.get("token_count_tokenizer", None), vertex_ai_audio_api=_model_info.get("vertex_ai_audio_api", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 61e45b648c7..e0af2f61448 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13405,7 +13405,9 @@ "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", "token_count_framing": 149, - "token_count_tool_framing": 151, + "token_count_tool_framing": 300, + "token_count_responses_framing": 145, + "token_count_responses_tool_framing": 104, "max_input_tokens": 128000, "max_output_tokens": 128000, "mode": "chat", @@ -53985,7 +53987,7 @@ "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", "token_count_framing": 185, - "token_count_tool_framing": 29, + "token_count_tool_framing": 214, "max_input_tokens": 1000000, "max_output_tokens": 1000000, "mode": "chat", diff --git a/tests/integration/providers/test_chat_template_tool_count_wire.py b/tests/integration/providers/test_chat_template_tool_count_wire.py index a8b97fa732e..552df3a87bc 100644 --- a/tests/integration/providers/test_chat_template_tool_count_wire.py +++ b/tests/integration/providers/test_chat_template_tool_count_wire.py @@ -39,7 +39,7 @@ _API_KEY: Final = "scripted-provider-key" _DEEPSEEK_V4: Final = "azure_ai/DeepSeek-V4-Flash-0731" _DEEPSEEK_R1: Final = "azure_ai/DeepSeek-R1-0528" _QWEN3_5: Final = "databricks/databricks-qwen35-122b-a10b" -_GROK: Final = "azure_ai/grok-4-1" +_GROK_3: Final = "azure_ai/grok-3" _DEEPSEEK_V4_TOOL_PREAMBLE: Final = 222 _QWEN3_5_TOOL_PREAMBLE: Final = 196 _QWEN3_5_OPENED_SYSTEM_TURN: Final = 5 @@ -250,7 +250,7 @@ def test_messages_count_tokens_skips_web_search_and_counts_unmapped_tool_keys_fo def test_messages_count_tokens_keeps_openai_tool_framing_for_other_models(gateway: Gateway) -> None: with wire_server(_completion) as wire, gateway.scenario() as scenario: - model: Final = _deployment(scenario, "count", _GROK, wire.url) + model: Final = _deployment(scenario, "count", _GROK_3, wire.url) without_tools: Final = _count(gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, ())) plain: Final = _count(gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, (_ANTHROPIC_TOOL,))) deferred: Final = _count( @@ -428,7 +428,7 @@ def test_project_itpm_reserves_deepseek_v4_tools_the_way_its_chat_template_rende def test_project_itpm_keeps_openai_tool_framing_for_other_models(gateway: Gateway) -> None: chat: Final = _ENDPOINTS["chat-completions"] with wire_server(_completion) as wire, gateway.scenario() as scenario: - group: Final = _deployment(scenario, "grok-4-1", _GROK, wire.url) + group: Final = _deployment(scenario, "grok-3", _GROK_3, wire.url) response: Final = gateway.request( "POST", chat.path, @@ -504,8 +504,8 @@ def _agent_body(group: str, marker: str, file_body: str, trigger: int) -> dict[s @pytest.mark.parametrize( ("model", "files_kept"), - ((_DEEPSEEK_V4, (False, False, True)), (_GROK, (True, True, True))), - ids=("deepseek-v4", "grok-4-1"), + ((_DEEPSEEK_V4, (False, False, True)), (_GROK_3, (True, True, True))), + ids=("deepseek-v4", "grok-3"), ) def test_messages_context_management_trigger_counts_tools_the_way_the_models_chat_template_renders_them( gateway: Gateway, model: str, files_kept: tuple[bool, ...] diff --git a/tests/unit/proxy/common_utils/test_token_count_settings.py b/tests/unit/proxy/common_utils/test_token_count_settings.py index fac1da5c6c4..a06605b714d 100644 --- a/tests/unit/proxy/common_utils/test_token_count_settings.py +++ b/tests/unit/proxy/common_utils/test_token_count_settings.py @@ -131,27 +131,70 @@ def test_shipped_australian_count_region_stays_with_an_australian_deployment() - assert settings.count_model == TokenCountModel(model="au.anthropic.claude-opus-5-5", region="ap-southeast-4") +_ABSENT: Final = object() + + +@pytest.mark.parametrize( + "key", ["token_count_tool_framing", "token_count_responses_framing", "token_count_responses_tool_framing"] +) @pytest.mark.parametrize( ("configured", "default", "expected"), [ - ({}, {"token_count_tool_framing": 151}, (None, 151)), - ({"token_count_tool_framing": 40}, {"token_count_tool_framing": 151}, (None, 40)), - ({"token_count_tool_framing": 0}, {"token_count_tool_framing": 151}, (None, 0)), - ({"token_count_tool_framing": 40.0}, {}, (None, 40)), - ({"token_count_tool_framing": -1}, {"token_count_tool_framing": 151}, (None, 151)), - ({"token_count_tool_framing": True}, {"token_count_tool_framing": 151}, (None, 151)), - ({"token_count_tool_framing": "40"}, {}, (None, None)), - ({"token_count_tool_framing": 40}, {"token_count_framing": 97}, (97, 40)), - ({"token_count_framing": 40}, {"token_count_tool_framing": 151}, (40, 151)), + (_ABSENT, 151, 151), + (40, 151, 40), + (0, 151, 0), + (40.0, _ABSENT, 40), + (-1, 151, 151), + (True, 151, 151), + ("40", _ABSENT, None), ], ) -def test_tool_framing_resolves_like_framing_and_apart_from_it( - configured: dict[str, object], default: dict[str, object], expected: tuple[int | None, int | None] +def test_each_request_kind_framing_resolves_like_framing_and_apart_from_the_others( + key: str, configured: object, default: object, expected: int | None ) -> None: settings: Final = resolve_token_count_settings( - configured=configured, default=default, deployment_region=_in_us_west_2 + configured={} if configured is _ABSENT else {key: configured}, + default={"token_count_framing": 97, **({} if default is _ABSENT else {key: default})}, + deployment_region=_in_us_west_2, ) - assert (settings.framing, settings.tool_framing) == expected + resolved: Final = { + "token_count_tool_framing": settings.tool_framing, + "token_count_responses_framing": settings.responses_framing, + "token_count_responses_tool_framing": settings.responses_tool_framing, + } + assert settings.framing == 97 + assert resolved == {name: expected if name == key else None for name in resolved} + + +_CHAT_ONLY: Final = TokenCountSettings(framing=185, count_model=None, tool_framing=214) +_PER_API: Final = TokenCountSettings( + framing=149, count_model=None, tool_framing=300, responses_framing=145, responses_tool_framing=104 +) + + +@pytest.mark.parametrize( + ("settings", "carries_tools", "responses_api", "expected"), + [ + (TokenCountSettings(framing=None, count_model=None), True, True, 0), + (TokenCountSettings(framing=23, count_model=None), True, False, 23), + (TokenCountSettings(framing=23, count_model=None), True, True, 23), + (_CHAT_ONLY, False, False, 185), + (_CHAT_ONLY, True, False, 214), + (_CHAT_ONLY, False, True, 185), + (_CHAT_ONLY, True, True, 214), + (_PER_API, False, False, 149), + (_PER_API, True, False, 300), + (_PER_API, False, True, 145), + (_PER_API, True, True, 104), + (TokenCountSettings(framing=149, count_model=None, tool_framing=0), True, False, 0), + (TokenCountSettings(framing=149, count_model=None, responses_tool_framing=0), True, True, 0), + (TokenCountSettings(framing=149, count_model=None, responses_framing=145), True, True, 149), + ], +) +def test_hidden_prompt_is_the_framing_set_for_the_request_api_and_tools_falling_back_to_chat( + settings: TokenCountSettings, carries_tools: bool, responses_api: bool, expected: int +) -> None: + assert settings.hidden_prompt_tokens(carries_tools=carries_tools, responses_api=responses_api) == expected def test_no_setting_anywhere_means_the_model_counts_itself_without_framing() -> None: diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index afab406478d..b60696c9490 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -4468,6 +4468,8 @@ class TestModelInfoCostMapEchoFilter: ("bedrock/us.anthropic.claude-sonnet-5-5", "token_count_model", {}), ("bedrock/us.moonshotai.kimi-k3", "token_count_framing", 0), ("azure_ai/grok-4-1-fast-reasoning", "token_count_tool_framing", 0), + ("azure_ai/grok-4-1-fast-reasoning", "token_count_responses_framing", 0), + ("azure_ai/grok-4-1-fast-reasoning", "token_count_responses_tool_framing", 0), ], ) def test_token_count_setting_is_stored_only_when_the_operator_changed_it( diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 04e1a812346..483fe6b6993 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -2653,7 +2653,7 @@ async def test_framing_comes_from_the_row_the_deployment_provider_or_base_model_ @pytest.mark.asyncio -async def test_tool_framing_is_added_once_and_only_when_the_request_carries_tools(monkeypatch: pytest.MonkeyPatch): +async def test_a_tool_request_counts_its_framing_in_place_of_the_plain_one_per_api(monkeypatch: pytest.MonkeyPatch): def deployment(name: str, model_info: dict[str, object]) -> dict[str, object]: return { "model_name": name, @@ -2667,28 +2667,41 @@ async def test_tool_framing_is_added_once_and_only_when_the_request_carries_tool Router( model_list=[ deployment("plain", {}), - deployment("tool-framed", {"token_count_tool_framing": 50}), - deployment("both", {"token_count_framing": 7, "token_count_tool_framing": 50}), + deployment("chat", {"token_count_framing": 7, "token_count_tool_framing": 50}), + deployment( + "per-api", + { + "token_count_framing": 7, + "token_count_tool_framing": 50, + "token_count_responses_framing": 5, + "token_count_responses_tool_framing": 2, + }, + ), ] ), ) messages: Final = [{"role": "user", "content": "What is the weather in Paris?"}] tools: Final = [_KIMI_WEATHER] - async def count(model: str, request_tools: list[dict[str, object]] | None) -> int: - response: Final = await token_counter( - request=TokenCountRequest(model=model, messages=messages, tools=request_tools) + async def count(model: str, request_tools: list[dict[str, object]] | None, responses: bool) -> int: + response: Final = await count_request_tokens( + request=TokenCountRequest(model=model, messages=messages, tools=request_tools), + responses_request={"model": model, "input": "What is the weather in Paris?"} if responses else None, ) return response.total_tokens - plain_with_tools: Final = await count("plain", tools) - plain_without_tools: Final = await count("plain", None) + plain_with_tools: Final = await count("plain", tools, responses=False) + plain_without_tools: Final = await count("plain", None, responses=False) - assert await count("tool-framed", tools) == plain_with_tools + 50 - assert await count("both", tools) == plain_with_tools + 57 - assert await count("tool-framed", None) == plain_without_tools - assert await count("tool-framed", []) == plain_without_tools - assert await count("both", None) == plain_without_tools + 7 + assert await count("plain", tools, responses=True) == plain_with_tools + assert await count("chat", tools, responses=False) == plain_with_tools + 50 + assert await count("chat", None, responses=False) == plain_without_tools + 7 + assert await count("chat", [], responses=False) == plain_without_tools + 7 + assert await count("chat", tools, responses=True) == plain_with_tools + 50 + assert await count("chat", None, responses=True) == plain_without_tools + 7 + assert await count("per-api", tools, responses=False) == plain_with_tools + 50 + assert await count("per-api", tools, responses=True) == plain_with_tools + 2 + assert await count("per-api", None, responses=True) == plain_without_tools + 5 _GET_WEATHER: Final = { @@ -2797,6 +2810,55 @@ async def test_local_estimate_matches_the_billed_prompt_with_the_shipped_framing assert abs(counted.total_tokens - billed_prompt_tokens) <= billed_prompt_tokens * 0.05 +# usage.input_tokens that POST /v1/responses billed for these requests with the input "What is the weather in Paris?" +# through Azure AI Foundry grok-4-1-fast-reasoning, Fireworks deepseek-v4p1-flash and xAI grok-4.3 on 2026-10-29 +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "system", "tools", "billed_prompt_tokens"), + [ + (_AZURE_GROK_4_1, None, (), 159), + (_AZURE_GROK_4_1, None, (_GET_WEATHER,), 159), + (_AZURE_GROK_4_1, None, _THREE_TOOLS, 257), + (_AZURE_GROK_4_1, None, (_FILL_FORM,), 352), + (_AZURE_GROK_4_1, _WEATHER_SYSTEM, (_GET_WEATHER,), 170), + (_FIREWORKS_DEEPSEEK_V4_1, None, (_GET_WEATHER,), 304), + (_XAI_GROK_4_3, None, (_GET_WEATHER,), 283), + ], +) +async def test_responses_estimate_matches_the_billed_responses_prompt_with_the_shipped_framing( + monkeypatch: pytest.MonkeyPatch, + model: str, + system: str | None, + tools: tuple[dict[str, object], ...], + billed_prompt_tokens: int, +): + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "billed", + "litellm_params": {"model": model, "api_key": "unused", "api_base": "https://unused.invalid"}, + } + ] + ), + ) + + counted: Final = await count_request_tokens( + request=TokenCountRequest( + model="billed", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=list(tools) or None, + system=system, + ), + responses_request={"model": "billed", "input": "What is the weather in Paris?"}, + ) + + assert counted.count_source == "local_estimate" + assert abs(counted.total_tokens - billed_prompt_tokens) <= billed_prompt_tokens * 0.05 + + _KIMI_WEATHER: Final = { "type": "function", "function": { diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 181e09a4ffd..d8bae6ac9ec 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -625,6 +625,8 @@ def test_token_count_settings_stay_on_the_deployment_that_set_them() -> None: "token_count_model": {"model": "anthropic.claude-opus-5-5", "region": "us-west-2"}, "token_count_framing": 7, "token_count_tool_framing": 3, + "token_count_responses_framing": 5, + "token_count_responses_tool_framing": 2, } builtin: Final = {field: cost_map_entry(backend_model).get(field) for field in configured} assert all(builtin[field] != value for field, value in configured.items()), builtin diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 371c06c6acb..08626f72572 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1173,6 +1173,8 @@ _COST_MAP_ROW_SCHEMA: Final = { "supports_web_search": {"type": "boolean"}, "token_count_framing": {"type": "integer"}, "token_count_tool_framing": {"type": "integer"}, + "token_count_responses_framing": {"type": "integer"}, + "token_count_responses_tool_framing": {"type": "integer"}, "token_count_model": { "type": "object", "properties": {"model": {"type": "string"}, "region": {"type": "string"}}, From 020ed0c3bd91ffec25171bc0f24957c3ab8900e5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 08:21:28 -0700 Subject: [PATCH 3/7] fix(count_tokens): keep operator framing on tool and Responses counts, frame bridged Responses like chat, follow base_model for tool rendering, count grok tool_choice --- .../prompt_templates/chat_template_tools.py | 8 ++- litellm/litellm_core_utils/token_counter.py | 6 +- .../common_utils/token_count_settings.py | 20 ++++-- litellm/proxy/proxy_server.py | 36 ++++++++-- .../litellm_core_utils/test_token_counter.py | 12 ++++ .../common_utils/test_token_count_settings.py | 34 ++++++++++ tests/unit/proxy/test_proxy_token_counter.py | 68 ++++++++++++++++++- 7 files changed, 170 insertions(+), 14 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py b/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py index 67ed6f82b41..65b8f35971b 100644 --- a/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py +++ b/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py @@ -10,6 +10,7 @@ class ChatTemplateToolSection: preamble_tokens: int implicit_system_turn_tokens: int render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str] + counts_tool_choice: bool = False def _json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: @@ -40,9 +41,12 @@ _TOOL_SECTIONS: Final = ( (re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)), ( re.compile(r"grok-4[-.]1(?![.\d])", re.IGNORECASE), - ChatTemplateToolSection(0, 0, _compact_json_line_tool_schemas), + ChatTemplateToolSection(0, 0, _compact_json_line_tool_schemas, counts_tool_choice=True), + ), + ( + re.compile(r"grok-4[-.]3(?![.\d])", re.IGNORECASE), + ChatTemplateToolSection(0, 0, _json_line_tool_schemas, counts_tool_choice=True), ), - (re.compile(r"grok-4[-.]3(?![.\d])", re.IGNORECASE), ChatTemplateToolSection(0, 0, _json_line_tool_schemas)), ) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 2fe840dc863..f25e0aa83f7 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -873,7 +873,11 @@ def _count_extra( tool_section (Optional[ChatTemplateToolSection]): How the model's chat template renders tools, if known. """ reply_primer_tokens: Final = 3 - tool_choice_tokens: Final = 0 if tool_section is not None else _count_tool_choice(count_function, tool_choice) + tool_choice_tokens: Final = ( + _count_tool_choice(count_function, tool_choice) + if tool_section is None or tool_section.counts_tool_choice + else 0 + ) return ( reply_primer_tokens + _count_tool_definitions(count_function, tools, includes_system_message, tool_section) diff --git a/litellm/proxy/common_utils/token_count_settings.py b/litellm/proxy/common_utils/token_count_settings.py index e9d8c93b0f1..7076cf89ddb 100644 --- a/litellm/proxy/common_utils/token_count_settings.py +++ b/litellm/proxy/common_utils/token_count_settings.py @@ -113,9 +113,15 @@ def _framing(value: object) -> int | None: return None -def _configured_framing(configured: Mapping[str, object], default: Mapping[str, object], key: str) -> int | None: +def _configured_framing( + configured: Mapping[str, object], default: Mapping[str, object], key: str, *broader_keys: str +) -> int | None: framing: Final = _framing(configured.get(key)) - return framing if framing is not None else _framing(default.get(key)) + if framing is not None: + return framing + if any(_framing(configured.get(broader)) is not None for broader in broader_keys): + return None + return _framing(default.get(key)) def _non_blank(value: object) -> str | None: @@ -194,9 +200,13 @@ def resolve_token_count_settings( configured.get("token_count_model"), default.get("token_count_model"), deployment_region ), tokenizer=_tokenizer(default.get("token_count_tokenizer")), - tool_framing=_configured_framing(configured, default, "token_count_tool_framing"), - responses_framing=_configured_framing(configured, default, "token_count_responses_framing"), - responses_tool_framing=_configured_framing(configured, default, "token_count_responses_tool_framing"), + tool_framing=_configured_framing(configured, default, "token_count_tool_framing", "token_count_framing"), + responses_framing=_configured_framing( + configured, default, "token_count_responses_framing", "token_count_framing" + ), + responses_tool_framing=_configured_framing( + configured, default, "token_count_responses_tool_framing", "token_count_tool_framing", "token_count_framing" + ), ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 12aca6af8fe..8585312ee27 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -976,6 +976,7 @@ from litellm.proxy.video_endpoints.endpoints import router as video_router from litellm.repositories.base_repository import SupportsModelDump from litellm.repositories.credentials_repository import CredentialsRepository from litellm.repositories.prisma_protocols import TableActions +from litellm.responses.main import provider_stores_the_conversation from litellm.responses.streaming_iterator import drain_logging_tasks from litellm.router import ( AssistantsTypedDict, @@ -15110,6 +15111,7 @@ async def _tokenizer_or_provider_error( async def _local_token_estimate( model: str, + template_model: str, prompt: str | None, messages: Sequence[AllMessageValues] | None, tools: list[ChatCompletionToolParam] | None, @@ -15122,7 +15124,7 @@ async def _local_token_estimate( else await _tokenizer_or_provider_error(model, custom_tokenizer, provider_failure) ) count: Final = offload_token_count(litellm.token_counter)( - model=model, + model=template_model, text=prompt, messages=messages, tools=tools, @@ -15185,10 +15187,15 @@ def _deployment_aws_region(deployment: _TokenCountDeployment) -> str: ) +def _deployment_base_model(deployment: _TokenCountDeployment) -> str | None: + return _non_empty_str(deployment.get("model_info", {}).get("base_model")) or _non_empty_str( + deployment.get("litellm_params", {}).get("base_model") + ) + + def _token_count_settings_model(deployment: _TokenCountDeployment, request_model: str) -> tuple[str, str | None]: - model_info: Final = deployment.get("model_info", {}) params: Final = deployment.get("litellm_params", {}) - base_model: Final = _non_empty_str(model_info.get("base_model")) or _non_empty_str(params.get("base_model")) + base_model: Final = _deployment_base_model(deployment) if base_model is not None: return base_model, None model: Final = _non_empty_str(params.get("model")) @@ -15280,6 +15287,22 @@ def _deployment_api_base(deployment: _TokenCountDeployment) -> str | None: return _non_empty_str(deployment.get("litellm_params", {}).get("api_base")) +def _serves_native_responses(deployment: _TokenCountDeployment, request_model: str) -> bool: + params: Final = deployment.get("litellm_params", {}) + try: + return provider_stores_the_conversation( + _non_empty_str(params.get("model")) or request_model, + _non_empty_str(params.get("custom_llm_provider")), + { + "use_chat_completions_api": params.get("use_chat_completions_api"), + "model_info": dict(deployment.get("model_info", {})), + "api_base": _deployment_api_base(deployment), + }, + ) + except litellm.BadRequestError: + return False + + def _system_message(system: object) -> ChatCompletionSystemMessage | None: if not isinstance(system, (str, list)) or not system: return None @@ -15534,7 +15557,8 @@ async def _count_request_tokens( 0 if counted_messages is None else settings.hidden_prompt_tokens( - carries_tools=bool(counted_tools), responses_api=responses_request is not None + carries_tools=bool(counted_tools), + responses_api=responses_request is not None and _serves_native_responses(deployment_view, request.model), ) ) kimi_k3_messages: Final = counted_messages if settings.tokenizer == "kimi_k3" else None @@ -15551,8 +15575,10 @@ async def _count_request_tokens( tokenizer_type="huggingface_tokenizer", count_source="local_estimate", ) + openai_model: Final = deployment_openai_model(deployment_view) estimate: Final = await _local_token_estimate( - model=deployment_openai_model(deployment_view) or model_to_use, + model=openai_model or model_to_use, + template_model=openai_model or _deployment_base_model(deployment_view) or model_to_use, prompt=prompt if counted_messages is None else None, messages=counted_messages, tools=counted_tools, diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 4eab59cb6b4..a5c96edd4ea 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -667,6 +667,18 @@ def test_token_counter_tool_choice_adds_nothing_for_deepseek_v4(tool_choice: obj ) == token_counter_new(model="DeepSeek-V4-Flash-0731", messages=list(_USER_MESSAGES), tools=tools) +# A request with a one-function tool billed 283 prompt tokens on xAI grok-4.3 without tool_choice and 297 with a +# named one, and 357 and 358 on Azure AI Foundry grok-4-1-fast-reasoning, on 2026-10-29 +@pytest.mark.parametrize("model", ["grok-4.3", "grok-4-1-fast-reasoning"]) +def test_token_counter_counts_a_named_tool_choice_for_grok(model: str): + tools: Final = [_openai_tool(_WEATHER_FUNCTION)] + named: Final = {"type": "function", "function": {"name": "get_weather"}} + + assert token_counter_new( + model=model, messages=list(_USER_MESSAGES), tools=tools, tool_choice=named + ) > token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=tools) + + @pytest.mark.parametrize( ("tool_choice", "expected_tool_choice"), [ diff --git a/tests/unit/proxy/common_utils/test_token_count_settings.py b/tests/unit/proxy/common_utils/test_token_count_settings.py index a06605b714d..858241b70ee 100644 --- a/tests/unit/proxy/common_utils/test_token_count_settings.py +++ b/tests/unit/proxy/common_utils/test_token_count_settings.py @@ -197,6 +197,40 @@ def test_hidden_prompt_is_the_framing_set_for_the_request_api_and_tools_falling_ assert settings.hidden_prompt_tokens(carries_tools=carries_tools, responses_api=responses_api) == expected +_PER_API_COST_MAP: Final = { + "token_count_framing": 149, + "token_count_tool_framing": 300, + "token_count_responses_framing": 145, + "token_count_responses_tool_framing": 104, +} + + +@pytest.mark.parametrize( + ("configured", "expected"), + [ + ({}, (149, 300, 145, 104)), + ({"token_count_framing": 500}, (500, 500, 500, 500)), + ({"token_count_framing": 0}, (0, 0, 0, 0)), + ({"token_count_tool_framing": 250}, (149, 250, 145, 250)), + ({"token_count_framing": 500, "token_count_tool_framing": 250}, (500, 250, 500, 250)), + ({"token_count_responses_framing": 120}, (149, 300, 120, 104)), + ({"token_count_framing": -1}, (149, 300, 145, 104)), + ], +) +def test_an_operator_framing_applies_to_every_request_kind_the_cost_map_splits_more_finely( + configured: dict[str, object], expected: tuple[int, int, int, int] +) -> None: + settings: Final = resolve_token_count_settings( + configured=configured, default=_PER_API_COST_MAP, deployment_region=_in_us_west_2 + ) + assert ( + settings.hidden_prompt_tokens(carries_tools=False, responses_api=False), + settings.hidden_prompt_tokens(carries_tools=True, responses_api=False), + settings.hidden_prompt_tokens(carries_tools=False, responses_api=True), + settings.hidden_prompt_tokens(carries_tools=True, responses_api=True), + ) == expected + + def test_no_setting_anywhere_means_the_model_counts_itself_without_framing() -> None: assert resolve_token_count_settings( configured={}, default={}, deployment_region=_in_us_west_2 diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 483fe6b6993..c5bb9c23b56 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -2751,6 +2751,11 @@ _WEATHER_SYSTEM: Final = "You are a helpful assistant that answers weather quest _AZURE_GROK_4_1: Final = "azure_ai/grok-4-1-fast-reasoning" _FIREWORKS_DEEPSEEK_V4_1: Final = "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash" _XAI_GROK_4_3: Final = "xai/grok-4.3" +_NATIVE_RESPONSES_BASE: Final = { + _AZURE_GROK_4_1: "https://unused.services.ai.azure.com/models", + _FIREWORKS_DEEPSEEK_V4_1: "https://unused.invalid", + _XAI_GROK_4_3: "https://unused.invalid", +} # usage.input_tokens plus cache_read_input_tokens that POST /v1/messages billed for these requests with the user turn @@ -2839,7 +2844,7 @@ async def test_responses_estimate_matches_the_billed_responses_prompt_with_the_s model_list=[ { "model_name": "billed", - "litellm_params": {"model": model, "api_key": "unused", "api_base": "https://unused.invalid"}, + "litellm_params": {"model": model, "api_key": "unused", "api_base": _NATIVE_RESPONSES_BASE[model]}, } ] ), @@ -2859,6 +2864,67 @@ async def test_responses_estimate_matches_the_billed_responses_prompt_with_the_s assert abs(counted.total_tokens - billed_prompt_tokens) <= billed_prompt_tokens * 0.05 +def _single_deployment_router(litellm_params: dict[str, object], model_info: dict[str, object]) -> Router: + return Router( + model_list=[ + { + "model_name": "counted", + "litellm_params": {"api_key": "unused", **litellm_params}, + "model_info": model_info, + } + ] + ) + + +async def _count_with_router( + monkeypatch: pytest.MonkeyPatch, router: Router, tools: tuple[dict[str, object], ...], responses: bool +) -> int: + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + counted: Final = await count_request_tokens( + request=TokenCountRequest( + model="counted", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=list(tools) or None, + ), + responses_request={"model": "counted", "input": "What is the weather in Paris?"} if responses else None, + ) + return counted.total_tokens + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tools", [(), (_GET_WEATHER,), _THREE_TOOLS, (_FILL_FORM,)]) +@pytest.mark.parametrize("responses", [False, True]) +async def test_a_custom_deployment_name_counts_like_the_base_model_it_names( + monkeypatch: pytest.MonkeyPatch, tools: tuple[dict[str, object], ...], responses: bool +): + api_base: Final = _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1] + custom_name: Final = _single_deployment_router( + {"model": "azure_ai/prod-grok", "api_base": api_base}, {"base_model": _AZURE_GROK_4_1} + ) + row_name: Final = _single_deployment_router({"model": _AZURE_GROK_4_1, "api_base": api_base}, {}) + + assert await _count_with_router(monkeypatch, custom_name, tools, responses) == await _count_with_router( + monkeypatch, row_name, tools, responses + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tools", [(), (_GET_WEATHER,), _THREE_TOOLS]) +async def test_a_responses_count_the_proxy_bridges_to_chat_completions_frames_like_the_chat_count( + monkeypatch: pytest.MonkeyPatch, tools: tuple[dict[str, object], ...] +): + bridged: Final = _single_deployment_router( + {"model": _AZURE_GROK_4_1, "api_base": "https://unused.cognitiveservices.azure.com/models"}, {} + ) + native: Final = _single_deployment_router( + {"model": _AZURE_GROK_4_1, "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, {} + ) + chat_count: Final = await _count_with_router(monkeypatch, bridged, tools, responses=False) + + assert await _count_with_router(monkeypatch, bridged, tools, responses=True) == chat_count + assert await _count_with_router(monkeypatch, native, tools, responses=True) != chat_count + + _KIMI_WEATHER: Final = { "type": "function", "function": { From e1f5e444c391566abc33872705d0f96f86a7e1a4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:50:16 -0700 Subject: [PATCH 4/7] fix(token_counter): count Azure Responses strict default, skip false strict and built-in-only tools, per-kind operator framing --- litellm/litellm_core_utils/token_counter.py | 16 ++- ...odel_prices_and_context_window_backup.json | 2 +- .../common_utils/token_count_settings.py | 40 +++++--- litellm/proxy/proxy_server.py | 35 ++++++- model_prices_and_context_window.json | 2 +- .../litellm_core_utils/test_token_counter.py | 14 +++ .../common_utils/test_token_count_settings.py | 81 ++++++++-------- tests/unit/proxy/test_proxy_token_counter.py | 97 ++++++++++++++++++- 8 files changed, 221 insertions(+), 66 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index f25e0aa83f7..aac689794c8 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -916,15 +916,25 @@ def _count_tool_choice( return 0 +def _named_function_definitions(tools: Sequence[object]) -> Iterable[Mapping[str, object]]: + return (function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name")) + + +def sends_function_tools(tools: Sequence[object] | None) -> bool: + return tools is not None and any(_named_function_definitions(tools)) + + +def _without_false_strict(function: Mapping[str, object]) -> Mapping[str, object]: + return {key: value for key, value in function.items() if key != "strict" or value} + + def _count_chat_template_tools( count_function: TokenCounterFunction, tools: Sequence[object], includes_system_message: bool, tool_section: ChatTemplateToolSection, ) -> int: - functions: Final = tuple( - function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name") - ) + functions: Final = tuple(map(_without_false_strict, _named_function_definitions(tools))) if not functions: return 0 system_turn_tokens: Final = 0 if includes_system_message else tool_section.implicit_system_turn_tokens diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e0af2f61448..0e5ef09bd09 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13407,7 +13407,7 @@ "token_count_framing": 149, "token_count_tool_framing": 300, "token_count_responses_framing": 145, - "token_count_responses_tool_framing": 104, + "token_count_responses_tool_framing": 100, "max_input_tokens": 128000, "max_output_tokens": 128000, "mode": "chat", diff --git a/litellm/proxy/common_utils/token_count_settings.py b/litellm/proxy/common_utils/token_count_settings.py index 7076cf89ddb..4e3059cfe26 100644 --- a/litellm/proxy/common_utils/token_count_settings.py +++ b/litellm/proxy/common_utils/token_count_settings.py @@ -60,10 +60,18 @@ class TokenCountSettings: responses_framing: int | None = None responses_tool_framing: int | None = None + @property + def frames_counts(self) -> bool: + return any( + framing is not None + for framing in (self.framing, self.tool_framing, self.responses_framing, self.responses_tool_framing) + ) + def hidden_prompt_tokens(self, carries_tools: bool, responses_api: bool) -> int: - chat: Final = self.tool_framing if carries_tools and self.tool_framing is not None else self.framing + chat: Final = self.tool_framing if carries_tools else None responses: Final = self.responses_tool_framing if carries_tools else self.responses_framing - return (responses if responses_api and responses is not None else chat) or 0 + narrower: Final = responses if responses_api else chat + return (narrower if narrower is not None else self.framing) or 0 @dataclass(frozen=True, slots=True) @@ -113,15 +121,13 @@ def _framing(value: object) -> int | None: return None -def _configured_framing( - configured: Mapping[str, object], default: Mapping[str, object], key: str, *broader_keys: str -) -> int | None: - framing: Final = _framing(configured.get(key)) - if framing is not None: - return framing - if any(_framing(configured.get(broader)) is not None for broader in broader_keys): - return None - return _framing(default.get(key)) +def _first_framing(source: Mapping[str, object], keys: tuple[str, ...]) -> int | None: + return next((framing for framing in map(_framing, map(source.get, keys)) if framing is not None), None) + + +def _configured_framing(configured: Mapping[str, object], default: Mapping[str, object], *keys: str) -> int | None: + operator_sets_one: Final = _first_framing(configured, (*keys, "token_count_framing")) is not None + return _first_framing(configured if operator_sets_one else default, keys) def _non_blank(value: object) -> str | None: @@ -200,12 +206,14 @@ def resolve_token_count_settings( configured.get("token_count_model"), default.get("token_count_model"), deployment_region ), tokenizer=_tokenizer(default.get("token_count_tokenizer")), - tool_framing=_configured_framing(configured, default, "token_count_tool_framing", "token_count_framing"), - responses_framing=_configured_framing( - configured, default, "token_count_responses_framing", "token_count_framing" - ), + tool_framing=_configured_framing(configured, default, "token_count_tool_framing"), + responses_framing=_configured_framing(configured, default, "token_count_responses_framing"), responses_tool_framing=_configured_framing( - configured, default, "token_count_responses_tool_framing", "token_count_tool_framing", "token_count_framing" + configured, + default, + "token_count_responses_tool_framing", + "token_count_responses_framing", + "token_count_tool_framing", ), ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8585312ee27..0cfc08e83eb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -104,6 +104,7 @@ from litellm.litellm_core_utils.token_counter import ( UnsupportedContentTypeError, deployment_openai_model, offload_token_count, + sends_function_tools, ) from litellm.llms.bedrock.count_tokens.kimi_k3 import ( HUGGING_FACE_SOURCE, @@ -14911,7 +14912,7 @@ async def _provider_token_count( framing_can_stand_in: Final = (count_model is None or not count_model.configured) and ( failure.cannot_count_model or (pinned_count_model is not None and failure.denied_in_region) ) - if not (framing_can_stand_in and settings.framing is not None and local_estimate_allowed): + if not (framing_can_stand_in and settings.frames_counts and local_estimate_allowed): raise failure.to_proxy_exception() if pinned_count_model is not None: verbose_proxy_logger.warning( @@ -15287,6 +15288,18 @@ def _deployment_api_base(deployment: _TokenCountDeployment) -> str | None: return _non_empty_str(deployment.get("litellm_params", {}).get("api_base")) +def _deployment_provider(deployment: _TokenCountDeployment, request_model: str) -> str | None: + params: Final = deployment.get("litellm_params", {}) + try: + return litellm.get_llm_provider( + model=_non_empty_str(params.get("model")) or request_model, + custom_llm_provider=_non_empty_str(params.get("custom_llm_provider")), + api_base=_deployment_api_base(deployment), + )[1] + except litellm.BadRequestError: + return None + + def _serves_native_responses(deployment: _TokenCountDeployment, request_model: str) -> bool: params: Final = deployment.get("litellm_params", {}) try: @@ -15303,6 +15316,13 @@ def _serves_native_responses(deployment: _TokenCountDeployment, request_model: s return False +def _with_strict_by_default(tools: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: + return tuple( + {**tool, "strict": True} if tool.get("type") == "function" and tool.get("strict") is None else tool + for tool in tools + ) + + def _system_message(system: object) -> ChatCompletionSystemMessage | None: if not isinstance(system, (str, list)) or not system: return None @@ -15550,15 +15570,22 @@ async def _count_request_tokens( counted_messages: Final = ( typed_messages if typed_messages is None or system_message is None else (system_message, *typed_messages) ) - counted_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats + native_responses: Final = responses_request is not None and _serves_native_responses(deployment_view, request.model) + sent_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats list[ChatCompletionToolParam] | None, tools if counted_messages is not None else None ) + counted_tools: Final = ( + cast(list[ChatCompletionToolParam], list(_with_strict_by_default(sent_tools))) # cast-ok: strict added + if sent_tools is not None + and native_responses + and _deployment_provider(deployment_view, request.model) == litellm.LlmProviders.AZURE_AI.value + else sent_tools + ) framing: Final = ( 0 if counted_messages is None else settings.hidden_prompt_tokens( - carries_tools=bool(counted_tools), - responses_api=responses_request is not None and _serves_native_responses(deployment_view, request.model), + carries_tools=sends_function_tools(counted_tools), responses_api=native_responses ) ) kimi_k3_messages: Final = counted_messages if settings.tokenizer == "kimi_k3" else None diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e0af2f61448..0e5ef09bd09 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13407,7 +13407,7 @@ "token_count_framing": 149, "token_count_tool_framing": 300, "token_count_responses_framing": 145, - "token_count_responses_tool_framing": 104, + "token_count_responses_tool_framing": 100, "max_input_tokens": 128000, "max_output_tokens": 128000, "mode": "chat", diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index a5c96edd4ea..9d5b582961a 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -679,6 +679,20 @@ def test_token_counter_counts_a_named_tool_choice_for_grok(model: str): ) > token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=tools) +# A one-function tool billed the same prompt tokens with strict unset and false on 2026-10-29: 357 through Azure AI +# Foundry grok-4-1-fast-reasoning chat, 283 through xAI grok-4.3 and 304 through Fireworks deepseek-v4p1-flash +@pytest.mark.parametrize( + "model", ["grok-4-1-fast-reasoning", "grok-4.3", "accounts/fireworks/models/deepseek-v4p1-flash"] +) +def test_token_counter_renders_a_false_strict_like_an_unset_one_for_chat_template_tools(model: str): + def counted(**strict: bool) -> int: + return token_counter_new( + model=model, messages=list(_USER_MESSAGES), tools=[_openai_tool({**_WEATHER_FUNCTION, **strict})] + ) + + assert counted(strict=False) == counted() < counted(strict=True) + + @pytest.mark.parametrize( ("tool_choice", "expected_tool_choice"), [ diff --git a/tests/unit/proxy/common_utils/test_token_count_settings.py b/tests/unit/proxy/common_utils/test_token_count_settings.py index 858241b70ee..68910f52b02 100644 --- a/tests/unit/proxy/common_utils/test_token_count_settings.py +++ b/tests/unit/proxy/common_utils/test_token_count_settings.py @@ -135,7 +135,12 @@ _ABSENT: Final = object() @pytest.mark.parametrize( - "key", ["token_count_tool_framing", "token_count_responses_framing", "token_count_responses_tool_framing"] + ("key", "field"), + [ + ("token_count_tool_framing", "tool_framing"), + ("token_count_responses_framing", "responses_framing"), + ("token_count_responses_tool_framing", "responses_tool_framing"), + ], ) @pytest.mark.parametrize( ("configured", "default", "expected"), @@ -149,52 +154,51 @@ _ABSENT: Final = object() ("40", _ABSENT, None), ], ) -def test_each_request_kind_framing_resolves_like_framing_and_apart_from_the_others( - key: str, configured: object, default: object, expected: int | None +def test_each_request_kind_framing_reads_its_value_like_framing( + key: str, field: str, configured: object, default: object, expected: int | None ) -> None: settings: Final = resolve_token_count_settings( configured={} if configured is _ABSENT else {key: configured}, - default={"token_count_framing": 97, **({} if default is _ABSENT else {key: default})}, + default={} if default is _ABSENT else {key: default}, deployment_region=_in_us_west_2, ) - resolved: Final = { - "token_count_tool_framing": settings.tool_framing, - "token_count_responses_framing": settings.responses_framing, - "token_count_responses_tool_framing": settings.responses_tool_framing, - } - assert settings.framing == 97 - assert resolved == {name: expected if name == key else None for name in resolved} + assert getattr(settings, field) == expected -_CHAT_ONLY: Final = TokenCountSettings(framing=185, count_model=None, tool_framing=214) -_PER_API: Final = TokenCountSettings( - framing=149, count_model=None, tool_framing=300, responses_framing=145, responses_tool_framing=104 -) +def _hidden_prompts(settings: TokenCountSettings) -> tuple[int, int, int, int]: + return ( + settings.hidden_prompt_tokens(carries_tools=False, responses_api=False), + settings.hidden_prompt_tokens(carries_tools=True, responses_api=False), + settings.hidden_prompt_tokens(carries_tools=False, responses_api=True), + settings.hidden_prompt_tokens(carries_tools=True, responses_api=True), + ) @pytest.mark.parametrize( - ("settings", "carries_tools", "responses_api", "expected"), + ("default", "expected"), [ - (TokenCountSettings(framing=None, count_model=None), True, True, 0), - (TokenCountSettings(framing=23, count_model=None), True, False, 23), - (TokenCountSettings(framing=23, count_model=None), True, True, 23), - (_CHAT_ONLY, False, False, 185), - (_CHAT_ONLY, True, False, 214), - (_CHAT_ONLY, False, True, 185), - (_CHAT_ONLY, True, True, 214), - (_PER_API, False, False, 149), - (_PER_API, True, False, 300), - (_PER_API, False, True, 145), - (_PER_API, True, True, 104), - (TokenCountSettings(framing=149, count_model=None, tool_framing=0), True, False, 0), - (TokenCountSettings(framing=149, count_model=None, responses_tool_framing=0), True, True, 0), - (TokenCountSettings(framing=149, count_model=None, responses_framing=145), True, True, 149), + ({}, (0, 0, 0, 0)), + ({"token_count_framing": 23}, (23, 23, 23, 23)), + ({"token_count_framing": 185, "token_count_tool_framing": 214}, (185, 214, 185, 214)), + ( + { + "token_count_framing": 149, + "token_count_tool_framing": 300, + "token_count_responses_framing": 145, + "token_count_responses_tool_framing": 104, + }, + (149, 300, 145, 104), + ), + ({"token_count_framing": 149, "token_count_tool_framing": 0}, (149, 0, 149, 0)), + ({"token_count_framing": 149, "token_count_responses_tool_framing": 0}, (149, 149, 149, 0)), + ({"token_count_framing": 149, "token_count_responses_framing": 145}, (149, 149, 145, 145)), ], ) -def test_hidden_prompt_is_the_framing_set_for_the_request_api_and_tools_falling_back_to_chat( - settings: TokenCountSettings, carries_tools: bool, responses_api: bool, expected: int +def test_hidden_prompt_is_the_most_specific_framing_set_for_the_request_api_and_tools( + default: dict[str, object], expected: tuple[int, int, int, int] ) -> None: - assert settings.hidden_prompt_tokens(carries_tools=carries_tools, responses_api=responses_api) == expected + settings: Final = resolve_token_count_settings(configured={}, default=default, deployment_region=_in_us_west_2) + assert _hidden_prompts(settings) == expected _PER_API_COST_MAP: Final = { @@ -213,7 +217,9 @@ _PER_API_COST_MAP: Final = { ({"token_count_framing": 0}, (0, 0, 0, 0)), ({"token_count_tool_framing": 250}, (149, 250, 145, 250)), ({"token_count_framing": 500, "token_count_tool_framing": 250}, (500, 250, 500, 250)), - ({"token_count_responses_framing": 120}, (149, 300, 120, 104)), + ({"token_count_responses_framing": 120}, (149, 300, 120, 120)), + ({"token_count_tool_framing": 250, "token_count_responses_framing": 120}, (149, 250, 120, 120)), + ({"token_count_responses_tool_framing": 90}, (149, 300, 145, 90)), ({"token_count_framing": -1}, (149, 300, 145, 104)), ], ) @@ -223,12 +229,7 @@ def test_an_operator_framing_applies_to_every_request_kind_the_cost_map_splits_m settings: Final = resolve_token_count_settings( configured=configured, default=_PER_API_COST_MAP, deployment_region=_in_us_west_2 ) - assert ( - settings.hidden_prompt_tokens(carries_tools=False, responses_api=False), - settings.hidden_prompt_tokens(carries_tools=True, responses_api=False), - settings.hidden_prompt_tokens(carries_tools=False, responses_api=True), - settings.hidden_prompt_tokens(carries_tools=True, responses_api=True), - ) == expected + assert _hidden_prompts(settings) == expected def test_no_setting_anywhere_means_the_model_counts_itself_without_framing() -> None: diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index c5bb9c23b56..28ca664734f 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -1849,6 +1849,23 @@ async def test_uncountable_model_without_framing_raises(): assert raised.value.code == "404" +@pytest.mark.parametrize( + "settings", + [ + TokenCountSettings(framing=None, count_model=None, tool_framing=300), + TokenCountSettings(framing=None, count_model=None, responses_framing=145), + TokenCountSettings(framing=None, count_model=None, responses_tool_framing=100), + ], +) +@pytest.mark.asyncio +async def test_uncountable_model_with_only_a_request_kind_framing_falls_through_to_the_local_estimate( + settings: TokenCountSettings, +): + counter: Final = _ScriptedCounter({"us.anthropic.claude-sonnet-5-5": _provider_reply("sonnet", status_code=404)}) + + assert isinstance(await _ladder(counter, settings), ProviderCountFailure) + + @pytest.mark.parametrize( ("count_model", "counted_as"), [ @@ -2758,6 +2775,20 @@ _NATIVE_RESPONSES_BASE: Final = { } +def _responses_function(tool: Mapping[str, object], **strict: bool) -> dict[str, object]: + return { + "type": "function", + "name": tool["name"], + "description": tool["description"], + "parameters": tool["input_schema"], + **strict, + } + + +def _get_weather_copies(count: int) -> tuple[dict[str, object], ...]: + return tuple({**_GET_WEATHER, "name": f"get_weather_{index}"} for index in range(count)) + + # usage.input_tokens plus cache_read_input_tokens that POST /v1/messages billed for these requests with the user turn # "What is the weather in Paris?" through Azure AI Foundry grok-4-1-fast-reasoning, Fireworks deepseek-v4p1-flash and # xAI grok-4.3 on 2026-10-29 @@ -2854,7 +2885,7 @@ async def test_responses_estimate_matches_the_billed_responses_prompt_with_the_s request=TokenCountRequest( model="billed", messages=[{"role": "user", "content": "What is the weather in Paris?"}], - tools=list(tools) or None, + tools=[_responses_function(tool) for tool in tools] or None, system=system, ), responses_request={"model": "billed", "input": "What is the weather in Paris?"}, @@ -2908,6 +2939,70 @@ async def test_a_custom_deployment_name_counts_like_the_base_model_it_names( ) +# usage.input_tokens that POST /v1/responses billed through Azure AI Foundry grok-4-1-fast-reasoning on 2026-10-29 for +# the input "What is the weather in Paris?" with these function tools, strict left unset or sent as false +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("tools", "strict", "billed_prompt_tokens"), + [ + ((_GET_WEATHER,), {}, 159), + ((_GET_WEATHER,), {"strict": False}, 155), + (_THREE_TOOLS, {}, 257), + (_get_weather_copies(5), {}, 345), + (_get_weather_copies(10), {}, 575), + (_get_weather_copies(10), {"strict": False}, 535), + ], +) +async def test_azure_responses_estimate_stays_at_or_just_over_the_billed_prompt_as_function_tools_are_added( + monkeypatch: pytest.MonkeyPatch, + tools: tuple[dict[str, object], ...], + strict: dict[str, bool], + billed_prompt_tokens: int, +): + router: Final = _single_deployment_router( + {"model": _AZURE_GROK_4_1, "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, {} + ) + + counted: Final = await _count_with_router( + monkeypatch, router, tuple(_responses_function(tool, **strict) for tool in tools), responses=True + ) + + assert billed_prompt_tokens <= counted <= billed_prompt_tokens * 1.02 + + +# POST /v1/responses billed the same usage.input_tokens with strict unset and false on 2026-10-29, 283 for one tool +# through xAI grok-4.3 and 304 through Fireworks deepseek-v4p1-flash +@pytest.mark.asyncio +@pytest.mark.parametrize("model", [_XAI_GROK_4_3, _FIREWORKS_DEEPSEEK_V4_1]) +async def test_a_responses_host_that_bills_an_unset_strict_like_false_counts_them_alike( + monkeypatch: pytest.MonkeyPatch, model: str +): + router: Final = _single_deployment_router({"model": model, "api_base": _NATIVE_RESPONSES_BASE[model]}, {}) + + unset: Final = await _count_with_router(monkeypatch, router, (_responses_function(_GET_WEATHER),), responses=True) + + assert unset == await _count_with_router( + monkeypatch, router, (_responses_function(_GET_WEATHER, strict=False),), responses=True + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("built_in_tool", "responses"), + [({"type": "web_search"}, True), ({"type": "web_search_20250305", "name": "web_search"}, False)], +) +async def test_a_request_whose_only_tools_are_built_in_counts_like_one_without_tools( + monkeypatch: pytest.MonkeyPatch, built_in_tool: dict[str, object], responses: bool +): + router: Final = _single_deployment_router( + {"model": _AZURE_GROK_4_1, "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, {} + ) + + assert await _count_with_router(monkeypatch, router, (built_in_tool,), responses) == await _count_with_router( + monkeypatch, router, (), responses + ) + + @pytest.mark.asyncio @pytest.mark.parametrize("tools", [(), (_GET_WEATHER,), _THREE_TOOLS]) async def test_a_responses_count_the_proxy_bridges_to_chat_completions_frames_like_the_chat_count( From 06c7c9fc2f46cc498db315f5cb267771f21457e6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:46:21 -0700 Subject: [PATCH 5/7] fix(token_counter): count an explicit null strict like false on Azure Responses --- litellm/proxy/proxy_server.py | 3 +-- tests/unit/proxy/test_proxy_token_counter.py | 8 +++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 88414e887a3..1df10e7d79e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15387,8 +15387,7 @@ def _serves_native_responses(deployment: _TokenCountDeployment, request_model: s def _with_strict_by_default(tools: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: return tuple( - {**tool, "strict": True} if tool.get("type") == "function" and tool.get("strict") is None else tool - for tool in tools + {**tool, "strict": True} if tool.get("type") == "function" and "strict" not in tool else tool for tool in tools ) diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 4a8cd031a39..679207de794 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -2813,7 +2813,7 @@ _NATIVE_RESPONSES_BASE: Final = { } -def _responses_function(tool: Mapping[str, object], **strict: bool) -> dict[str, object]: +def _responses_function(tool: Mapping[str, object], **strict: bool | None) -> dict[str, object]: return { "type": "function", "name": tool["name"], @@ -2978,23 +2978,25 @@ async def test_a_custom_deployment_name_counts_like_the_base_model_it_names( # usage.input_tokens that POST /v1/responses billed through Azure AI Foundry grok-4-1-fast-reasoning on 2026-10-29 for -# the input "What is the weather in Paris?" with these function tools, strict left unset or sent as false +# the input "What is the weather in Paris?" with these function tools, strict left unset or sent as false or null @pytest.mark.asyncio @pytest.mark.parametrize( ("tools", "strict", "billed_prompt_tokens"), [ ((_GET_WEATHER,), {}, 159), ((_GET_WEATHER,), {"strict": False}, 155), + ((_GET_WEATHER,), {"strict": None}, 155), (_THREE_TOOLS, {}, 257), (_get_weather_copies(5), {}, 345), (_get_weather_copies(10), {}, 575), (_get_weather_copies(10), {"strict": False}, 535), + (_get_weather_copies(10), {"strict": None}, 535), ], ) async def test_azure_responses_estimate_stays_at_or_just_over_the_billed_prompt_as_function_tools_are_added( monkeypatch: pytest.MonkeyPatch, tools: tuple[dict[str, object], ...], - strict: dict[str, bool], + strict: dict[str, bool | None], billed_prompt_tokens: int, ): router: Final = _single_deployment_router( From e6e9462860d1e4f9cf3e51e54d26622f35c3686b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:37:56 -0700 Subject: [PATCH 6/7] fix(count_tokens): follow the chat completions bridge flag and Azure's strict default on azure/ deployments Keep Azure grok-4-1 chat tool counts at or over the bill by raising its tool framing to 302 --- ...odel_prices_and_context_window_backup.json | 2 +- litellm/proxy/proxy_server.py | 28 +++++++--- model_prices_and_context_window.json | 2 +- tests/unit/proxy/test_proxy_token_counter.py | 56 +++++++++++++++++-- 4 files changed, 74 insertions(+), 14 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 84204271533..53cabca88d8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13456,7 +13456,7 @@ "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", "token_count_framing": 149, - "token_count_tool_framing": 300, + "token_count_tool_framing": 302, "token_count_responses_framing": 145, "token_count_responses_tool_framing": 100, "max_input_tokens": 128000, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1df10e7d79e..b0075aae084 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15325,12 +15325,15 @@ def _responses_routing_kwargs(responses_request: Mapping[str, object] | None) -> return {"previous_response_id": previous_response_id} if isinstance(previous_response_id, str) else {} +def _inference_params( + litellm_params: Mapping[str, object], responses_request: Mapping[str, object] +) -> Mapping[str, object]: + return {**litellm_params, **_router_default_litellm_params(llm_router), **responses_request} + + def _count_drop_policy(deployment: Mapping[str, object] | None, responses_request: Mapping[str, object]) -> DropPolicy: litellm_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {} - router_defaults: Final[Mapping[str, object]] = ( - {} if llm_router is None else _router_default_litellm_params(llm_router) - ) - return drop_policy({**litellm_params, **router_defaults, **responses_request}) + return drop_policy(_inference_params(litellm_params, responses_request)) def _native_responses_count( @@ -15369,14 +15372,18 @@ def _deployment_provider(deployment: _TokenCountDeployment, request_model: str) return None -def _serves_native_responses(deployment: _TokenCountDeployment, request_model: str) -> bool: +def _serves_native_responses( + deployment: _TokenCountDeployment, request_model: str, responses_request: Mapping[str, object] +) -> bool: params: Final = deployment.get("litellm_params", {}) try: return provider_stores_the_conversation( _non_empty_str(params.get("model")) or request_model, _non_empty_str(params.get("custom_llm_provider")), { - "use_chat_completions_api": params.get("use_chat_completions_api"), + "use_chat_completions_api": _inference_params(params, responses_request).get( + "use_chat_completions_api" + ), "model_info": dict(deployment.get("model_info", {})), "api_base": _deployment_api_base(deployment), }, @@ -15385,6 +15392,9 @@ def _serves_native_responses(deployment: _TokenCountDeployment, request_model: s return False +_AZURE_PROVIDERS: Final = frozenset((litellm.LlmProviders.AZURE.value, litellm.LlmProviders.AZURE_AI.value)) + + def _with_strict_by_default(tools: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: return tuple( {**tool, "strict": True} if tool.get("type") == "function" and "strict" not in tool else tool for tool in tools @@ -15636,7 +15646,9 @@ async def _count_request_tokens( counted_messages: Final = ( typed_messages if typed_messages is None or system_message is None else (system_message, *typed_messages) ) - native_responses: Final = responses_request is not None and _serves_native_responses(deployment_view, request.model) + native_responses: Final = responses_request is not None and _serves_native_responses( + deployment_view, request.model, responses_request + ) sent_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats list[ChatCompletionToolParam] | None, tools if counted_messages is not None else None ) @@ -15644,7 +15656,7 @@ async def _count_request_tokens( cast(list[ChatCompletionToolParam], list(_with_strict_by_default(sent_tools))) # cast-ok: strict added if sent_tools is not None and native_responses - and _deployment_provider(deployment_view, request.model) == litellm.LlmProviders.AZURE_AI.value + and _deployment_provider(deployment_view, request.model) in _AZURE_PROVIDERS else sent_tools ) framing: Final = ( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 84204271533..53cabca88d8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13456,7 +13456,7 @@ "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", "token_count_framing": 149, - "token_count_tool_framing": 300, + "token_count_tool_framing": 302, "token_count_responses_framing": 145, "token_count_responses_tool_framing": 100, "max_input_tokens": 128000, diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 679207de794..f03711f83dc 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -2978,7 +2978,8 @@ async def test_a_custom_deployment_name_counts_like_the_base_model_it_names( # usage.input_tokens that POST /v1/responses billed through Azure AI Foundry grok-4-1-fast-reasoning on 2026-10-29 for -# the input "What is the weather in Paris?" with these function tools, strict left unset or sent as false or null +# the input "What is the weather in Paris?" with these function tools, strict left unset or sent as false or null, +# the same on the .services.ai.azure.com and .openai.azure.com hosts @pytest.mark.asyncio @pytest.mark.parametrize( ("tools", "strict", "billed_prompt_tokens"), @@ -2993,15 +2994,25 @@ async def test_a_custom_deployment_name_counts_like_the_base_model_it_names( (_get_weather_copies(10), {"strict": None}, 535), ], ) +@pytest.mark.parametrize( + ("litellm_params", "model_info"), + [ + ({"model": _AZURE_GROK_4_1, "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, {}), + ( + {"model": "azure/grok-4-1-fast-reasoning", "api_base": "https://unused.openai.azure.com/openai/v1"}, + {"base_model": _AZURE_GROK_4_1}, + ), + ], +) async def test_azure_responses_estimate_stays_at_or_just_over_the_billed_prompt_as_function_tools_are_added( monkeypatch: pytest.MonkeyPatch, tools: tuple[dict[str, object], ...], strict: dict[str, bool | None], billed_prompt_tokens: int, + litellm_params: dict[str, object], + model_info: dict[str, object], ): - router: Final = _single_deployment_router( - {"model": _AZURE_GROK_4_1, "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, {} - ) + router: Final = _single_deployment_router(litellm_params, model_info) counted: Final = await _count_with_router( monkeypatch, router, tuple(_responses_function(tool, **strict) for tool in tools), responses=True @@ -3060,6 +3071,43 @@ async def test_a_responses_count_the_proxy_bridges_to_chat_completions_frames_li assert await _count_with_router(monkeypatch, native, tools, responses=True) != chat_count +@pytest.mark.asyncio +@pytest.mark.parametrize("from_router_defaults", [False, True]) +async def test_a_responses_count_that_asks_for_the_chat_completions_bridge_frames_like_the_chat_count( + monkeypatch: pytest.MonkeyPatch, from_router_defaults: bool +): + router: Final = Router( + model_list=[ + { + "model_name": "counted", + "litellm_params": { + "api_key": "unused", + "model": _AZURE_GROK_4_1, + "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1], + }, + "model_info": {}, + } + ], + default_litellm_params={"use_chat_completions_api": True} if from_router_defaults else None, + ) + chat_count: Final = await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=False) + + counted: Final = await count_request_tokens( + request=TokenCountRequest( + model="counted", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_GET_WEATHER], + ), + responses_request={ + "model": "counted", + "input": "What is the weather in Paris?", + **({} if from_router_defaults else {"use_chat_completions_api": True}), + }, + ) + + assert counted.total_tokens == chat_count + + _KIMI_WEATHER: Final = { "type": "function", "function": { From a2e1d374310ecc37b5c91dead6c53c524e50d58d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 02:30:18 -0700 Subject: [PATCH 7/7] fix(token_counter): keep strict false for non-grok templates, coerce the bridge flag, frame DeepSeek API counts --- .../prompt_templates/chat_template_tools.py | 9 ++- litellm/litellm_core_utils/token_counter.py | 7 ++- ...odel_prices_and_context_window_backup.json | 8 ++- litellm/proxy/proxy_server.py | 13 ++-- model_prices_and_context_window.json | 8 ++- .../litellm_core_utils/test_token_counter.py | 38 ++++++++---- tests/unit/proxy/test_proxy_token_counter.py | 60 +++++++++++++++++-- 7 files changed, 116 insertions(+), 27 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py b/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py index 65b8f35971b..49ffae03c40 100644 --- a/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py +++ b/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py @@ -11,6 +11,7 @@ class ChatTemplateToolSection: implicit_system_turn_tokens: int render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str] counts_tool_choice: bool = False + renders_strict_only_when_true: bool = False def _json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: @@ -41,11 +42,15 @@ _TOOL_SECTIONS: Final = ( (re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)), ( re.compile(r"grok-4[-.]1(?![.\d])", re.IGNORECASE), - ChatTemplateToolSection(0, 0, _compact_json_line_tool_schemas, counts_tool_choice=True), + ChatTemplateToolSection( + 0, 0, _compact_json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True + ), ), ( re.compile(r"grok-4[-.]3(?![.\d])", re.IGNORECASE), - ChatTemplateToolSection(0, 0, _json_line_tool_schemas, counts_tool_choice=True), + ChatTemplateToolSection( + 0, 0, _json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True + ), ), ) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 5731d1a6082..2a6ef2378b9 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -934,7 +934,12 @@ def _count_chat_template_tools( includes_system_message: bool, tool_section: ChatTemplateToolSection, ) -> int: - functions: Final = tuple(map(_without_false_strict, _named_function_definitions(tools))) + named_functions: Final = tuple(_named_function_definitions(tools)) + functions: Final = ( + tuple(map(_without_false_strict, named_functions)) + if tool_section.renders_strict_only_when_true + else named_functions + ) if not functions: return 0 system_turn_tokens: Final = 0 if includes_system_message else tool_section.implicit_system_turn_tokens diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 53cabca88d8..56ca4ae0d22 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -62208,7 +62208,9 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_framing": 23, + "token_count_tool_framing": 12 }, "deepseek/deepseek-v4-flash-vision-exp": { "cache_creation_input_token_cost": 0.0, @@ -62314,7 +62316,9 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "token_count_framing": 76, + "token_count_tool_framing": 75 }, "tencent/deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b0075aae084..793e40f71be 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -100,6 +100,7 @@ from litellm.litellm_core_utils.litellm_logging import ( initialize_sentry, ) from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER, LoggingWorker +from litellm.litellm_core_utils.prompt_templates.chat_template_tools import chat_template_tool_section from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.token_counter import ( @@ -15381,8 +15382,8 @@ def _serves_native_responses( _non_empty_str(params.get("model")) or request_model, _non_empty_str(params.get("custom_llm_provider")), { - "use_chat_completions_api": _inference_params(params, responses_request).get( - "use_chat_completions_api" + "use_chat_completions_api": bool( + _inference_params(params, responses_request).get("use_chat_completions_api") ), "model_info": dict(deployment.get("model_info", {})), "api_base": _deployment_api_base(deployment), @@ -15652,11 +15653,16 @@ async def _count_request_tokens( sent_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats list[ChatCompletionToolParam] | None, tools if counted_messages is not None else None ) + openai_model: Final = deployment_openai_model(deployment_view) + template_model: Final = openai_model or _deployment_base_model(deployment_view) or model_to_use + tool_section: Final = chat_template_tool_section(template_model) counted_tools: Final = ( cast(list[ChatCompletionToolParam], list(_with_strict_by_default(sent_tools))) # cast-ok: strict added if sent_tools is not None and native_responses and _deployment_provider(deployment_view, request.model) in _AZURE_PROVIDERS + and tool_section is not None + and tool_section.renders_strict_only_when_true else sent_tools ) framing: Final = ( @@ -15680,10 +15686,9 @@ async def _count_request_tokens( tokenizer_type="huggingface_tokenizer", count_source="local_estimate", ) - openai_model: Final = deployment_openai_model(deployment_view) estimate: Final = await _local_token_estimate( model=openai_model or model_to_use, - template_model=openai_model or _deployment_base_model(deployment_view) or model_to_use, + template_model=template_model, prompt=prompt if counted_messages is None else None, messages=counted_messages, tools=counted_tools, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 53cabca88d8..56ca4ae0d22 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -62208,7 +62208,9 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_framing": 23, + "token_count_tool_framing": 12 }, "deepseek/deepseek-v4-flash-vision-exp": { "cache_creation_input_token_cost": 0.0, @@ -62314,7 +62316,9 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "token_count_framing": 76, + "token_count_tool_framing": 75 }, "tencent/deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 497f81d59bf..b670676b398 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -681,18 +681,34 @@ def test_token_counter_counts_a_named_tool_choice_for_grok(model: str): ) > token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=tools) -# A one-function tool billed the same prompt tokens with strict unset and false on 2026-10-29: 357 through Azure AI -# Foundry grok-4-1-fast-reasoning chat, 283 through xAI grok-4.3 and 304 through Fireworks deepseek-v4p1-flash -@pytest.mark.parametrize( - "model", ["grok-4-1-fast-reasoning", "grok-4.3", "accounts/fireworks/models/deepseek-v4p1-flash"] -) -def test_token_counter_renders_a_false_strict_like_an_unset_one_for_chat_template_tools(model: str): - def counted(**strict: bool) -> int: - return token_counter_new( - model=model, messages=list(_USER_MESSAGES), tools=[_openai_tool({**_WEATHER_FUNCTION, **strict})] - ) +def _counted_with_strict(model: str, **strict: bool) -> int: + return token_counter_new( + model=model, messages=list(_USER_MESSAGES), tools=[_openai_tool({**_WEATHER_FUNCTION, **strict})] + ) - assert counted(strict=False) == counted() < counted(strict=True) + +# A one-function tool billed the same prompt tokens with strict unset and false on 2026-10-29: 357 through Azure AI +# Foundry grok-4-1-fast-reasoning chat and 283 through xAI grok-4.3 +@pytest.mark.parametrize("model", ["grok-4-1-fast-reasoning", "grok-4.3"]) +def test_token_counter_renders_a_false_strict_like_an_unset_one_for_grok(model: str): + assert ( + _counted_with_strict(model, strict=False) + == _counted_with_strict(model) + < _counted_with_strict(model, strict=True) + ) + + +# A one-function tool billed 5 prompt tokens more with strict false than unset, the same as with strict true, on +# 2026-10-30: 310 against 305 through Baseten DeepSeek-V4.1-Flash and 309 against 304 through Together +@pytest.mark.parametrize( + "model", ["accounts/fireworks/models/deepseek-v4p1-flash", "deepseek-v4-flash", "qwen3.5-plus"] +) +def test_token_counter_renders_a_false_strict_like_a_true_one_for_other_chat_templates(model: str): + assert ( + _counted_with_strict(model, strict=False) + == _counted_with_strict(model, strict=True) + > _counted_with_strict(model) + ) @pytest.mark.parametrize( diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index f03711f83dc..4211cfc4f13 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -2829,7 +2829,8 @@ def _get_weather_copies(count: int) -> tuple[dict[str, object], ...]: # usage.input_tokens plus cache_read_input_tokens that POST /v1/messages billed for these requests with the user turn # "What is the weather in Paris?" through Azure AI Foundry grok-4-1-fast-reasoning, Fireworks deepseek-v4p1-flash and -# xAI grok-4.3 on 2026-10-29 +# xAI grok-4.3 on 2026-10-29, and usage.prompt_tokens that POST /v1/chat/completions billed through DeepSeek's API on +# 2026-10-30 @pytest.mark.asyncio @pytest.mark.parametrize( ("model", "system", "tools", "billed_prompt_tokens"), @@ -2849,6 +2850,14 @@ def _get_weather_copies(count: int) -> tuple[dict[str, object], ...]: (_XAI_GROK_4_3, None, _THREE_TOOLS, 402), (_XAI_GROK_4_3, None, (_FILL_FORM,), 530), (_XAI_GROK_4_3, _WEATHER_SYSTEM, (_GET_WEATHER,), 297), + ("deepseek/deepseek-v4-flash", None, (), 37), + ("deepseek/deepseek-v4-flash", None, (_GET_WEATHER,), 304), + ("deepseek/deepseek-v4-flash", None, _THREE_TOOLS, 430), + ("deepseek/deepseek-v4-flash", _WEATHER_SYSTEM, (_GET_WEATHER,), 315), + ("deepseek/deepseek-v4-pro", None, (), 90), + ("deepseek/deepseek-v4-pro", None, (_GET_WEATHER,), 367), + ("deepseek/deepseek-v4-pro", None, _THREE_TOOLS, 493), + ("deepseek/deepseek-v4-pro", _WEATHER_SYSTEM, (_GET_WEATHER,), 378), ], ) async def test_local_estimate_matches_the_billed_prompt_with_the_shipped_framing( @@ -3022,13 +3031,14 @@ async def test_azure_responses_estimate_stays_at_or_just_over_the_billed_prompt_ # POST /v1/responses billed the same usage.input_tokens with strict unset and false on 2026-10-29, 283 for one tool -# through xAI grok-4.3 and 304 through Fireworks deepseek-v4p1-flash +# through xAI grok-4.3 @pytest.mark.asyncio -@pytest.mark.parametrize("model", [_XAI_GROK_4_3, _FIREWORKS_DEEPSEEK_V4_1]) async def test_a_responses_host_that_bills_an_unset_strict_like_false_counts_them_alike( - monkeypatch: pytest.MonkeyPatch, model: str + monkeypatch: pytest.MonkeyPatch, ): - router: Final = _single_deployment_router({"model": model, "api_base": _NATIVE_RESPONSES_BASE[model]}, {}) + router: Final = _single_deployment_router( + {"model": _XAI_GROK_4_3, "api_base": _NATIVE_RESPONSES_BASE[_XAI_GROK_4_3]}, {} + ) unset: Final = await _count_with_router(monkeypatch, router, (_responses_function(_GET_WEATHER),), responses=True) @@ -3108,6 +3118,46 @@ async def test_a_responses_count_that_asks_for_the_chat_completions_bridge_frame assert counted.total_tokens == chat_count +@pytest.mark.asyncio +@pytest.mark.parametrize("use_chat_completions_api", ["maybe", {"a": 1}]) +async def test_a_responses_count_with_a_malformed_bridge_flag_reads_it_the_way_the_responses_route_does( + monkeypatch: pytest.MonkeyPatch, use_chat_completions_api: object +): + router: Final = _single_deployment_router( + {"model": _AZURE_GROK_4_1, "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, {} + ) + chat_count: Final = await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=False) + + counted: Final = await count_request_tokens( + request=TokenCountRequest( + model="counted", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_GET_WEATHER], + ), + responses_request={ + "model": "counted", + "input": "What is the weather in Paris?", + "use_chat_completions_api": use_chat_completions_api, + }, + ) + + assert counted.total_tokens == chat_count + + +@pytest.mark.asyncio +async def test_an_azure_responses_count_leaves_strict_unset_for_templates_not_measured_there( + monkeypatch: pytest.MonkeyPatch, +): + router: Final = _single_deployment_router( + {"model": "azure_ai/deepseek-v4.1-flash", "api_base": "https://unused.services.ai.azure.com/models"}, {} + ) + + async def counted(**strict: bool) -> int: + return await _count_with_router(monkeypatch, router, (_responses_function(_GET_WEATHER, **strict),), True) + + assert await counted() < await counted(strict=True) + + _KIMI_WEATHER: Final = { "type": "function", "function": {