diff --git a/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py b/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py index 65b8f35971b..49ffae03c40 100644 --- a/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py +++ b/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py @@ -11,6 +11,7 @@ class ChatTemplateToolSection: implicit_system_turn_tokens: int render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str] counts_tool_choice: bool = False + renders_strict_only_when_true: bool = False def _json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: @@ -41,11 +42,15 @@ _TOOL_SECTIONS: Final = ( (re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)), ( re.compile(r"grok-4[-.]1(?![.\d])", re.IGNORECASE), - ChatTemplateToolSection(0, 0, _compact_json_line_tool_schemas, counts_tool_choice=True), + ChatTemplateToolSection( + 0, 0, _compact_json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True + ), ), ( re.compile(r"grok-4[-.]3(?![.\d])", re.IGNORECASE), - ChatTemplateToolSection(0, 0, _json_line_tool_schemas, counts_tool_choice=True), + ChatTemplateToolSection( + 0, 0, _json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True + ), ), ) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 5731d1a6082..2a6ef2378b9 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -934,7 +934,12 @@ def _count_chat_template_tools( includes_system_message: bool, tool_section: ChatTemplateToolSection, ) -> int: - functions: Final = tuple(map(_without_false_strict, _named_function_definitions(tools))) + named_functions: Final = tuple(_named_function_definitions(tools)) + functions: Final = ( + tuple(map(_without_false_strict, named_functions)) + if tool_section.renders_strict_only_when_true + else named_functions + ) if not functions: return 0 system_turn_tokens: Final = 0 if includes_system_message else tool_section.implicit_system_turn_tokens diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 53cabca88d8..56ca4ae0d22 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -62208,7 +62208,9 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_framing": 23, + "token_count_tool_framing": 12 }, "deepseek/deepseek-v4-flash-vision-exp": { "cache_creation_input_token_cost": 0.0, @@ -62314,7 +62316,9 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "token_count_framing": 76, + "token_count_tool_framing": 75 }, "tencent/deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b0075aae084..793e40f71be 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -100,6 +100,7 @@ from litellm.litellm_core_utils.litellm_logging import ( initialize_sentry, ) from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER, LoggingWorker +from litellm.litellm_core_utils.prompt_templates.chat_template_tools import chat_template_tool_section from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.token_counter import ( @@ -15381,8 +15382,8 @@ def _serves_native_responses( _non_empty_str(params.get("model")) or request_model, _non_empty_str(params.get("custom_llm_provider")), { - "use_chat_completions_api": _inference_params(params, responses_request).get( - "use_chat_completions_api" + "use_chat_completions_api": bool( + _inference_params(params, responses_request).get("use_chat_completions_api") ), "model_info": dict(deployment.get("model_info", {})), "api_base": _deployment_api_base(deployment), @@ -15652,11 +15653,16 @@ async def _count_request_tokens( sent_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats list[ChatCompletionToolParam] | None, tools if counted_messages is not None else None ) + openai_model: Final = deployment_openai_model(deployment_view) + template_model: Final = openai_model or _deployment_base_model(deployment_view) or model_to_use + tool_section: Final = chat_template_tool_section(template_model) counted_tools: Final = ( cast(list[ChatCompletionToolParam], list(_with_strict_by_default(sent_tools))) # cast-ok: strict added if sent_tools is not None and native_responses and _deployment_provider(deployment_view, request.model) in _AZURE_PROVIDERS + and tool_section is not None + and tool_section.renders_strict_only_when_true else sent_tools ) framing: Final = ( @@ -15680,10 +15686,9 @@ async def _count_request_tokens( tokenizer_type="huggingface_tokenizer", count_source="local_estimate", ) - openai_model: Final = deployment_openai_model(deployment_view) estimate: Final = await _local_token_estimate( model=openai_model or model_to_use, - template_model=openai_model or _deployment_base_model(deployment_view) or model_to_use, + template_model=template_model, prompt=prompt if counted_messages is None else None, messages=counted_messages, tools=counted_tools, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 53cabca88d8..56ca4ae0d22 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -62208,7 +62208,9 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_framing": 23, + "token_count_tool_framing": 12 }, "deepseek/deepseek-v4-flash-vision-exp": { "cache_creation_input_token_cost": 0.0, @@ -62314,7 +62316,9 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "token_count_framing": 76, + "token_count_tool_framing": 75 }, "tencent/deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 497f81d59bf..b670676b398 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -681,18 +681,34 @@ def test_token_counter_counts_a_named_tool_choice_for_grok(model: str): ) > token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=tools) -# A one-function tool billed the same prompt tokens with strict unset and false on 2026-10-29: 357 through Azure AI -# Foundry grok-4-1-fast-reasoning chat, 283 through xAI grok-4.3 and 304 through Fireworks deepseek-v4p1-flash -@pytest.mark.parametrize( - "model", ["grok-4-1-fast-reasoning", "grok-4.3", "accounts/fireworks/models/deepseek-v4p1-flash"] -) -def test_token_counter_renders_a_false_strict_like_an_unset_one_for_chat_template_tools(model: str): - def counted(**strict: bool) -> int: - return token_counter_new( - model=model, messages=list(_USER_MESSAGES), tools=[_openai_tool({**_WEATHER_FUNCTION, **strict})] - ) +def _counted_with_strict(model: str, **strict: bool) -> int: + return token_counter_new( + model=model, messages=list(_USER_MESSAGES), tools=[_openai_tool({**_WEATHER_FUNCTION, **strict})] + ) - assert counted(strict=False) == counted() < counted(strict=True) + +# A one-function tool billed the same prompt tokens with strict unset and false on 2026-10-29: 357 through Azure AI +# Foundry grok-4-1-fast-reasoning chat and 283 through xAI grok-4.3 +@pytest.mark.parametrize("model", ["grok-4-1-fast-reasoning", "grok-4.3"]) +def test_token_counter_renders_a_false_strict_like_an_unset_one_for_grok(model: str): + assert ( + _counted_with_strict(model, strict=False) + == _counted_with_strict(model) + < _counted_with_strict(model, strict=True) + ) + + +# A one-function tool billed 5 prompt tokens more with strict false than unset, the same as with strict true, on +# 2026-10-30: 310 against 305 through Baseten DeepSeek-V4.1-Flash and 309 against 304 through Together +@pytest.mark.parametrize( + "model", ["accounts/fireworks/models/deepseek-v4p1-flash", "deepseek-v4-flash", "qwen3.5-plus"] +) +def test_token_counter_renders_a_false_strict_like_a_true_one_for_other_chat_templates(model: str): + assert ( + _counted_with_strict(model, strict=False) + == _counted_with_strict(model, strict=True) + > _counted_with_strict(model) + ) @pytest.mark.parametrize( diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index f03711f83dc..4211cfc4f13 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -2829,7 +2829,8 @@ def _get_weather_copies(count: int) -> tuple[dict[str, object], ...]: # usage.input_tokens plus cache_read_input_tokens that POST /v1/messages billed for these requests with the user turn # "What is the weather in Paris?" through Azure AI Foundry grok-4-1-fast-reasoning, Fireworks deepseek-v4p1-flash and -# xAI grok-4.3 on 2026-10-29 +# xAI grok-4.3 on 2026-10-29, and usage.prompt_tokens that POST /v1/chat/completions billed through DeepSeek's API on +# 2026-10-30 @pytest.mark.asyncio @pytest.mark.parametrize( ("model", "system", "tools", "billed_prompt_tokens"), @@ -2849,6 +2850,14 @@ def _get_weather_copies(count: int) -> tuple[dict[str, object], ...]: (_XAI_GROK_4_3, None, _THREE_TOOLS, 402), (_XAI_GROK_4_3, None, (_FILL_FORM,), 530), (_XAI_GROK_4_3, _WEATHER_SYSTEM, (_GET_WEATHER,), 297), + ("deepseek/deepseek-v4-flash", None, (), 37), + ("deepseek/deepseek-v4-flash", None, (_GET_WEATHER,), 304), + ("deepseek/deepseek-v4-flash", None, _THREE_TOOLS, 430), + ("deepseek/deepseek-v4-flash", _WEATHER_SYSTEM, (_GET_WEATHER,), 315), + ("deepseek/deepseek-v4-pro", None, (), 90), + ("deepseek/deepseek-v4-pro", None, (_GET_WEATHER,), 367), + ("deepseek/deepseek-v4-pro", None, _THREE_TOOLS, 493), + ("deepseek/deepseek-v4-pro", _WEATHER_SYSTEM, (_GET_WEATHER,), 378), ], ) async def test_local_estimate_matches_the_billed_prompt_with_the_shipped_framing( @@ -3022,13 +3031,14 @@ async def test_azure_responses_estimate_stays_at_or_just_over_the_billed_prompt_ # POST /v1/responses billed the same usage.input_tokens with strict unset and false on 2026-10-29, 283 for one tool -# through xAI grok-4.3 and 304 through Fireworks deepseek-v4p1-flash +# through xAI grok-4.3 @pytest.mark.asyncio -@pytest.mark.parametrize("model", [_XAI_GROK_4_3, _FIREWORKS_DEEPSEEK_V4_1]) async def test_a_responses_host_that_bills_an_unset_strict_like_false_counts_them_alike( - monkeypatch: pytest.MonkeyPatch, model: str + monkeypatch: pytest.MonkeyPatch, ): - router: Final = _single_deployment_router({"model": model, "api_base": _NATIVE_RESPONSES_BASE[model]}, {}) + router: Final = _single_deployment_router( + {"model": _XAI_GROK_4_3, "api_base": _NATIVE_RESPONSES_BASE[_XAI_GROK_4_3]}, {} + ) unset: Final = await _count_with_router(monkeypatch, router, (_responses_function(_GET_WEATHER),), responses=True) @@ -3108,6 +3118,46 @@ async def test_a_responses_count_that_asks_for_the_chat_completions_bridge_frame assert counted.total_tokens == chat_count +@pytest.mark.asyncio +@pytest.mark.parametrize("use_chat_completions_api", ["maybe", {"a": 1}]) +async def test_a_responses_count_with_a_malformed_bridge_flag_reads_it_the_way_the_responses_route_does( + monkeypatch: pytest.MonkeyPatch, use_chat_completions_api: object +): + router: Final = _single_deployment_router( + {"model": _AZURE_GROK_4_1, "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, {} + ) + chat_count: Final = await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=False) + + counted: Final = await count_request_tokens( + request=TokenCountRequest( + model="counted", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_GET_WEATHER], + ), + responses_request={ + "model": "counted", + "input": "What is the weather in Paris?", + "use_chat_completions_api": use_chat_completions_api, + }, + ) + + assert counted.total_tokens == chat_count + + +@pytest.mark.asyncio +async def test_an_azure_responses_count_leaves_strict_unset_for_templates_not_measured_there( + monkeypatch: pytest.MonkeyPatch, +): + router: Final = _single_deployment_router( + {"model": "azure_ai/deepseek-v4.1-flash", "api_base": "https://unused.services.ai.azure.com/models"}, {} + ) + + async def counted(**strict: bool) -> int: + return await _count_with_router(monkeypatch, router, (_responses_function(_GET_WEATHER, **strict),), True) + + assert await counted() < await counted(strict=True) + + _KIMI_WEATHER: Final = { "type": "function", "function": {