fix(token_counter): keep strict false for non-grok templates, coerce the bridge flag, frame DeepSeek API counts

This commit is contained in:
mateo-berri 2026-10-30 02:30:18 -07:00
parent e6e9462860
commit a2e1d37431
7 changed files with 116 additions and 27 deletions

View file

@ -11,6 +11,7 @@ class ChatTemplateToolSection:
implicit_system_turn_tokens: int
render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str]
counts_tool_choice: bool = False
renders_strict_only_when_true: bool = False
def _json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
@ -41,11 +42,15 @@ _TOOL_SECTIONS: Final = (
(re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)),
(
re.compile(r"grok-4[-.]1(?![.\d])", re.IGNORECASE),
ChatTemplateToolSection(0, 0, _compact_json_line_tool_schemas, counts_tool_choice=True),
ChatTemplateToolSection(
0, 0, _compact_json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True
),
),
(
re.compile(r"grok-4[-.]3(?![.\d])", re.IGNORECASE),
ChatTemplateToolSection(0, 0, _json_line_tool_schemas, counts_tool_choice=True),
ChatTemplateToolSection(
0, 0, _json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True
),
),
)

View file

@ -934,7 +934,12 @@ def _count_chat_template_tools(
includes_system_message: bool,
tool_section: ChatTemplateToolSection,
) -> int:
functions: Final = tuple(map(_without_false_strict, _named_function_definitions(tools)))
named_functions: Final = tuple(_named_function_definitions(tools))
functions: Final = (
tuple(map(_without_false_strict, named_functions))
if tool_section.renders_strict_only_when_true
else named_functions
)
if not functions:
return 0
system_turn_tokens: Final = 0 if includes_system_message else tool_section.implicit_system_turn_tokens

View file

@ -62208,7 +62208,9 @@
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"token_count_framing": 23,
"token_count_tool_framing": 12
},
"deepseek/deepseek-v4-flash-vision-exp": {
"cache_creation_input_token_cost": 0.0,
@ -62314,7 +62316,9 @@
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": false
"supports_vision": false,
"token_count_framing": 76,
"token_count_tool_framing": 75
},
"tencent/deepseek-v4-pro": {
"cache_creation_input_token_cost": 0.0,

View file

@ -100,6 +100,7 @@ from litellm.litellm_core_utils.litellm_logging import (
initialize_sentry,
)
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER, LoggingWorker
from litellm.litellm_core_utils.prompt_templates.chat_template_tools import chat_template_tool_section
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.litellm_core_utils.token_counter import (
@ -15381,8 +15382,8 @@ def _serves_native_responses(
_non_empty_str(params.get("model")) or request_model,
_non_empty_str(params.get("custom_llm_provider")),
{
"use_chat_completions_api": _inference_params(params, responses_request).get(
"use_chat_completions_api"
"use_chat_completions_api": bool(
_inference_params(params, responses_request).get("use_chat_completions_api")
),
"model_info": dict(deployment.get("model_info", {})),
"api_base": _deployment_api_base(deployment),
@ -15652,11 +15653,16 @@ async def _count_request_tokens(
sent_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats
list[ChatCompletionToolParam] | None, tools if counted_messages is not None else None
)
openai_model: Final = deployment_openai_model(deployment_view)
template_model: Final = openai_model or _deployment_base_model(deployment_view) or model_to_use
tool_section: Final = chat_template_tool_section(template_model)
counted_tools: Final = (
cast(list[ChatCompletionToolParam], list(_with_strict_by_default(sent_tools))) # cast-ok: strict added
if sent_tools is not None
and native_responses
and _deployment_provider(deployment_view, request.model) in _AZURE_PROVIDERS
and tool_section is not None
and tool_section.renders_strict_only_when_true
else sent_tools
)
framing: Final = (
@ -15680,10 +15686,9 @@ async def _count_request_tokens(
tokenizer_type="huggingface_tokenizer",
count_source="local_estimate",
)
openai_model: Final = deployment_openai_model(deployment_view)
estimate: Final = await _local_token_estimate(
model=openai_model or model_to_use,
template_model=openai_model or _deployment_base_model(deployment_view) or model_to_use,
template_model=template_model,
prompt=prompt if counted_messages is None else None,
messages=counted_messages,
tools=counted_tools,

View file

@ -62208,7 +62208,9 @@
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"token_count_framing": 23,
"token_count_tool_framing": 12
},
"deepseek/deepseek-v4-flash-vision-exp": {
"cache_creation_input_token_cost": 0.0,
@ -62314,7 +62316,9 @@
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": false
"supports_vision": false,
"token_count_framing": 76,
"token_count_tool_framing": 75
},
"tencent/deepseek-v4-pro": {
"cache_creation_input_token_cost": 0.0,

View file

@ -681,18 +681,34 @@ def test_token_counter_counts_a_named_tool_choice_for_grok(model: str):
) > token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=tools)
# A one-function tool billed the same prompt tokens with strict unset and false on 2026-10-29: 357 through Azure AI
# Foundry grok-4-1-fast-reasoning chat, 283 through xAI grok-4.3 and 304 through Fireworks deepseek-v4p1-flash
@pytest.mark.parametrize(
"model", ["grok-4-1-fast-reasoning", "grok-4.3", "accounts/fireworks/models/deepseek-v4p1-flash"]
)
def test_token_counter_renders_a_false_strict_like_an_unset_one_for_chat_template_tools(model: str):
def counted(**strict: bool) -> int:
return token_counter_new(
model=model, messages=list(_USER_MESSAGES), tools=[_openai_tool({**_WEATHER_FUNCTION, **strict})]
)
def _counted_with_strict(model: str, **strict: bool) -> int:
return token_counter_new(
model=model, messages=list(_USER_MESSAGES), tools=[_openai_tool({**_WEATHER_FUNCTION, **strict})]
)
assert counted(strict=False) == counted() < counted(strict=True)
# A one-function tool billed the same prompt tokens with strict unset and false on 2026-10-29: 357 through Azure AI
# Foundry grok-4-1-fast-reasoning chat and 283 through xAI grok-4.3
@pytest.mark.parametrize("model", ["grok-4-1-fast-reasoning", "grok-4.3"])
def test_token_counter_renders_a_false_strict_like_an_unset_one_for_grok(model: str):
assert (
_counted_with_strict(model, strict=False)
== _counted_with_strict(model)
< _counted_with_strict(model, strict=True)
)
# A one-function tool billed 5 prompt tokens more with strict false than unset, the same as with strict true, on
# 2026-10-30: 310 against 305 through Baseten DeepSeek-V4.1-Flash and 309 against 304 through Together
@pytest.mark.parametrize(
"model", ["accounts/fireworks/models/deepseek-v4p1-flash", "deepseek-v4-flash", "qwen3.5-plus"]
)
def test_token_counter_renders_a_false_strict_like_a_true_one_for_other_chat_templates(model: str):
assert (
_counted_with_strict(model, strict=False)
== _counted_with_strict(model, strict=True)
> _counted_with_strict(model)
)
@pytest.mark.parametrize(

View file

@ -2829,7 +2829,8 @@ def _get_weather_copies(count: int) -> tuple[dict[str, object], ...]:
# usage.input_tokens plus cache_read_input_tokens that POST /v1/messages billed for these requests with the user turn
# "What is the weather in Paris?" through Azure AI Foundry grok-4-1-fast-reasoning, Fireworks deepseek-v4p1-flash and
# xAI grok-4.3 on 2026-10-29
# xAI grok-4.3 on 2026-10-29, and usage.prompt_tokens that POST /v1/chat/completions billed through DeepSeek's API on
# 2026-10-30
@pytest.mark.asyncio
@pytest.mark.parametrize(
("model", "system", "tools", "billed_prompt_tokens"),
@ -2849,6 +2850,14 @@ def _get_weather_copies(count: int) -> tuple[dict[str, object], ...]:
(_XAI_GROK_4_3, None, _THREE_TOOLS, 402),
(_XAI_GROK_4_3, None, (_FILL_FORM,), 530),
(_XAI_GROK_4_3, _WEATHER_SYSTEM, (_GET_WEATHER,), 297),
("deepseek/deepseek-v4-flash", None, (), 37),
("deepseek/deepseek-v4-flash", None, (_GET_WEATHER,), 304),
("deepseek/deepseek-v4-flash", None, _THREE_TOOLS, 430),
("deepseek/deepseek-v4-flash", _WEATHER_SYSTEM, (_GET_WEATHER,), 315),
("deepseek/deepseek-v4-pro", None, (), 90),
("deepseek/deepseek-v4-pro", None, (_GET_WEATHER,), 367),
("deepseek/deepseek-v4-pro", None, _THREE_TOOLS, 493),
("deepseek/deepseek-v4-pro", _WEATHER_SYSTEM, (_GET_WEATHER,), 378),
],
)
async def test_local_estimate_matches_the_billed_prompt_with_the_shipped_framing(
@ -3022,13 +3031,14 @@ async def test_azure_responses_estimate_stays_at_or_just_over_the_billed_prompt_
# POST /v1/responses billed the same usage.input_tokens with strict unset and false on 2026-10-29, 283 for one tool
# through xAI grok-4.3 and 304 through Fireworks deepseek-v4p1-flash
# through xAI grok-4.3
@pytest.mark.asyncio
@pytest.mark.parametrize("model", [_XAI_GROK_4_3, _FIREWORKS_DEEPSEEK_V4_1])
async def test_a_responses_host_that_bills_an_unset_strict_like_false_counts_them_alike(
monkeypatch: pytest.MonkeyPatch, model: str
monkeypatch: pytest.MonkeyPatch,
):
router: Final = _single_deployment_router({"model": model, "api_base": _NATIVE_RESPONSES_BASE[model]}, {})
router: Final = _single_deployment_router(
{"model": _XAI_GROK_4_3, "api_base": _NATIVE_RESPONSES_BASE[_XAI_GROK_4_3]}, {}
)
unset: Final = await _count_with_router(monkeypatch, router, (_responses_function(_GET_WEATHER),), responses=True)
@ -3108,6 +3118,46 @@ async def test_a_responses_count_that_asks_for_the_chat_completions_bridge_frame
assert counted.total_tokens == chat_count
@pytest.mark.asyncio
@pytest.mark.parametrize("use_chat_completions_api", ["maybe", {"a": 1}])
async def test_a_responses_count_with_a_malformed_bridge_flag_reads_it_the_way_the_responses_route_does(
monkeypatch: pytest.MonkeyPatch, use_chat_completions_api: object
):
router: Final = _single_deployment_router(
{"model": _AZURE_GROK_4_1, "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, {}
)
chat_count: Final = await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=False)
counted: Final = await count_request_tokens(
request=TokenCountRequest(
model="counted",
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
tools=[_GET_WEATHER],
),
responses_request={
"model": "counted",
"input": "What is the weather in Paris?",
"use_chat_completions_api": use_chat_completions_api,
},
)
assert counted.total_tokens == chat_count
@pytest.mark.asyncio
async def test_an_azure_responses_count_leaves_strict_unset_for_templates_not_measured_there(
monkeypatch: pytest.MonkeyPatch,
):
router: Final = _single_deployment_router(
{"model": "azure_ai/deepseek-v4.1-flash", "api_base": "https://unused.services.ai.azure.com/models"}, {}
)
async def counted(**strict: bool) -> int:
return await _count_with_router(monkeypatch, router, (_responses_function(_GET_WEATHER, **strict),), True)
assert await counted() < await counted(strict=True)
_KIMI_WEATHER: Final = {
"type": "function",
"function": {