mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(token_counter): keep strict false for non-grok templates, coerce the bridge flag, frame DeepSeek API counts
This commit is contained in:
parent
e6e9462860
commit
a2e1d37431
7 changed files with 116 additions and 27 deletions
|
|
@ -11,6 +11,7 @@ class ChatTemplateToolSection:
|
|||
implicit_system_turn_tokens: int
|
||||
render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str]
|
||||
counts_tool_choice: bool = False
|
||||
renders_strict_only_when_true: bool = False
|
||||
|
||||
|
||||
def _json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
|
||||
|
|
@ -41,11 +42,15 @@ _TOOL_SECTIONS: Final = (
|
|||
(re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)),
|
||||
(
|
||||
re.compile(r"grok-4[-.]1(?![.\d])", re.IGNORECASE),
|
||||
ChatTemplateToolSection(0, 0, _compact_json_line_tool_schemas, counts_tool_choice=True),
|
||||
ChatTemplateToolSection(
|
||||
0, 0, _compact_json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True
|
||||
),
|
||||
),
|
||||
(
|
||||
re.compile(r"grok-4[-.]3(?![.\d])", re.IGNORECASE),
|
||||
ChatTemplateToolSection(0, 0, _json_line_tool_schemas, counts_tool_choice=True),
|
||||
ChatTemplateToolSection(
|
||||
0, 0, _json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -934,7 +934,12 @@ def _count_chat_template_tools(
|
|||
includes_system_message: bool,
|
||||
tool_section: ChatTemplateToolSection,
|
||||
) -> int:
|
||||
functions: Final = tuple(map(_without_false_strict, _named_function_definitions(tools)))
|
||||
named_functions: Final = tuple(_named_function_definitions(tools))
|
||||
functions: Final = (
|
||||
tuple(map(_without_false_strict, named_functions))
|
||||
if tool_section.renders_strict_only_when_true
|
||||
else named_functions
|
||||
)
|
||||
if not functions:
|
||||
return 0
|
||||
system_turn_tokens: Final = 0 if includes_system_message else tool_section.implicit_system_turn_tokens
|
||||
|
|
|
|||
|
|
@ -62208,7 +62208,9 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"token_count_framing": 23,
|
||||
"token_count_tool_framing": 12
|
||||
},
|
||||
"deepseek/deepseek-v4-flash-vision-exp": {
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
|
|
@ -62314,7 +62316,9 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
"supports_vision": false,
|
||||
"token_count_framing": 76,
|
||||
"token_count_tool_framing": 75
|
||||
},
|
||||
"tencent/deepseek-v4-pro": {
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
|
|
|
|||
|
|
@ -100,6 +100,7 @@ from litellm.litellm_core_utils.litellm_logging import (
|
|||
initialize_sentry,
|
||||
)
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER, LoggingWorker
|
||||
from litellm.litellm_core_utils.prompt_templates.chat_template_tools import chat_template_tool_section
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.litellm_core_utils.token_counter import (
|
||||
|
|
@ -15381,8 +15382,8 @@ def _serves_native_responses(
|
|||
_non_empty_str(params.get("model")) or request_model,
|
||||
_non_empty_str(params.get("custom_llm_provider")),
|
||||
{
|
||||
"use_chat_completions_api": _inference_params(params, responses_request).get(
|
||||
"use_chat_completions_api"
|
||||
"use_chat_completions_api": bool(
|
||||
_inference_params(params, responses_request).get("use_chat_completions_api")
|
||||
),
|
||||
"model_info": dict(deployment.get("model_info", {})),
|
||||
"api_base": _deployment_api_base(deployment),
|
||||
|
|
@ -15652,11 +15653,16 @@ async def _count_request_tokens(
|
|||
sent_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats
|
||||
list[ChatCompletionToolParam] | None, tools if counted_messages is not None else None
|
||||
)
|
||||
openai_model: Final = deployment_openai_model(deployment_view)
|
||||
template_model: Final = openai_model or _deployment_base_model(deployment_view) or model_to_use
|
||||
tool_section: Final = chat_template_tool_section(template_model)
|
||||
counted_tools: Final = (
|
||||
cast(list[ChatCompletionToolParam], list(_with_strict_by_default(sent_tools))) # cast-ok: strict added
|
||||
if sent_tools is not None
|
||||
and native_responses
|
||||
and _deployment_provider(deployment_view, request.model) in _AZURE_PROVIDERS
|
||||
and tool_section is not None
|
||||
and tool_section.renders_strict_only_when_true
|
||||
else sent_tools
|
||||
)
|
||||
framing: Final = (
|
||||
|
|
@ -15680,10 +15686,9 @@ async def _count_request_tokens(
|
|||
tokenizer_type="huggingface_tokenizer",
|
||||
count_source="local_estimate",
|
||||
)
|
||||
openai_model: Final = deployment_openai_model(deployment_view)
|
||||
estimate: Final = await _local_token_estimate(
|
||||
model=openai_model or model_to_use,
|
||||
template_model=openai_model or _deployment_base_model(deployment_view) or model_to_use,
|
||||
template_model=template_model,
|
||||
prompt=prompt if counted_messages is None else None,
|
||||
messages=counted_messages,
|
||||
tools=counted_tools,
|
||||
|
|
|
|||
|
|
@ -62208,7 +62208,9 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"token_count_framing": 23,
|
||||
"token_count_tool_framing": 12
|
||||
},
|
||||
"deepseek/deepseek-v4-flash-vision-exp": {
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
|
|
@ -62314,7 +62316,9 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
"supports_vision": false,
|
||||
"token_count_framing": 76,
|
||||
"token_count_tool_framing": 75
|
||||
},
|
||||
"tencent/deepseek-v4-pro": {
|
||||
"cache_creation_input_token_cost": 0.0,
|
||||
|
|
|
|||
|
|
@ -681,18 +681,34 @@ def test_token_counter_counts_a_named_tool_choice_for_grok(model: str):
|
|||
) > token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=tools)
|
||||
|
||||
|
||||
# A one-function tool billed the same prompt tokens with strict unset and false on 2026-10-29: 357 through Azure AI
|
||||
# Foundry grok-4-1-fast-reasoning chat, 283 through xAI grok-4.3 and 304 through Fireworks deepseek-v4p1-flash
|
||||
@pytest.mark.parametrize(
|
||||
"model", ["grok-4-1-fast-reasoning", "grok-4.3", "accounts/fireworks/models/deepseek-v4p1-flash"]
|
||||
)
|
||||
def test_token_counter_renders_a_false_strict_like_an_unset_one_for_chat_template_tools(model: str):
|
||||
def counted(**strict: bool) -> int:
|
||||
return token_counter_new(
|
||||
model=model, messages=list(_USER_MESSAGES), tools=[_openai_tool({**_WEATHER_FUNCTION, **strict})]
|
||||
)
|
||||
def _counted_with_strict(model: str, **strict: bool) -> int:
|
||||
return token_counter_new(
|
||||
model=model, messages=list(_USER_MESSAGES), tools=[_openai_tool({**_WEATHER_FUNCTION, **strict})]
|
||||
)
|
||||
|
||||
assert counted(strict=False) == counted() < counted(strict=True)
|
||||
|
||||
# A one-function tool billed the same prompt tokens with strict unset and false on 2026-10-29: 357 through Azure AI
|
||||
# Foundry grok-4-1-fast-reasoning chat and 283 through xAI grok-4.3
|
||||
@pytest.mark.parametrize("model", ["grok-4-1-fast-reasoning", "grok-4.3"])
|
||||
def test_token_counter_renders_a_false_strict_like_an_unset_one_for_grok(model: str):
|
||||
assert (
|
||||
_counted_with_strict(model, strict=False)
|
||||
== _counted_with_strict(model)
|
||||
< _counted_with_strict(model, strict=True)
|
||||
)
|
||||
|
||||
|
||||
# A one-function tool billed 5 prompt tokens more with strict false than unset, the same as with strict true, on
|
||||
# 2026-10-30: 310 against 305 through Baseten DeepSeek-V4.1-Flash and 309 against 304 through Together
|
||||
@pytest.mark.parametrize(
|
||||
"model", ["accounts/fireworks/models/deepseek-v4p1-flash", "deepseek-v4-flash", "qwen3.5-plus"]
|
||||
)
|
||||
def test_token_counter_renders_a_false_strict_like_a_true_one_for_other_chat_templates(model: str):
|
||||
assert (
|
||||
_counted_with_strict(model, strict=False)
|
||||
== _counted_with_strict(model, strict=True)
|
||||
> _counted_with_strict(model)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
|
|||
|
|
@ -2829,7 +2829,8 @@ def _get_weather_copies(count: int) -> tuple[dict[str, object], ...]:
|
|||
|
||||
# usage.input_tokens plus cache_read_input_tokens that POST /v1/messages billed for these requests with the user turn
|
||||
# "What is the weather in Paris?" through Azure AI Foundry grok-4-1-fast-reasoning, Fireworks deepseek-v4p1-flash and
|
||||
# xAI grok-4.3 on 2026-10-29
|
||||
# xAI grok-4.3 on 2026-10-29, and usage.prompt_tokens that POST /v1/chat/completions billed through DeepSeek's API on
|
||||
# 2026-10-30
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("model", "system", "tools", "billed_prompt_tokens"),
|
||||
|
|
@ -2849,6 +2850,14 @@ def _get_weather_copies(count: int) -> tuple[dict[str, object], ...]:
|
|||
(_XAI_GROK_4_3, None, _THREE_TOOLS, 402),
|
||||
(_XAI_GROK_4_3, None, (_FILL_FORM,), 530),
|
||||
(_XAI_GROK_4_3, _WEATHER_SYSTEM, (_GET_WEATHER,), 297),
|
||||
("deepseek/deepseek-v4-flash", None, (), 37),
|
||||
("deepseek/deepseek-v4-flash", None, (_GET_WEATHER,), 304),
|
||||
("deepseek/deepseek-v4-flash", None, _THREE_TOOLS, 430),
|
||||
("deepseek/deepseek-v4-flash", _WEATHER_SYSTEM, (_GET_WEATHER,), 315),
|
||||
("deepseek/deepseek-v4-pro", None, (), 90),
|
||||
("deepseek/deepseek-v4-pro", None, (_GET_WEATHER,), 367),
|
||||
("deepseek/deepseek-v4-pro", None, _THREE_TOOLS, 493),
|
||||
("deepseek/deepseek-v4-pro", _WEATHER_SYSTEM, (_GET_WEATHER,), 378),
|
||||
],
|
||||
)
|
||||
async def test_local_estimate_matches_the_billed_prompt_with_the_shipped_framing(
|
||||
|
|
@ -3022,13 +3031,14 @@ async def test_azure_responses_estimate_stays_at_or_just_over_the_billed_prompt_
|
|||
|
||||
|
||||
# POST /v1/responses billed the same usage.input_tokens with strict unset and false on 2026-10-29, 283 for one tool
|
||||
# through xAI grok-4.3 and 304 through Fireworks deepseek-v4p1-flash
|
||||
# through xAI grok-4.3
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model", [_XAI_GROK_4_3, _FIREWORKS_DEEPSEEK_V4_1])
|
||||
async def test_a_responses_host_that_bills_an_unset_strict_like_false_counts_them_alike(
|
||||
monkeypatch: pytest.MonkeyPatch, model: str
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
router: Final = _single_deployment_router({"model": model, "api_base": _NATIVE_RESPONSES_BASE[model]}, {})
|
||||
router: Final = _single_deployment_router(
|
||||
{"model": _XAI_GROK_4_3, "api_base": _NATIVE_RESPONSES_BASE[_XAI_GROK_4_3]}, {}
|
||||
)
|
||||
|
||||
unset: Final = await _count_with_router(monkeypatch, router, (_responses_function(_GET_WEATHER),), responses=True)
|
||||
|
||||
|
|
@ -3108,6 +3118,46 @@ async def test_a_responses_count_that_asks_for_the_chat_completions_bridge_frame
|
|||
assert counted.total_tokens == chat_count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_chat_completions_api", ["maybe", {"a": 1}])
|
||||
async def test_a_responses_count_with_a_malformed_bridge_flag_reads_it_the_way_the_responses_route_does(
|
||||
monkeypatch: pytest.MonkeyPatch, use_chat_completions_api: object
|
||||
):
|
||||
router: Final = _single_deployment_router(
|
||||
{"model": _AZURE_GROK_4_1, "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, {}
|
||||
)
|
||||
chat_count: Final = await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=False)
|
||||
|
||||
counted: Final = await count_request_tokens(
|
||||
request=TokenCountRequest(
|
||||
model="counted",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris?"}],
|
||||
tools=[_GET_WEATHER],
|
||||
),
|
||||
responses_request={
|
||||
"model": "counted",
|
||||
"input": "What is the weather in Paris?",
|
||||
"use_chat_completions_api": use_chat_completions_api,
|
||||
},
|
||||
)
|
||||
|
||||
assert counted.total_tokens == chat_count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_azure_responses_count_leaves_strict_unset_for_templates_not_measured_there(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
router: Final = _single_deployment_router(
|
||||
{"model": "azure_ai/deepseek-v4.1-flash", "api_base": "https://unused.services.ai.azure.com/models"}, {}
|
||||
)
|
||||
|
||||
async def counted(**strict: bool) -> int:
|
||||
return await _count_with_router(monkeypatch, router, (_responses_function(_GET_WEATHER, **strict),), True)
|
||||
|
||||
assert await counted() < await counted(strict=True)
|
||||
|
||||
|
||||
_KIMI_WEATHER: Final = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue