diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs index 8f7549c1c6d..141702bf857 100644 --- a/litellm-rust/crates/model-catalog/src/model_info.rs +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -186,6 +186,9 @@ pub struct ModelInfoFields { pub supports_image_size: Option, pub supported_audio_formats: Option, pub token_count_framing: Option, + pub token_count_tool_framing: Option, + pub token_count_responses_framing: Option, + pub token_count_responses_tool_framing: Option, pub token_count_model: Option, pub token_count_tokenizer: Option, pub vertex_ai_audio_api: Option, diff --git a/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py b/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py index 60134456577..49ffae03c40 100644 --- a/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py +++ b/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py @@ -10,12 +10,18 @@ class ChatTemplateToolSection: preamble_tokens: int implicit_system_turn_tokens: int render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str] + counts_tool_choice: bool = False + renders_strict_only_when_true: bool = False -def _deepseek_v4_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: +def _json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: return "\n".join(json.dumps(function, ensure_ascii=False) for function in functions) +def _compact_json_line_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: + return "\n".join(json.dumps(function, ensure_ascii=False, separators=(",", ":")) for function in functions) + + def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: return "".join( "\n" + json.dumps({"type": "function", "function": function}, ensure_ascii=False) for function in functions @@ -23,13 +29,29 @@ def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: _TOOL_SECTIONS: Final = ( + ( + re.compile(r"deepseek-v4(?:[-._]|p)1(?!\d)", re.IGNORECASE), + ChatTemplateToolSection(211, 0, _json_line_tool_schemas), + ), ( re.compile( r"deepseek-v4(?![.\d]|p\d)|^(?:deepseek/)?deepseek-(?:chat|reasoner|flash)(?::free)?$", re.IGNORECASE ), - ChatTemplateToolSection(222, 0, _deepseek_v4_tool_schemas), + ChatTemplateToolSection(222, 0, _json_line_tool_schemas), ), (re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)), + ( + re.compile(r"grok-4[-.]1(?![.\d])", re.IGNORECASE), + ChatTemplateToolSection( + 0, 0, _compact_json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True + ), + ), + ( + re.compile(r"grok-4[-.]3(?![.\d])", re.IGNORECASE), + ChatTemplateToolSection( + 0, 0, _json_line_tool_schemas, counts_tool_choice=True, renders_strict_only_when_true=True + ), + ), ) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 323af72940a..df8038e756c 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -990,7 +990,11 @@ def _count_extra( tool_section (Optional[ChatTemplateToolSection]): How the model's chat template renders tools, if known. """ reply_primer_tokens: Final = 3 - tool_choice_tokens: Final = 0 if tool_section is not None else _count_tool_choice(count_function, tool_choice) + tool_choice_tokens: Final = ( + _count_tool_choice(count_function, tool_choice) + if tool_section is None or tool_section.counts_tool_choice + else 0 + ) return ( reply_primer_tokens + _count_tool_definitions(count_function, tools, includes_system_message, tool_section) @@ -1029,14 +1033,29 @@ def _count_tool_choice( return 0 +def _named_function_definitions(tools: Sequence[object]) -> Iterable[Mapping[str, object]]: + return (function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name")) + + +def sends_function_tools(tools: Sequence[object] | None) -> bool: + return tools is not None and any(_named_function_definitions(tools)) + + +def _without_false_strict(function: Mapping[str, object]) -> Mapping[str, object]: + return {key: value for key, value in function.items() if key != "strict" or value} + + def _count_chat_template_tools( count_function: TokenCounterFunction, tools: Sequence[object], includes_system_message: bool, tool_section: ChatTemplateToolSection, ) -> int: - functions: Final = tuple( - function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name") + named_functions: Final = tuple(_named_function_definitions(tools)) + functions: Final = ( + tuple(map(_without_false_strict, named_functions)) + if tool_section.renders_strict_only_when_true + else named_functions ) if not functions: return 0 diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 02bf7dedcef..e6f65d6b416 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13478,6 +13478,10 @@ "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", + "token_count_framing": 149, + "token_count_tool_framing": 302, + "token_count_responses_framing": 145, + "token_count_responses_tool_framing": 100, "max_input_tokens": 128000, "max_output_tokens": 128000, "mode": "chat", @@ -54457,6 +54461,8 @@ "input_cost_per_token_above_200k_tokens_batches": 2e-06, "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", + "token_count_framing": 185, + "token_count_tool_framing": 214, "max_input_tokens": 1000000, "max_output_tokens": 1000000, "mode": "chat", @@ -62640,7 +62646,9 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_framing": 23, + "token_count_tool_framing": 12 }, "deepseek/deepseek-v4-flash-vision-exp": { "cache_creation_input_token_cost": 0.0, @@ -62746,7 +62754,9 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "token_count_framing": 76, + "token_count_tool_framing": 75 }, "tencent/deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, @@ -64032,6 +64042,7 @@ "input_cost_per_token": 3e-07, "input_cost_per_token_priority": 3.75e-07, "litellm_provider": "fireworks_ai", + "token_count_framing": 23, "max_input_tokens": 1048576, "max_output_tokens": 393216, "mode": "chat", @@ -64133,6 +64144,7 @@ "input_cost_per_token": 3e-07, "input_cost_per_token_priority": 3.75e-07, "litellm_provider": "fireworks_ai", + "token_count_framing": 23, "max_input_tokens": 1048576, "max_output_tokens": 393216, "mode": "chat", diff --git a/litellm/proxy/common_utils/token_count_settings.py b/litellm/proxy/common_utils/token_count_settings.py index dd04e531f2d..4e3059cfe26 100644 --- a/litellm/proxy/common_utils/token_count_settings.py +++ b/litellm/proxy/common_utils/token_count_settings.py @@ -56,6 +56,22 @@ class TokenCountSettings: framing: int | None count_model: TokenCountModel | None tokenizer: TokenCountTokenizer | None = None + tool_framing: int | None = None + responses_framing: int | None = None + responses_tool_framing: int | None = None + + @property + def frames_counts(self) -> bool: + return any( + framing is not None + for framing in (self.framing, self.tool_framing, self.responses_framing, self.responses_tool_framing) + ) + + def hidden_prompt_tokens(self, carries_tools: bool, responses_api: bool) -> int: + chat: Final = self.tool_framing if carries_tools else None + responses: Final = self.responses_tool_framing if carries_tools else self.responses_framing + narrower: Final = responses if responses_api else chat + return (narrower if narrower is not None else self.framing) or 0 @dataclass(frozen=True, slots=True) @@ -105,6 +121,15 @@ def _framing(value: object) -> int | None: return None +def _first_framing(source: Mapping[str, object], keys: tuple[str, ...]) -> int | None: + return next((framing for framing in map(_framing, map(source.get, keys)) if framing is not None), None) + + +def _configured_framing(configured: Mapping[str, object], default: Mapping[str, object], *keys: str) -> int | None: + operator_sets_one: Final = _first_framing(configured, (*keys, "token_count_framing")) is not None + return _first_framing(configured if operator_sets_one else default, keys) + + def _non_blank(value: object) -> str | None: return value if isinstance(value, str) and value.strip() else None @@ -175,13 +200,21 @@ def _tokenizer(value: object) -> TokenCountTokenizer | None: def resolve_token_count_settings( configured: Mapping[str, object], default: Mapping[str, object], deployment_region: Callable[[], str] ) -> TokenCountSettings: - framing: Final = _framing(configured.get("token_count_framing")) return TokenCountSettings( - framing=framing if framing is not None else _framing(default.get("token_count_framing")), + framing=_configured_framing(configured, default, "token_count_framing"), count_model=_resolved_count_model( configured.get("token_count_model"), default.get("token_count_model"), deployment_region ), tokenizer=_tokenizer(default.get("token_count_tokenizer")), + tool_framing=_configured_framing(configured, default, "token_count_tool_framing"), + responses_framing=_configured_framing(configured, default, "token_count_responses_framing"), + responses_tool_framing=_configured_framing( + configured, + default, + "token_count_responses_tool_framing", + "token_count_responses_framing", + "token_count_tool_framing", + ), ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index fb08272e687..0e35560f30f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -107,6 +107,7 @@ from litellm.litellm_core_utils.litellm_logging import ( initialize_sentry, ) from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER, LoggingWorker +from litellm.litellm_core_utils.prompt_templates.chat_template_tools import chat_template_tool_section from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias from litellm.litellm_core_utils.routing_groups import parse_routing_groups from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -115,6 +116,7 @@ from litellm.litellm_core_utils.token_counter import ( UnsupportedContentTypeError, deployment_openai_model, offload_token_count, + sends_function_tools, ) from litellm.llms.bedrock.count_tokens.kimi_k3 import ( HUGGING_FACE_SOURCE, @@ -1007,6 +1009,7 @@ from litellm.proxy.video_endpoints.endpoints import router as video_router from litellm.repositories.base_repository import SupportsModelDump from litellm.repositories.credentials_repository import CredentialsRepository from litellm.repositories.prisma_protocols import TableActions +from litellm.responses.main import provider_stores_the_conversation from litellm.responses.streaming_iterator import drain_logging_tasks from litellm.router import ( AssistantsTypedDict, @@ -15037,7 +15040,7 @@ async def _provider_token_count( framing_can_stand_in: Final = (count_model is None or not count_model.configured) and ( failure.cannot_count_model or (pinned_count_model is not None and failure.denied_in_region) ) - if not (framing_can_stand_in and settings.framing is not None and local_estimate_allowed): + if not (framing_can_stand_in and settings.frames_counts and local_estimate_allowed): raise failure.to_proxy_exception() if pinned_count_model is not None: verbose_proxy_logger.warning( @@ -15237,6 +15240,7 @@ async def _tokenizer_or_provider_error( async def _local_token_estimate( model: str, + template_model: str, prompt: str | None, messages: Sequence[AllMessageValues] | None, tools: list[ChatCompletionToolParam] | None, @@ -15249,7 +15253,7 @@ async def _local_token_estimate( else await _tokenizer_or_provider_error(model, custom_tokenizer, provider_failure) ) count: Final = offload_token_count(litellm.token_counter)( - model=model, + model=template_model, text=prompt, messages=messages, tools=tools, @@ -15315,10 +15319,15 @@ def _deployment_aws_region(deployment: _TokenCountDeployment) -> str: ) +def _deployment_base_model(deployment: _TokenCountDeployment) -> str | None: + return _non_empty_str(deployment.get("model_info", {}).get("base_model")) or _non_empty_str( + deployment.get("litellm_params", {}).get("base_model") + ) + + def _token_count_settings_model(deployment: _TokenCountDeployment, request_model: str) -> tuple[str, str | None]: - model_info: Final = deployment.get("model_info", {}) params: Final = deployment.get("litellm_params", {}) - base_model: Final = _non_empty_str(model_info.get("base_model")) or _non_empty_str(params.get("base_model")) + base_model: Final = _deployment_base_model(deployment) if base_model is not None: return base_model, None model: Final = _non_empty_str(params.get("model")) @@ -15470,12 +15479,15 @@ def _responses_routing_kwargs(responses_request: Mapping[str, object] | None) -> return {"previous_response_id": previous_response_id} if isinstance(previous_response_id, str) else {} +def _inference_params( + litellm_params: Mapping[str, object], responses_request: Mapping[str, object] +) -> Mapping[str, object]: + return {**litellm_params, **_router_default_litellm_params(llm_router), **responses_request} + + def _count_drop_policy(deployment: Mapping[str, object] | None, responses_request: Mapping[str, object]) -> DropPolicy: litellm_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params")) or {} - router_defaults: Final[Mapping[str, object]] = ( - {} if llm_router is None else _router_default_litellm_params(llm_router) - ) - return drop_policy({**litellm_params, **router_defaults, **responses_request}) + return drop_policy(_inference_params(litellm_params, responses_request)) def _native_responses_count( @@ -15498,8 +15510,54 @@ def _native_responses_count( ) -def _deployment_api_base(deployment: _TokenCountDeployment) -> str | None: - return _non_empty_str(deployment.get("litellm_params", {}).get("api_base")) +def _deployment_api_base( + deployment: _TokenCountDeployment, responses_request: Mapping[str, object] | None = None +) -> str | None: + return _non_empty_str( + _inference_params(deployment.get("litellm_params", {}), responses_request or {}).get("api_base") + ) + + +def _deployment_provider( + deployment: _TokenCountDeployment, request_model: str, responses_request: Mapping[str, object] | None +) -> str | None: + params: Final = deployment.get("litellm_params", {}) + try: + return litellm.get_llm_provider( + model=_non_empty_str(params.get("model")) or request_model, + custom_llm_provider=_non_empty_str(params.get("custom_llm_provider")), + api_base=_deployment_api_base(deployment, responses_request), + )[1] + except litellm.BadRequestError: + return None + + +def _serves_native_responses( + deployment: _TokenCountDeployment, request_model: str, responses_request: Mapping[str, object] +) -> bool: + params: Final = deployment.get("litellm_params", {}) + inference: Final = _inference_params(params, responses_request) + try: + return provider_stores_the_conversation( + _non_empty_str(params.get("model")) or request_model, + _non_empty_str(params.get("custom_llm_provider")), + { + "use_chat_completions_api": bool(inference.get("use_chat_completions_api")), + "model_info": dict(deployment.get("model_info", {})), + "api_base": _non_empty_str(inference.get("api_base")), + }, + ) + except litellm.BadRequestError: + return False + + +_AZURE_PROVIDERS: Final = frozenset((litellm.LlmProviders.AZURE.value, litellm.LlmProviders.AZURE_AI.value)) + + +def _with_strict_by_default(tools: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]: + return tuple( + {**tool, "strict": True} if tool.get("type") == "function" and "strict" not in tool else tool for tool in tools + ) def _system_message(system: object) -> ChatCompletionSystemMessage | None: @@ -15704,7 +15762,7 @@ async def _count_request_tokens( else await chat_count_request( model_to_use, custom_llm_provider, - _deployment_api_base(deployment_view), + _deployment_api_base(deployment_view, routed_request), routed_request, _count_drop_policy(routed_deployment, routed_request), ) @@ -15802,10 +15860,31 @@ async def _count_request_tokens( counted_messages: Final = ( typed_messages if typed_messages is None or system_message is None else (system_message, *typed_messages) ) - counted_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats + native_responses: Final = responses_request is not None and _serves_native_responses( + deployment_view, request.model, responses_request + ) + sent_tools: Final = cast( # cast-ok: raw OpenAI or Anthropic tool dicts, both of which token_counter formats list[ChatCompletionToolParam] | None, tools if counted_messages is not None else None ) - framing: Final = settings.framing if counted_messages is not None and settings.framing is not None else 0 + openai_model: Final = deployment_openai_model(deployment_view) + template_model: Final = openai_model or _deployment_base_model(deployment_view) or model_to_use + tool_section: Final = chat_template_tool_section(template_model) + counted_tools: Final = ( + cast(list[ChatCompletionToolParam], list(_with_strict_by_default(sent_tools))) # cast-ok: strict added + if sent_tools is not None + and native_responses + and _deployment_provider(deployment_view, request.model, responses_request) in _AZURE_PROVIDERS + and tool_section is not None + and tool_section.renders_strict_only_when_true + else sent_tools + ) + framing: Final = ( + 0 + if counted_messages is None + else settings.hidden_prompt_tokens( + carries_tools=sends_function_tools(counted_tools), responses_api=native_responses + ) + ) kimi_k3_messages: Final = counted_messages if settings.tokenizer == "kimi_k3" else None exact: Final = ( await _kimi_k3_count( @@ -15829,7 +15908,8 @@ async def _count_request_tokens( count_source="local_estimate", ) estimate: Final = await _local_token_estimate( - model=deployment_openai_model(deployment_view) or model_to_use, + model=openai_model or model_to_use, + template_model=template_model, prompt=prompt if counted_messages is None else None, messages=counted_messages, tools=counted_tools, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index da1d99957a7..bf69445fdd1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -246,6 +246,9 @@ class ProviderSpecificModelInfo(TypedDict, total=False): bedrock_converse_supports_strict_tools: bool | None supports_regex_lookaround: ReadOnly[bool | None] token_count_framing: ReadOnly[int | None] + token_count_tool_framing: ReadOnly[int | None] + token_count_responses_framing: ReadOnly[int | None] + token_count_responses_tool_framing: ReadOnly[int | None] token_count_model: ReadOnly[TokenCountModelInfo | None] token_count_tokenizer: ReadOnly[TokenCountTokenizer | None] @@ -4236,7 +4239,15 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( } ) -DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset({"token_count_framing", "token_count_model"}) +DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset( + { + "token_count_framing", + "token_count_tool_framing", + "token_count_responses_framing", + "token_count_responses_tool_framing", + "token_count_model", + } +) SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = ( frozenset(ModelInfoBase.__required_keys__ | ModelInfoBase.__optional_keys__) diff --git a/litellm/utils.py b/litellm/utils.py index b0826f84006..5f40b64e79f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7398,6 +7398,9 @@ def get_model_info_helper( supports_image_size=_model_info.get("supports_image_size", None), supported_audio_formats=_model_info.get("supported_audio_formats", None), token_count_framing=_model_info.get("token_count_framing", None), + token_count_tool_framing=_model_info.get("token_count_tool_framing", None), + token_count_responses_framing=_model_info.get("token_count_responses_framing", None), + token_count_responses_tool_framing=_model_info.get("token_count_responses_tool_framing", None), token_count_model=_model_info.get("token_count_model", None), token_count_tokenizer=_model_info.get("token_count_tokenizer", None), vertex_ai_audio_api=_model_info.get("vertex_ai_audio_api", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 02bf7dedcef..e6f65d6b416 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13478,6 +13478,10 @@ "input_cost_per_token": 2e-07, "output_cost_per_token": 5e-07, "litellm_provider": "azure_ai", + "token_count_framing": 149, + "token_count_tool_framing": 302, + "token_count_responses_framing": 145, + "token_count_responses_tool_framing": 100, "max_input_tokens": 128000, "max_output_tokens": 128000, "mode": "chat", @@ -54457,6 +54461,8 @@ "input_cost_per_token_above_200k_tokens_batches": 2e-06, "input_cost_per_token_batches": 1e-06, "litellm_provider": "xai", + "token_count_framing": 185, + "token_count_tool_framing": 214, "max_input_tokens": 1000000, "max_output_tokens": 1000000, "mode": "chat", @@ -62640,7 +62646,9 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "token_count_framing": 23, + "token_count_tool_framing": 12 }, "deepseek/deepseek-v4-flash-vision-exp": { "cache_creation_input_token_cost": 0.0, @@ -62746,7 +62754,9 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": false + "supports_vision": false, + "token_count_framing": 76, + "token_count_tool_framing": 75 }, "tencent/deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, @@ -64032,6 +64042,7 @@ "input_cost_per_token": 3e-07, "input_cost_per_token_priority": 3.75e-07, "litellm_provider": "fireworks_ai", + "token_count_framing": 23, "max_input_tokens": 1048576, "max_output_tokens": 393216, "mode": "chat", @@ -64133,6 +64144,7 @@ "input_cost_per_token": 3e-07, "input_cost_per_token_priority": 3.75e-07, "litellm_provider": "fireworks_ai", + "token_count_framing": 23, "max_input_tokens": 1048576, "max_output_tokens": 393216, "mode": "chat", diff --git a/tests/integration/providers/test_chat_template_tool_count_wire.py b/tests/integration/providers/test_chat_template_tool_count_wire.py index a8b97fa732e..552df3a87bc 100644 --- a/tests/integration/providers/test_chat_template_tool_count_wire.py +++ b/tests/integration/providers/test_chat_template_tool_count_wire.py @@ -39,7 +39,7 @@ _API_KEY: Final = "scripted-provider-key" _DEEPSEEK_V4: Final = "azure_ai/DeepSeek-V4-Flash-0731" _DEEPSEEK_R1: Final = "azure_ai/DeepSeek-R1-0528" _QWEN3_5: Final = "databricks/databricks-qwen35-122b-a10b" -_GROK: Final = "azure_ai/grok-4-1" +_GROK_3: Final = "azure_ai/grok-3" _DEEPSEEK_V4_TOOL_PREAMBLE: Final = 222 _QWEN3_5_TOOL_PREAMBLE: Final = 196 _QWEN3_5_OPENED_SYSTEM_TURN: Final = 5 @@ -250,7 +250,7 @@ def test_messages_count_tokens_skips_web_search_and_counts_unmapped_tool_keys_fo def test_messages_count_tokens_keeps_openai_tool_framing_for_other_models(gateway: Gateway) -> None: with wire_server(_completion) as wire, gateway.scenario() as scenario: - model: Final = _deployment(scenario, "count", _GROK, wire.url) + model: Final = _deployment(scenario, "count", _GROK_3, wire.url) without_tools: Final = _count(gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, ())) plain: Final = _count(gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, (_ANTHROPIC_TOOL,))) deferred: Final = _count( @@ -428,7 +428,7 @@ def test_project_itpm_reserves_deepseek_v4_tools_the_way_its_chat_template_rende def test_project_itpm_keeps_openai_tool_framing_for_other_models(gateway: Gateway) -> None: chat: Final = _ENDPOINTS["chat-completions"] with wire_server(_completion) as wire, gateway.scenario() as scenario: - group: Final = _deployment(scenario, "grok-4-1", _GROK, wire.url) + group: Final = _deployment(scenario, "grok-3", _GROK_3, wire.url) response: Final = gateway.request( "POST", chat.path, @@ -504,8 +504,8 @@ def _agent_body(group: str, marker: str, file_body: str, trigger: int) -> dict[s @pytest.mark.parametrize( ("model", "files_kept"), - ((_DEEPSEEK_V4, (False, False, True)), (_GROK, (True, True, True))), - ids=("deepseek-v4", "grok-4-1"), + ((_DEEPSEEK_V4, (False, False, True)), (_GROK_3, (True, True, True))), + ids=("deepseek-v4", "grok-3"), ) def test_messages_context_management_trigger_counts_tools_the_way_the_models_chat_template_renders_them( gateway: Gateway, model: str, files_kept: tuple[bool, ...] diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_chat_template_tools.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_chat_template_tools.py index 51249e01d4d..a8162860684 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_chat_template_tools.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_chat_template_tools.py @@ -9,7 +9,10 @@ _BARE_FUNCTION: Final = {"name": "g", "parameters": {}} # Tool schemas as rendered by render_tools in encoding/encoding_dsv4.py (deepseek-ai/DeepSeek-V4-Flash) and by -# chat_template.jinja (Qwen/Qwen3.5-122B-A10B), both read on Hugging Face on 2026-10-17 +# chat_template.jinja (Qwen/Qwen3.5-122B-A10B), both read on Hugging Face on 2026-10-17, and by chat_template.jinja +# (deepseek-ai/DeepSeek-V4.1-Flash) read on 2026-10-29. Grok's template is closed, so its two renderings are the ones +# whose token counts left a constant preamble across seven tool shapes billed by Azure AI Foundry +# grok-4-1-fast-reasoning and xAI grok-4.3 on 2026-10-29 @pytest.mark.parametrize( ("model", "rendered"), [ @@ -17,6 +20,18 @@ _BARE_FUNCTION: Final = {"name": "g", "parameters": {}} "DeepSeek-V4-Flash-0731", '{"name": "f", "description": "é", "parameters": {}}\n{"name": "g", "parameters": {}}', ), + ( + "DeepSeek-V4.1-Flash", + '{"name": "f", "description": "é", "parameters": {}}\n{"name": "g", "parameters": {}}', + ), + ( + "grok-4-1-fast-reasoning", + '{"name":"f","description":"é","parameters":{}}\n{"name":"g","parameters":{}}', + ), + ( + "grok-4.3", + '{"name": "f", "description": "é", "parameters": {}}\n{"name": "g", "parameters": {}}', + ), ( "databricks-qwen35-122b-a10b", '\n{"type": "function", "function": {"name": "f", "description": "é", "parameters": {}}}' diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 697ac76b4e5..8ef118154d1 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -638,8 +638,14 @@ def test_token_counter_tools_match_billed_prompt_for_json_tool_chat_templates( ("deepseek/deepseek-chat:free", "DeepSeek-V4-Flash-0731"), ("deepseek-reasoner", "DeepSeek-V4-Flash-0731"), ("deepseek/deepseek-flash", "DeepSeek-V4-Flash-0731"), - ("DeepSeek-V4.1-Flash", "deepseek-v3.2"), - ("accounts/fireworks/models/deepseek-v4p1-flash", "deepseek-v3.2"), + ("DeepSeek-V4.1-Flash", "accounts/fireworks/models/deepseek-v4p1-flash"), + ("databricks-deepseek-v4-1-flash", "accounts/fireworks/models/deepseek-v4p1-flash"), + ("deepseek-ai/deepseek-v4.1-flash-maas", "accounts/fireworks/models/deepseek-v4p1-flash"), + ("deepseek-v4p1-flash-1017", "accounts/fireworks/models/deepseek-v4p1-flash"), + ("grok-4.1-fast-non-reasoning", "grok-4-1-fast-reasoning"), + ("xai.grok-4.1-fast", "grok-4-1-fast-reasoning"), + ("grok-4.3-latest", "grok-4.3"), + ("xai.grok-4-3", "grok-4.3"), ("Qwen/Qwen3.5-122B-A10B", "databricks-qwen35-122b-a10b"), ("qwen3-235b-a22b", "mistral-large-latest"), ], @@ -704,6 +710,48 @@ def test_token_counter_tool_choice_adds_nothing_for_deepseek_v4(tool_choice: obj ) == token_counter_new(model="DeepSeek-V4-Flash-0731", messages=list(_USER_MESSAGES), tools=tools) +# A request with a one-function tool billed 283 prompt tokens on xAI grok-4.3 without tool_choice and 297 with a +# named one, and 357 and 358 on Azure AI Foundry grok-4-1-fast-reasoning, on 2026-10-29 +@pytest.mark.parametrize("model", ["grok-4.3", "grok-4-1-fast-reasoning"]) +def test_token_counter_counts_a_named_tool_choice_for_grok(model: str): + tools: Final = [_openai_tool(_WEATHER_FUNCTION)] + named: Final = {"type": "function", "function": {"name": "get_weather"}} + + assert token_counter_new( + model=model, messages=list(_USER_MESSAGES), tools=tools, tool_choice=named + ) > token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=tools) + + +def _counted_with_strict(model: str, **strict: bool) -> int: + return token_counter_new( + model=model, messages=list(_USER_MESSAGES), tools=[_openai_tool({**_WEATHER_FUNCTION, **strict})] + ) + + +# A one-function tool billed the same prompt tokens with strict unset and false on 2026-10-29: 357 through Azure AI +# Foundry grok-4-1-fast-reasoning chat and 283 through xAI grok-4.3 +@pytest.mark.parametrize("model", ["grok-4-1-fast-reasoning", "grok-4.3"]) +def test_token_counter_renders_a_false_strict_like_an_unset_one_for_grok(model: str): + assert ( + _counted_with_strict(model, strict=False) + == _counted_with_strict(model) + < _counted_with_strict(model, strict=True) + ) + + +# A one-function tool billed 5 prompt tokens more with strict false than unset, the same as with strict true, on +# 2026-10-30: 310 against 305 through Baseten DeepSeek-V4.1-Flash and 309 against 304 through Together +@pytest.mark.parametrize( + "model", ["accounts/fireworks/models/deepseek-v4p1-flash", "deepseek-v4-flash", "qwen3.5-plus"] +) +def test_token_counter_renders_a_false_strict_like_a_true_one_for_other_chat_templates(model: str): + assert ( + _counted_with_strict(model, strict=False) + == _counted_with_strict(model, strict=True) + > _counted_with_strict(model) + ) + + @pytest.mark.parametrize( ("tool_choice", "expected_tool_choice"), [ @@ -759,9 +807,17 @@ def test_token_counter_counts_anthropic_tools_as_sent_to_chat_template_families( # Qwen3.5's chat_template.jinja opens a system turn for the tool schemas when the request has none, 5 tokens that a # request with a system message already pays, while DeepSeek V4's encoding_dsv4.py appends them to the first turn -# either way (both read on Hugging Face on 2026-10-17) +# either way (both read on Hugging Face on 2026-10-17). A system message cost the same with and without a tool on +# Fireworks deepseek-v4p1-flash, Azure AI Foundry grok-4-1-fast-reasoning and xAI grok-4.3 on 2026-10-29 @pytest.mark.parametrize( - ("model", "implicit_system_turn_tokens"), [("DeepSeek-V4-Flash-0731", 0), ("databricks-qwen35-122b-a10b", 5)] + ("model", "implicit_system_turn_tokens"), + [ + ("DeepSeek-V4-Flash-0731", 0), + ("databricks-qwen35-122b-a10b", 5), + ("accounts/fireworks/models/deepseek-v4p1-flash", 0), + ("grok-4-1-fast-reasoning", 0), + ("grok-4.3", 0), + ], ) def test_token_counter_counts_the_tools_system_turn_only_without_a_system_message( model: str, implicit_system_turn_tokens: int diff --git a/tests/unit/proxy/common_utils/test_token_count_settings.py b/tests/unit/proxy/common_utils/test_token_count_settings.py index 43edc7e449f..68910f52b02 100644 --- a/tests/unit/proxy/common_utils/test_token_count_settings.py +++ b/tests/unit/proxy/common_utils/test_token_count_settings.py @@ -131,6 +131,107 @@ def test_shipped_australian_count_region_stays_with_an_australian_deployment() - assert settings.count_model == TokenCountModel(model="au.anthropic.claude-opus-5-5", region="ap-southeast-4") +_ABSENT: Final = object() + + +@pytest.mark.parametrize( + ("key", "field"), + [ + ("token_count_tool_framing", "tool_framing"), + ("token_count_responses_framing", "responses_framing"), + ("token_count_responses_tool_framing", "responses_tool_framing"), + ], +) +@pytest.mark.parametrize( + ("configured", "default", "expected"), + [ + (_ABSENT, 151, 151), + (40, 151, 40), + (0, 151, 0), + (40.0, _ABSENT, 40), + (-1, 151, 151), + (True, 151, 151), + ("40", _ABSENT, None), + ], +) +def test_each_request_kind_framing_reads_its_value_like_framing( + key: str, field: str, configured: object, default: object, expected: int | None +) -> None: + settings: Final = resolve_token_count_settings( + configured={} if configured is _ABSENT else {key: configured}, + default={} if default is _ABSENT else {key: default}, + deployment_region=_in_us_west_2, + ) + assert getattr(settings, field) == expected + + +def _hidden_prompts(settings: TokenCountSettings) -> tuple[int, int, int, int]: + return ( + settings.hidden_prompt_tokens(carries_tools=False, responses_api=False), + settings.hidden_prompt_tokens(carries_tools=True, responses_api=False), + settings.hidden_prompt_tokens(carries_tools=False, responses_api=True), + settings.hidden_prompt_tokens(carries_tools=True, responses_api=True), + ) + + +@pytest.mark.parametrize( + ("default", "expected"), + [ + ({}, (0, 0, 0, 0)), + ({"token_count_framing": 23}, (23, 23, 23, 23)), + ({"token_count_framing": 185, "token_count_tool_framing": 214}, (185, 214, 185, 214)), + ( + { + "token_count_framing": 149, + "token_count_tool_framing": 300, + "token_count_responses_framing": 145, + "token_count_responses_tool_framing": 104, + }, + (149, 300, 145, 104), + ), + ({"token_count_framing": 149, "token_count_tool_framing": 0}, (149, 0, 149, 0)), + ({"token_count_framing": 149, "token_count_responses_tool_framing": 0}, (149, 149, 149, 0)), + ({"token_count_framing": 149, "token_count_responses_framing": 145}, (149, 149, 145, 145)), + ], +) +def test_hidden_prompt_is_the_most_specific_framing_set_for_the_request_api_and_tools( + default: dict[str, object], expected: tuple[int, int, int, int] +) -> None: + settings: Final = resolve_token_count_settings(configured={}, default=default, deployment_region=_in_us_west_2) + assert _hidden_prompts(settings) == expected + + +_PER_API_COST_MAP: Final = { + "token_count_framing": 149, + "token_count_tool_framing": 300, + "token_count_responses_framing": 145, + "token_count_responses_tool_framing": 104, +} + + +@pytest.mark.parametrize( + ("configured", "expected"), + [ + ({}, (149, 300, 145, 104)), + ({"token_count_framing": 500}, (500, 500, 500, 500)), + ({"token_count_framing": 0}, (0, 0, 0, 0)), + ({"token_count_tool_framing": 250}, (149, 250, 145, 250)), + ({"token_count_framing": 500, "token_count_tool_framing": 250}, (500, 250, 500, 250)), + ({"token_count_responses_framing": 120}, (149, 300, 120, 120)), + ({"token_count_tool_framing": 250, "token_count_responses_framing": 120}, (149, 250, 120, 120)), + ({"token_count_responses_tool_framing": 90}, (149, 300, 145, 90)), + ({"token_count_framing": -1}, (149, 300, 145, 104)), + ], +) +def test_an_operator_framing_applies_to_every_request_kind_the_cost_map_splits_more_finely( + configured: dict[str, object], expected: tuple[int, int, int, int] +) -> None: + settings: Final = resolve_token_count_settings( + configured=configured, default=_PER_API_COST_MAP, deployment_region=_in_us_west_2 + ) + assert _hidden_prompts(settings) == expected + + def test_no_setting_anywhere_means_the_model_counts_itself_without_framing() -> None: assert resolve_token_count_settings( configured={}, default={}, deployment_region=_in_us_west_2 diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index ab6cfcfebfe..44c43713438 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -4896,6 +4896,9 @@ class TestModelInfoCostMapEchoFilter: [ ("bedrock/us.anthropic.claude-sonnet-5-5", "token_count_model", {}), ("bedrock/us.moonshotai.kimi-k3", "token_count_framing", 0), + ("azure_ai/grok-4-1-fast-reasoning", "token_count_tool_framing", 0), + ("azure_ai/grok-4-1-fast-reasoning", "token_count_responses_framing", 0), + ("azure_ai/grok-4-1-fast-reasoning", "token_count_responses_tool_framing", 0), ], ) def test_token_count_setting_is_stored_only_when_the_operator_changed_it( diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index 608efe20c50..2f85d1c7516 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -1925,6 +1925,23 @@ async def test_uncountable_model_without_framing_raises(): assert raised.value.code == "404" +@pytest.mark.parametrize( + "settings", + [ + TokenCountSettings(framing=None, count_model=None, tool_framing=300), + TokenCountSettings(framing=None, count_model=None, responses_framing=145), + TokenCountSettings(framing=None, count_model=None, responses_tool_framing=100), + ], +) +@pytest.mark.asyncio +async def test_uncountable_model_with_only_a_request_kind_framing_falls_through_to_the_local_estimate( + settings: TokenCountSettings, +): + counter: Final = _ScriptedCounter({"us.anthropic.claude-sonnet-5-5": _provider_reply("sonnet", status_code=404)}) + + assert isinstance(await _ladder(counter, settings), ProviderCountFailure) + + @pytest.mark.parametrize( ("count_model", "counted_as"), [ @@ -2770,6 +2787,480 @@ async def test_framing_comes_from_the_row_the_deployment_provider_or_base_model_ assert aliased.total_tokens == litellm.token_counter(model="gpt-5.4-mini", messages=messages) + framing +@pytest.mark.asyncio +async def test_a_tool_request_counts_its_framing_in_place_of_the_plain_one_per_api(monkeypatch: pytest.MonkeyPatch): + def deployment(name: str, model_info: dict[str, object]) -> dict[str, object]: + return { + "model_name": name, + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": f"sk-unused-{name}"}, + "model_info": model_info, + } + + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router( + model_list=[ + deployment("plain", {}), + deployment("chat", {"token_count_framing": 7, "token_count_tool_framing": 50}), + deployment( + "per-api", + { + "token_count_framing": 7, + "token_count_tool_framing": 50, + "token_count_responses_framing": 5, + "token_count_responses_tool_framing": 2, + }, + ), + ] + ), + ) + messages: Final = [{"role": "user", "content": "What is the weather in Paris?"}] + tools: Final = [_KIMI_WEATHER] + + async def count(model: str, request_tools: list[dict[str, object]] | None, responses: bool) -> int: + response: Final = await count_request_tokens( + request=TokenCountRequest(model=model, messages=messages, tools=request_tools), + responses_request={"model": model, "input": "What is the weather in Paris?"} if responses else None, + ) + return response.total_tokens + + plain_with_tools: Final = await count("plain", tools, responses=False) + plain_without_tools: Final = await count("plain", None, responses=False) + + assert await count("plain", tools, responses=True) == plain_with_tools + assert await count("chat", tools, responses=False) == plain_with_tools + 50 + assert await count("chat", None, responses=False) == plain_without_tools + 7 + assert await count("chat", [], responses=False) == plain_without_tools + 7 + assert await count("chat", tools, responses=True) == plain_with_tools + 50 + assert await count("chat", None, responses=True) == plain_without_tools + 7 + assert await count("per-api", tools, responses=False) == plain_with_tools + 50 + assert await count("per-api", tools, responses=True) == plain_with_tools + 2 + assert await count("per-api", None, responses=True) == plain_without_tools + 5 + + +_GET_WEATHER: Final = { + "name": "get_weather", + "description": "Get the current weather for a city", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string", "description": "City name"}}, + "required": ["city"], + }, +} +_THREE_TOOLS: Final = ( + _GET_WEATHER, + { + "name": "get_time", + "description": "Get the local time in a timezone", + "input_schema": { + "type": "object", + "properties": {"timezone": {"type": "string", "description": "IANA timezone"}}, + "required": ["timezone"], + }, + }, + { + "name": "search_docs", + "description": "Search internal documents and return the top matches", + "input_schema": { + "type": "object", + "properties": {"query": {"type": "string"}, "limit": {"type": "integer", "description": "Max results"}}, + "required": ["query"], + }, + }, +) +_FORM_FIELDS: Final = tuple(f"field_{index}" for index in range(10)) +_FILL_FORM: Final = { + "name": "fill_form", + "description": "Fill a form", + "input_schema": { + "type": "object", + "properties": { + field: {"type": "string", "description": f"Value for field number {index}"} + for index, field in enumerate(_FORM_FIELDS) + }, + "required": list(_FORM_FIELDS), + }, +} +_WEATHER_SYSTEM: Final = "You are a helpful assistant that answers weather questions briefly." +_AZURE_GROK_4_1: Final = "azure_ai/grok-4-1-fast-reasoning" +_FIREWORKS_DEEPSEEK_V4_1: Final = "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash" +_XAI_GROK_4_3: Final = "xai/grok-4.3" +_NATIVE_RESPONSES_BASE: Final = { + _AZURE_GROK_4_1: "https://unused.services.ai.azure.com/models", + _FIREWORKS_DEEPSEEK_V4_1: "https://unused.invalid", + _XAI_GROK_4_3: "https://unused.invalid", +} + + +def _responses_function(tool: Mapping[str, object], **strict: bool | None) -> dict[str, object]: + return { + "type": "function", + "name": tool["name"], + "description": tool["description"], + "parameters": tool["input_schema"], + **strict, + } + + +def _get_weather_copies(count: int) -> tuple[dict[str, object], ...]: + return tuple({**_GET_WEATHER, "name": f"get_weather_{index}"} for index in range(count)) + + +# usage.input_tokens plus cache_read_input_tokens that POST /v1/messages billed for these requests with the user turn +# "What is the weather in Paris?" through Azure AI Foundry grok-4-1-fast-reasoning, Fireworks deepseek-v4p1-flash and +# xAI grok-4.3 on 2026-10-29, and usage.prompt_tokens that POST /v1/chat/completions billed through DeepSeek's API on +# 2026-10-30 +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "system", "tools", "billed_prompt_tokens"), + [ + (_AZURE_GROK_4_1, None, (), 163), + (_AZURE_GROK_4_1, None, (_GET_WEATHER,), 357), + (_AZURE_GROK_4_1, None, _THREE_TOOLS, 445), + (_AZURE_GROK_4_1, None, (_FILL_FORM,), 550), + (_AZURE_GROK_4_1, _WEATHER_SYSTEM, (_GET_WEATHER,), 368), + (_FIREWORKS_DEEPSEEK_V4_1, None, (), 37), + (_FIREWORKS_DEEPSEEK_V4_1, None, (_GET_WEATHER,), 304), + (_FIREWORKS_DEEPSEEK_V4_1, None, _THREE_TOOLS, 430), + (_FIREWORKS_DEEPSEEK_V4_1, None, (_FILL_FORM,), 550), + (_FIREWORKS_DEEPSEEK_V4_1, _WEATHER_SYSTEM, (_GET_WEATHER,), 315), + (_XAI_GROK_4_3, None, (), 199), + (_XAI_GROK_4_3, None, (_GET_WEATHER,), 283), + (_XAI_GROK_4_3, None, _THREE_TOOLS, 402), + (_XAI_GROK_4_3, None, (_FILL_FORM,), 530), + (_XAI_GROK_4_3, _WEATHER_SYSTEM, (_GET_WEATHER,), 297), + ("deepseek/deepseek-v4-flash", None, (), 37), + ("deepseek/deepseek-v4-flash", None, (_GET_WEATHER,), 304), + ("deepseek/deepseek-v4-flash", None, _THREE_TOOLS, 430), + ("deepseek/deepseek-v4-flash", _WEATHER_SYSTEM, (_GET_WEATHER,), 315), + ("deepseek/deepseek-v4-pro", None, (), 90), + ("deepseek/deepseek-v4-pro", None, (_GET_WEATHER,), 367), + ("deepseek/deepseek-v4-pro", None, _THREE_TOOLS, 493), + ("deepseek/deepseek-v4-pro", _WEATHER_SYSTEM, (_GET_WEATHER,), 378), + ], +) +async def test_local_estimate_matches_the_billed_prompt_with_the_shipped_framing( + monkeypatch: pytest.MonkeyPatch, + model: str, + system: str | None, + tools: tuple[dict[str, object], ...], + billed_prompt_tokens: int, +): + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "billed", + "litellm_params": {"model": model, "api_key": "unused", "api_base": "https://unused.invalid"}, + } + ] + ), + ) + + counted: Final = await token_counter( + request=TokenCountRequest( + model="billed", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=list(tools) or None, + system=system, + ) + ) + + assert counted.count_source == "local_estimate" + assert abs(counted.total_tokens - billed_prompt_tokens) <= billed_prompt_tokens * 0.05 + + +# usage.input_tokens that POST /v1/responses billed for these requests with the input "What is the weather in Paris?" +# through Azure AI Foundry grok-4-1-fast-reasoning, Fireworks deepseek-v4p1-flash and xAI grok-4.3 on 2026-10-29 +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "system", "tools", "billed_prompt_tokens"), + [ + (_AZURE_GROK_4_1, None, (), 159), + (_AZURE_GROK_4_1, None, (_GET_WEATHER,), 159), + (_AZURE_GROK_4_1, None, _THREE_TOOLS, 257), + (_AZURE_GROK_4_1, None, (_FILL_FORM,), 352), + (_AZURE_GROK_4_1, _WEATHER_SYSTEM, (_GET_WEATHER,), 170), + (_FIREWORKS_DEEPSEEK_V4_1, None, (_GET_WEATHER,), 304), + (_XAI_GROK_4_3, None, (_GET_WEATHER,), 283), + ], +) +async def test_responses_estimate_matches_the_billed_responses_prompt_with_the_shipped_framing( + monkeypatch: pytest.MonkeyPatch, + model: str, + system: str | None, + tools: tuple[dict[str, object], ...], + billed_prompt_tokens: int, +): + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "billed", + "litellm_params": {"model": model, "api_key": "unused", "api_base": _NATIVE_RESPONSES_BASE[model]}, + } + ] + ), + ) + + counted: Final = await count_request_tokens( + request=TokenCountRequest( + model="billed", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_responses_function(tool) for tool in tools] or None, + system=system, + ), + responses_request={"model": "billed", "input": "What is the weather in Paris?"}, + ) + + assert counted.count_source == "local_estimate" + assert abs(counted.total_tokens - billed_prompt_tokens) <= billed_prompt_tokens * 0.05 + + +def _single_deployment_router(litellm_params: dict[str, object], model_info: dict[str, object]) -> Router: + return Router( + model_list=[ + { + "model_name": "counted", + "litellm_params": {"api_key": "unused", **litellm_params}, + "model_info": model_info, + } + ] + ) + + +async def _count_with_router( + monkeypatch: pytest.MonkeyPatch, router: Router, tools: tuple[dict[str, object], ...], responses: bool +) -> int: + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + counted: Final = await count_request_tokens( + request=TokenCountRequest( + model="counted", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=list(tools) or None, + ), + responses_request={"model": "counted", "input": "What is the weather in Paris?"} if responses else None, + ) + return counted.total_tokens + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tools", [(), (_GET_WEATHER,), _THREE_TOOLS, (_FILL_FORM,)]) +@pytest.mark.parametrize("responses", [False, True]) +async def test_a_custom_deployment_name_counts_like_the_base_model_it_names( + monkeypatch: pytest.MonkeyPatch, tools: tuple[dict[str, object], ...], responses: bool +): + api_base: Final = _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1] + custom_name: Final = _single_deployment_router( + {"model": "azure_ai/prod-grok", "api_base": api_base}, {"base_model": _AZURE_GROK_4_1} + ) + row_name: Final = _single_deployment_router({"model": _AZURE_GROK_4_1, "api_base": api_base}, {}) + + assert await _count_with_router(monkeypatch, custom_name, tools, responses) == await _count_with_router( + monkeypatch, row_name, tools, responses + ) + + +# usage.input_tokens that POST /v1/responses billed through Azure AI Foundry grok-4-1-fast-reasoning on 2026-10-29 for +# the input "What is the weather in Paris?" with these function tools, strict left unset or sent as false or null, +# the same on the .services.ai.azure.com and .openai.azure.com hosts +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("tools", "strict", "billed_prompt_tokens"), + [ + ((_GET_WEATHER,), {}, 159), + ((_GET_WEATHER,), {"strict": False}, 155), + ((_GET_WEATHER,), {"strict": None}, 155), + (_THREE_TOOLS, {}, 257), + (_get_weather_copies(5), {}, 345), + (_get_weather_copies(10), {}, 575), + (_get_weather_copies(10), {"strict": False}, 535), + (_get_weather_copies(10), {"strict": None}, 535), + ], +) +@pytest.mark.parametrize( + ("litellm_params", "model_info"), + [ + ({"model": _AZURE_GROK_4_1, "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, {}), + ( + {"model": "azure/grok-4-1-fast-reasoning", "api_base": "https://unused.openai.azure.com/openai/v1"}, + {"base_model": _AZURE_GROK_4_1}, + ), + ], +) +async def test_azure_responses_estimate_stays_at_or_just_over_the_billed_prompt_as_function_tools_are_added( + monkeypatch: pytest.MonkeyPatch, + tools: tuple[dict[str, object], ...], + strict: dict[str, bool | None], + billed_prompt_tokens: int, + litellm_params: dict[str, object], + model_info: dict[str, object], +): + router: Final = _single_deployment_router(litellm_params, model_info) + + counted: Final = await _count_with_router( + monkeypatch, router, tuple(_responses_function(tool, **strict) for tool in tools), responses=True + ) + + assert billed_prompt_tokens <= counted <= billed_prompt_tokens * 1.02 + + +# POST /v1/responses billed the same usage.input_tokens with strict unset and false on 2026-10-29, 283 for one tool +# through xAI grok-4.3 +@pytest.mark.asyncio +async def test_a_responses_host_that_bills_an_unset_strict_like_false_counts_them_alike( + monkeypatch: pytest.MonkeyPatch, +): + router: Final = _single_deployment_router( + {"model": _XAI_GROK_4_3, "api_base": _NATIVE_RESPONSES_BASE[_XAI_GROK_4_3]}, {} + ) + + unset: Final = await _count_with_router(monkeypatch, router, (_responses_function(_GET_WEATHER),), responses=True) + + assert unset == await _count_with_router( + monkeypatch, router, (_responses_function(_GET_WEATHER, strict=False),), responses=True + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("built_in_tool", "responses"), + [({"type": "web_search"}, True), ({"type": "web_search_20250305", "name": "web_search"}, False)], +) +async def test_a_request_whose_only_tools_are_built_in_counts_like_one_without_tools( + monkeypatch: pytest.MonkeyPatch, built_in_tool: dict[str, object], responses: bool +): + router: Final = _single_deployment_router( + {"model": _AZURE_GROK_4_1, "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, {} + ) + + assert await _count_with_router(monkeypatch, router, (built_in_tool,), responses) == await _count_with_router( + monkeypatch, router, (), responses + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tools", [(), (_GET_WEATHER,), _THREE_TOOLS]) +async def test_a_responses_count_the_proxy_bridges_to_chat_completions_frames_like_the_chat_count( + monkeypatch: pytest.MonkeyPatch, tools: tuple[dict[str, object], ...] +): + bridged: Final = _single_deployment_router( + {"model": _AZURE_GROK_4_1, "api_base": "https://unused.cognitiveservices.azure.com/models"}, {} + ) + native: Final = _single_deployment_router( + {"model": _AZURE_GROK_4_1, "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, {} + ) + chat_count: Final = await _count_with_router(monkeypatch, bridged, tools, responses=False) + + assert await _count_with_router(monkeypatch, bridged, tools, responses=True) == chat_count + assert await _count_with_router(monkeypatch, native, tools, responses=True) != chat_count + + +@pytest.mark.asyncio +@pytest.mark.parametrize("from_router_defaults", [False, True]) +async def test_a_responses_count_that_asks_for_the_chat_completions_bridge_frames_like_the_chat_count( + monkeypatch: pytest.MonkeyPatch, from_router_defaults: bool +): + router: Final = Router( + model_list=[ + { + "model_name": "counted", + "litellm_params": { + "api_key": "unused", + "model": _AZURE_GROK_4_1, + "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1], + }, + "model_info": {}, + } + ], + default_litellm_params={"use_chat_completions_api": True} if from_router_defaults else None, + ) + chat_count: Final = await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=False) + + counted: Final = await count_request_tokens( + request=TokenCountRequest( + model="counted", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_GET_WEATHER], + ), + responses_request={ + "model": "counted", + "input": "What is the weather in Paris?", + **({} if from_router_defaults else {"use_chat_completions_api": True}), + }, + ) + + assert counted.total_tokens == chat_count + + +@pytest.mark.asyncio +async def test_a_responses_count_frames_for_the_api_base_router_defaults_send_inference_to( + monkeypatch: pytest.MonkeyPatch, +): + router: Final = Router( + model_list=[ + { + "model_name": "counted", + "litellm_params": { + "api_key": "unused", + "model": _AZURE_GROK_4_1, + "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1], + }, + "model_info": {}, + } + ], + default_litellm_params={"api_base": "https://unused.cognitiveservices.azure.com/models"}, + ) + chat_count: Final = await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=False) + + assert await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=True) == chat_count + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_chat_completions_api", ["maybe", {"a": 1}]) +async def test_a_responses_count_with_a_malformed_bridge_flag_reads_it_the_way_the_responses_route_does( + monkeypatch: pytest.MonkeyPatch, use_chat_completions_api: object +): + router: Final = _single_deployment_router( + {"model": _AZURE_GROK_4_1, "api_base": _NATIVE_RESPONSES_BASE[_AZURE_GROK_4_1]}, {} + ) + chat_count: Final = await _count_with_router(monkeypatch, router, (_GET_WEATHER,), responses=False) + + counted: Final = await count_request_tokens( + request=TokenCountRequest( + model="counted", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_GET_WEATHER], + ), + responses_request={ + "model": "counted", + "input": "What is the weather in Paris?", + "use_chat_completions_api": use_chat_completions_api, + }, + ) + + assert counted.total_tokens == chat_count + + +@pytest.mark.asyncio +async def test_an_azure_responses_count_leaves_strict_unset_for_templates_not_measured_there( + monkeypatch: pytest.MonkeyPatch, +): + router: Final = _single_deployment_router( + {"model": "azure_ai/deepseek-v4.1-flash", "api_base": "https://unused.services.ai.azure.com/models"}, {} + ) + + async def counted(**strict: bool) -> int: + return await _count_with_router(monkeypatch, router, (_responses_function(_GET_WEATHER, **strict),), True) + + assert await counted() < await counted(strict=True) + + _KIMI_WEATHER: Final = { "type": "function", "function": { diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 91f0b2339c7..447a4dfb81d 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -672,7 +672,7 @@ def test_pdf_opt_out_on_one_deployment_does_not_block_its_sibling(respx_mock: re def test_token_count_settings_stay_on_the_deployment_that_set_them() -> None: - """A deployment's ``token_count_model`` and ``token_count_framing`` must not land on + """A deployment's ``token_count_model`` and framing settings must not land on the shared ``{provider}/{model}`` key, or a sibling deployment of that model would count with them.""" from litellm.proxy.common_utils.token_count_settings import cost_map_entry @@ -681,6 +681,9 @@ def test_token_count_settings_stay_on_the_deployment_that_set_them() -> None: configured: Final = { "token_count_model": {"model": "anthropic.claude-opus-5-5", "region": "us-west-2"}, "token_count_framing": 7, + "token_count_tool_framing": 3, + "token_count_responses_framing": 5, + "token_count_responses_tool_framing": 2, } builtin: Final = {field: cost_map_entry(backend_model).get(field) for field in configured} assert all(builtin[field] != value for field, value in configured.items()), builtin diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 5cf872f4ea7..61825e463f8 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1178,6 +1178,9 @@ _COST_MAP_ROW_SCHEMA: Final = { "supports_vision": {"type": "boolean"}, "supports_web_search": {"type": "boolean"}, "token_count_framing": {"type": "integer"}, + "token_count_tool_framing": {"type": "integer"}, + "token_count_responses_framing": {"type": "integer"}, + "token_count_responses_tool_framing": {"type": "integer"}, "token_count_model": { "type": "object", "properties": {"model": {"type": "string"}, "region": {"type": "string"}},