fix(cost): make cost-breakdown headers respect service tier

The breakdown priced reasoning tokens at the flat standard rate while the
total billed them tier-aware, so on flex requests the reasoning sub-cost
header could exceed the whole response cost. Route the breakdown's
reasoning rate through the same tier-aware resolver as the total.

On /v1/messages the response is a TypedDict that can never carry hidden
params, yet the client wrapper still recomputed cost on it, clobbering the
already-correct breakdown with a tier-less, reasoning-less one. Skip the
metadata pass for results that cannot hold hidden params, since apply()
discarded it anyway.
This commit is contained in:
mateo-berri 2026-08-26 17:09:30 -07:00
parent 8a9d5b15b4
commit c0f9af0802
4 changed files with 81 additions and 6 deletions

View file

@ -1063,15 +1063,17 @@ def get_token_type_cost_breakdown(
reasoning_tokens = _coerce_token_count(getattr(usage, "reasoning_tokens", 0))
# Reasoning is billed at the selected tier's reasoning rate for tiered models,
# else at the explicit per-reasoning-token rate when the model defines one,
# otherwise at the standard output-token rate - this mirrors how the total
# completion cost is computed, so the breakdown can never diverge from it.
# else at the service-tier-aware per-reasoning-token rate - this mirrors how the
# total completion cost is computed, so the breakdown can never diverge from it.
tiered_reasoning_rate: Final = _get_tiered_reasoning_rate(model_info=model_info, usage=usage)
flat_reasoning_rate: Final = _get_cost_per_unit(model_info, "output_cost_per_reasoning_token", None)
reasoning_rate: Final = (
tiered_reasoning_rate
if tiered_reasoning_rate is not None
else (flat_reasoning_rate if flat_reasoning_rate is not None else completion_base_cost)
else _resolve_reasoning_token_cost(
model_info=model_info,
service_tier=service_tier,
completion_base_cost=completion_base_cost,
)
)
reasoning_cost = float(reasoning_tokens) * reasoning_rate

View file

@ -178,7 +178,7 @@ def update_response_metadata(
- response._hidden_params["litellm_overhead_time_ms"]
- response.response_time_ms
"""
if result is None:
if result is None or not hasattr(result, "_hidden_params"):
return
metadata: Final = ResponseMetadata(result)

View file

@ -2764,6 +2764,46 @@ def test_token_type_cost_breakdown_matches_real_gemini_numbers(_local_model_cost
assert breakdown.cache_creation_cost == 0.0
def test_token_type_cost_breakdown_flex_tier_prices_reasoning_at_flex_rate(_local_model_cost_map):
"""Regression for the flex-tier breakdown drift: gemini-3.5-flash defines a flat
output_cost_per_reasoning_token (9e-06, the standard output rate) but no _flex
variant, so the breakdown priced reasoning at the standard rate on flex requests
while the total billed it at the flex output rate (4.5e-06). The reasoning
sub-cost then exceeded the entire flex completion cost."""
usage = Usage(
prompt_tokens=7,
completion_tokens=320,
total_tokens=327,
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=315, text_tokens=5),
)
breakdown = get_token_type_cost_breakdown(
model="gemini-3.5-flash",
custom_llm_provider="vertex_ai",
usage=usage,
service_tier="flex",
)
assert breakdown.reasoning_cost == pytest.approx(315 * 4.5e-06)
_, flex_completion_cost = generic_cost_per_token(
model="gemini-3.5-flash",
usage=usage,
custom_llm_provider="vertex_ai",
service_tier="flex",
)
assert breakdown.reasoning_cost <= flex_completion_cost
standard_breakdown = get_token_type_cost_breakdown(
model="gemini-3.5-flash",
custom_llm_provider="vertex_ai",
usage=usage,
service_tier=None,
)
assert standard_breakdown.reasoning_cost == pytest.approx(315 * 9e-06)
def test_token_type_cost_breakdown_xai_at_exactly_200k_uses_higher_tier_rates(_local_model_cost_map):
usage = Usage(

View file

@ -92,6 +92,39 @@ class TestCallbackDurationMs:
assert hidden.get("litellm_overhead_time_ms") is not None
class TestDictResultsSkipMetadataUpdate:
"""Regression for /v1/messages cost-breakdown clobbering: AnthropicMessagesResponse
is a TypedDict, so apply() can never attach _hidden_params to it and the whole
metadata pass is discarded - except the cost recompute, whose only observable
effect was overwriting the logging object's already-correct cost breakdown with a
service-tier-less, reasoning-less recompute on the adapted response."""
def test_update_response_metadata_skips_cost_recompute_for_dict_results(self):
anthropic_response = {
"id": "msg_123",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "hi"}],
"usage": {"input_tokens": 7, "output_tokens": 320},
}
logging_obj = MagicMock()
logging_obj.model_call_details = {}
logging_obj.caching_details = None
logging_obj.litellm_call_id = "test-call-id"
update_response_metadata(
result=anthropic_response,
logging_obj=logging_obj,
model="vertex_ai/gemini-3.5-flash",
kwargs={},
start_time=datetime.datetime(2025, 1, 1, 0, 0, 0),
end_time=datetime.datetime(2025, 1, 1, 0, 0, 1),
)
logging_obj._response_cost_calculator.assert_not_called()
assert "_hidden_params" not in anthropic_response
class TestCallbackDurationInCustomHeaders:
"""Test that callback_duration_ms flows into get_custom_headers."""