mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
fix(cost): make cost-breakdown headers respect service tier
The breakdown priced reasoning tokens at the flat standard rate while the total billed them tier-aware, so on flex requests the reasoning sub-cost header could exceed the whole response cost. Route the breakdown's reasoning rate through the same tier-aware resolver as the total. On /v1/messages the response is a TypedDict that can never carry hidden params, yet the client wrapper still recomputed cost on it, clobbering the already-correct breakdown with a tier-less, reasoning-less one. Skip the metadata pass for results that cannot hold hidden params, since apply() discarded it anyway.
This commit is contained in:
parent
8a9d5b15b4
commit
c0f9af0802
4 changed files with 81 additions and 6 deletions
|
|
@ -1063,15 +1063,17 @@ def get_token_type_cost_breakdown(
|
|||
reasoning_tokens = _coerce_token_count(getattr(usage, "reasoning_tokens", 0))
|
||||
|
||||
# Reasoning is billed at the selected tier's reasoning rate for tiered models,
|
||||
# else at the explicit per-reasoning-token rate when the model defines one,
|
||||
# otherwise at the standard output-token rate - this mirrors how the total
|
||||
# completion cost is computed, so the breakdown can never diverge from it.
|
||||
# else at the service-tier-aware per-reasoning-token rate - this mirrors how the
|
||||
# total completion cost is computed, so the breakdown can never diverge from it.
|
||||
tiered_reasoning_rate: Final = _get_tiered_reasoning_rate(model_info=model_info, usage=usage)
|
||||
flat_reasoning_rate: Final = _get_cost_per_unit(model_info, "output_cost_per_reasoning_token", None)
|
||||
reasoning_rate: Final = (
|
||||
tiered_reasoning_rate
|
||||
if tiered_reasoning_rate is not None
|
||||
else (flat_reasoning_rate if flat_reasoning_rate is not None else completion_base_cost)
|
||||
else _resolve_reasoning_token_cost(
|
||||
model_info=model_info,
|
||||
service_tier=service_tier,
|
||||
completion_base_cost=completion_base_cost,
|
||||
)
|
||||
)
|
||||
reasoning_cost = float(reasoning_tokens) * reasoning_rate
|
||||
|
||||
|
|
|
|||
|
|
@ -178,7 +178,7 @@ def update_response_metadata(
|
|||
- response._hidden_params["litellm_overhead_time_ms"]
|
||||
- response.response_time_ms
|
||||
"""
|
||||
if result is None:
|
||||
if result is None or not hasattr(result, "_hidden_params"):
|
||||
return
|
||||
|
||||
metadata: Final = ResponseMetadata(result)
|
||||
|
|
|
|||
|
|
@ -2764,6 +2764,46 @@ def test_token_type_cost_breakdown_matches_real_gemini_numbers(_local_model_cost
|
|||
assert breakdown.cache_creation_cost == 0.0
|
||||
|
||||
|
||||
def test_token_type_cost_breakdown_flex_tier_prices_reasoning_at_flex_rate(_local_model_cost_map):
|
||||
"""Regression for the flex-tier breakdown drift: gemini-3.5-flash defines a flat
|
||||
output_cost_per_reasoning_token (9e-06, the standard output rate) but no _flex
|
||||
variant, so the breakdown priced reasoning at the standard rate on flex requests
|
||||
while the total billed it at the flex output rate (4.5e-06). The reasoning
|
||||
sub-cost then exceeded the entire flex completion cost."""
|
||||
|
||||
usage = Usage(
|
||||
prompt_tokens=7,
|
||||
completion_tokens=320,
|
||||
total_tokens=327,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=315, text_tokens=5),
|
||||
)
|
||||
|
||||
breakdown = get_token_type_cost_breakdown(
|
||||
model="gemini-3.5-flash",
|
||||
custom_llm_provider="vertex_ai",
|
||||
usage=usage,
|
||||
service_tier="flex",
|
||||
)
|
||||
|
||||
assert breakdown.reasoning_cost == pytest.approx(315 * 4.5e-06)
|
||||
|
||||
_, flex_completion_cost = generic_cost_per_token(
|
||||
model="gemini-3.5-flash",
|
||||
usage=usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
service_tier="flex",
|
||||
)
|
||||
assert breakdown.reasoning_cost <= flex_completion_cost
|
||||
|
||||
standard_breakdown = get_token_type_cost_breakdown(
|
||||
model="gemini-3.5-flash",
|
||||
custom_llm_provider="vertex_ai",
|
||||
usage=usage,
|
||||
service_tier=None,
|
||||
)
|
||||
assert standard_breakdown.reasoning_cost == pytest.approx(315 * 9e-06)
|
||||
|
||||
|
||||
def test_token_type_cost_breakdown_xai_at_exactly_200k_uses_higher_tier_rates(_local_model_cost_map):
|
||||
|
||||
usage = Usage(
|
||||
|
|
|
|||
|
|
@ -92,6 +92,39 @@ class TestCallbackDurationMs:
|
|||
assert hidden.get("litellm_overhead_time_ms") is not None
|
||||
|
||||
|
||||
class TestDictResultsSkipMetadataUpdate:
|
||||
"""Regression for /v1/messages cost-breakdown clobbering: AnthropicMessagesResponse
|
||||
is a TypedDict, so apply() can never attach _hidden_params to it and the whole
|
||||
metadata pass is discarded - except the cost recompute, whose only observable
|
||||
effect was overwriting the logging object's already-correct cost breakdown with a
|
||||
service-tier-less, reasoning-less recompute on the adapted response."""
|
||||
|
||||
def test_update_response_metadata_skips_cost_recompute_for_dict_results(self):
|
||||
anthropic_response = {
|
||||
"id": "msg_123",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "text", "text": "hi"}],
|
||||
"usage": {"input_tokens": 7, "output_tokens": 320},
|
||||
}
|
||||
logging_obj = MagicMock()
|
||||
logging_obj.model_call_details = {}
|
||||
logging_obj.caching_details = None
|
||||
logging_obj.litellm_call_id = "test-call-id"
|
||||
|
||||
update_response_metadata(
|
||||
result=anthropic_response,
|
||||
logging_obj=logging_obj,
|
||||
model="vertex_ai/gemini-3.5-flash",
|
||||
kwargs={},
|
||||
start_time=datetime.datetime(2025, 1, 1, 0, 0, 0),
|
||||
end_time=datetime.datetime(2025, 1, 1, 0, 0, 1),
|
||||
)
|
||||
|
||||
logging_obj._response_cost_calculator.assert_not_called()
|
||||
assert "_hidden_params" not in anthropic_response
|
||||
|
||||
|
||||
class TestCallbackDurationInCustomHeaders:
|
||||
"""Test that callback_duration_ms flows into get_custom_headers."""
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue