diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 0a52e1d283e..3f533de4f44 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1063,15 +1063,17 @@ def get_token_type_cost_breakdown( reasoning_tokens = _coerce_token_count(getattr(usage, "reasoning_tokens", 0)) # Reasoning is billed at the selected tier's reasoning rate for tiered models, - # else at the explicit per-reasoning-token rate when the model defines one, - # otherwise at the standard output-token rate - this mirrors how the total - # completion cost is computed, so the breakdown can never diverge from it. + # else at the service-tier-aware per-reasoning-token rate - this mirrors how the + # total completion cost is computed, so the breakdown can never diverge from it. tiered_reasoning_rate: Final = _get_tiered_reasoning_rate(model_info=model_info, usage=usage) - flat_reasoning_rate: Final = _get_cost_per_unit(model_info, "output_cost_per_reasoning_token", None) reasoning_rate: Final = ( tiered_reasoning_rate if tiered_reasoning_rate is not None - else (flat_reasoning_rate if flat_reasoning_rate is not None else completion_base_cost) + else _resolve_reasoning_token_cost( + model_info=model_info, + service_tier=service_tier, + completion_base_cost=completion_base_cost, + ) ) reasoning_cost = float(reasoning_tokens) * reasoning_rate diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 44fed944d2a..a375560288f 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -178,7 +178,7 @@ def update_response_metadata( - response._hidden_params["litellm_overhead_time_ms"] - response.response_time_ms """ - if result is None: + if result is None or not hasattr(result, "_hidden_params"): return metadata: Final = ResponseMetadata(result) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index e13643ed6ce..8565a098211 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2764,6 +2764,46 @@ def test_token_type_cost_breakdown_matches_real_gemini_numbers(_local_model_cost assert breakdown.cache_creation_cost == 0.0 +def test_token_type_cost_breakdown_flex_tier_prices_reasoning_at_flex_rate(_local_model_cost_map): + """Regression for the flex-tier breakdown drift: gemini-3.5-flash defines a flat + output_cost_per_reasoning_token (9e-06, the standard output rate) but no _flex + variant, so the breakdown priced reasoning at the standard rate on flex requests + while the total billed it at the flex output rate (4.5e-06). The reasoning + sub-cost then exceeded the entire flex completion cost.""" + + usage = Usage( + prompt_tokens=7, + completion_tokens=320, + total_tokens=327, + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=315, text_tokens=5), + ) + + breakdown = get_token_type_cost_breakdown( + model="gemini-3.5-flash", + custom_llm_provider="vertex_ai", + usage=usage, + service_tier="flex", + ) + + assert breakdown.reasoning_cost == pytest.approx(315 * 4.5e-06) + + _, flex_completion_cost = generic_cost_per_token( + model="gemini-3.5-flash", + usage=usage, + custom_llm_provider="vertex_ai", + service_tier="flex", + ) + assert breakdown.reasoning_cost <= flex_completion_cost + + standard_breakdown = get_token_type_cost_breakdown( + model="gemini-3.5-flash", + custom_llm_provider="vertex_ai", + usage=usage, + service_tier=None, + ) + assert standard_breakdown.reasoning_cost == pytest.approx(315 * 9e-06) + + def test_token_type_cost_breakdown_xai_at_exactly_200k_uses_higher_tier_rates(_local_model_cost_map): usage = Usage( diff --git a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_response_metadata.py b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_response_metadata.py index 203c6d3da0d..996530daa2e 100644 --- a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_response_metadata.py +++ b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_response_metadata.py @@ -92,6 +92,39 @@ class TestCallbackDurationMs: assert hidden.get("litellm_overhead_time_ms") is not None +class TestDictResultsSkipMetadataUpdate: + """Regression for /v1/messages cost-breakdown clobbering: AnthropicMessagesResponse + is a TypedDict, so apply() can never attach _hidden_params to it and the whole + metadata pass is discarded - except the cost recompute, whose only observable + effect was overwriting the logging object's already-correct cost breakdown with a + service-tier-less, reasoning-less recompute on the adapted response.""" + + def test_update_response_metadata_skips_cost_recompute_for_dict_results(self): + anthropic_response = { + "id": "msg_123", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "hi"}], + "usage": {"input_tokens": 7, "output_tokens": 320}, + } + logging_obj = MagicMock() + logging_obj.model_call_details = {} + logging_obj.caching_details = None + logging_obj.litellm_call_id = "test-call-id" + + update_response_metadata( + result=anthropic_response, + logging_obj=logging_obj, + model="vertex_ai/gemini-3.5-flash", + kwargs={}, + start_time=datetime.datetime(2025, 1, 1, 0, 0, 0), + end_time=datetime.datetime(2025, 1, 1, 0, 0, 1), + ) + + logging_obj._response_cost_calculator.assert_not_called() + assert "_hidden_params" not in anthropic_response + + class TestCallbackDurationInCustomHeaders: """Test that callback_duration_ms flows into get_custom_headers."""