From e0833c4ba361c5873de8f1b3485331bbe5fc7137 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:15:52 -0700 Subject: [PATCH] fix(cost): bill reasoning tokens at the service tier output rate A tier request against a model that publishes tier output pricing but no tier reasoning key (every current Gemini flash entry) billed reasoning tokens at the standard output_cost_per_reasoning_token, undercounting priority and fast traffic where thinking tokens dominate completions generic_cost_per_token now resolves the reasoning rate with explicit precedence: an explicit output_cost_per_reasoning_token_ key wins, then the tier-resolved output rate when the model prices that tier, then the standard reasoning key, then the output base cost. The two tier reasoning keys are wired through ModelInfo so providers can publish real tiered reasoning prices when they exist --- .../litellm_core_utils/llm_cost_calc/utils.py | 24 +++- litellm/types/utils.py | 4 + litellm/utils.py | 4 + .../llm_cost_calc/test_llm_cost_calc_utils.py | 117 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 8 ++ 5 files changed, 154 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index aeef604510b..5c923e5a8fa 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -681,6 +681,23 @@ def _get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | return 1.0 +def _resolve_reasoning_token_cost( + model_info: ModelInfo, + service_tier: str | None, + completion_base_cost: float, +) -> float: + tier_reasoning_key: Final = _get_service_tier_cost_key("output_cost_per_reasoning_token", service_tier) + if model_info.get(tier_reasoning_key) is not None: + tier_reasoning_cost: Final = _get_cost_per_unit(model_info, tier_reasoning_key, None) + if tier_reasoning_cost is not None: + return tier_reasoning_cost + tier_output_key: Final = _get_service_tier_cost_key("output_cost_per_token", service_tier) + if tier_output_key != "output_cost_per_token" and model_info.get(tier_output_key) is not None: + return completion_base_cost + standard_reasoning_cost: Final = _get_cost_per_unit(model_info, "output_cost_per_reasoning_token", None) + return standard_reasoning_cost if standard_reasoning_cost is not None else completion_base_cost + + def generic_cost_per_token( model: str, usage: Usage, @@ -817,9 +834,10 @@ def generic_cost_per_token( ## REASONING COST if not is_text_tokens_total and reasoning_tokens and reasoning_tokens > 0: - _output_cost_per_reasoning_token = _get_cost_per_unit(model_info, "output_cost_per_reasoning_token", None) - _output_cost_per_reasoning_token = ( - _output_cost_per_reasoning_token if _output_cost_per_reasoning_token is not None else completion_base_cost + _output_cost_per_reasoning_token = _resolve_reasoning_token_cost( + model_info=model_info, + service_tier=service_tier, + completion_base_cost=completion_base_cost, ) completion_cost += float(reasoning_tokens) * _output_cost_per_reasoning_token diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 77f83c5b6f8..8e23a380792 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -263,6 +263,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): output_cost_per_video_token: Optional[float] # for gemini omni models with video output output_vector_size: Optional[int] output_cost_per_reasoning_token: Optional[float] + output_cost_per_reasoning_token_flex: Optional[float] + output_cost_per_reasoning_token_priority: Optional[float] output_cost_per_video_per_second: Optional[float] # only for vertex ai models output_cost_per_audio_per_second: Optional[float] # only for vertex ai models output_cost_per_second: Optional[float] # for OpenAI Speech models @@ -3308,6 +3310,8 @@ class CustomPricingLiteLLMParams(BaseModel): output_cost_per_image_token: Optional[float] = None output_cost_per_video_token: Optional[float] = None output_cost_per_reasoning_token: Optional[float] = None + output_cost_per_reasoning_token_flex: Optional[float] = None + output_cost_per_reasoning_token_priority: Optional[float] = None output_cost_per_video_per_second: Optional[float] = None output_cost_per_audio_per_second: Optional[float] = None search_context_cost_per_query: Optional[Dict[str, Any]] = None diff --git a/litellm/utils.py b/litellm/utils.py index d24a4dc928f..3dbb35926dd 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5482,6 +5482,10 @@ def _get_model_info_helper( output_cost_per_audio_token=_model_info.get("output_cost_per_audio_token", None), output_cost_per_character=_model_info.get("output_cost_per_character", None), output_cost_per_reasoning_token=_model_info.get("output_cost_per_reasoning_token", None), + output_cost_per_reasoning_token_flex=_model_info.get("output_cost_per_reasoning_token_flex", None), + output_cost_per_reasoning_token_priority=_model_info.get( + "output_cost_per_reasoning_token_priority", None + ), output_cost_per_token_above_128k_tokens=_model_info.get( "output_cost_per_token_above_128k_tokens", None ), diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 9145e5dc76d..f5b9ddfa7d0 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -2553,3 +2553,120 @@ def test_fast_service_tier_matches_priority_above_the_context_threshold(_local_m assert fast == priority assert fast[0] == pytest.approx(300_000 * 1e-05, rel=1e-9) assert fast[1] == pytest.approx(1_000 * 4.5e-05, rel=1e-9) + + +def test_priority_reasoning_tokens_bill_at_the_priority_output_rate(_local_model_cost_map): + """Regression: gemini-3.5-flash publishes priority output pricing but no priority + reasoning key, so reasoning tokens under priority/fast were billed at the standard + output_cost_per_reasoning_token instead of following the tier's output rate.""" + from litellm.types.utils import Usage + + usage = Usage( + prompt_tokens=1_000, + completion_tokens=5_000, + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=4_000), + ) + + model_info = litellm.get_model_info(model="gemini-3.5-flash", custom_llm_provider="gemini") + standard_output_rate = model_info["output_cost_per_token"] + standard_reasoning_rate = model_info["output_cost_per_reasoning_token"] + priority_output_rate = model_info["output_cost_per_token_priority"] + assert priority_output_rate is not None + assert priority_output_rate != standard_reasoning_rate + + standard = generic_cost_per_token( + model="gemini-3.5-flash", usage=usage, custom_llm_provider="gemini", service_tier=None + ) + priority = generic_cost_per_token( + model="gemini-3.5-flash", usage=usage, custom_llm_provider="gemini", service_tier="priority" + ) + fast = generic_cost_per_token( + model="gemini-3.5-flash", usage=usage, custom_llm_provider="gemini", service_tier="fast" + ) + + assert standard[1] == pytest.approx(1_000 * standard_output_rate + 4_000 * standard_reasoning_rate, rel=1e-9) + assert priority[1] == pytest.approx(5_000 * priority_output_rate, rel=1e-9) + assert fast == priority + + +def test_explicit_tier_reasoning_key_wins_over_the_tier_output_rate(): + from litellm.types.utils import Usage + + model_info = { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "output_cost_per_reasoning_token": 6e-06, + "input_cost_per_token_priority": 2e-06, + "output_cost_per_token_priority": 8e-06, + "output_cost_per_reasoning_token_priority": 1.2e-05, + } + usage = Usage( + prompt_tokens=100, + completion_tokens=1_000, + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=600), + ) + + _, completion_cost = generic_cost_per_token( + model="synthetic-model", + usage=usage, + custom_llm_provider="openai", + service_tier="priority", + model_info=model_info, + ) + + assert completion_cost == pytest.approx(400 * 8e-06 + 600 * 1.2e-05, rel=1e-9) + + +def test_null_tier_reasoning_key_falls_back_to_the_tier_output_rate(): + """get_model_info dumps every ModelInfo field, so an unpublished tier reasoning key + arrives as an explicit None and must not shadow the tier output rate.""" + from litellm.types.utils import Usage + + model_info = { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "output_cost_per_reasoning_token": 6e-06, + "output_cost_per_reasoning_token_priority": None, + "input_cost_per_token_priority": 2e-06, + "output_cost_per_token_priority": 8e-06, + } + usage = Usage( + prompt_tokens=100, + completion_tokens=1_000, + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=600), + ) + + _, completion_cost = generic_cost_per_token( + model="synthetic-model", + usage=usage, + custom_llm_provider="openai", + service_tier="priority", + model_info=model_info, + ) + + assert completion_cost == pytest.approx(1_000 * 8e-06, rel=1e-9) + + +def test_tier_request_without_tier_pricing_keeps_the_standard_reasoning_rate(): + from litellm.types.utils import Usage + + model_info = { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "output_cost_per_reasoning_token": 6e-06, + } + usage = Usage( + prompt_tokens=100, + completion_tokens=1_000, + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=600), + ) + + _, completion_cost = generic_cost_per_token( + model="synthetic-model", + usage=usage, + custom_llm_provider="openai", + service_tier="priority", + model_info=model_info, + ) + + assert completion_cost == pytest.approx(400 * 4e-06 + 600 * 6e-06, rel=1e-9) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index df85decc676..d70a11dc1b9 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -26561,6 +26561,10 @@ export interface components { output_cost_per_pixel?: number | null; /** Output Cost Per Reasoning Token */ output_cost_per_reasoning_token?: number | null; + /** Output Cost Per Reasoning Token Flex */ + output_cost_per_reasoning_token_flex?: number | null; + /** Output Cost Per Reasoning Token Priority */ + output_cost_per_reasoning_token_priority?: number | null; /** Output Cost Per Second */ output_cost_per_second?: number | null; /** Output Cost Per Second 1080P */ @@ -35120,6 +35124,10 @@ export interface components { output_cost_per_pixel?: number | null; /** Output Cost Per Reasoning Token */ output_cost_per_reasoning_token?: number | null; + /** Output Cost Per Reasoning Token Flex */ + output_cost_per_reasoning_token_flex?: number | null; + /** Output Cost Per Reasoning Token Priority */ + output_cost_per_reasoning_token_priority?: number | null; /** Output Cost Per Second */ output_cost_per_second?: number | null; /** Output Cost Per Second 1080P */