From 6271e1e7bddf59a4a95200eb69fd4cb79f5ada09 Mon Sep 17 00:00:00 2001 From: LH-kevin Date: Sat, 22 Aug 2026 15:54:21 +0800 Subject: [PATCH] fix(cost): prefer matching tier at equal threshold --- .../litellm_core_utils/llm_cost_calc/utils.py | 4 +- .../test_threshold_cost_selection.py | 127 ++++++++++++++++++ 2 files changed, 130 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 3c382f29755..1da57fb9356 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -347,8 +347,10 @@ def _get_token_base_cost( base_key = match.group("base_key") selected = selected_threshold_keys.get(base_key) + selected_specificity = 0 if selected is None or selected[2] is None else 1 + candidate_specificity = 0 if threshold_key_tier is None else 1 - if selected is None or threshold > selected[0]: + if selected is None or (threshold, candidate_specificity) > (selected[0], selected_specificity): selected_threshold_keys[base_key] = ( threshold, key, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py index 5f4e1490b92..fc322eee14e 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py @@ -70,6 +70,133 @@ def test_cost_types_select_thresholds_independently() -> None: assert rates[1] == pytest.approx(18e-6) +@pytest.mark.parametrize( + "model_info", + [ + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + }, + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + }, + ], + ids=["standard-first", "priority-first"], +) +def test_equal_threshold_matching_tier_wins_for_output(model_info): + """At an equal threshold the matching service-tier rate must win regardless of the + order the standard and tier-qualified keys appear in model_info.""" + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost(model_info=model_info, usage=usage, service_tier="priority") + + assert rates[1] == pytest.approx(1.5e-6) + + +@pytest.mark.parametrize( + "model_info", + [ + { + "input_cost_per_token": 1e-6, + "cache_read_input_token_cost": 2e-7, + "cache_read_input_token_cost_above_200k_tokens": 3e-7, + "cache_read_input_token_cost_above_200k_tokens_priority": 4e-7, + }, + { + "input_cost_per_token": 1e-6, + "cache_read_input_token_cost": 2e-7, + "cache_read_input_token_cost_above_200k_tokens_priority": 4e-7, + "cache_read_input_token_cost_above_200k_tokens": 3e-7, + }, + ], + ids=["standard-first", "priority-first"], +) +def test_equal_threshold_matching_tier_wins_for_cache_read(model_info): + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost(model_info=model_info, usage=usage, service_tier="priority") + + assert rates[4] == pytest.approx(4e-7) + + +def test_equal_threshold_default_tier_keeps_standard(): + """service_tier=None must ignore the tier-qualified key and keep the standard rate.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost(model_info=model_info, usage=usage) + + assert rates[1] == pytest.approx(1e-6) + + +def test_equal_threshold_wrong_tier_keeps_standard(): + """A priority-qualified key must not win under a different service tier.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier="flex", + ) + + assert rates[1] == pytest.approx(1e-6) + + +def test_unequal_threshold_highest_standard_still_wins(): + """A higher standard threshold governs over a lower matching-tier threshold.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + "output_cost_per_token_above_128k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier="priority", + ) + + assert rates[1] == pytest.approx(1e-6) + + @pytest.mark.parametrize( ("tier_field", "tier_value", "rate_index"), [