From e79ab817fa593e3af843660f679749ce04a10638 Mon Sep 17 00:00:00 2001 From: LH-kevin Date: Fri, 21 Aug 2026 19:12:43 +0800 Subject: [PATCH] fix(cost): zero custom threshold rates for PTU deployments --- litellm/litellm_core_utils/ptu_pricing.py | 18 ++++++++ .../litellm_core_utils/test_ptu_pricing.py | 43 +++++++++++++++++++ .../test_router_model_cost_isolation.py | 21 ++++++++- 3 files changed, 80 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/ptu_pricing.py b/litellm/litellm_core_utils/ptu_pricing.py index 6923e6beb96..b599beb2dee 100644 --- a/litellm/litellm_core_utils/ptu_pricing.py +++ b/litellm/litellm_core_utils/ptu_pricing.py @@ -6,6 +6,7 @@ together because they have to agree: a deployment the rollup declines to charge router prices at zero serves its traffic for free. """ +import re from collections.abc import Mapping from dataclasses import dataclass from datetime import datetime, timezone @@ -42,6 +43,19 @@ SEARCH_CONTEXT_SIZES: Final = ("search_context_size_low", "search_context_size_m # and zeroing one of those would destroy the deployment's configuration rather than stop a # charge. CUSTOM_PRICING_FIELDS: Final = frozenset(f for f in CustomPricingLiteLLMParams.model_fields if "cost" in f) +# Custom threshold rates the router copies from litellm_params into the cost map (e.g. +# input_cost_per_token_above_32k_tokens) are not enumerated in CustomPricingLiteLLMParams, +# so they are matched here and zeroed like every other declared rate. Only the cost +# calculator's base keys qualify, so a param that merely ends in _above_k_tokens (e.g. +# api_key_above_32k_tokens) is not treated as a charge. +_THRESHOLD_RATE_KEY: Final[re.Pattern[str]] = re.compile( + r"^(?:" + r"input_cost_per_token|" + r"output_cost_per_token|" + r"cache_creation_input_token_cost(?:_above_1hr)?|" + r"cache_read_input_token_cost" + r")_above_\d+k?_tokens$" +) PTU_ZEROED_PRICING: Final[Mapping[str, float | tuple[()] | Mapping[str, float]]] = MappingProxyType( { **dict.fromkeys(PTU_ZEROED_PRICING_FIELDS, 0.0), @@ -175,6 +189,9 @@ def zeroed_ptu_pricing( return None if not is_ptu_cost_attribution_enabled(): return None + declared_threshold_rates: Final = frozenset( + key for key in declared if _THRESHOLD_RATE_KEY.fullmatch(key) is not None + ) return MappingProxyType( { **PTU_ZEROED_PRICING, @@ -184,5 +201,6 @@ def zeroed_ptu_pricing( .difference(PTU_EMPTIED_PRICING_FIELDS), 0.0, ), + **dict.fromkeys(declared_threshold_rates, 0.0), } ) diff --git a/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py b/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py index b953bfaa565..673599b0e82 100644 --- a/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py +++ b/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py @@ -155,6 +155,49 @@ def test_a_rate_the_deployment_declares_itself_is_zeroed_too(): assert override[extra] == 0.0 +@pytest.mark.parametrize( + "threshold_field", + [ + "input_cost_per_token_above_32k_tokens", + "output_cost_per_token_above_32k_tokens", + "cache_read_input_token_cost_above_32k_tokens", + "cache_creation_input_token_cost_above_32k_tokens", + "cache_creation_input_token_cost_above_1hr_above_32k_tokens", + ], +) +def test_arbitrary_threshold_rate_the_deployment_declares_is_zeroed_too(threshold_field): + """Threshold rates a deployment can declare but CustomPricingLiteLLMParams does not + enumerate (e.g. input_cost_per_token_above_32k_tokens) must be zeroed like any other + declared rate, or a PTU deployment bills per token past the threshold on top of the + hourly charge.""" + assert threshold_field not in CUSTOM_PRICING_FIELDS + assert threshold_field not in PTU_ZEROED_PRICING_FIELDS + + override = _with_flag(_VALID, declared={threshold_field: 9e-06}) + + assert override is not None + assert override.get(threshold_field, 9e-06) == 0.0 + + +@pytest.mark.parametrize( + "param_field", + [ + "api_key_above_32k_tokens", + "secret_above_32k_tokens", + "credential_above_32k_tokens", + "custom_provider_param_above_32k_tokens", + ], +) +def test_threshold_like_non_pricing_params_are_left_alone(param_field): + """The threshold matcher must only cover the cost calculator's pricing base keys: a + deployment param that merely ends in _above_k_tokens is not a charge and must keep + its value.""" + override = _with_flag(_VALID, declared={param_field: "keep-me"}) + + assert override is not None + assert param_field not in override + + def test_a_setting_that_is_not_a_charge_is_left_alone(): """CustomPricingLiteLLMParams also carries configuration, and zeroing one of those would break the deployment rather than stop a charge.""" diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index 7aace172611..20c4facaab2 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -1825,8 +1825,7 @@ def test_price_data_reload_preserves_arbitrary_above_threshold_pricing(monkeypat assert rebuilt["cache_read_input_token_cost_above_32k_tokens"] == 4e-6 assert router.model_list finally: - litellm.model_cost = saved_model_cost - _invalidate_model_cost_lowercase_map() + _restore_model_cost_entries(saved_model_cost) def test_strategy_router_alias_pricing_never_enters_model_cost(monkeypatch): @@ -1985,6 +1984,24 @@ def test_a_config_ptu_deployment_bills_nothing_per_token(): assert litellm.model_cost[entry["model_info"]["id"]]["input_cost_per_token"] == 0.0 +def test_a_config_ptu_deployment_zeroes_custom_threshold_rates(): + """A custom threshold rate the deployment declares (e.g. + input_cost_per_token_above_32k_tokens) must be zeroed like every other declared rate, + or the PTU deployment bills per token past the threshold on top of the hourly charge.""" + router = _ptu_router( + litellm_params={ + "input_cost_per_token": 5e-06, + "input_cost_per_token_above_32k_tokens": 9e-06, + } + ) + entry = router.model_list[0] + + assert entry["litellm_params"]["input_cost_per_token"] == 0.0 + assert entry["litellm_params"]["input_cost_per_token_above_32k_tokens"] == 0.0 + assert litellm.model_cost[entry["model_info"]["id"]]["input_cost_per_token"] == 0.0 + assert litellm.model_cost[entry["model_info"]["id"]]["input_cost_per_token_above_32k_tokens"] == 0.0 + + @pytest.mark.parametrize( "backend", ["anthropic/claude-sonnet-4-5-20250929", "azure/gpt-4o", "gemini/gemini-2.5-flash"],