From a886f32f29243c00a1d0dbd7b3e20e59b938e369 Mon Sep 17 00:00:00 2001 From: LH-kevin Date: Fri, 24 Jul 2026 19:43:26 +0800 Subject: [PATCH 01/14] fix(router): preserve arbitrary threshold pricing fields --- litellm/router.py | 26 ++-- .../test_router_model_cost_isolation.py | 114 ++++++++++++++++++ 2 files changed, 133 insertions(+), 7 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 3ecaef591f3..faadeff61b9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -210,6 +210,7 @@ from litellm.types.utils import ( Usage, ) from litellm.utils import ( + _ABOVE_THRESHOLD_COST_KEY, CustomStreamWrapper, EmbeddingResponse, ModelResponse, @@ -269,6 +270,16 @@ def _cost_value_as_float(value: Union[str, int, float, None]) -> Optional[float] return None +def _copy_custom_pricing_fields( + model_info: Dict[str, Any], + litellm_params: LiteLLM_Params, +) -> None: + """Copy declared and arbitrary threshold pricing into a model-cost entry.""" + for field, value in litellm_params.model_dump(exclude_none=True).items(): + if field in CustomPricingLiteLLMParams.model_fields or _ABOVE_THRESHOLD_COST_KEY.search(field) is not None: + model_info[field] = value + + _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT") @@ -7471,9 +7482,10 @@ class Router: litellm_params=litellm_params, model_info=_model_info, ) - for field in CustomPricingLiteLLMParams.model_fields.keys(): - if deployment.litellm_params.get(field) is not None: - _model_info[field] = deployment.litellm_params[field] + _copy_custom_pricing_fields( + model_info=_model_info, + litellm_params=deployment.litellm_params, + ) if _model_info.get("input_cost_per_token") is not None: Router._inherit_builtin_cache_pricing( @@ -8194,10 +8206,10 @@ class Router: self._add_deployment(deployment=deployment) _model_info_dict: dict = deployment.model_info.model_dump(exclude_none=True) - for field in CustomPricingLiteLLMParams.model_fields.keys(): - field_value = deployment.litellm_params.get(field) - if field_value is not None: - _model_info_dict[field] = field_value + _copy_custom_pricing_fields( + model_info=_model_info_dict, + litellm_params=deployment.litellm_params, + ) if _model_info_dict.get("input_cost_per_token") is not None: Router._inherit_builtin_cache_pricing( diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index 672b5b36197..914423abef7 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -944,3 +944,117 @@ def test_wildcard_zero_cost_request_does_not_poison_named_deployment_pricing(): assert named_cost == pytest.approx(10 * builtin_input_cost) finally: _restore_model_cost_entries(model_keys) + + +def test_router_registers_arbitrary_above_threshold_pricing_from_litellm_params(): + """Regression for #34378. + + Arbitrary *_above__tokens pricing fields accepted in litellm_params + must be registered under the deployment model ID. + """ + backend_model = "openai/gpt-4o-mini" + model_id = "router-custom-above-32k-pricing-34378" + shared_keys = ("gpt-4o-mini", backend_model) + + original_entries = { + key: copy.deepcopy(litellm.model_cost.get(key)) + for key in (*shared_keys, model_id) + } + + try: + Router( + model_list=[ + { + "model_name": "custom-above-32k-model", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_token_above_32k_tokens": 9e-6, + "output_cost_per_token_above_32k_tokens": 18e-6, + "cache_read_input_token_cost_above_32k_tokens": 4e-6, + }, + "model_info": { + "id": model_id, + }, + } + ] + ) + + registered = litellm.model_cost.get(model_id) + assert registered is not None + + assert registered["input_cost_per_token"] == 1e-6 + assert registered["output_cost_per_token"] == 2e-6 + assert registered["input_cost_per_token_above_32k_tokens"] == 9e-6 + assert registered["output_cost_per_token_above_32k_tokens"] == 18e-6 + assert registered["cache_read_input_token_cost_above_32k_tokens"] == 4e-6 + + from litellm.litellm_core_utils.llm_cost_calc.utils import ( + generic_cost_per_token, + ) + from litellm.types.utils import Usage + + prompt_tokens = 40_000 + completion_tokens = 1_000 + usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model=model_id, + usage=usage, + custom_llm_provider="openai", + ) + + assert prompt_cost == pytest.approx(9e-6 * prompt_tokens) + assert completion_cost == pytest.approx(18e-6 * completion_tokens) + finally: + _restore_model_cost_entries(original_entries) + + +def test_add_deployment_registers_arbitrary_above_threshold_pricing(): + """Dynamic add_deployment must preserve arbitrary pricing thresholds.""" + backend_model = "openai/gpt-4o-mini" + model_id = "add-deployment-custom-above-32k-pricing-34378" + shared_keys = ("gpt-4o-mini", backend_model) + + original_entries = { + key: copy.deepcopy(litellm.model_cost.get(key)) + for key in (*shared_keys, model_id) + } + + try: + router = Router(model_list=[]) + router.add_deployment( + deployment=Deployment( + model_name="dynamic-custom-above-32k-model", + litellm_params=LiteLLM_Params( + model=backend_model, + api_key="fake-key", + input_cost_per_token=1e-6, + output_cost_per_token=2e-6, + input_cost_per_token_above_32k_tokens=9e-6, + output_cost_per_token_above_32k_tokens=18e-6, + cache_read_input_token_cost_above_32k_tokens=4e-6, + ), + model_info=ModelInfo(id=model_id), + ) + ) + + registered = litellm.model_cost.get(model_id) + assert registered is not None + assert registered["input_cost_per_token_above_32k_tokens"] == 9e-6 + assert registered["output_cost_per_token_above_32k_tokens"] == 18e-6 + assert registered["cache_read_input_token_cost_above_32k_tokens"] == 4e-6 + + for shared_key in shared_keys: + shared_entry = litellm.model_cost.get(shared_key) or {} + assert "input_cost_per_token_above_32k_tokens" not in shared_entry + assert "output_cost_per_token_above_32k_tokens" not in shared_entry + assert "cache_read_input_token_cost_above_32k_tokens" not in shared_entry + finally: + _restore_model_cost_entries(original_entries) From d54b126545f87b86c52149685e105792cebe3a81 Mon Sep 17 00:00:00 2001 From: LH-kevin Date: Fri, 24 Jul 2026 23:16:57 +0800 Subject: [PATCH 02/14] test(router): cover custom pricing helper --- .../test_router_model_cost_isolation.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index 914423abef7..562dd6e4261 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -20,6 +20,7 @@ sys.path.insert( import litellm from litellm import Router +from litellm.router import _copy_custom_pricing_fields from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo from litellm.utils import _invalidate_model_cost_lowercase_map @@ -33,6 +34,27 @@ def _restore_model_cost_entries(original_entries): _invalidate_model_cost_lowercase_map() +def test_copy_custom_pricing_fields_preserves_declared_and_arbitrary_threshold_fields(): + """Copy supported pricing fields without copying unrelated LiteLLM params.""" + model_info = {} + litellm_params = LiteLLM_Params( + model="openai/gpt-4o-mini", + api_key="fake-key", + input_cost_per_token=1e-6, + input_cost_per_token_above_32k_tokens=9e-6, + ) + + _copy_custom_pricing_fields( + model_info=model_info, + litellm_params=litellm_params, + ) + + assert model_info == { + "input_cost_per_token": 1e-6, + "input_cost_per_token_above_32k_tokens": 9e-6, + } + + def test_should_not_pollute_shared_key_with_zero_cost_pricing(): """ When deployment A has input_cost_per_token=0 and deployment B has no From 1c2786e6bdfcc3fcb9a5d4728d5667f42acd1394 Mon Sep 17 00:00:00 2001 From: LH-kevin Date: Sat, 25 Jul 2026 00:38:09 +0800 Subject: [PATCH 03/14] fix(cost): select standalone threshold pricing tiers --- .../litellm_core_utils/llm_cost_calc/utils.py | 197 +++++++----------- .../test_threshold_cost_selection.py | 70 +++++++ 2 files changed, 146 insertions(+), 121 deletions(-) create mode 100644 tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 85ed0665ebf..a8c98bf617e 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1,6 +1,7 @@ # What is this? ## Helper utilities for cost_per_token() +import re from dataclasses import dataclass from typing import Any, Literal, Optional, Tuple, TypedDict, cast @@ -39,6 +40,15 @@ _VALID_DATA_RESIDENCIES = frozenset(r.value for r in DataResidency) # of being rebuilt for every model_info key on every call. _SERVICE_TIER_SUFFIXES: tuple[str, ...] = tuple(f"_{st.value}" for st in ServiceTier) +_ABOVE_TOKEN_THRESHOLD_COST_KEY = re.compile( + r"^(?P" + r"input_cost_per_token|" + r"output_cost_per_token|" + r"cache_creation_input_token_cost(?:_above_1hr)?|" + r"cache_read_input_token_cost" + r")_above_\d+k?_tokens$" +) + def _get_token_detail_value(details: object, key: str) -> Optional[int]: if isinstance(details, dict): @@ -194,7 +204,7 @@ def _get_service_tier_cost_key(base_key: str, service_tier: Optional[str]) -> st def _parse_above_token_threshold(key: str) -> float: - threshold_str = key.split("_above_")[1].split("_tokens")[0] + threshold_str = key.rsplit("_above_", 1)[1].split("_tokens")[0] return float(threshold_str.replace("k", "")) * (1000 if "k" in threshold_str else 1) @@ -204,8 +214,9 @@ def _get_token_base_cost( """ Return prompt cost, completion cost, and cache costs for a given model and usage. - If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set, - then we use the corresponding threshold cost for all token types. + For each token cost type, use its highest configured + `*_above_[x]k_tokens` or `*_above_[x]_tokens` rate when prompt tokens + exceed that threshold. Returns: Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost) @@ -233,127 +244,71 @@ def _get_token_base_cost( cache_read_cost = cast(float, _get_cost_per_unit(model_info, cache_read_cost_key)) ## CHECK IF ABOVE THRESHOLD - # Optimization: collect threshold keys first to avoid sorting all model_info keys. - # Most models don't have threshold pricing, so we can return early. - # Exclude service_tier-specific variants (e.g. input_cost_per_token_above_200k_tokens_priority) - # so that the threshold detection loop only processes standard keys. The - # service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key. - threshold_keys = [ - k for k in model_info if k.startswith("input_cost_per_token_above_") and not k.endswith(_SERVICE_TIER_SUFFIXES) - ] - if not threshold_keys: - return ( - prompt_base_cost, - completion_base_cost, - cache_creation_cost, - cache_creation_cost_above_1hr, - cache_read_cost, + selected_threshold_keys: dict[str, tuple[float, str]] = {} + + for key in model_info: + if "_above_" not in key or not key.endswith("_tokens"): + continue + + match = _ABOVE_TOKEN_THRESHOLD_COST_KEY.fullmatch(key) + if match is None or model_info.get(key) is None: + continue + + try: + threshold = _parse_above_token_threshold(key) + except (IndexError, ValueError): + continue + + if usage.prompt_tokens <= threshold: + continue + + base_key = match.group("base_key") + selected = selected_threshold_keys.get(base_key) + + if selected is None or threshold > selected[0]: + selected_threshold_keys[base_key] = ( + threshold, + key, + ) + + costs_by_base_key = { + "input_cost_per_token": prompt_base_cost, + "output_cost_per_token": completion_base_cost, + "cache_creation_input_token_cost": cache_creation_cost, + "cache_creation_input_token_cost_above_1hr": ( + cache_creation_cost_above_1hr + ), + "cache_read_input_token_cost": cache_read_cost, + } + + for base_key, (_, threshold_key) in ( + selected_threshold_keys.items() + ): + tiered_key = _get_service_tier_cost_key( + threshold_key, + service_tier, + ) + costs_by_base_key[base_key] = cast( + float, + _get_cost_per_unit( + model_info, + tiered_key, + costs_by_base_key[base_key], + ), ) - # Only sort the threshold keys (typically 1-2 keys instead of 66+) - threshold: Optional[float] = None - for key in sorted(threshold_keys, key=_parse_above_token_threshold, reverse=True): - value = model_info.get(key) - if value is not None: - try: - # Handle both formats: _above_128k_tokens and _above_128_tokens - threshold_str = key.split("_above_")[1].split("_tokens")[0] - threshold = _parse_above_token_threshold(key) - if usage.prompt_tokens > threshold: - # Prefer a service_tier-specific above-threshold key when available, - # e.g. input_cost_per_token_priority_above_200k_tokens for Gemini - # ON_DEMAND_PRIORITY. Falls back to the standard key automatically - # via _get_cost_per_unit's service_tier fallback logic. - tiered_input_key = ( - _get_service_tier_cost_key( - f"input_cost_per_token_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else key - ) - prompt_base_cost = cast( - float, - _get_cost_per_unit(model_info, tiered_input_key, prompt_base_cost), - ) - tiered_output_key = ( - _get_service_tier_cost_key( - f"output_cost_per_token_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"output_cost_per_token_above_{threshold_str}_tokens" - ) - completion_base_cost = cast( - float, - _get_cost_per_unit( - model_info, - tiered_output_key, - completion_base_cost, - ), - ) - - # Apply tiered pricing to cache costs - cache_creation_tiered_key = ( - _get_service_tier_cost_key( - f"cache_creation_input_token_cost_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"cache_creation_input_token_cost_above_{threshold_str}_tokens" - ) - cache_creation_1hr_tiered_key = ( - _get_service_tier_cost_key( - f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens" - ) - cache_read_tiered_key = ( - _get_service_tier_cost_key( - f"cache_read_input_token_cost_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"cache_read_input_token_cost_above_{threshold_str}_tokens" - ) - - cache_creation_cost = cast( - float, - _get_cost_per_unit( - model_info, - cache_creation_tiered_key, - cache_creation_cost, - ), - ) - - cache_creation_cost_above_1hr = cast( - float, - _get_cost_per_unit( - model_info, - cache_creation_1hr_tiered_key, - cache_creation_cost_above_1hr, - ), - ) - - cache_read_cost = cast( - float, - _get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost), - ) - - break - except (IndexError, ValueError): - continue - except Exception: - continue - return ( - prompt_base_cost, - completion_base_cost, - cache_creation_cost, - cache_creation_cost_above_1hr, - cache_read_cost, + costs_by_base_key["input_cost_per_token"], + costs_by_base_key["output_cost_per_token"], + costs_by_base_key[ + "cache_creation_input_token_cost" + ], + costs_by_base_key[ + "cache_creation_input_token_cost_above_1hr" + ], + costs_by_base_key[ + "cache_read_input_token_cost" + ], ) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py new file mode 100644 index 00000000000..af893cccc49 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py @@ -0,0 +1,70 @@ +import pytest + +from litellm.litellm_core_utils.llm_cost_calc.utils import ( + _get_token_base_cost, +) +from litellm.types.utils import Usage + + +@pytest.mark.parametrize( + ("tier_field", "tier_value", "rate_index"), + [ + ( + "output_cost_per_token_above_32k_tokens", + 18e-6, + 1, + ), + ( + "cache_read_input_token_cost_above_32k_tokens", + 4e-6, + 4, + ), + ], +) +def test_standalone_threshold_field_selects_tier( + tier_field: str, + tier_value: float, + rate_index: int, +) -> None: + """Output and cache fields can independently define a tier.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_read_input_token_cost": 5e-7, + tier_field: tier_value, + } + usage = Usage( + prompt_tokens=40_000, + completion_tokens=1_000, + total_tokens=41_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + ) + + assert rates[rate_index] == pytest.approx(tier_value) + + +def test_cost_types_select_thresholds_independently() -> None: + """An inactive input tier must not suppress an active output tier.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_token_above_128k_tokens": 9e-6, + "output_cost_per_token_above_32k_tokens": 18e-6, + } + usage = Usage( + prompt_tokens=40_000, + completion_tokens=1_000, + total_tokens=41_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + ) + + assert rates[0] == pytest.approx(1e-6) + assert rates[1] == pytest.approx(18e-6) From f555e5263740f6ccbcef699a9feeff44946865b8 Mon Sep 17 00:00:00 2001 From: LH-kevin Date: Sat, 25 Jul 2026 01:34:59 +0800 Subject: [PATCH 04/14] style(cost): format threshold pricing selection --- .../litellm_core_utils/llm_cost_calc/utils.py | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index a8c98bf617e..1225f542cff 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -275,15 +275,11 @@ def _get_token_base_cost( "input_cost_per_token": prompt_base_cost, "output_cost_per_token": completion_base_cost, "cache_creation_input_token_cost": cache_creation_cost, - "cache_creation_input_token_cost_above_1hr": ( - cache_creation_cost_above_1hr - ), + "cache_creation_input_token_cost_above_1hr": (cache_creation_cost_above_1hr), "cache_read_input_token_cost": cache_read_cost, } - for base_key, (_, threshold_key) in ( - selected_threshold_keys.items() - ): + for base_key, (_, threshold_key) in selected_threshold_keys.items(): tiered_key = _get_service_tier_cost_key( threshold_key, service_tier, @@ -300,15 +296,9 @@ def _get_token_base_cost( return ( costs_by_base_key["input_cost_per_token"], costs_by_base_key["output_cost_per_token"], - costs_by_base_key[ - "cache_creation_input_token_cost" - ], - costs_by_base_key[ - "cache_creation_input_token_cost_above_1hr" - ], - costs_by_base_key[ - "cache_read_input_token_cost" - ], + costs_by_base_key["cache_creation_input_token_cost"], + costs_by_base_key["cache_creation_input_token_cost_above_1hr"], + costs_by_base_key["cache_read_input_token_cost"], ) From 1f036fd842e7ffee32d98a42fc16f698e250e993 Mon Sep 17 00:00:00 2001 From: LH-kevin Date: Mon, 3 Aug 2026 00:21:03 +0800 Subject: [PATCH 05/14] fix(ci): satisfy type discipline gate --- litellm/litellm_core_utils/llm_cost_calc/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 5979cae16ca..117aca765ff 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -252,7 +252,7 @@ def _get_token_base_cost( cache_read_cost = cast(float, _get_cost_per_unit(model_info, cache_read_cost_key)) ## CHECK IF ABOVE THRESHOLD - selected_threshold_keys: dict[str, tuple[float, str]] = {} + selected_threshold_keys: dict[str, tuple[float, str]] = {} # mutable-ok: threshold accumulator for key in model_info: if "_above_" not in key or not key.endswith("_tokens"): @@ -279,7 +279,7 @@ def _get_token_base_cost( key, ) - costs_by_base_key = { + costs_by_base_key = { # mutable-ok: selected-rate updates "input_cost_per_token": prompt_base_cost, "output_cost_per_token": completion_base_cost, "cache_creation_input_token_cost": cache_creation_cost, From b64d0a6eee85da7947ab4901f7ba9b0821fd4edd Mon Sep 17 00:00:00 2001 From: LH-kevin Date: Mon, 3 Aug 2026 01:00:01 +0800 Subject: [PATCH 06/14] fix(ci): annotate mutable pricing map --- litellm/router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index 92a15fd6733..6fe1975a297 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -269,7 +269,7 @@ def _cost_value_as_float(value: str | float | None) -> float | None: def _copy_custom_pricing_fields( - model_info: dict[str, Any], + model_info: dict[str, Any], # mutable-ok: updates model-cost entry in place litellm_params: LiteLLM_Params, ) -> None: """Copy declared and arbitrary threshold pricing into a model-cost entry.""" From 96f967eb44402a180ea6f59b6cdab99dcd64e3b1 Mon Sep 17 00:00:00 2001 From: LH-kevin Date: Sat, 15 Aug 2026 17:03:00 +0800 Subject: [PATCH 07/14] fix(router): satisfy pricing helper CI gates --- litellm/router.py | 8 ++++++-- .../test_litellm/test_router_model_cost_isolation.py | 12 +++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index a700d526916..8b4ded108a0 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -212,7 +212,6 @@ from litellm.types.utils import ( ) from litellm.types.utils import ModelInfo as ModelMapInfo from litellm.utils import ( - _ABOVE_THRESHOLD_COST_KEY, CustomStreamWrapper, EmbeddingResponse, ModelResponse, @@ -281,8 +280,13 @@ def _cost_value_as_float(value: str | float | None) -> float | None: return None +_CUSTOM_PRICING_THRESHOLD_COST_KEY: Final[re.Pattern[str]] = re.compile(r"_above_\d+k?_tokens$") + + def _is_custom_pricing_field(field: str) -> bool: - return field in CustomPricingLiteLLMParams.model_fields or _ABOVE_THRESHOLD_COST_KEY.search(field) is not None + return ( + field in CustomPricingLiteLLMParams.model_fields or _CUSTOM_PRICING_THRESHOLD_COST_KEY.search(field) is not None + ) def _copy_custom_pricing_fields( diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index f32bd9762ae..d935734cdbd 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -20,7 +20,10 @@ sys.path.insert( import litellm from litellm import Router -from litellm.router import _copy_custom_pricing_fields +from litellm.router import ( + _copy_custom_pricing_fields, + _is_custom_pricing_field, +) from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo from litellm.utils import ( _invalidate_model_cost_lowercase_map, @@ -52,6 +55,13 @@ def _restore_model_cost_entries(original_entries): _invalidate_model_cost_lowercase_map() +def test_is_custom_pricing_field_recognizes_declared_and_arbitrary_threshold_fields(): + assert _is_custom_pricing_field("input_cost_per_token") + assert _is_custom_pricing_field("input_cost_per_token_above_32k_tokens") + assert not _is_custom_pricing_field("api_key") + assert not _is_custom_pricing_field("input_cost_per_token_above_32k_tokens_extra") + + def test_copy_custom_pricing_fields_preserves_declared_and_arbitrary_threshold_fields(): """Copy supported pricing fields without copying unrelated LiteLLM params.""" model_info = {} From 5dd86bc7591694e2f846aaad34c4ed7fa1134c69 Mon Sep 17 00:00:00 2001 From: LH-kevin Date: Tue, 18 Aug 2026 18:32:13 +0800 Subject: [PATCH 08/14] fix(router): restrict custom pricing threshold fields --- litellm/router.py | 9 +- .../test_router_model_cost_isolation.py | 98 +++++++++++++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index 8b4ded108a0..226811f25ed 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -280,7 +280,14 @@ def _cost_value_as_float(value: str | float | None) -> float | None: return None -_CUSTOM_PRICING_THRESHOLD_COST_KEY: Final[re.Pattern[str]] = re.compile(r"_above_\d+k?_tokens$") +_CUSTOM_PRICING_THRESHOLD_COST_KEY: Final[re.Pattern[str]] = re.compile( + r"^(?:" + r"input_cost_per_token|" + r"output_cost_per_token|" + r"cache_creation_input_token_cost(?:_above_1hr)?|" + r"cache_read_input_token_cost" + r")_above_\d+k?_tokens$" +) def _is_custom_pricing_field(field: str) -> bool: diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index d935734cdbd..bd2c07ba84e 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -83,6 +83,54 @@ def test_copy_custom_pricing_fields_preserves_declared_and_arbitrary_threshold_f } +def test_is_custom_pricing_field_accepts_supported_threshold_base_keys(): + """Every threshold base key the cost calculator supports stays accepted.""" + for field in ( + "input_cost_per_token_above_32k_tokens", + "output_cost_per_token_above_32k_tokens", + "cache_read_input_token_cost_above_32k_tokens", + "cache_creation_input_token_cost_above_32k_tokens", + "cache_creation_input_token_cost_above_1hr_above_32k_tokens", + ): + assert _is_custom_pricing_field(field), field + + +def test_is_custom_pricing_field_rejects_non_pricing_threshold_like_params(): + """Deployment params that merely look like *_above__tokens are not pricing.""" + for field in ( + "api_key_above_32k_tokens", + "secret_above_32k_tokens", + "credential_above_32k_tokens", + "custom_provider_param_above_32k_tokens", + ): + assert not _is_custom_pricing_field(field), field + + +def test_copy_custom_pricing_fields_ignores_non_pricing_threshold_like_params(): + """Only supported pricing fields are copied; threshold-like deployment params stay out of model_info.""" + model_info = {} + litellm_params = LiteLLM_Params( + model="openai/gpt-4o-mini", + api_key="fake-key", + input_cost_per_token=1e-6, + input_cost_per_token_above_32k_tokens=9e-6, + api_key_above_32k_tokens="leaked", + secret_above_32k_tokens="leaked", + credential_above_32k_tokens="leaked", + custom_provider_param_above_32k_tokens="leaked", + ) + + _copy_custom_pricing_fields( + model_info=model_info, + litellm_params=litellm_params, + ) + + assert model_info == { + "input_cost_per_token": 1e-6, + "input_cost_per_token_above_32k_tokens": 9e-6, + } + + def test_should_not_pollute_shared_key_with_zero_cost_pricing(): """ When deployment A has input_cost_per_token=0 and deployment B has no @@ -1574,6 +1622,56 @@ def test_router_registers_arbitrary_above_threshold_pricing_from_litellm_params( _restore_model_cost_entries(original_entries) +def test_router_registration_never_registers_non_pricing_threshold_like_params(): + """Non-pricing *_above__tokens deployment params must not reach litellm.model_cost.""" + backend_model = "openai/gpt-4o-mini" + model_id = "router-rejects-non-pricing-above-32k-34504" + shared_keys = ("gpt-4o-mini", backend_model) + + original_entries = { + key: copy.deepcopy(litellm.model_cost.get(key)) + for key in (*shared_keys, model_id) + } + + try: + Router( + model_list=[ + { + "model_name": "custom-above-32k-model", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_token_above_32k_tokens": 9e-6, + "api_key_above_32k_tokens": "leaked", + "secret_above_32k_tokens": "leaked", + "credential_above_32k_tokens": "leaked", + "custom_provider_param_above_32k_tokens": "leaked", + }, + "model_info": { + "id": model_id, + }, + } + ] + ) + + registered = litellm.model_cost.get(model_id) + assert registered is not None + + assert registered["input_cost_per_token"] == 1e-6 + assert registered["output_cost_per_token"] == 2e-6 + assert registered["input_cost_per_token_above_32k_tokens"] == 9e-6 + for junk in ( + "api_key_above_32k_tokens", + "secret_above_32k_tokens", + "credential_above_32k_tokens", + "custom_provider_param_above_32k_tokens", + ): + assert junk not in registered, junk + finally: + _restore_model_cost_entries(original_entries) + def test_add_deployment_registers_arbitrary_above_threshold_pricing(): """Dynamic add_deployment must preserve arbitrary pricing thresholds.""" From e79ab817fa593e3af843660f679749ce04a10638 Mon Sep 17 00:00:00 2001 From: LH-kevin Date: Fri, 21 Aug 2026 19:12:43 +0800 Subject: [PATCH 09/14] fix(cost): zero custom threshold rates for PTU deployments --- litellm/litellm_core_utils/ptu_pricing.py | 18 ++++++++ .../litellm_core_utils/test_ptu_pricing.py | 43 +++++++++++++++++++ .../test_router_model_cost_isolation.py | 21 ++++++++- 3 files changed, 80 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/ptu_pricing.py b/litellm/litellm_core_utils/ptu_pricing.py index 6923e6beb96..b599beb2dee 100644 --- a/litellm/litellm_core_utils/ptu_pricing.py +++ b/litellm/litellm_core_utils/ptu_pricing.py @@ -6,6 +6,7 @@ together because they have to agree: a deployment the rollup declines to charge router prices at zero serves its traffic for free. """ +import re from collections.abc import Mapping from dataclasses import dataclass from datetime import datetime, timezone @@ -42,6 +43,19 @@ SEARCH_CONTEXT_SIZES: Final = ("search_context_size_low", "search_context_size_m # and zeroing one of those would destroy the deployment's configuration rather than stop a # charge. CUSTOM_PRICING_FIELDS: Final = frozenset(f for f in CustomPricingLiteLLMParams.model_fields if "cost" in f) +# Custom threshold rates the router copies from litellm_params into the cost map (e.g. +# input_cost_per_token_above_32k_tokens) are not enumerated in CustomPricingLiteLLMParams, +# so they are matched here and zeroed like every other declared rate. Only the cost +# calculator's base keys qualify, so a param that merely ends in _above_k_tokens (e.g. +# api_key_above_32k_tokens) is not treated as a charge. +_THRESHOLD_RATE_KEY: Final[re.Pattern[str]] = re.compile( + r"^(?:" + r"input_cost_per_token|" + r"output_cost_per_token|" + r"cache_creation_input_token_cost(?:_above_1hr)?|" + r"cache_read_input_token_cost" + r")_above_\d+k?_tokens$" +) PTU_ZEROED_PRICING: Final[Mapping[str, float | tuple[()] | Mapping[str, float]]] = MappingProxyType( { **dict.fromkeys(PTU_ZEROED_PRICING_FIELDS, 0.0), @@ -175,6 +189,9 @@ def zeroed_ptu_pricing( return None if not is_ptu_cost_attribution_enabled(): return None + declared_threshold_rates: Final = frozenset( + key for key in declared if _THRESHOLD_RATE_KEY.fullmatch(key) is not None + ) return MappingProxyType( { **PTU_ZEROED_PRICING, @@ -184,5 +201,6 @@ def zeroed_ptu_pricing( .difference(PTU_EMPTIED_PRICING_FIELDS), 0.0, ), + **dict.fromkeys(declared_threshold_rates, 0.0), } ) diff --git a/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py b/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py index b953bfaa565..673599b0e82 100644 --- a/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py +++ b/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py @@ -155,6 +155,49 @@ def test_a_rate_the_deployment_declares_itself_is_zeroed_too(): assert override[extra] == 0.0 +@pytest.mark.parametrize( + "threshold_field", + [ + "input_cost_per_token_above_32k_tokens", + "output_cost_per_token_above_32k_tokens", + "cache_read_input_token_cost_above_32k_tokens", + "cache_creation_input_token_cost_above_32k_tokens", + "cache_creation_input_token_cost_above_1hr_above_32k_tokens", + ], +) +def test_arbitrary_threshold_rate_the_deployment_declares_is_zeroed_too(threshold_field): + """Threshold rates a deployment can declare but CustomPricingLiteLLMParams does not + enumerate (e.g. input_cost_per_token_above_32k_tokens) must be zeroed like any other + declared rate, or a PTU deployment bills per token past the threshold on top of the + hourly charge.""" + assert threshold_field not in CUSTOM_PRICING_FIELDS + assert threshold_field not in PTU_ZEROED_PRICING_FIELDS + + override = _with_flag(_VALID, declared={threshold_field: 9e-06}) + + assert override is not None + assert override.get(threshold_field, 9e-06) == 0.0 + + +@pytest.mark.parametrize( + "param_field", + [ + "api_key_above_32k_tokens", + "secret_above_32k_tokens", + "credential_above_32k_tokens", + "custom_provider_param_above_32k_tokens", + ], +) +def test_threshold_like_non_pricing_params_are_left_alone(param_field): + """The threshold matcher must only cover the cost calculator's pricing base keys: a + deployment param that merely ends in _above_k_tokens is not a charge and must keep + its value.""" + override = _with_flag(_VALID, declared={param_field: "keep-me"}) + + assert override is not None + assert param_field not in override + + def test_a_setting_that_is_not_a_charge_is_left_alone(): """CustomPricingLiteLLMParams also carries configuration, and zeroing one of those would break the deployment rather than stop a charge.""" diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index 7aace172611..20c4facaab2 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -1825,8 +1825,7 @@ def test_price_data_reload_preserves_arbitrary_above_threshold_pricing(monkeypat assert rebuilt["cache_read_input_token_cost_above_32k_tokens"] == 4e-6 assert router.model_list finally: - litellm.model_cost = saved_model_cost - _invalidate_model_cost_lowercase_map() + _restore_model_cost_entries(saved_model_cost) def test_strategy_router_alias_pricing_never_enters_model_cost(monkeypatch): @@ -1985,6 +1984,24 @@ def test_a_config_ptu_deployment_bills_nothing_per_token(): assert litellm.model_cost[entry["model_info"]["id"]]["input_cost_per_token"] == 0.0 +def test_a_config_ptu_deployment_zeroes_custom_threshold_rates(): + """A custom threshold rate the deployment declares (e.g. + input_cost_per_token_above_32k_tokens) must be zeroed like every other declared rate, + or the PTU deployment bills per token past the threshold on top of the hourly charge.""" + router = _ptu_router( + litellm_params={ + "input_cost_per_token": 5e-06, + "input_cost_per_token_above_32k_tokens": 9e-06, + } + ) + entry = router.model_list[0] + + assert entry["litellm_params"]["input_cost_per_token"] == 0.0 + assert entry["litellm_params"]["input_cost_per_token_above_32k_tokens"] == 0.0 + assert litellm.model_cost[entry["model_info"]["id"]]["input_cost_per_token"] == 0.0 + assert litellm.model_cost[entry["model_info"]["id"]]["input_cost_per_token_above_32k_tokens"] == 0.0 + + @pytest.mark.parametrize( "backend", ["anthropic/claude-sonnet-4-5-20250929", "azure/gpt-4o", "gemini/gemini-2.5-flash"], From 9cb966ef78b144605e041c7506d6afee4ebca15e Mon Sep 17 00:00:00 2001 From: LH-kevin Date: Sat, 22 Aug 2026 01:25:25 +0800 Subject: [PATCH 10/14] fix(cost): enforce threshold pricing for DB-backed PTU --- litellm/litellm_core_utils/ptu_pricing.py | 11 ++- .../model_management_endpoints.py | 15 +++- .../litellm_core_utils/test_ptu_pricing.py | 31 ++++++++ .../test_ptu_model_settings.py | 72 +++++++++++++++++++ 4 files changed, 124 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/ptu_pricing.py b/litellm/litellm_core_utils/ptu_pricing.py index b599beb2dee..5df0870ac3c 100644 --- a/litellm/litellm_core_utils/ptu_pricing.py +++ b/litellm/litellm_core_utils/ptu_pricing.py @@ -56,6 +56,13 @@ _THRESHOLD_RATE_KEY: Final[re.Pattern[str]] = re.compile( r"cache_read_input_token_cost" r")_above_\d+k?_tokens$" ) + + +def is_threshold_rate_key(field: str) -> bool: + """Whether ``field`` is a custom above-threshold rate key the cost calculator reads.""" + return _THRESHOLD_RATE_KEY.fullmatch(field) is not None + + PTU_ZEROED_PRICING: Final[Mapping[str, float | tuple[()] | Mapping[str, float]]] = MappingProxyType( { **dict.fromkeys(PTU_ZEROED_PRICING_FIELDS, 0.0), @@ -189,9 +196,7 @@ def zeroed_ptu_pricing( return None if not is_ptu_cost_attribution_enabled(): return None - declared_threshold_rates: Final = frozenset( - key for key in declared if _THRESHOLD_RATE_KEY.fullmatch(key) is not None - ) + declared_threshold_rates: Final = frozenset(key for key in declared if is_threshold_rate_key(key)) return MappingProxyType( { **PTU_ZEROED_PRICING, diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index ec98d7d65f1..3a46aedf09d 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -30,6 +30,7 @@ from litellm.litellm_core_utils.ptu_pricing import ( PTU_ZEROED_PRICING_FIELDS, PTU_ZEROED_TABLE_FIELDS, SEARCH_CONTEXT_SIZES, + is_threshold_rate_key, ptu_config_error, ) from litellm.proxy._types import ( @@ -379,6 +380,9 @@ def _raise_if_ptu_deployment_is_priced(*, model_info: Mapping[str, object], supp sorted( tuple(field for field in _CUSTOM_PRICING_FIELDS if _is_nonzero_price(supplied.get(field))) + tuple(field for field in _PTU_EMPTIED_PRICING_FIELDS if supplied.get(field)) + + tuple( + field for field in supplied if is_threshold_rate_key(field) and _is_nonzero_price(supplied.get(field)) + ) ) ) if not priced: @@ -418,9 +422,12 @@ def _ptu_zeroed_pricing( if model_info.get("ptu_count") is None or model_info.get("cost_per_ptu_per_hour") is None: return _NO_PRICING_OVERRIDE _raise_if_ptu_deployment_is_priced(model_info=model_info, supplied=supplied) + candidates: Final = _CUSTOM_PRICING_FIELDS | frozenset( + field for field in (*model_info.keys(), *litellm_params.keys()) if is_threshold_rate_key(field) + ) stored: Final = frozenset( field - for field in _CUSTOM_PRICING_FIELDS + for field in candidates if _is_nonzero_price(model_info.get(field)) or _is_nonzero_price(litellm_params.get(field)) ) return MappingProxyType( @@ -460,7 +467,11 @@ def _ptu_pricing_delta( return _NO_PRICING_OVERRIDE, frozenset() return _NO_PRICING_OVERRIDE, frozenset( field - for field in _CUSTOM_PRICING_FIELDS.union(_PTU_ZEROED_PRICING_FIELDS, _PTU_EMPTIED_PRICING_FIELDS) + for field in _CUSTOM_PRICING_FIELDS.union( + _PTU_ZEROED_PRICING_FIELDS, + _PTU_EMPTIED_PRICING_FIELDS, + frozenset(field for field in (*model_info.keys(), *litellm_params.keys()) if is_threshold_rate_key(field)), + ) if _is_zero_price(model_info.get(field)) or _is_zero_price(litellm_params.get(field)) ) diff --git a/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py b/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py index 673599b0e82..0aa3ce9e62f 100644 --- a/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py +++ b/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py @@ -7,6 +7,7 @@ from unittest.mock import patch import pytest from litellm.litellm_core_utils.ptu_pricing import ( + is_threshold_rate_key, ptu_config_error, CUSTOM_PRICING_FIELDS, PTU_EMPTIED_PRICING_FIELDS, @@ -198,6 +199,36 @@ def test_threshold_like_non_pricing_params_are_left_alone(param_field): assert param_field not in override +@pytest.mark.parametrize( + "field", + [ + "input_cost_per_token_above_32k_tokens", + "output_cost_per_token_above_64k_tokens", + "cache_read_input_token_cost_above_128k_tokens", + "cache_creation_input_token_cost_above_200k_tokens", + "cache_creation_input_token_cost_above_1hr_above_200k_tokens", + ], +) +def test_is_threshold_rate_key_accepts_supported_threshold_rates(field): + assert is_threshold_rate_key(field) + + +@pytest.mark.parametrize( + "field", + [ + "api_key_above_32k_tokens", + "secret_above_32k_tokens", + "credential_above_32k_tokens", + "custom_provider_param_above_32k_tokens", + "input_cost_per_token_above_32k_tokens_priority", + "input_cost_per_token", + "tiered_pricing", + ], +) +def test_is_threshold_rate_key_rejects_non_pricing_fields(field): + assert not is_threshold_rate_key(field) + + def test_a_setting_that_is_not_a_charge_is_left_alone(): """CustomPricingLiteLLMParams also carries configuration, and zeroing one of those would break the deployment rather than stop a charge.""" diff --git a/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py b/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py index a1c38d26b9d..1df6ec1dd29 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py @@ -794,6 +794,15 @@ class TestPtuDeploymentsAreNotBilledPerToken: assert exc.value.status_code == 400 assert "tiered_pricing" in str(exc.value.detail) + @pytest.mark.parametrize("field", ["input_cost_per_token_above_32k_tokens", "output_cost_per_token_above_32k_tokens"]) + def test_an_arbitrary_threshold_rate_the_caller_supplies_is_refused(self, field): + """The router lets deployments declare arbitrary above-threshold rates; those bill a + PTU deployment past the threshold just as surely as a flat rate, so the refusal must + cover them too.""" + with pytest.raises(HTTPException) as exc: + self._zeroed(model_info=self.PTU, supplied={field: 9e-06}) + assert exc.value.status_code == 400 + assert field in str(exc.value.detail) def test_a_search_context_price_the_caller_supplies_is_refused(self): """The rates sit in a table keyed by context size, so a guard that only reads numbers lets a per-request charge onto a deployment its reserved capacity already pays for.""" @@ -880,6 +889,12 @@ class TestPtuDeploymentsAreNotBilledPerToken: assert zeroed["input_cost_per_second"] == 0 assert zeroed["input_cost_per_token"] == 0 + @pytest.mark.parametrize("field", ["input_cost_per_token_above_32k_tokens", "output_cost_per_token_above_32k_tokens"]) + def test_an_arbitrary_threshold_rate_already_on_the_row_is_zeroed(self, field): + """A row priced through a path this rule does not cover must heal on its next save.""" + zeroed = self._zeroed(model_info={**self.PTU, field: 9e-06}, litellm_params={}) + assert zeroed[field] == 0 + assert zeroed["input_cost_per_token"] == 0 @pytest.mark.asyncio async def test_a_refused_price_does_not_leave_the_team_changed(self): """The team ACL write autocommits, so the refusal has to run before it. Otherwise a @@ -944,6 +959,32 @@ class TestPtuDeploymentsAreNotBilledPerToken: assert priced.litellm_params.get("regional_processing_uplift_multiplier_eu") == 1.15 assert priced.litellm_params.get("input_cost_per_token") == 0 + @pytest.mark.parametrize( + "param", + [ + "api_key_above_32k_tokens", + "secret_above_32k_tokens", + "credential_above_32k_tokens", + "custom_provider_param_above_32k_tokens", + ], + ) + def test_a_threshold_like_setting_that_is_not_a_price_is_left_alone(self, param): + """Only the cost calculator's base keys qualify as threshold rates: a deployment param + that merely ends in _above_k_tokens is not a charge and must keep its value.""" + priced = _ptu_priced_deployment( + Deployment( + model_name="settings", + litellm_params=LiteLLM_Params(model="openai/gpt-4o-mini", **{param: "keep-me"}), + model_info=ModelInfo( + id="dep-settings", + team_id="t", + ptu_effective_from=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc), + **self.PTU, + ), + ) + ) + assert priced.litellm_params.get(param) == "keep-me" + assert priced.litellm_params.get("input_cost_per_token") == 0 def test_removing_ptu_config_releases_every_rate_it_zeroed(self): """The zeroing covers any stored rate, so a release that only spans the mirrored fields leaves a per-second deployment billing nothing for that dimension forever.""" @@ -976,6 +1017,37 @@ class TestPtuDeploymentsAreNotBilledPerToken: ) assert "input_cost_per_second" not in json.loads(off["litellm_params"]) + def test_removing_ptu_config_releases_a_zeroed_threshold_rate(self): + """The zeroing spans threshold rates too, so a release that only spans the enumeration + sets would leave one billing nothing past the threshold forever.""" + on = update_db_model( + db_model=Deployment( + model_name="threshold", + litellm_params=LiteLLM_Params(model="openai/gpt-4o-mini", input_cost_per_token_above_32k_tokens=9e-06), + model_info=ModelInfo(id="dep-thr", team_id="t"), + ), + updated_patch=updateDeployment( + model_info=ModelInfo( + id="dep-thr", + team_id="t", + ptu_effective_from=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc), + **self.PTU, + ) + ), + ) + assert json.loads(on["litellm_params"])["input_cost_per_token_above_32k_tokens"] == 0 + + off = update_db_model( + db_model=Deployment( + model_name="threshold", + litellm_params=LiteLLM_Params(**json.loads(on["litellm_params"])), + model_info=ModelInfo(**json.loads(on["model_info"])), + ), + updated_patch=updateDeployment( + model_info=ModelInfo(id="dep-thr", ptu_count=None, cost_per_ptu_per_hour=None) + ), + ) + assert "input_cost_per_token_above_32k_tokens" not in json.loads(off["litellm_params"]) def test_removing_ptu_config_releases_a_zeroed_search_context_table(self): """The all-zero table exists only to stop the double charge, so a deployment taken off PTU has to give it up or it keeps serving grounded requests for free forever.""" From c68da92debe5c38843aa5fa14bcf3b0112f29dfc Mon Sep 17 00:00:00 2001 From: LH-kevin Date: Sat, 22 Aug 2026 02:01:23 +0800 Subject: [PATCH 11/14] fix(cost): support custom service-tier thresholds --- .../litellm_core_utils/llm_cost_calc/utils.py | 20 +++-- litellm/router.py | 2 +- .../test_threshold_cost_selection.py | 75 +++++++++++++++++++ .../test_router_model_cost_isolation.py | 35 +++++++++ 4 files changed, 123 insertions(+), 9 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 6c48381f0e0..3c382f29755 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -55,7 +55,7 @@ _ABOVE_TOKEN_THRESHOLD_COST_KEY: Final[re.Pattern[str]] = re.compile( r"output_cost_per_token|" r"cache_creation_input_token_cost(?:_above_1hr)?|" r"cache_read_input_token_cost" - r")_above_\d+k?_tokens$" + r")_above_\d+k?_tokens(?:_(?Ppriority|flex|ultrafast))?$" ) _SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType( @@ -323,16 +323,20 @@ def _get_token_base_cost( cache_read_cost = cast(float, _get_cost_per_unit(model_info, cache_read_cost_key)) ## CHECK IF ABOVE THRESHOLD - selected_threshold_keys: dict[str, tuple[float, str]] = {} # mutable-ok: threshold accumulator + selected_threshold_keys: dict[str, tuple[float, str, str | None]] = {} # mutable-ok: threshold accumulator for key in model_info: - if "_above_" not in key or not key.endswith("_tokens"): + if "_above_" not in key or not key.endswith(("_tokens", "_priority", "_flex", "_ultrafast")): continue match = _ABOVE_TOKEN_THRESHOLD_COST_KEY.fullmatch(key) if match is None or model_info.get(key) is None: continue + threshold_key_tier = match.group("tier") + if threshold_key_tier is not None and threshold_key_tier != service_tier: + continue + try: threshold = _parse_above_token_threshold(key) except (IndexError, ValueError): @@ -348,6 +352,7 @@ def _get_token_base_cost( selected_threshold_keys[base_key] = ( threshold, key, + threshold_key_tier, ) costs_by_base_key = { # mutable-ok: selected-rate updates @@ -358,16 +363,15 @@ def _get_token_base_cost( "cache_read_input_token_cost": cache_read_cost, } - for base_key, (_, threshold_key) in selected_threshold_keys.items(): - tiered_key = _get_service_tier_cost_key( - threshold_key, - service_tier, + for base_key, (_, threshold_key, threshold_key_tier) in selected_threshold_keys.items(): + cost_key = ( + threshold_key if threshold_key_tier is not None else _get_service_tier_cost_key(threshold_key, service_tier) ) costs_by_base_key[base_key] = cast( float, _get_cost_per_unit( model_info, - tiered_key, + cost_key, costs_by_base_key[base_key], ), ) diff --git a/litellm/router.py b/litellm/router.py index 13eb0038dde..56ff872d2ed 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -293,7 +293,7 @@ _CUSTOM_PRICING_THRESHOLD_COST_KEY: Final[re.Pattern[str]] = re.compile( r"output_cost_per_token|" r"cache_creation_input_token_cost(?:_above_1hr)?|" r"cache_read_input_token_cost" - r")_above_\d+k?_tokens$" + r")_above_\d+k?_tokens(?:_(?:priority|flex|ultrafast))?$" ) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py index af893cccc49..5f4e1490b92 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py @@ -68,3 +68,78 @@ def test_cost_types_select_thresholds_independently() -> None: assert rates[0] == pytest.approx(1e-6) assert rates[1] == pytest.approx(18e-6) + + +@pytest.mark.parametrize( + ("tier_field", "tier_value", "rate_index"), + [ + ("output_cost_per_token_above_200k_tokens_priority", 1.5e-6, 1), + ("cache_read_input_token_cost_above_200k_tokens_priority", 4e-7, 4), + ], +) +def test_tier_qualified_threshold_field_selects_tier_for_matching_service_tier( + tier_field: str, tier_value: float, rate_index: int +) -> None: + """A tier-qualified threshold key with no standard sibling must apply when the request's + service tier matches.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_read_input_token_cost": 5e-7, + tier_field: tier_value, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier="priority", + ) + + assert rates[rate_index] == pytest.approx(tier_value) + + +def test_tier_qualified_threshold_is_ignored_under_the_default_tier() -> None: + """service_tier=None must keep billing the flat rate: the priority key is inactive.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost(model_info=model_info, usage=usage) + + assert rates[1] == pytest.approx(2e-6) + + +def test_tier_qualified_threshold_is_ignored_under_a_different_service_tier() -> None: + """A priority-qualified key must not bill a flex request.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier="flex", + ) + + assert rates[1] == pytest.approx(2e-6) diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index 20c4facaab2..8824eb4509e 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -73,6 +73,41 @@ def test_is_custom_pricing_field_recognizes_declared_and_arbitrary_threshold_fie assert not _is_custom_pricing_field("input_cost_per_token_above_32k_tokens_extra") +def test_is_custom_pricing_field_recognizes_tier_qualified_threshold_fields(): + assert _is_custom_pricing_field("input_cost_per_token_above_200k_tokens_priority") + assert _is_custom_pricing_field("output_cost_per_token_above_200k_tokens_flex") + assert _is_custom_pricing_field("cache_read_input_token_cost_above_200k_tokens_priority") + assert _is_custom_pricing_field("cache_creation_input_token_cost_above_1hr_above_200k_tokens_ultrafast") + assert not _is_custom_pricing_field("api_key_above_32k_tokens_priority") + assert not _is_custom_pricing_field("secret_above_32k_tokens_flex") + assert not _is_custom_pricing_field("credential_above_200k_tokens_ultrafast") + assert not _is_custom_pricing_field("custom_provider_param_above_32k_tokens_priority") + + +def test_custom_tier_qualified_threshold_key_is_registered(): + """A deployment declaring only a tier-qualified threshold rate (e.g. + output_cost_per_token_above_200k_tokens_priority, which is not enumerated in + CustomPricingLiteLLMParams) must keep it in its model_cost entry so matching-tier + requests bill it.""" + model_id = "custom-tier-only-threshold" + Router( + model_list=[ + { + "model_name": "tier-only-model", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "fake-key", + "input_cost_per_token": 1e-6, + "output_cost_per_token_above_32k_tokens_priority": 1.5e-6, + }, + "model_info": {"id": model_id}, + } + ] + ) + + registered = litellm.model_cost[model_id] + assert registered["output_cost_per_token_above_32k_tokens_priority"] == 1.5e-6 + def test_copy_custom_pricing_fields_preserves_declared_and_arbitrary_threshold_fields(): """Copy supported pricing fields without copying unrelated LiteLLM params.""" model_info = {} From c04a8b7014edc09c780322d3418f14a0b0473739 Mon Sep 17 00:00:00 2001 From: LH-kevin Date: Sat, 22 Aug 2026 02:50:12 +0800 Subject: [PATCH 12/14] fix(cost): enforce PTU zeroing for tier thresholds --- litellm/litellm_core_utils/ptu_pricing.py | 2 +- .../litellm_core_utils/test_ptu_pricing.py | 38 ++++++++++++- .../test_ptu_model_settings.py | 56 ++++++++++++++++++- 3 files changed, 92 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/ptu_pricing.py b/litellm/litellm_core_utils/ptu_pricing.py index 5df0870ac3c..0ec4a6bf3ff 100644 --- a/litellm/litellm_core_utils/ptu_pricing.py +++ b/litellm/litellm_core_utils/ptu_pricing.py @@ -54,7 +54,7 @@ _THRESHOLD_RATE_KEY: Final[re.Pattern[str]] = re.compile( r"output_cost_per_token|" r"cache_creation_input_token_cost(?:_above_1hr)?|" r"cache_read_input_token_cost" - r")_above_\d+k?_tokens$" + r")_above_\d+k?_tokens(?:_(?:priority|flex|ultrafast))?$" ) diff --git a/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py b/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py index 0aa3ce9e62f..3faa8b63cd8 100644 --- a/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py +++ b/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py @@ -180,6 +180,25 @@ def test_arbitrary_threshold_rate_the_deployment_declares_is_zeroed_too(threshol assert override.get(threshold_field, 9e-06) == 0.0 +@pytest.mark.parametrize( + "threshold_field", + [ + "output_cost_per_token_above_32k_tokens_priority", + "cache_read_input_token_cost_above_32k_tokens_priority", + ], +) +def test_service_tier_qualified_threshold_rate_declared_is_zeroed_too(threshold_field): + """A tier-qualified threshold rate the deployment declares must be zeroed like every + other declared rate, or a PTU deployment bills per token past the threshold.""" + assert threshold_field not in CUSTOM_PRICING_FIELDS + assert threshold_field not in PTU_ZEROED_PRICING_FIELDS + + override = _with_flag(_VALID, declared={threshold_field: 9e-06}) + + assert override is not None + assert override.get(threshold_field, 9e-06) == 0.0 + + @pytest.mark.parametrize( "param_field", [ @@ -213,6 +232,20 @@ def test_is_threshold_rate_key_accepts_supported_threshold_rates(field): assert is_threshold_rate_key(field) +@pytest.mark.parametrize( + "field", + [ + "input_cost_per_token_above_32k_tokens_priority", + "output_cost_per_token_above_32k_tokens_flex", + "cache_read_input_token_cost_above_200k_tokens_priority", + "cache_creation_input_token_cost_above_32k_tokens_ultrafast", + "cache_creation_input_token_cost_above_1hr_above_200k_tokens_priority", + ], +) +def test_is_threshold_rate_key_accepts_service_tier_qualified_threshold_rates(field): + assert is_threshold_rate_key(field) + + @pytest.mark.parametrize( "field", [ @@ -220,7 +253,10 @@ def test_is_threshold_rate_key_accepts_supported_threshold_rates(field): "secret_above_32k_tokens", "credential_above_32k_tokens", "custom_provider_param_above_32k_tokens", - "input_cost_per_token_above_32k_tokens_priority", + "api_key_above_32k_tokens_priority", + "secret_above_32k_tokens_flex", + "credential_above_200k_tokens_ultrafast", + "custom_provider_param_above_32k_tokens_priority", "input_cost_per_token", "tiered_pricing", ], diff --git a/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py b/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py index 1df6ec1dd29..8a3ddd0eae7 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py @@ -794,7 +794,14 @@ class TestPtuDeploymentsAreNotBilledPerToken: assert exc.value.status_code == 400 assert "tiered_pricing" in str(exc.value.detail) - @pytest.mark.parametrize("field", ["input_cost_per_token_above_32k_tokens", "output_cost_per_token_above_32k_tokens"]) + @pytest.mark.parametrize( + "field", + [ + "input_cost_per_token_above_32k_tokens", + "output_cost_per_token_above_32k_tokens", + "input_cost_per_token_above_32k_tokens_priority", + ], + ) def test_an_arbitrary_threshold_rate_the_caller_supplies_is_refused(self, field): """The router lets deployments declare arbitrary above-threshold rates; those bill a PTU deployment past the threshold just as surely as a flat rate, so the refusal must @@ -889,7 +896,14 @@ class TestPtuDeploymentsAreNotBilledPerToken: assert zeroed["input_cost_per_second"] == 0 assert zeroed["input_cost_per_token"] == 0 - @pytest.mark.parametrize("field", ["input_cost_per_token_above_32k_tokens", "output_cost_per_token_above_32k_tokens"]) + @pytest.mark.parametrize( + "field", + [ + "input_cost_per_token_above_32k_tokens", + "output_cost_per_token_above_32k_tokens", + "output_cost_per_token_above_32k_tokens_priority", + ], + ) def test_an_arbitrary_threshold_rate_already_on_the_row_is_zeroed(self, field): """A row priced through a path this rule does not cover must heal on its next save.""" zeroed = self._zeroed(model_info={**self.PTU, field: 9e-06}, litellm_params={}) @@ -966,6 +980,8 @@ class TestPtuDeploymentsAreNotBilledPerToken: "secret_above_32k_tokens", "credential_above_32k_tokens", "custom_provider_param_above_32k_tokens", + "api_key_above_32k_tokens_priority", + "custom_provider_param_above_32k_tokens_flex", ], ) def test_a_threshold_like_setting_that_is_not_a_price_is_left_alone(self, param): @@ -1048,6 +1064,42 @@ class TestPtuDeploymentsAreNotBilledPerToken: ), ) assert "input_cost_per_token_above_32k_tokens" not in json.loads(off["litellm_params"]) + + def test_removing_ptu_config_releases_a_zeroed_tier_qualified_threshold_rate(self): + """The zeroing spans tier-qualified threshold rates too, so a release that only + spans the enumeration sets would leave one billing nothing past the threshold + forever.""" + on = update_db_model( + db_model=Deployment( + model_name="tier-threshold", + litellm_params=LiteLLM_Params( + model="openai/gpt-4o-mini", output_cost_per_token_above_32k_tokens_priority=9e-06 + ), + model_info=ModelInfo(id="dep-tier-thr", team_id="t"), + ), + updated_patch=updateDeployment( + model_info=ModelInfo( + id="dep-tier-thr", + team_id="t", + ptu_effective_from=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc), + **self.PTU, + ) + ), + ) + assert json.loads(on["litellm_params"])["output_cost_per_token_above_32k_tokens_priority"] == 0 + + off = update_db_model( + db_model=Deployment( + model_name="tier-threshold", + litellm_params=LiteLLM_Params(**json.loads(on["litellm_params"])), + model_info=ModelInfo(**json.loads(on["model_info"])), + ), + updated_patch=updateDeployment( + model_info=ModelInfo(id="dep-tier-thr", ptu_count=None, cost_per_ptu_per_hour=None) + ), + ) + assert "output_cost_per_token_above_32k_tokens_priority" not in json.loads(off["litellm_params"]) + def test_removing_ptu_config_releases_a_zeroed_search_context_table(self): """The all-zero table exists only to stop the double charge, so a deployment taken off PTU has to give it up or it keeps serving grounded requests for free forever.""" From 6271e1e7bddf59a4a95200eb69fd4cb79f5ada09 Mon Sep 17 00:00:00 2001 From: LH-kevin Date: Sat, 22 Aug 2026 15:54:21 +0800 Subject: [PATCH 13/14] fix(cost): prefer matching tier at equal threshold --- .../litellm_core_utils/llm_cost_calc/utils.py | 4 +- .../test_threshold_cost_selection.py | 127 ++++++++++++++++++ 2 files changed, 130 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 3c382f29755..1da57fb9356 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -347,8 +347,10 @@ def _get_token_base_cost( base_key = match.group("base_key") selected = selected_threshold_keys.get(base_key) + selected_specificity = 0 if selected is None or selected[2] is None else 1 + candidate_specificity = 0 if threshold_key_tier is None else 1 - if selected is None or threshold > selected[0]: + if selected is None or (threshold, candidate_specificity) > (selected[0], selected_specificity): selected_threshold_keys[base_key] = ( threshold, key, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py index 5f4e1490b92..fc322eee14e 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py @@ -70,6 +70,133 @@ def test_cost_types_select_thresholds_independently() -> None: assert rates[1] == pytest.approx(18e-6) +@pytest.mark.parametrize( + "model_info", + [ + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + }, + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + }, + ], + ids=["standard-first", "priority-first"], +) +def test_equal_threshold_matching_tier_wins_for_output(model_info): + """At an equal threshold the matching service-tier rate must win regardless of the + order the standard and tier-qualified keys appear in model_info.""" + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost(model_info=model_info, usage=usage, service_tier="priority") + + assert rates[1] == pytest.approx(1.5e-6) + + +@pytest.mark.parametrize( + "model_info", + [ + { + "input_cost_per_token": 1e-6, + "cache_read_input_token_cost": 2e-7, + "cache_read_input_token_cost_above_200k_tokens": 3e-7, + "cache_read_input_token_cost_above_200k_tokens_priority": 4e-7, + }, + { + "input_cost_per_token": 1e-6, + "cache_read_input_token_cost": 2e-7, + "cache_read_input_token_cost_above_200k_tokens_priority": 4e-7, + "cache_read_input_token_cost_above_200k_tokens": 3e-7, + }, + ], + ids=["standard-first", "priority-first"], +) +def test_equal_threshold_matching_tier_wins_for_cache_read(model_info): + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost(model_info=model_info, usage=usage, service_tier="priority") + + assert rates[4] == pytest.approx(4e-7) + + +def test_equal_threshold_default_tier_keeps_standard(): + """service_tier=None must ignore the tier-qualified key and keep the standard rate.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost(model_info=model_info, usage=usage) + + assert rates[1] == pytest.approx(1e-6) + + +def test_equal_threshold_wrong_tier_keeps_standard(): + """A priority-qualified key must not win under a different service tier.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier="flex", + ) + + assert rates[1] == pytest.approx(1e-6) + + +def test_unequal_threshold_highest_standard_still_wins(): + """A higher standard threshold governs over a lower matching-tier threshold.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + "output_cost_per_token_above_128k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier="priority", + ) + + assert rates[1] == pytest.approx(1e-6) + + @pytest.mark.parametrize( ("tier_field", "tier_value", "rate_index"), [ From 11937bf88dd6aef23bf392ca97a5740a9009a96b Mon Sep 17 00:00:00 2001 From: LH-kevin Date: Sat, 22 Aug 2026 21:05:28 +0800 Subject: [PATCH 14/14] fix(cost): apply fast pricing alias to thresholds --- .../litellm_core_utils/llm_cost_calc/utils.py | 5 +- .../test_threshold_cost_selection.py | 146 ++++++++++++++++++ 2 files changed, 150 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 1da57fb9356..868f3e49d07 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -334,7 +334,10 @@ def _get_token_base_cost( continue threshold_key_tier = match.group("tier") - if threshold_key_tier is not None and threshold_key_tier != service_tier: + active_pricing_tier = ( + None if service_tier is None else _SERVICE_TIER_TO_COST_KEY_SUFFIX.get(service_tier.lower()) + ) + if threshold_key_tier is not None and threshold_key_tier != active_pricing_tier: continue try: diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py index fc322eee14e..a2243f04b04 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py @@ -197,6 +197,152 @@ def test_unequal_threshold_highest_standard_still_wins(): assert rates[1] == pytest.approx(1e-6) +@pytest.mark.parametrize( + ("tier_field", "tier_value", "rate_index"), + [ + ("output_cost_per_token_above_200k_tokens_priority", 1.5e-6, 1), + ("cache_read_input_token_cost_above_200k_tokens_priority", 4e-7, 4), + ], +) +def test_fast_service_tier_uses_priority_qualified_threshold(tier_field, tier_value, rate_index): + """fast aliases to priority pricing, so a priority-qualified threshold with no standard + sibling must apply to a fast request too.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 1e-6, + "cache_read_input_token_cost": 2e-7, + tier_field: tier_value, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier="fast", + ) + + assert rates[rate_index] == pytest.approx(tier_value) + + +@pytest.mark.parametrize( + ("service_tier", "expected"), + [ + ("priority", 1.5e-6), + ("flex", 1e-6), + (None, 1e-6), + ], + ids=["priority-guard", "flex-guard", "default-guard"], +) +def test_fast_alias_guard_tiers_for_priority_only_output_threshold(service_tier, expected): + """priority must keep working under the alias; flex and default must not pick up the + priority rate.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier=service_tier, + ) + + assert rates[1] == pytest.approx(expected) + + +@pytest.mark.parametrize("service_tier", ["priority", "fast"], ids=["priority", "fast"]) +def test_fast_builtin_standard_plus_priority_threshold_resolves_priority(service_tier): + """The builtin shape (standard sibling present) keeps resolving the priority variant via + the existing standard-key alias lookup.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 1e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier=service_tier, + ) + + assert rates[1] == pytest.approx(1.5e-6) + + +@pytest.mark.parametrize( + "model_info", + [ + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 1e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + }, + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + }, + ], + ids=["standard-first", "priority-first"], +) +def test_fast_equal_threshold_matching_tier_wins_both_orders(model_info): + """The Phase D equal-threshold tie-break must hold under the fast alias, independent of + insertion order.""" + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier="fast", + ) + + assert rates[1] == pytest.approx(1.5e-6) + + +def test_fast_service_tier_uppercase_uses_priority_qualified_threshold(): + """The scanner must be as case-insensitive as the lookup path.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier="FAST", + ) + + assert rates[1] == pytest.approx(1.5e-6) + + @pytest.mark.parametrize( ("tier_field", "tier_value", "rate_index"), [