diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 9d782cf7a4d..32fb57c19c4 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1,6 +1,7 @@ # What is this? ## Helper utilities for cost_per_token() +import re from collections.abc import Mapping from dataclasses import dataclass from types import MappingProxyType @@ -48,6 +49,15 @@ _SERVICE_TIER_SUFFIXES: Final[tuple[str, ...]] = tuple( sorted((f"_{st.value}" for st in ServiceTier), key=len, reverse=True) ) +_ABOVE_TOKEN_THRESHOLD_COST_KEY: Final[re.Pattern[str]] = re.compile( + r"^(?P" + r"input_cost_per_token|" + r"output_cost_per_token|" + r"cache_creation_input_token_cost(?:_above_1hr)?|" + r"cache_read_input_token_cost" + r")_above_\d+k?_tokens(?:_(?Ppriority|flex|ultrafast))?$" +) + _SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType( { ServiceTier.FLEX.value: ServiceTier.FLEX.value, @@ -211,7 +221,7 @@ def _get_service_tier_cost_key(base_key: str, service_tier: str | None) -> str: def _parse_above_token_threshold(key: str) -> float: - threshold_str: Final = key.split("_above_")[1].split("_tokens")[0] + threshold_str: Final = key.rsplit("_above_", 1)[1].split("_tokens")[0] return float(threshold_str.replace("k", "")) * (1000 if "k" in threshold_str else 1) @@ -276,8 +286,9 @@ def _get_token_base_cost( """ Return prompt cost, completion cost, and cache costs for a given model and usage. - If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set, - then we use the corresponding threshold cost for all token types. + For each token cost type, use its highest configured + `*_above_[x]k_tokens` or `*_above_[x]_tokens` rate when prompt tokens + exceed that threshold. `threshold_is_inclusive` switches that comparison to >=, for providers such as xAI that bill the higher tier once the prompt reaches the threshold. @@ -312,127 +323,70 @@ def _get_token_base_cost( cache_read_cost = cast(float, _get_cost_per_unit(model_info, cache_read_cost_key)) ## CHECK IF ABOVE THRESHOLD - # Optimization: collect threshold keys first to avoid sorting all model_info keys. - # Most models don't have threshold pricing, so we can return early. - # Exclude service_tier-specific variants (e.g. input_cost_per_token_above_200k_tokens_priority) - # so that the threshold detection loop only processes standard keys. The - # service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key. - threshold_keys: Final = [ - k for k in model_info if k.startswith("input_cost_per_token_above_") and not k.endswith(_SERVICE_TIER_SUFFIXES) - ] - if not threshold_keys: - return ( - prompt_base_cost, - completion_base_cost, - cache_creation_cost, - cache_creation_cost_above_1hr, - cache_read_cost, + selected_threshold_keys: dict[str, tuple[float, str, str | None]] = {} # mutable-ok: threshold accumulator + + for key in model_info: + if "_above_" not in key or not key.endswith(("_tokens", "_priority", "_flex", "_ultrafast")): + continue + + match = _ABOVE_TOKEN_THRESHOLD_COST_KEY.fullmatch(key) + if match is None or model_info.get(key) is None: + continue + + threshold_key_tier = match.group("tier") + active_pricing_tier = ( + None if service_tier is None else _SERVICE_TIER_TO_COST_KEY_SUFFIX.get(service_tier.lower()) + ) + if threshold_key_tier is not None and threshold_key_tier != active_pricing_tier: + continue + + try: + threshold = _parse_above_token_threshold(key) + except (IndexError, ValueError): + continue + + if usage.prompt_tokens < threshold or (usage.prompt_tokens == threshold and not threshold_is_inclusive): + continue + + base_key = match.group("base_key") + selected = selected_threshold_keys.get(base_key) + selected_specificity = 0 if selected is None or selected[2] is None else 1 + candidate_specificity = 0 if threshold_key_tier is None else 1 + + if selected is None or (threshold, candidate_specificity) > (selected[0], selected_specificity): + selected_threshold_keys[base_key] = ( + threshold, + key, + threshold_key_tier, + ) + + costs_by_base_key = { # mutable-ok: selected-rate updates + "input_cost_per_token": prompt_base_cost, + "output_cost_per_token": completion_base_cost, + "cache_creation_input_token_cost": cache_creation_cost, + "cache_creation_input_token_cost_above_1hr": (cache_creation_cost_above_1hr), + "cache_read_input_token_cost": cache_read_cost, + } + + for base_key, (_, threshold_key, threshold_key_tier) in selected_threshold_keys.items(): + cost_key = ( + threshold_key if threshold_key_tier is not None else _get_service_tier_cost_key(threshold_key, service_tier) + ) + costs_by_base_key[base_key] = cast( + float, + _get_cost_per_unit( + model_info, + cost_key, + costs_by_base_key[base_key], + ), ) - # Only sort the threshold keys (typically 1-2 keys instead of 66+) - threshold: float | None = None - for key in sorted(threshold_keys, key=_parse_above_token_threshold, reverse=True): - value = model_info.get(key) - if value is not None: - try: - # Handle both formats: _above_128k_tokens and _above_128_tokens - threshold_str = key.split("_above_")[1].split("_tokens")[0] - threshold = _parse_above_token_threshold(key) - if usage.prompt_tokens > threshold or (threshold_is_inclusive and usage.prompt_tokens == threshold): - # Prefer a service_tier-specific above-threshold key when available, - # e.g. input_cost_per_token_priority_above_200k_tokens for Gemini - # ON_DEMAND_PRIORITY. Falls back to the standard key automatically - # via _get_cost_per_unit's service_tier fallback logic. - tiered_input_key = ( - _get_service_tier_cost_key( - f"input_cost_per_token_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else key - ) - prompt_base_cost = cast( - float, - _get_cost_per_unit(model_info, tiered_input_key, prompt_base_cost), - ) - tiered_output_key = ( - _get_service_tier_cost_key( - f"output_cost_per_token_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"output_cost_per_token_above_{threshold_str}_tokens" - ) - completion_base_cost = cast( - float, - _get_cost_per_unit( - model_info, - tiered_output_key, - completion_base_cost, - ), - ) - - # Apply tiered pricing to cache costs - cache_creation_tiered_key = ( - _get_service_tier_cost_key( - f"cache_creation_input_token_cost_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"cache_creation_input_token_cost_above_{threshold_str}_tokens" - ) - cache_creation_1hr_tiered_key = ( - _get_service_tier_cost_key( - f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens" - ) - cache_read_tiered_key = ( - _get_service_tier_cost_key( - f"cache_read_input_token_cost_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"cache_read_input_token_cost_above_{threshold_str}_tokens" - ) - - cache_creation_cost = cast( - float, - _get_cost_per_unit( - model_info, - cache_creation_tiered_key, - cache_creation_cost, - ), - ) - - cache_creation_cost_above_1hr = cast( - float, - _get_cost_per_unit( - model_info, - cache_creation_1hr_tiered_key, - cache_creation_cost_above_1hr, - ), - ) - - cache_read_cost = cast( - float, - _get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost), - ) - - break - except (IndexError, ValueError): - continue - except Exception: - continue - return ( - prompt_base_cost, - completion_base_cost, - cache_creation_cost, - cache_creation_cost_above_1hr, - cache_read_cost, + costs_by_base_key["input_cost_per_token"], + costs_by_base_key["output_cost_per_token"], + costs_by_base_key["cache_creation_input_token_cost"], + costs_by_base_key["cache_creation_input_token_cost_above_1hr"], + costs_by_base_key["cache_read_input_token_cost"], ) diff --git a/litellm/litellm_core_utils/ptu_pricing.py b/litellm/litellm_core_utils/ptu_pricing.py index f545ba4aa3b..1ef15b6ca4e 100644 --- a/litellm/litellm_core_utils/ptu_pricing.py +++ b/litellm/litellm_core_utils/ptu_pricing.py @@ -6,6 +6,7 @@ together because they have to agree: a deployment the rollup declines to charge router prices at zero serves its traffic for free. """ +import re from collections.abc import Mapping from dataclasses import dataclass from datetime import date, datetime, time, timezone @@ -43,6 +44,26 @@ SEARCH_CONTEXT_SIZES: Final = ("search_context_size_low", "search_context_size_m # and zeroing one of those would destroy the deployment's configuration rather than stop a # charge. CUSTOM_PRICING_FIELDS: Final = frozenset(f for f in CustomPricingLiteLLMParams.model_fields if "cost" in f) +# Custom threshold rates the router copies from litellm_params into the cost map (e.g. +# input_cost_per_token_above_32k_tokens) are not enumerated in CustomPricingLiteLLMParams, +# so they are matched here and zeroed like every other declared rate. Only the cost +# calculator's base keys qualify, so a param that merely ends in _above_k_tokens (e.g. +# api_key_above_32k_tokens) is not treated as a charge. +_THRESHOLD_RATE_KEY: Final[re.Pattern[str]] = re.compile( + r"^(?:" + r"input_cost_per_token|" + r"output_cost_per_token|" + r"cache_creation_input_token_cost(?:_above_1hr)?|" + r"cache_read_input_token_cost" + r")_above_\d+k?_tokens(?:_(?:priority|flex|ultrafast))?$" +) + + +def is_threshold_rate_key(field: str) -> bool: + """Whether ``field`` is a custom above-threshold rate key the cost calculator reads.""" + return _THRESHOLD_RATE_KEY.fullmatch(field) is not None + + PTU_ZEROED_PRICING: Final[Mapping[str, float | tuple[()] | Mapping[str, float]]] = MappingProxyType( { **dict.fromkeys(PTU_ZEROED_PRICING_FIELDS, 0.0), @@ -224,6 +245,7 @@ def zeroed_ptu_pricing( return None if not is_ptu_cost_attribution_enabled(): return None + declared_threshold_rates: Final = frozenset(key for key in declared if is_threshold_rate_key(key)) return MappingProxyType( { **PTU_ZEROED_PRICING, @@ -233,5 +255,6 @@ def zeroed_ptu_pricing( .difference(PTU_EMPTIED_PRICING_FIELDS), 0.0, ), + **dict.fromkeys(declared_threshold_rates, 0.0), } ) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 9ea0796b680..300fc52459c 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -31,6 +31,7 @@ from litellm.litellm_core_utils.ptu_pricing import ( PTU_ZEROED_PRICING_FIELDS, PTU_ZEROED_TABLE_FIELDS, SEARCH_CONTEXT_SIZES, + is_threshold_rate_key, ptu_config_error, ) from litellm.proxy._types import ( @@ -420,6 +421,9 @@ def _raise_if_ptu_deployment_is_priced(*, model_info: Mapping[str, object], supp sorted( tuple(field for field in _CUSTOM_PRICING_FIELDS if _is_nonzero_price(supplied.get(field))) + tuple(field for field in _PTU_EMPTIED_PRICING_FIELDS if supplied.get(field)) + + tuple( + field for field in supplied if is_threshold_rate_key(field) and _is_nonzero_price(supplied.get(field)) + ) ) ) if not priced: @@ -459,9 +463,12 @@ def _ptu_zeroed_pricing( if model_info.get("ptu_count") is None or model_info.get("cost_per_ptu_per_hour") is None: return _NO_PRICING_OVERRIDE _raise_if_ptu_deployment_is_priced(model_info=model_info, supplied=supplied) + candidates: Final = _CUSTOM_PRICING_FIELDS | frozenset( + field for field in (*model_info.keys(), *litellm_params.keys()) if is_threshold_rate_key(field) + ) stored: Final = frozenset( field - for field in _CUSTOM_PRICING_FIELDS + for field in candidates if _is_nonzero_price(model_info.get(field)) or _is_nonzero_price(litellm_params.get(field)) ) return MappingProxyType( @@ -501,7 +508,11 @@ def _ptu_pricing_delta( return _NO_PRICING_OVERRIDE, frozenset() return _NO_PRICING_OVERRIDE, frozenset( field - for field in _CUSTOM_PRICING_FIELDS.union(_PTU_ZEROED_PRICING_FIELDS, _PTU_EMPTIED_PRICING_FIELDS) + for field in _CUSTOM_PRICING_FIELDS.union( + _PTU_ZEROED_PRICING_FIELDS, + _PTU_EMPTIED_PRICING_FIELDS, + frozenset(field for field in (*model_info.keys(), *litellm_params.keys()) if is_threshold_rate_key(field)), + ) if _is_zero_price(model_info.get(field)) or _is_zero_price(litellm_params.get(field)) ) diff --git a/litellm/router.py b/litellm/router.py index f0ebb539bb7..8e792cc58c1 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -304,6 +304,36 @@ def _cost_value_as_float(value: str | float | None) -> float | None: return None +_CUSTOM_PRICING_THRESHOLD_COST_KEY: Final[re.Pattern[str]] = re.compile( + r"^(?:" + r"input_cost_per_token|" + r"output_cost_per_token|" + r"cache_creation_input_token_cost(?:_above_1hr)?|" + r"cache_read_input_token_cost" + r")_above_\d+k?_tokens(?:_(?:priority|flex|ultrafast))?$" +) + + +def _is_custom_pricing_field(field: str) -> bool: + return ( + field in CustomPricingLiteLLMParams.model_fields or _CUSTOM_PRICING_THRESHOLD_COST_KEY.search(field) is not None + ) + + +def _copy_custom_pricing_fields( + model_info: dict[str, Any], # mutable-ok: updates model-cost entry in place + litellm_params: LiteLLM_Params, +) -> None: + """Copy declared and arbitrary threshold pricing into a model-cost entry.""" + model_info.update( + { + field: value + for field, value in litellm_params.model_dump(exclude_none=True).items() + if _is_custom_pricing_field(field) + } + ) + + def model_info_is_active_for_environment(model_info: Mapping[str, object] | None) -> bool: """Single owner of the environment-gating rule: a deployment whose model_info names `supported_environments` loads only on pods whose LITELLM_ENVIRONMENT is in that list. @@ -8170,9 +8200,10 @@ class Router: litellm_params=litellm_params, model_info=_model_info, ) - for field in CustomPricingLiteLLMParams.model_fields: - if deployment.litellm_params.get(field) is not None: - _model_info[field] = deployment.litellm_params[field] + _copy_custom_pricing_fields( + model_info=_model_info, + litellm_params=deployment.litellm_params, + ) if _model_info.get("input_cost_per_token") is not None: Router._inherit_builtin_cache_pricing( @@ -8910,10 +8941,10 @@ class Router: self._add_deployment(deployment=deployment) _model_info_dict: Final[dict] = deployment.model_info.model_dump(exclude_none=True) - for field in CustomPricingLiteLLMParams.model_fields: - field_value = deployment.litellm_params.get(field) - if field_value is not None: - _model_info_dict[field] = field_value + _copy_custom_pricing_fields( + model_info=_model_info_dict, + litellm_params=deployment.litellm_params, + ) if _model_info_dict.get("input_cost_per_token") is not None: Router._inherit_builtin_cache_pricing( @@ -9165,10 +9196,10 @@ class Router: deployment alone, which is what lets a refresh rebuild the same entries. """ model_info: Final[dict] = deployment.model_info.model_dump(exclude_none=True) # mutable-ok: built in place - for field in CustomPricingLiteLLMParams.model_fields: - field_value = deployment.litellm_params.get(field) - if field_value is not None: - model_info[field] = field_value + _copy_custom_pricing_fields( + model_info=model_info, + litellm_params=deployment.litellm_params, + ) if model_info.get("input_cost_per_token") is not None: Router._inherit_builtin_cache_pricing( model_info=model_info, @@ -9206,10 +9237,16 @@ class Router: """ if classify_strategy_router_model(model) is not None: model_info = { # mutable-ok: filtered copy of the caller's entry, handed straight to register_model - k: v for k, v in model_info.items() if k not in CustomPricingLiteLLMParams.model_fields + k: v for k, v in model_info.items() if not _is_custom_pricing_field(k) } if model_id is not None: + existing_model_info = litellm.model_cost.get(model_id) + if existing_model_info is not None: + for field in tuple(existing_model_info): + if _is_custom_pricing_field(field) and field not in model_info: + existing_model_info.pop(field, None) + litellm.register_model(model_cost={model_id: model_info}, persist_across_reloads=False) ## OLD MODEL REGISTRATION ## Kept to prevent breaking changes diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py new file mode 100644 index 00000000000..a2243f04b04 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_threshold_cost_selection.py @@ -0,0 +1,418 @@ +import pytest + +from litellm.litellm_core_utils.llm_cost_calc.utils import ( + _get_token_base_cost, +) +from litellm.types.utils import Usage + + +@pytest.mark.parametrize( + ("tier_field", "tier_value", "rate_index"), + [ + ( + "output_cost_per_token_above_32k_tokens", + 18e-6, + 1, + ), + ( + "cache_read_input_token_cost_above_32k_tokens", + 4e-6, + 4, + ), + ], +) +def test_standalone_threshold_field_selects_tier( + tier_field: str, + tier_value: float, + rate_index: int, +) -> None: + """Output and cache fields can independently define a tier.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_read_input_token_cost": 5e-7, + tier_field: tier_value, + } + usage = Usage( + prompt_tokens=40_000, + completion_tokens=1_000, + total_tokens=41_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + ) + + assert rates[rate_index] == pytest.approx(tier_value) + + +def test_cost_types_select_thresholds_independently() -> None: + """An inactive input tier must not suppress an active output tier.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_token_above_128k_tokens": 9e-6, + "output_cost_per_token_above_32k_tokens": 18e-6, + } + usage = Usage( + prompt_tokens=40_000, + completion_tokens=1_000, + total_tokens=41_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + ) + + assert rates[0] == pytest.approx(1e-6) + assert rates[1] == pytest.approx(18e-6) + + +@pytest.mark.parametrize( + "model_info", + [ + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + }, + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + }, + ], + ids=["standard-first", "priority-first"], +) +def test_equal_threshold_matching_tier_wins_for_output(model_info): + """At an equal threshold the matching service-tier rate must win regardless of the + order the standard and tier-qualified keys appear in model_info.""" + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost(model_info=model_info, usage=usage, service_tier="priority") + + assert rates[1] == pytest.approx(1.5e-6) + + +@pytest.mark.parametrize( + "model_info", + [ + { + "input_cost_per_token": 1e-6, + "cache_read_input_token_cost": 2e-7, + "cache_read_input_token_cost_above_200k_tokens": 3e-7, + "cache_read_input_token_cost_above_200k_tokens_priority": 4e-7, + }, + { + "input_cost_per_token": 1e-6, + "cache_read_input_token_cost": 2e-7, + "cache_read_input_token_cost_above_200k_tokens_priority": 4e-7, + "cache_read_input_token_cost_above_200k_tokens": 3e-7, + }, + ], + ids=["standard-first", "priority-first"], +) +def test_equal_threshold_matching_tier_wins_for_cache_read(model_info): + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost(model_info=model_info, usage=usage, service_tier="priority") + + assert rates[4] == pytest.approx(4e-7) + + +def test_equal_threshold_default_tier_keeps_standard(): + """service_tier=None must ignore the tier-qualified key and keep the standard rate.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost(model_info=model_info, usage=usage) + + assert rates[1] == pytest.approx(1e-6) + + +def test_equal_threshold_wrong_tier_keeps_standard(): + """A priority-qualified key must not win under a different service tier.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier="flex", + ) + + assert rates[1] == pytest.approx(1e-6) + + +def test_unequal_threshold_highest_standard_still_wins(): + """A higher standard threshold governs over a lower matching-tier threshold.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + "output_cost_per_token_above_128k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier="priority", + ) + + assert rates[1] == pytest.approx(1e-6) + + +@pytest.mark.parametrize( + ("tier_field", "tier_value", "rate_index"), + [ + ("output_cost_per_token_above_200k_tokens_priority", 1.5e-6, 1), + ("cache_read_input_token_cost_above_200k_tokens_priority", 4e-7, 4), + ], +) +def test_fast_service_tier_uses_priority_qualified_threshold(tier_field, tier_value, rate_index): + """fast aliases to priority pricing, so a priority-qualified threshold with no standard + sibling must apply to a fast request too.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 1e-6, + "cache_read_input_token_cost": 2e-7, + tier_field: tier_value, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier="fast", + ) + + assert rates[rate_index] == pytest.approx(tier_value) + + +@pytest.mark.parametrize( + ("service_tier", "expected"), + [ + ("priority", 1.5e-6), + ("flex", 1e-6), + (None, 1e-6), + ], + ids=["priority-guard", "flex-guard", "default-guard"], +) +def test_fast_alias_guard_tiers_for_priority_only_output_threshold(service_tier, expected): + """priority must keep working under the alias; flex and default must not pick up the + priority rate.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier=service_tier, + ) + + assert rates[1] == pytest.approx(expected) + + +@pytest.mark.parametrize("service_tier", ["priority", "fast"], ids=["priority", "fast"]) +def test_fast_builtin_standard_plus_priority_threshold_resolves_priority(service_tier): + """The builtin shape (standard sibling present) keeps resolving the priority variant via + the existing standard-key alias lookup.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 1e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier=service_tier, + ) + + assert rates[1] == pytest.approx(1.5e-6) + + +@pytest.mark.parametrize( + "model_info", + [ + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 1e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + }, + { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + "output_cost_per_token_above_200k_tokens": 1e-6, + }, + ], + ids=["standard-first", "priority-first"], +) +def test_fast_equal_threshold_matching_tier_wins_both_orders(model_info): + """The Phase D equal-threshold tie-break must hold under the fast alias, independent of + insertion order.""" + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier="fast", + ) + + assert rates[1] == pytest.approx(1.5e-6) + + +def test_fast_service_tier_uppercase_uses_priority_qualified_threshold(): + """The scanner must be as case-insensitive as the lookup path.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 1e-6, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier="FAST", + ) + + assert rates[1] == pytest.approx(1.5e-6) + + +@pytest.mark.parametrize( + ("tier_field", "tier_value", "rate_index"), + [ + ("output_cost_per_token_above_200k_tokens_priority", 1.5e-6, 1), + ("cache_read_input_token_cost_above_200k_tokens_priority", 4e-7, 4), + ], +) +def test_tier_qualified_threshold_field_selects_tier_for_matching_service_tier( + tier_field: str, tier_value: float, rate_index: int +) -> None: + """A tier-qualified threshold key with no standard sibling must apply when the request's + service tier matches.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_read_input_token_cost": 5e-7, + tier_field: tier_value, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier="priority", + ) + + assert rates[rate_index] == pytest.approx(tier_value) + + +def test_tier_qualified_threshold_is_ignored_under_the_default_tier() -> None: + """service_tier=None must keep billing the flat rate: the priority key is inactive.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost(model_info=model_info, usage=usage) + + assert rates[1] == pytest.approx(2e-6) + + +def test_tier_qualified_threshold_is_ignored_under_a_different_service_tier() -> None: + """A priority-qualified key must not bill a flex request.""" + model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_token_above_200k_tokens_priority": 1.5e-6, + } + usage = Usage( + prompt_tokens=250_000, + completion_tokens=1_000, + total_tokens=251_000, + ) + + rates = _get_token_base_cost( + model_info=model_info, + usage=usage, + service_tier="flex", + ) + + assert rates[1] == pytest.approx(2e-6) diff --git a/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py b/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py index b8fb372d537..588096daa3a 100644 --- a/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py +++ b/tests/test_litellm/litellm_core_utils/test_ptu_pricing.py @@ -7,6 +7,7 @@ from unittest.mock import patch import pytest from litellm.litellm_core_utils.ptu_pricing import ( + is_threshold_rate_key, ptu_config_error, ptu_identity_error, CUSTOM_PRICING_FIELDS, @@ -165,6 +166,115 @@ def test_a_rate_the_deployment_declares_itself_is_zeroed_too(): assert override[extra] == 0.0 +@pytest.mark.parametrize( + "threshold_field", + [ + "input_cost_per_token_above_32k_tokens", + "output_cost_per_token_above_32k_tokens", + "cache_read_input_token_cost_above_32k_tokens", + "cache_creation_input_token_cost_above_32k_tokens", + "cache_creation_input_token_cost_above_1hr_above_32k_tokens", + ], +) +def test_arbitrary_threshold_rate_the_deployment_declares_is_zeroed_too(threshold_field): + """Threshold rates a deployment can declare but CustomPricingLiteLLMParams does not + enumerate (e.g. input_cost_per_token_above_32k_tokens) must be zeroed like any other + declared rate, or a PTU deployment bills per token past the threshold on top of the + hourly charge.""" + assert threshold_field not in CUSTOM_PRICING_FIELDS + assert threshold_field not in PTU_ZEROED_PRICING_FIELDS + + override = _with_flag(_VALID, declared={threshold_field: 9e-06}) + + assert override is not None + assert override.get(threshold_field, 9e-06) == 0.0 + + +@pytest.mark.parametrize( + "threshold_field", + [ + "output_cost_per_token_above_32k_tokens_priority", + "cache_read_input_token_cost_above_32k_tokens_priority", + ], +) +def test_service_tier_qualified_threshold_rate_declared_is_zeroed_too(threshold_field): + """A tier-qualified threshold rate the deployment declares must be zeroed like every + other declared rate, or a PTU deployment bills per token past the threshold.""" + assert threshold_field not in CUSTOM_PRICING_FIELDS + assert threshold_field not in PTU_ZEROED_PRICING_FIELDS + + override = _with_flag(_VALID, declared={threshold_field: 9e-06}) + + assert override is not None + assert override.get(threshold_field, 9e-06) == 0.0 + + +@pytest.mark.parametrize( + "param_field", + [ + "api_key_above_32k_tokens", + "secret_above_32k_tokens", + "credential_above_32k_tokens", + "custom_provider_param_above_32k_tokens", + ], +) +def test_threshold_like_non_pricing_params_are_left_alone(param_field): + """The threshold matcher must only cover the cost calculator's pricing base keys: a + deployment param that merely ends in _above_k_tokens is not a charge and must keep + its value.""" + override = _with_flag(_VALID, declared={param_field: "keep-me"}) + + assert override is not None + assert param_field not in override + + +@pytest.mark.parametrize( + "field", + [ + "input_cost_per_token_above_32k_tokens", + "output_cost_per_token_above_64k_tokens", + "cache_read_input_token_cost_above_128k_tokens", + "cache_creation_input_token_cost_above_200k_tokens", + "cache_creation_input_token_cost_above_1hr_above_200k_tokens", + ], +) +def test_is_threshold_rate_key_accepts_supported_threshold_rates(field): + assert is_threshold_rate_key(field) + + +@pytest.mark.parametrize( + "field", + [ + "input_cost_per_token_above_32k_tokens_priority", + "output_cost_per_token_above_32k_tokens_flex", + "cache_read_input_token_cost_above_200k_tokens_priority", + "cache_creation_input_token_cost_above_32k_tokens_ultrafast", + "cache_creation_input_token_cost_above_1hr_above_200k_tokens_priority", + ], +) +def test_is_threshold_rate_key_accepts_service_tier_qualified_threshold_rates(field): + assert is_threshold_rate_key(field) + + +@pytest.mark.parametrize( + "field", + [ + "api_key_above_32k_tokens", + "secret_above_32k_tokens", + "credential_above_32k_tokens", + "custom_provider_param_above_32k_tokens", + "api_key_above_32k_tokens_priority", + "secret_above_32k_tokens_flex", + "credential_above_200k_tokens_ultrafast", + "custom_provider_param_above_32k_tokens_priority", + "input_cost_per_token", + "tiered_pricing", + ], +) +def test_is_threshold_rate_key_rejects_non_pricing_fields(field): + assert not is_threshold_rate_key(field) + + def test_a_setting_that_is_not_a_charge_is_left_alone(): """CustomPricingLiteLLMParams also carries configuration, and zeroing one of those would break the deployment rather than stop a charge.""" diff --git a/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py b/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py index a1c38d26b9d..8a3ddd0eae7 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py @@ -794,6 +794,22 @@ class TestPtuDeploymentsAreNotBilledPerToken: assert exc.value.status_code == 400 assert "tiered_pricing" in str(exc.value.detail) + @pytest.mark.parametrize( + "field", + [ + "input_cost_per_token_above_32k_tokens", + "output_cost_per_token_above_32k_tokens", + "input_cost_per_token_above_32k_tokens_priority", + ], + ) + def test_an_arbitrary_threshold_rate_the_caller_supplies_is_refused(self, field): + """The router lets deployments declare arbitrary above-threshold rates; those bill a + PTU deployment past the threshold just as surely as a flat rate, so the refusal must + cover them too.""" + with pytest.raises(HTTPException) as exc: + self._zeroed(model_info=self.PTU, supplied={field: 9e-06}) + assert exc.value.status_code == 400 + assert field in str(exc.value.detail) def test_a_search_context_price_the_caller_supplies_is_refused(self): """The rates sit in a table keyed by context size, so a guard that only reads numbers lets a per-request charge onto a deployment its reserved capacity already pays for.""" @@ -880,6 +896,19 @@ class TestPtuDeploymentsAreNotBilledPerToken: assert zeroed["input_cost_per_second"] == 0 assert zeroed["input_cost_per_token"] == 0 + @pytest.mark.parametrize( + "field", + [ + "input_cost_per_token_above_32k_tokens", + "output_cost_per_token_above_32k_tokens", + "output_cost_per_token_above_32k_tokens_priority", + ], + ) + def test_an_arbitrary_threshold_rate_already_on_the_row_is_zeroed(self, field): + """A row priced through a path this rule does not cover must heal on its next save.""" + zeroed = self._zeroed(model_info={**self.PTU, field: 9e-06}, litellm_params={}) + assert zeroed[field] == 0 + assert zeroed["input_cost_per_token"] == 0 @pytest.mark.asyncio async def test_a_refused_price_does_not_leave_the_team_changed(self): """The team ACL write autocommits, so the refusal has to run before it. Otherwise a @@ -944,6 +973,34 @@ class TestPtuDeploymentsAreNotBilledPerToken: assert priced.litellm_params.get("regional_processing_uplift_multiplier_eu") == 1.15 assert priced.litellm_params.get("input_cost_per_token") == 0 + @pytest.mark.parametrize( + "param", + [ + "api_key_above_32k_tokens", + "secret_above_32k_tokens", + "credential_above_32k_tokens", + "custom_provider_param_above_32k_tokens", + "api_key_above_32k_tokens_priority", + "custom_provider_param_above_32k_tokens_flex", + ], + ) + def test_a_threshold_like_setting_that_is_not_a_price_is_left_alone(self, param): + """Only the cost calculator's base keys qualify as threshold rates: a deployment param + that merely ends in _above_k_tokens is not a charge and must keep its value.""" + priced = _ptu_priced_deployment( + Deployment( + model_name="settings", + litellm_params=LiteLLM_Params(model="openai/gpt-4o-mini", **{param: "keep-me"}), + model_info=ModelInfo( + id="dep-settings", + team_id="t", + ptu_effective_from=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc), + **self.PTU, + ), + ) + ) + assert priced.litellm_params.get(param) == "keep-me" + assert priced.litellm_params.get("input_cost_per_token") == 0 def test_removing_ptu_config_releases_every_rate_it_zeroed(self): """The zeroing covers any stored rate, so a release that only spans the mirrored fields leaves a per-second deployment billing nothing for that dimension forever.""" @@ -976,6 +1033,73 @@ class TestPtuDeploymentsAreNotBilledPerToken: ) assert "input_cost_per_second" not in json.loads(off["litellm_params"]) + def test_removing_ptu_config_releases_a_zeroed_threshold_rate(self): + """The zeroing spans threshold rates too, so a release that only spans the enumeration + sets would leave one billing nothing past the threshold forever.""" + on = update_db_model( + db_model=Deployment( + model_name="threshold", + litellm_params=LiteLLM_Params(model="openai/gpt-4o-mini", input_cost_per_token_above_32k_tokens=9e-06), + model_info=ModelInfo(id="dep-thr", team_id="t"), + ), + updated_patch=updateDeployment( + model_info=ModelInfo( + id="dep-thr", + team_id="t", + ptu_effective_from=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc), + **self.PTU, + ) + ), + ) + assert json.loads(on["litellm_params"])["input_cost_per_token_above_32k_tokens"] == 0 + + off = update_db_model( + db_model=Deployment( + model_name="threshold", + litellm_params=LiteLLM_Params(**json.loads(on["litellm_params"])), + model_info=ModelInfo(**json.loads(on["model_info"])), + ), + updated_patch=updateDeployment( + model_info=ModelInfo(id="dep-thr", ptu_count=None, cost_per_ptu_per_hour=None) + ), + ) + assert "input_cost_per_token_above_32k_tokens" not in json.loads(off["litellm_params"]) + + def test_removing_ptu_config_releases_a_zeroed_tier_qualified_threshold_rate(self): + """The zeroing spans tier-qualified threshold rates too, so a release that only + spans the enumeration sets would leave one billing nothing past the threshold + forever.""" + on = update_db_model( + db_model=Deployment( + model_name="tier-threshold", + litellm_params=LiteLLM_Params( + model="openai/gpt-4o-mini", output_cost_per_token_above_32k_tokens_priority=9e-06 + ), + model_info=ModelInfo(id="dep-tier-thr", team_id="t"), + ), + updated_patch=updateDeployment( + model_info=ModelInfo( + id="dep-tier-thr", + team_id="t", + ptu_effective_from=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc), + **self.PTU, + ) + ), + ) + assert json.loads(on["litellm_params"])["output_cost_per_token_above_32k_tokens_priority"] == 0 + + off = update_db_model( + db_model=Deployment( + model_name="tier-threshold", + litellm_params=LiteLLM_Params(**json.loads(on["litellm_params"])), + model_info=ModelInfo(**json.loads(on["model_info"])), + ), + updated_patch=updateDeployment( + model_info=ModelInfo(id="dep-tier-thr", ptu_count=None, cost_per_ptu_per_hour=None) + ), + ) + assert "output_cost_per_token_above_32k_tokens_priority" not in json.loads(off["litellm_params"]) + def test_removing_ptu_config_releases_a_zeroed_search_context_table(self): """The all-zero table exists only to stop the double charge, so a deployment taken off PTU has to give it up or it keeps serving grounded requests for free forever.""" diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index b580b03574e..f0b772b7d0a 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -19,6 +19,10 @@ import pytest import litellm from litellm import Router from litellm.litellm_core_utils.ptu_pricing import ptu_config_error +from litellm.router import ( + _copy_custom_pricing_fields, + _is_custom_pricing_field, +) from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo from litellm.utils import ( _invalidate_model_cost_lowercase_map, @@ -60,6 +64,117 @@ def _restore_model_cost_entries(original_entries): _invalidate_model_cost_lowercase_map() +def test_is_custom_pricing_field_recognizes_declared_and_arbitrary_threshold_fields(): + assert _is_custom_pricing_field("input_cost_per_token") + assert _is_custom_pricing_field("input_cost_per_token_above_32k_tokens") + assert not _is_custom_pricing_field("api_key") + assert not _is_custom_pricing_field("input_cost_per_token_above_32k_tokens_extra") + + +def test_is_custom_pricing_field_recognizes_tier_qualified_threshold_fields(): + assert _is_custom_pricing_field("input_cost_per_token_above_200k_tokens_priority") + assert _is_custom_pricing_field("output_cost_per_token_above_200k_tokens_flex") + assert _is_custom_pricing_field("cache_read_input_token_cost_above_200k_tokens_priority") + assert _is_custom_pricing_field("cache_creation_input_token_cost_above_1hr_above_200k_tokens_ultrafast") + assert not _is_custom_pricing_field("api_key_above_32k_tokens_priority") + assert not _is_custom_pricing_field("secret_above_32k_tokens_flex") + assert not _is_custom_pricing_field("credential_above_200k_tokens_ultrafast") + assert not _is_custom_pricing_field("custom_provider_param_above_32k_tokens_priority") + + +def test_custom_tier_qualified_threshold_key_is_registered(): + """A deployment declaring only a tier-qualified threshold rate (e.g. + output_cost_per_token_above_200k_tokens_priority, which is not enumerated in + CustomPricingLiteLLMParams) must keep it in its model_cost entry so matching-tier + requests bill it.""" + model_id = "custom-tier-only-threshold" + Router( + model_list=[ + { + "model_name": "tier-only-model", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "fake-key", + "input_cost_per_token": 1e-6, + "output_cost_per_token_above_32k_tokens_priority": 1.5e-6, + }, + "model_info": {"id": model_id}, + } + ] + ) + + registered = litellm.model_cost[model_id] + assert registered["output_cost_per_token_above_32k_tokens_priority"] == 1.5e-6 + +def test_copy_custom_pricing_fields_preserves_declared_and_arbitrary_threshold_fields(): + """Copy supported pricing fields without copying unrelated LiteLLM params.""" + model_info = {} + litellm_params = LiteLLM_Params( + model="openai/gpt-4o-mini", + api_key="fake-key", + input_cost_per_token=1e-6, + input_cost_per_token_above_32k_tokens=9e-6, + ) + + _copy_custom_pricing_fields( + model_info=model_info, + litellm_params=litellm_params, + ) + + assert model_info == { + "input_cost_per_token": 1e-6, + "input_cost_per_token_above_32k_tokens": 9e-6, + } + + +def test_is_custom_pricing_field_accepts_supported_threshold_base_keys(): + """Every threshold base key the cost calculator supports stays accepted.""" + for field in ( + "input_cost_per_token_above_32k_tokens", + "output_cost_per_token_above_32k_tokens", + "cache_read_input_token_cost_above_32k_tokens", + "cache_creation_input_token_cost_above_32k_tokens", + "cache_creation_input_token_cost_above_1hr_above_32k_tokens", + ): + assert _is_custom_pricing_field(field), field + + +def test_is_custom_pricing_field_rejects_non_pricing_threshold_like_params(): + """Deployment params that merely look like *_above__tokens are not pricing.""" + for field in ( + "api_key_above_32k_tokens", + "secret_above_32k_tokens", + "credential_above_32k_tokens", + "custom_provider_param_above_32k_tokens", + ): + assert not _is_custom_pricing_field(field), field + + +def test_copy_custom_pricing_fields_ignores_non_pricing_threshold_like_params(): + """Only supported pricing fields are copied; threshold-like deployment params stay out of model_info.""" + model_info = {} + litellm_params = LiteLLM_Params( + model="openai/gpt-4o-mini", + api_key="fake-key", + input_cost_per_token=1e-6, + input_cost_per_token_above_32k_tokens=9e-6, + api_key_above_32k_tokens="leaked", + secret_above_32k_tokens="leaked", + credential_above_32k_tokens="leaked", + custom_provider_param_above_32k_tokens="leaked", + ) + + _copy_custom_pricing_fields( + model_info=model_info, + litellm_params=litellm_params, + ) + + assert model_info == { + "input_cost_per_token": 1e-6, + "input_cost_per_token_above_32k_tokens": 9e-6, + } + + def test_should_not_pollute_shared_key_with_zero_cost_pricing(): """ When deployment A has input_cost_per_token=0 and deployment B has no @@ -1426,6 +1541,270 @@ def test_replay_live_router_model_cost_rebuilds_every_live_router(): litellm.model_cost = saved_model_cost _invalidate_model_cost_lowercase_map() +def test_router_registers_arbitrary_above_threshold_pricing_from_litellm_params(): + """Regression for #34378. + + Arbitrary *_above__tokens pricing fields accepted in litellm_params + must be registered under the deployment model ID. + """ + backend_model = "openai/gpt-4o-mini" + model_id = "router-custom-above-32k-pricing-34378" + shared_keys = ("gpt-4o-mini", backend_model) + + original_entries = { + key: copy.deepcopy(litellm.model_cost.get(key)) + for key in (*shared_keys, model_id) + } + + try: + Router( + model_list=[ + { + "model_name": "custom-above-32k-model", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_token_above_32k_tokens": 9e-6, + "output_cost_per_token_above_32k_tokens": 18e-6, + "cache_read_input_token_cost_above_32k_tokens": 4e-6, + }, + "model_info": { + "id": model_id, + }, + } + ] + ) + + registered = litellm.model_cost.get(model_id) + assert registered is not None + + assert registered["input_cost_per_token"] == 1e-6 + assert registered["output_cost_per_token"] == 2e-6 + assert registered["input_cost_per_token_above_32k_tokens"] == 9e-6 + assert registered["output_cost_per_token_above_32k_tokens"] == 18e-6 + assert registered["cache_read_input_token_cost_above_32k_tokens"] == 4e-6 + + from litellm.litellm_core_utils.llm_cost_calc.utils import ( + generic_cost_per_token, + ) + from litellm.types.utils import Usage + + prompt_tokens = 40_000 + completion_tokens = 1_000 + usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model=model_id, + usage=usage, + custom_llm_provider="openai", + ) + + assert prompt_cost == pytest.approx(9e-6 * prompt_tokens) + assert completion_cost == pytest.approx(18e-6 * completion_tokens) + finally: + _restore_model_cost_entries(original_entries) + + +def test_router_registration_never_registers_non_pricing_threshold_like_params(): + """Non-pricing *_above__tokens deployment params must not reach litellm.model_cost.""" + backend_model = "openai/gpt-4o-mini" + model_id = "router-rejects-non-pricing-above-32k-34504" + shared_keys = ("gpt-4o-mini", backend_model) + + original_entries = { + key: copy.deepcopy(litellm.model_cost.get(key)) + for key in (*shared_keys, model_id) + } + + try: + Router( + model_list=[ + { + "model_name": "custom-above-32k-model", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_token_above_32k_tokens": 9e-6, + "api_key_above_32k_tokens": "leaked", + "secret_above_32k_tokens": "leaked", + "credential_above_32k_tokens": "leaked", + "custom_provider_param_above_32k_tokens": "leaked", + }, + "model_info": { + "id": model_id, + }, + } + ] + ) + + registered = litellm.model_cost.get(model_id) + assert registered is not None + + assert registered["input_cost_per_token"] == 1e-6 + assert registered["output_cost_per_token"] == 2e-6 + assert registered["input_cost_per_token_above_32k_tokens"] == 9e-6 + for junk in ( + "api_key_above_32k_tokens", + "secret_above_32k_tokens", + "credential_above_32k_tokens", + "custom_provider_param_above_32k_tokens", + ): + assert junk not in registered, junk + finally: + _restore_model_cost_entries(original_entries) + + +def test_add_deployment_registers_arbitrary_above_threshold_pricing(): + """Dynamic add_deployment must preserve arbitrary pricing thresholds.""" + backend_model = "openai/gpt-4o-mini" + model_id = "add-deployment-custom-above-32k-pricing-34378" + shared_keys = ("gpt-4o-mini", backend_model) + + original_entries = { + key: copy.deepcopy(litellm.model_cost.get(key)) + for key in (*shared_keys, model_id) + } + + try: + router = Router(model_list=[]) + router.add_deployment( + deployment=Deployment( + model_name="dynamic-custom-above-32k-model", + litellm_params=LiteLLM_Params( + model=backend_model, + api_key="fake-key", + input_cost_per_token=1e-6, + output_cost_per_token=2e-6, + input_cost_per_token_above_32k_tokens=9e-6, + output_cost_per_token_above_32k_tokens=18e-6, + cache_read_input_token_cost_above_32k_tokens=4e-6, + ), + model_info=ModelInfo(id=model_id), + ) + ) + + registered = litellm.model_cost.get(model_id) + assert registered is not None + assert registered["input_cost_per_token_above_32k_tokens"] == 9e-6 + assert registered["output_cost_per_token_above_32k_tokens"] == 18e-6 + assert registered["cache_read_input_token_cost_above_32k_tokens"] == 4e-6 + + for shared_key in shared_keys: + shared_entry = litellm.model_cost.get(shared_key) or {} + assert "input_cost_per_token_above_32k_tokens" not in shared_entry + assert "output_cost_per_token_above_32k_tokens" not in shared_entry + assert "cache_read_input_token_cost_above_32k_tokens" not in shared_entry + finally: + _restore_model_cost_entries(original_entries) + + +def test_upsert_deployment_removes_withdrawn_arbitrary_above_threshold_pricing(): + """Updating a deployment must not retain arbitrary pricing tiers that were removed.""" + backend_model = "openai/gpt-4o-mini" + model_id = "upsert-withdrawn-above-32k-pricing-34378" + shared_keys = ("gpt-4o-mini", backend_model) + + original_entries = { + key: copy.deepcopy(litellm.model_cost.get(key)) + for key in (*shared_keys, model_id) + } + + try: + router = Router(model_list=[]) + + router.add_deployment( + deployment=Deployment( + model_name="upsert-custom-above-32k-model", + litellm_params=LiteLLM_Params( + model=backend_model, + api_key="fake-key", + input_cost_per_token=1e-6, + output_cost_per_token=2e-6, + input_cost_per_token_above_32k_tokens=9e-6, + ), + model_info=ModelInfo(id=model_id), + ) + ) + + assert litellm.model_cost[model_id]["input_cost_per_token_above_32k_tokens"] == 9e-6 + + router.upsert_deployment( + deployment=Deployment( + model_name="upsert-custom-above-32k-model", + litellm_params=LiteLLM_Params( + model=backend_model, + api_key="fake-key", + input_cost_per_token=1e-6, + output_cost_per_token=2e-6, + ), + model_info=ModelInfo(id=model_id), + ) + ) + + registered = litellm.model_cost[model_id] + assert registered["input_cost_per_token"] == 1e-6 + assert "input_cost_per_token_above_32k_tokens" not in registered + finally: + _restore_model_cost_entries(original_entries) + + +def test_price_data_reload_preserves_arbitrary_above_threshold_pricing(monkeypatch): + """Arbitrary pricing thresholds must survive Router cost-map replay.""" + from litellm import utils as litellm_utils + + monkeypatch.setattr( + litellm_utils, + "_runtime_registered_model_cost", + dict(litellm_utils._runtime_registered_model_cost), + ) + + model_id = "reload-custom-above-32k-pricing-34378" + saved_model_cost = copy.deepcopy(litellm.model_cost) + + try: + router = Router( + model_list=[ + { + "model_name": "reload-custom-above-32k-model", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "fake-key", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_token_above_32k_tokens": 9e-6, + "output_cost_per_token_above_32k_tokens": 18e-6, + "cache_read_input_token_cost_above_32k_tokens": 4e-6, + }, + "model_info": {"id": model_id}, + } + ] + ) + + before = litellm.model_cost[model_id] + assert before["input_cost_per_token_above_32k_tokens"] == 9e-6 + assert before["output_cost_per_token_above_32k_tokens"] == 18e-6 + assert before["cache_read_input_token_cost_above_32k_tokens"] == 4e-6 + + _simulate_price_data_reload( + {"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}}, + ) + + rebuilt = litellm.model_cost[model_id] + assert rebuilt["input_cost_per_token_above_32k_tokens"] == 9e-6 + assert rebuilt["output_cost_per_token_above_32k_tokens"] == 18e-6 + assert rebuilt["cache_read_input_token_cost_above_32k_tokens"] == 4e-6 + assert router.model_list + finally: + _restore_model_cost_entries(saved_model_cost) + def test_strategy_router_alias_pricing_never_enters_model_cost(monkeypatch): """ @@ -1453,6 +1832,7 @@ def test_strategy_router_alias_pricing_never_enters_model_cost(monkeypatch): "complexity_router_default_model": "paid-model", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, + "input_cost_per_token_above_32k_tokens": 9e-6, "complexity_router_config": {"tiers": {"simple": "paid-model"}}, }, "model_info": {"id": "strategy-alias-id", "max_input_tokens": 128000}, @@ -1471,6 +1851,7 @@ def test_strategy_router_alias_pricing_never_enters_model_cost(monkeypatch): assert entry["max_input_tokens"] == 128000 assert "input_cost_per_token" not in entry assert "output_cost_per_token" not in entry + assert "input_cost_per_token_above_32k_tokens" not in entry _assert_alias_unpriced() @@ -1583,6 +1964,24 @@ def test_a_config_ptu_deployment_bills_nothing_per_token(): assert litellm.model_cost[entry["model_info"]["id"]]["input_cost_per_token"] == 0.0 +def test_a_config_ptu_deployment_zeroes_custom_threshold_rates(): + """A custom threshold rate the deployment declares (e.g. + input_cost_per_token_above_32k_tokens) must be zeroed like every other declared rate, + or the PTU deployment bills per token past the threshold on top of the hourly charge.""" + router = _ptu_router( + litellm_params={ + "input_cost_per_token": 5e-06, + "input_cost_per_token_above_32k_tokens": 9e-06, + } + ) + entry = router.model_list[0] + + assert entry["litellm_params"]["input_cost_per_token"] == 0.0 + assert entry["litellm_params"]["input_cost_per_token_above_32k_tokens"] == 0.0 + assert litellm.model_cost[entry["model_info"]["id"]]["input_cost_per_token"] == 0.0 + assert litellm.model_cost[entry["model_info"]["id"]]["input_cost_per_token_above_32k_tokens"] == 0.0 + + @pytest.mark.parametrize( "backend", ["anthropic/claude-sonnet-4-5-20250929", "azure/gpt-4o", "gemini/gemini-2.5-flash"],