From 19264830f2354ec7778d861f45eb27e39ea3985a Mon Sep 17 00:00:00 2001 From: Mrinal Chanshetty Date: Tue, 20 Oct 2026 16:55:54 -0700 Subject: [PATCH] fix(pricing): port pricing helpers needed by #47419 Prerequisite for the #47419 backport to stable/1.103.x, taken from the stable/1.104.x hand-port 49cacdb4b2. Minimal port of helpers from #41523/#43890/#46647 needed by #47419; behavior changes of those PRs not included. Registrations are tracked in a _runtime_registered_fields side table and cost-map entries stay plain dicts. --- .../litellm_core_utils/llm_cost_calc/utils.py | 9 +- litellm/router.py | 46 +++++++ litellm/utils.py | 121 +++++++++++++++++- 3 files changed, 174 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 0c85729a307..366da9b5539 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -67,6 +67,9 @@ _SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType( ServiceTier.ULTRAFAST.value: ServiceTier.ULTRAFAST.value, } ) +SERVICE_TIER_COST_KEY_SUFFIXES: Final[tuple[str, ...]] = tuple( + sorted(frozenset(f"_{suffix}" for suffix in _SERVICE_TIER_TO_COST_KEY_SUFFIX.values())) +) _INCLUSIVE_THRESHOLD_PROVIDERS: Final = frozenset({"xai"}) @@ -577,10 +580,14 @@ def _get_token_base_cost( # Exclude service_tier-specific variants (e.g. input_cost_per_token_above_200k_tokens_priority) # so that the threshold detection loop only processes standard keys. The # service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key. + tier_key_suffix: Final = _get_service_tier_cost_key("", service_tier) threshold_keys: Final = [ k for k in model_info - if k.startswith("input_cost_per_token_above_") and not k.endswith(_NON_STANDARD_THRESHOLD_SUFFIXES) + if k.startswith("input_cost_per_token_above_") + and ( + not k.endswith(_NON_STANDARD_THRESHOLD_SUFFIXES) or (tier_key_suffix != "" and k.endswith(tier_key_suffix)) + ) ] # Only sort the threshold keys (typically 1-2 keys instead of 66+) diff --git a/litellm/router.py b/litellm/router.py index 8eca83f9d5e..fa2ba644a21 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -85,6 +85,7 @@ from litellm.litellm_core_utils.get_llm_provider_logic import ( is_registered_custom_provider, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES from litellm.litellm_core_utils.ptu_pricing import ( PTU_COST_ATTRIBUTION_ENV_VAR, declares_ptu, @@ -8786,6 +8787,11 @@ class Router: backend_model=deployment.litellm_params.model, custom_llm_provider=deployment.litellm_params.custom_llm_provider, ) + Router._inherit_builtin_service_tier_pricing( + model_info=_model_info, + backend_model=Router._cost_map_backend_model(deployment), + custom_llm_provider=deployment.litellm_params.custom_llm_provider, + ) Router._inherit_builtin_tiered_output_rate( model_info=_model_info, backend_model=deployment.litellm_params.model, @@ -9801,6 +9807,41 @@ class Router: return (backend_key, backend_key.replace("responses/", "")) return (backend_key,) + @staticmethod + def _inherit_builtin_service_tier_pricing( + model_info: dict, # mutable-ok: deployment cost-map entry filled in place + backend_model: str, + custom_llm_provider: str | None, + ) -> None: + """Inherit missing tier rates so a standalone entry does not fall back to custom standard rates.""" + if ptu_terms(model_info) is not None and is_ptu_cost_attribution_enabled(): + return + if all(model_info.get(field) is None for field in ("input_cost_per_token", "output_cost_per_token")): + return + try: + backend_info: Final = litellm.get_model_info(model=backend_model, custom_llm_provider=custom_llm_provider) + except Exception: # noqa: BLE001 # get_model_info raises plain Exception for an unmapped backend model + return + backend_entry: Final = litellm.model_cost.get(backend_info.get("key") or "") + if not isinstance(backend_entry, dict): + return + for field, backend_value in backend_entry.items(): + if not field.endswith(SERVICE_TIER_COST_KEY_SUFFIXES): + continue + if model_info.get(field) is not None or backend_value is None: + continue + model_info[field] = copy.deepcopy(backend_value) + + @staticmethod + def _cost_map_backend_model(deployment: Deployment) -> str: + model_info_base_model: Final = deployment.model_info.base_model + if isinstance(model_info_base_model, str) and model_info_base_model: + return model_info_base_model + params_base_model: Final = deployment.litellm_params.get("base_model") + if isinstance(params_base_model, str) and params_base_model: + return params_base_model + return deployment.litellm_params.model + @staticmethod def _deployment_model_cost_payload(deployment: Deployment) -> dict: # mutable-ok: cost-map entry """The ``model_info`` a deployment contributes to ``litellm.model_cost``. @@ -9826,6 +9867,11 @@ class Router: backend_model=deployment.litellm_params.model, custom_llm_provider=deployment.litellm_params.custom_llm_provider, ) + Router._inherit_builtin_service_tier_pricing( + model_info=model_info, + backend_model=Router._cost_map_backend_model(deployment), + custom_llm_provider=deployment.litellm_params.custom_llm_provider, + ) Router._inherit_builtin_tiered_output_rate( model_info=model_info, backend_model=deployment.litellm_params.model, diff --git a/litellm/utils.py b/litellm/utils.py index 9eed0e8345f..0b10ed65107 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -45,7 +45,7 @@ from httpx import Proxy from httpx._utils import get_environment_proxies from openai.lib import _parsing, _pydantic from openai.types.chat.completion_create_params import ResponseFormat -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, with_config from tiktoken import Encoding from tokenizers import Tokenizer @@ -3121,6 +3121,121 @@ def _get_builtin_model_info_for_registration(model: str) -> ModelInfo | None: _runtime_registered_model_cost: Final[dict[str, dict[str, object]]] = {} # mutable-ok: replayed on reload +_runtime_registered_fields: Final[dict[str, frozenset[str]]] = {} # mutable-ok: fields each registration set + + +_RATE_TABLE: Final = TypeAdapter(Mapping[str, object]) +_RATE_TIERS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +_TOKEN_COST_FIELDS: Final = ("input_cost_per_token", "output_cost_per_token") +_DIRECTION_RATE_FIELDS: Final = MappingProxyType( + { + "input_cost_per_token": ("input_cost_per_token",), + "output_cost_per_token": ("output_cost_per_token", "output_cost_per_reasoning_token"), + } +) + + +def _is_overridden_rate(name: str, overridden_fields: tuple[str, ...]) -> bool: + is_variant: Final = name.startswith(tuple(f"{rate_field}_" for rate_field in overridden_fields)) + return name in overridden_fields or (is_variant and "_cache_" not in name) + + +def _without_overridden_rates(rates: Mapping[str, object], overridden_fields: tuple[str, ...]) -> Mapping[str, object]: + return MappingProxyType( + {name: rate for name, rate in rates.items() if not _is_overridden_rate(name, overridden_fields)} + ) + + +def _rate_table_with_flat_prices( + name: str, table: object, overridden_fields: tuple[str, ...], flat_prices: Mapping[str, object] +) -> object: + try: + if name == "tiered_pricing": + return [ + {**_without_overridden_rates(tier, overridden_fields), **flat_prices} + for tier in _RATE_TIERS.validate_python(table) + ] + if name == "off_peak_pricing": + return dict(_without_overridden_rates(_RATE_TABLE.validate_python(table), overridden_fields)) + except ValidationError: + return table + return table + + +def _without_overridden_price_variants( + entry: Mapping[str, object], value: Mapping[str, object] +) -> Mapping[str, object]: + flat_prices: Final = MappingProxyType( + {cost_field: value[cost_field] for cost_field in _TOKEN_COST_FIELDS if value.get(cost_field) is not None} + ) + overridden_fields: Final = tuple( + itertools.chain.from_iterable(_DIRECTION_RATE_FIELDS[cost_field] for cost_field in flat_prices) + ) + if not overridden_fields: + return entry + return MappingProxyType( + { + name: _rate_table_with_flat_prices(name, field_value, overridden_fields, flat_prices) + for name, field_value in entry.items() + if not _is_overridden_rate(name, overridden_fields) + } + ) + + +@with_config(ConfigDict(extra="allow")) +class _PricingModelInfo(ModelInfo): + pass + + +_PRICING_MODEL_INFO: Final = TypeAdapter(_PricingModelInfo) + + +def model_info_with_pricing_overrides(model_info: ModelInfo, registration: Mapping[str, object]) -> ModelInfo: + from litellm.types.utils import is_server_derived_pricing_key + + overrides: Final = { + name: value for name, value in registration.items() if value is not None and is_server_derived_pricing_key(name) + } + return _PRICING_MODEL_INFO.validate_python( + _update_dictionary(dict(_without_overridden_price_variants(model_info, overrides)), overrides) + ) + + +def runtime_model_cost_registrations( + model: str, custom_llm_provider: str +) -> tuple[tuple[str, Mapping[str, object]], ...]: + names: Final = _get_potential_model_names(model, custom_llm_provider) + candidates: Final = ( + model, + names["combined_model_name"], + f"{custom_llm_provider}/{names['split_model']}", + names["split_model"], + names["stripped_model_name"], + ) + registered: Final = ( + (key, litellm.model_cost.get(key), _runtime_registered_fields.get(key, frozenset())) + for key in dict.fromkeys(candidates) + ) + return tuple( + (key, MappingProxyType({name: value for name, value in entry.items() if name in fields})) + for key, entry, fields in registered + if isinstance(entry, Mapping) and fields + ) + + +@dataclass(frozen=True, slots=True) +class ModelInfoLookupMiss: + tripped: bool + + +def lookup_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | ModelInfoLookupMiss: + try: + info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + except ModelNotMappedError: + return ModelInfoLookupMiss(tripped=False) + except Exception: + return ModelInfoLookupMiss(tripped=True) + return ModelInfoLookupMiss(tripped=False) if is_generalized_model_info(info) else info class _LiveDeploymentReplay: @@ -3162,6 +3277,7 @@ def reapply_runtime_model_cost_registrations() -> None: have no such owner are recorded and replayed, and a registration describing a single request opts out of even that. """ + _runtime_registered_fields.clear() if _LiveDeploymentReplay.callback is not None: _LiveDeploymentReplay.callback() if _runtime_registered_model_cost: @@ -3291,6 +3407,9 @@ def register_model( ## override / add new keys to the existing model cost dictionary updated_dictionary = _update_dictionary(existing_model, value) litellm.model_cost.setdefault(model_cost_key, {}).update(updated_dictionary) + _runtime_registered_fields[str(model_cost_key)] = _runtime_registered_fields.get( + str(model_cost_key), frozenset() + ) | frozenset(name for name, field_value in value.items() if field_value is not None) # Invalidate case-insensitive lookup map since model_cost was modified _invalidate_model_cost_lowercase_map()