From 69aef480f42c5d2aea6177788df91d0899470922 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 06:11:33 +0000 Subject: [PATCH] refactor(catalog): register caller keys through native layers --- litellm/_catalog/_install.py | 55 +-- litellm/_catalog/_register.py | 173 +++++++++ litellm/utils.py | 334 +----------------- tests/unit/conftest.py | 8 +- .../test_get_model_cost_map.py | 4 +- tests/unit/test_conftest_isolation.py | 8 +- tests/unit/test_cost_calculator.py | 25 +- .../test_register_model_custom_pricing.py | 171 +++++---- .../unit/test_router_model_cost_isolation.py | 84 +---- tests/unit/test_utils.py | 280 +++++++++++---- 10 files changed, 573 insertions(+), 569 deletions(-) create mode 100644 litellm/_catalog/_register.py diff --git a/litellm/_catalog/_install.py b/litellm/_catalog/_install.py index c12a3b7144d..6e11a65d839 100644 --- a/litellm/_catalog/_install.py +++ b/litellm/_catalog/_install.py @@ -2,7 +2,7 @@ from __future__ import annotations import copy import threading -from collections.abc import Callable, Mapping +from collections.abc import Mapping from dataclasses import dataclass, field from datetime import datetime, timezone from itertools import groupby @@ -101,39 +101,50 @@ def remember_written_rows(keys: frozenset[str]) -> None: def registration_write( - target: dict[str, Mapping[str, object]], # mutable-ok: retain the live public dict and subclass write behavior key: str, - entry: Mapping[str, object], - registration: Mapping[str, object], + base: Mapping[str, object], + fields: Mapping[str, object], *, persist: bool, inherits_from: str | None, - replace: Callable[ # mutable-ok: legacy registration publishes into the live dict - [dict[str, Mapping[str, object]], str, Mapping[str, object], Mapping[str, object]], None - ], -) -> None: + generation: int, +) -> bool: from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry with LOCK: - current: Final = dict[str, Mapping[str, object]].get(target, key) + target: Final = _live_rows() + current: Final = dict[str, object].get(target, key) names: Final = current.names if isinstance(current, AliasedCostEntry) else () keys: Final = frozenset((key, *(name for name in names if target.get(name) is current))) record_pending_writes(keys) - generation: Final = STATE.generation - fields: Final = {name: entry.get(name) for name, value in registration.items() if value is not None} - erased: Final = {name: None for name, value in entry.items() if value is None} - overrides: Final = _ROW.validate_python({**fields, **erased}) - replace(target, key, entry, registration) - if STATE.generation != generation: - refreshed: Final = { - alias: AliasedCostEntry({**STATE.rows.get(alias, {}), **overrides}, tuple(keys), frozenset(fields)) - for alias in keys - } - write_rows(_live_rows(), dict(refreshed), []) + encoded: Final = _encoded_row(fields) + local: Final = ModelCostLayers() + local.diff({key: ({}, None)}) + local.diff({key: (None, _encoded_row(base))}) + local.record_registration(key, None, encoded) + registry: Final = STATE.registry or ModelRegistry.bundled() + composed: Final = local.compose_keys(registry, frozenset({key})) + entry: Final = composed[0][1] + if entry is None: + return True + previous_fields: Final[frozenset[str]] = ( + current.registered_fields if isinstance(current, AliasedCostEntry) else frozenset() + ) + published: Final = AliasedCostEntry( + {name: _decoded_value(value) for name, value in entry.items()}, + tuple(keys), + previous_fields | frozenset(fields), + ) + snapshot: Final = copy.deepcopy(dict(published)) + if generation != STATE.generation: + return False if persist: for alias in keys: - LAYERS.record_registration(alias, inherits_from, _encoded_row(overrides)) - remember_written_rows(keys) + LAYERS.record_registration(alias, inherits_from, encoded) + write_rows(target, dict.fromkeys(keys, published), []) + for alias in keys: + STATE.rows[alias] = snapshot + return True def accepted_catalog(catalog: LoadedCatalog) -> LoadedCatalog | None: diff --git a/litellm/_catalog/_register.py b/litellm/_catalog/_register.py new file mode 100644 index 00000000000..3feee46cfde --- /dev/null +++ b/litellm/_catalog/_register.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +from collections.abc import ItemsView, Iterator, Mapping +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter + +from litellm._catalog import _install +from litellm._logging import verbose_logger +from litellm.rust_bridge._native import RegistrationTarget, registration_target + +_ROW: Final = TypeAdapter(Mapping[str, object]) + + +def _base_fields( + target: RegistrationTarget, + inherited: Mapping[str, object], + value: Mapping[str, object], + custom_llm_provider: str | None, +) -> Mapping[str, object]: + from litellm import utils + + lookups: Final = (utils.lookup_model_info_for_registration(target.write, custom_llm_provider) for _ in range(3)) + read: Final = next( + (lookup for lookup in lookups if not (isinstance(lookup, utils.ModelInfoLookupMiss) and lookup.tripped)), + utils.ModelInfoLookupMiss(tripped=True), + ) + primary: Final = not isinstance(read, utils.ModelInfoLookupMiss) and read.get("key") == target.inherit_from + if primary: + return {name: field for name, field in inherited.items() if name != "litellm_provider" or field is not None} + rates: Final = ( + inherited + if target.names_the_row + else utils._without_overridden_price_variants( # pyright: ignore[reportPrivateUsage] # shared rate policy + inherited, value + ) + ) + generalized: Final = _ROW.validate_python( + utils.match_capability_generalizations( # pyright: ignore[reportUnknownMemberType] # validate legacy rules + target.write + ) + or {} + ) + own_fields: Final = { + name: field + for name, field in rates.items() + if name != "litellm_provider" or (target.write == target.inherit_from and field is not None) + } + return {**generalized, **own_fields} + + +def registration_row(key: str, provider: str | None) -> Mapping[str, object]: + with _install.LOCK: + live: Final = _install._live_rows() # pyright: ignore[reportPrivateUsage] # retain public dict identity + target: Final = registration_target(live, key, provider=provider) + return _ROW.validate_python(live.get(target.inherit_from, {}) if target.inherit_from is not None else {}) + + +def _base_fields_replaced_by(value: Mapping[str, object]) -> frozenset[str]: + return ( + frozenset({"explicit_cache_read_input_token_cost"}) + if value.get("cache_read_input_token_cost") is not None + and value.get("explicit_cache_read_input_token_cost") is None + else frozenset() + ) + + +def _register_entry( + key: str, + value: Mapping[str, object], + *, + persist: bool, + custom_llm_provider: str | None, + warning_display_name: str | None, +) -> bool: + from litellm import utils + + generation: Final = _install.STATE.generation + live: Final = _install._live_rows() # pyright: ignore[reportPrivateUsage] # retain public dict identity + provider: Final = value.get("litellm_provider") + target: Final = registration_target( + live, key, provider=custom_llm_provider or (provider if isinstance(provider, str) else None) + ) + inherited: Final = _ROW.validate_python( + live.get(target.inherit_from, {}) if target.inherit_from is not None else {} + ) + base: Final = _base_fields(target, inherited, value, custom_llm_provider) + merged: Final = utils.update_dictionary(dict(base), value) + fields: Final = { + name: field + for name, field in merged.items() + if value.get(name) is not None or name not in inherited or field != inherited[name] + } + erased: Final = _base_fields_replaced_by(value) + previous: Final = _ROW.validate_python(live.get(key, {})) + removed: Final = (frozenset(inherited) | frozenset(previous)) - frozenset(merged) + overrides: Final = {**fields, **dict.fromkeys(removed | erased)} + if generation != _install.STATE.generation: + return False + if not _install.registration_write( + target.write, + base, + overrides, + persist=persist, + inherits_from=target.inherit_from if target.inherit_from != target.write else None, + generation=generation, + ): + return False + if target.inherit_from is None and ( + value.get("cache_creation_input_token_cost") is None + and value.get("cache_read_input_token_cost") is None + and value.get("tiered_pricing") is None + and (value.get("input_cost_per_token") is not None or value.get("output_cost_per_token") is not None) + ): + verbose_logger.warning( + "register_model: model=%s has custom pricing but not in built-in cost map and no prefix/region variant " + "matched; cache_creation_input_token_cost and cache_read_input_token_cost will default to 0 for this " + "model (input/output cost tracking is unaffected). To track cache cost, add them to model_info", + warning_display_name or key, + ) + return True + + +def register( + entries: Mapping[str, Mapping[str, object]], + *, + persist: bool, + custom_llm_provider: str | None, + warning_display_name: str | None, +) -> None: + for key, value in entries.items(): + with _install.LOCK: + while not _register_entry( + key, + value, + persist=persist, + custom_llm_provider=custom_llm_provider, + warning_display_name=warning_display_name, + ): + pass + + +class RuntimeRegistrations(Mapping[str, Mapping[str, object]]): + @staticmethod + def _snapshot() -> Mapping[str, Mapping[str, object]]: + with _install.LOCK: + entries: Final[dict[str, Mapping[str, object]]] = {} # mutable-ok: fold ordered native sparse records + for key, _, fields in _install.LAYERS.registrations(): + entries[key] = MappingProxyType( + { + **entries.get(key, {}), + **{ + name: _install._decoded_value( # pyright: ignore[reportPrivateUsage] # opaque metadata + value + ) + for name, value in fields.items() + }, + } + ) + return MappingProxyType(entries) + + def __getitem__(self, key: str) -> Mapping[str, object]: + return self._snapshot()[key] + + def __iter__(self) -> Iterator[str]: + return iter(self._snapshot()) + + def __len__(self) -> int: + return len(self._snapshot()) + + def items(self) -> ItemsView[str, Mapping[str, object]]: + return self._snapshot().items() diff --git a/litellm/utils.py b/litellm/utils.py index 88e730468d9..974a34943d8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -79,7 +79,6 @@ from litellm.constants import ( MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE, NON_INFERENCE_CALL_TYPES, OPENAI_EMBEDDING_PARAMS, - PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) from litellm.litellm_core_utils.core_helpers import ( @@ -3486,17 +3485,6 @@ def _convert_stringified_numbers(value): return value -_BEDROCK_REGION_PREFIXES: Final = ( - "us.", - "eu.", - "apac.", - "jp.", - "au.", - "us-gov.", - "global.", - "ap-northeast-1.", -) - _TOKEN_COST_FIELDS: Final = ("input_cost_per_token", "output_cost_per_token") @@ -3504,39 +3492,6 @@ def _builtin_cost_map() -> dict[str, Mapping[str, object]]: return cast(dict[str, Mapping[str, object]], litellm.model_cost) -def _replace_cost_map_entry( - cost_map: dict[str, Mapping[str, object]], key: str, entry: Mapping[str, object], registration: Mapping[str, object] -) -> None: - current: Final = cost_map.get(key) - merged: Final = {**(current or {}), **entry} - names: Final = current.names if isinstance(current, AliasedCostEntry) else () - previous_fields: Final[frozenset[str]] = ( - current.registered_fields if isinstance(current, AliasedCostEntry) else frozenset() - ) - registered_fields: Final = previous_fields | frozenset( - name for name, value in registration.items() if value is not None - ) - sharing_names: Final = tuple(name for name in names if cost_map.get(name) is current) - cost_map.update(dict.fromkeys((key, *sharing_names), AliasedCostEntry(merged, names, registered_fields))) - - -def _without_synthesized_costs( - model_info: Mapping[str, object], key: str, value: Mapping[str, object] -) -> Mapping[str, object]: - """``_get_model_info_helper`` synthesizes ``input_cost_per_token`` / ``output_cost_per_token`` = 0 when the raw - entry has neither (#30198). Persisting those zeros would turn a name-priced entry into a free one and make - ``_is_cost_explicitly_configured`` disable budget enforcement on the next re-registration.""" - builtin_cost_map: Final = _builtin_cost_map() - raw_fields: Final = frozenset(builtin_cost_map.get(str(model_info["key"]), builtin_cost_map.get(key, {}))) - return MappingProxyType( - { - name: field_value - for name, field_value in model_info.items() - if name not in _TOKEN_COST_FIELDS or name in raw_fields or name in value - } - ) - - _DIRECTION_RATE_FIELDS: Final = MappingProxyType( { "input_cost_per_token": ("input_cost_per_token",), @@ -3617,84 +3572,11 @@ def model_info_with_pricing_overrides(model_info: ModelInfo, registration: Mappi ) -@dataclass(frozen=True, slots=True) -class _BuiltinCostMatch: - key: str - entry: Mapping[str, object] - names_the_row: bool - - @dataclass(frozen=True, slots=True) class ModelInfoLookupMiss: tripped: bool -@dataclass(frozen=True, slots=True) -class _FallbackRegistration: - entry: Mapping[str, object] - matched_builtin: bool - - -def _fallback_registration(key: str, provider: str, value: Mapping[str, object]) -> _FallbackRegistration: - """The key keeps its own entry, seeded from the capability rules an unregistered lookup resolves it to, with the - matched built-in entry on top. The matched row's provider stays behind, since the key's own lookup can reject it - (an ``azure_ai/`` key never resolves to an ``openai`` entry), and the key keeps the provider its caller gave.""" - match: Final = _resolve_builtin_model_cost_entry(key=key, provider=provider) - generalized: Final[Mapping[str, object]] = match_capability_generalizations(key) or {} - if match is None: - return _FallbackRegistration(entry=generalized, matched_builtin=False) - inherited: Final = match.entry if match.names_the_row else _without_overridden_price_variants(match.entry, value) - own_fields: Final = {name: field_value for name, field_value in inherited.items() if name != "litellm_provider"} - return _FallbackRegistration(entry={**generalized, **own_fields}, matched_builtin=True) - - -def _resolve_builtin_model_cost_entry(key: str, provider: str) -> _BuiltinCostMatch | None: - """Best-effort lookup of a built-in ``model_cost`` entry for a custom key - whose shape ``get_model_info`` cannot resolve (repeated provider prefixes - like ``bedrock/bedrock/bedrock/us.anthropic.claude-sonnet-4-6`` or region - aliases), or whose lookup tripped. - - The key itself is tried first, then the key behind its provider prefixes - and bedrock route, so a catalog row whose lookup tripped inherits from - itself rather than from a region-stripped sibling priced at the base rate. - ``names_the_row`` is set when the match is the row the key names, which - ``get_model_info`` resolves it to, so the caller keeps that row's rates as - an untripped registration does. Returns ``None`` when no candidate exists - under a provider ``get_model_info`` would accept for ``provider``, or, when - the value names none, for the provider prefix the key itself carries, - which is what ``get_model_info`` derives when it reads the key back. - """ - from litellm.llms.bedrock.common_utils import strip_bedrock_routing_prefix - - segments: Final = key.split("/") - match_provider: Final = provider or (segments[0] if len(segments) > 1 and segments[0] in LlmProvidersSet else "") - provider_prefix_count: Final = next( - (idx for idx, segment in enumerate(segments[:-1]) if segment not in LlmProvidersSet), len(segments) - 1 - ) - prefix_stripped: Final = tuple("/".join(segments[idx:]) for idx in range(1, provider_prefix_count + 1)) - prefix_free: Final = prefix_stripped[-1] if prefix_stripped else key - base: Final = ( - strip_bedrock_routing_prefix(prefix_free) if match_provider in ("bedrock", "bedrock_converse") else prefix_free - ) - region_stripped: Final = tuple( - base[len(region_prefix) :] for region_prefix in _BEDROCK_REGION_PREFIXES if base.startswith(region_prefix) - ) - stripped: Final = _strip_model_name(model=base, custom_llm_provider=match_provider) if match_provider else base - row_names: Final = (key, *prefix_stripped, base, stripped) - candidates: Final = (key, *prefix_stripped, base, *region_stripped, stripped) - - builtin_cost_map: Final = _builtin_cost_map() - for candidate in candidates: - entry = builtin_cost_map.get(candidate) - if ( - entry is not None - and entry.get("litellm_provider") is not None - and _check_provider_match(dict(entry), match_provider) - ): - return _BuiltinCostMatch(key=candidate, entry=dict(entry), names_the_row=candidate in row_names) - return None - - def is_generalized_model_info(model_info: ModelInfo) -> bool: """Whether ``model_info`` came from a fallback-generalization capability rule. @@ -3709,9 +3591,6 @@ def is_generalized_model_info(model_info: ModelInfo) -> bool: return key not in litellm.model_cost and match_capability_generalizations(key) is not None -_REGISTRATION_ATTEMPTS: Final = 3 - - def lookup_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | ModelInfoLookupMiss: try: info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) @@ -3722,162 +3601,15 @@ def lookup_model_info_for_registration(model: str, custom_llm_provider: str | No return ModelInfoLookupMiss(tripped=False) if is_generalized_model_info(info) else info -def _get_builtin_model_info_for_registration( - model: str, custom_llm_provider: str | None -) -> ModelInfo | ModelInfoLookupMiss: - """Resolve ``model`` to its built-in cost-map entry for registration merging. - - Misses when the lookup cannot map the model or resolved it via a - fallback-generalization capability rule, since a rule-derived entry carries no - pricing. A lookup that raised anything else, such as a concurrent registration - changing the map mid-walk, is retried, so a transient failure registers exactly - where the untripped lookup would. - """ - lookups: Final = ( - lookup_model_info_for_registration(model, custom_llm_provider) for _ in range(_REGISTRATION_ATTEMPTS) - ) - return next( - (lookup for lookup in lookups if not (isinstance(lookup, ModelInfoLookupMiss) and lookup.tripped)), - ModelInfoLookupMiss(tripped=True), - ) - - -def _skips_builtin_registration_lookup(key: str, provider: object) -> bool: - return provider in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO or any( - key.startswith(f"{p}/") for p in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO - ) - - -@dataclass(frozen=True, slots=True) -class _RegistrationWrite: - key: str - entry: Mapping[str, object] - - def model_cost_registration_row(key: str, provider: object) -> Mapping[str, object]: - """The row a ``register_model`` write of a ``key`` entry with this ``litellm_provider`` merges onto: the - ``litellm.model_cost`` row it resolves to (``openai/o3-pro`` lands on ``o3-pro``), else the row it seeds ``key`` - with from the built-in entry it falls back to (an OpenAI fine-tune takes its base model's row).""" - base: Final = _registration_base( - key, {"litellm_provider": provider}, custom_llm_provider=None, warning_display_name=None - ) - return _builtin_cost_map().get(base.key) or base.entry + from litellm._catalog._register import registration_row + + return registration_row(key, provider if isinstance(provider, str) else None) -def _registration_base( - key: str, value: Mapping[str, object], custom_llm_provider: str | None, warning_display_name: str | None -) -> _RegistrationWrite: - provider: Final = str(value.get("litellm_provider") or "") - if _skips_builtin_registration_lookup(key, provider): - return _RegistrationWrite(key=key, entry=dict(_builtin_cost_map().get(key, {}))) - builtin_model_info: Final = _get_builtin_model_info_for_registration( - model=key, custom_llm_provider=custom_llm_provider - ) - if not isinstance(builtin_model_info, ModelInfoLookupMiss): - return _RegistrationWrite( - key=builtin_model_info["key"], - entry=dict(_without_synthesized_costs(builtin_model_info, key=key, value=value)), - ) - fallback: Final = _fallback_registration(key=key, provider=provider, value=value) - if not fallback.matched_builtin and ( - value.get("cache_creation_input_token_cost") is None - and value.get("cache_read_input_token_cost") is None - and value.get("tiered_pricing") is None - and (value.get("input_cost_per_token") is not None or value.get("output_cost_per_token") is not None) - ): - verbose_logger.warning( - "register_model: model=%s has custom pricing but not in built-in cost map and no prefix/region variant matched; cache_creation_input_token_cost and cache_read_input_token_cost will default to 0 for this model (input/output cost tracking is unaffected). To track cache cost, add them to model_info", - warning_display_name or key, - ) - return _RegistrationWrite(key=key, entry=dict(fallback.entry)) +from litellm._catalog._register import RuntimeRegistrations - -def _base_fields_replaced_by(value: Mapping[str, object]) -> frozenset[str]: - return ( - frozenset({"explicit_cache_read_input_token_cost"}) - if value.get("cache_read_input_token_cost") is not None - and value.get("explicit_cache_read_input_token_cost") is None - else frozenset() - ) - - -def _registration_write( - key: str, value: Mapping[str, object], custom_llm_provider: str | None, warning_display_name: str | None -) -> _RegistrationWrite: - base: Final = _registration_base(key, value, custom_llm_provider, warning_display_name) - # ``get_model_info`` returns ``litellm_provider: None`` when the - # provider is unknown (e.g. custom deployments registered via - # ``Router.add_deployment``). Persisting that None into - # ``litellm.model_cost`` causes ``_check_provider_match`` to drop - # custom pricing on subsequent cost lookups. - existing_model: Final = { - name: field_value - for name, field_value in base.entry.items() - if name != "litellm_provider" or field_value is not None - } - return _RegistrationWrite( - key=base.key, - entry={**update_dictionary(existing_model, value), **dict.fromkeys(_base_fields_replaced_by(value))}, - ) - - -def _registration_attempt( - key: str, - value: Mapping[str, object], - custom_llm_provider: str | None, - warning_display_name: str | None, - final_attempt: bool, - persist_across_reloads: bool = True, -) -> _RegistrationWrite | None: - from litellm._catalog._install import registration_write - - generation: Final = get_model_cost_mutation_generation() - source: Final = _builtin_cost_map() - write: Final = _registration_write(key, value, custom_llm_provider, warning_display_name) - if not final_attempt and get_model_cost_mutation_generation() != generation: - return None - provider: Final = value.get("litellm_provider") - match: Final = _resolve_builtin_model_cost_entry( - key=write.key, provider=provider if isinstance(provider, str) else "" - ) - registration_write( - _builtin_cost_map() if final_attempt else source, - write.key, - write.entry, - value, - persist=persist_across_reloads, - inherits_from=match.key if match is not None and match.key != write.key else None, - replace=_replace_cost_map_entry, - ) - return write if final_attempt or get_model_cost_mutation_generation() == generation else None - - -def _register_on_current_map( - key: str, - value: Mapping[str, object], - custom_llm_provider: str | None, - warning_display_name: str | None, - persist_across_reloads: bool = True, -) -> _RegistrationWrite: - """A price data reload that swaps the map while the entry is being built or written would otherwise get the old - map's prices written over its own, or drop the entry with the old map, so each attempt writes into the map it was - built from and the entry is rebuilt when a reload replaced that map before the write finished. The last attempt - writes into the current map, so the entry is never lost.""" - attempts: Final = ( - _registration_attempt( - key, - value, - custom_llm_provider, - warning_display_name, - final_attempt=attempt == _REGISTRATION_ATTEMPTS - 1, - persist_across_reloads=persist_across_reloads, - ) - for attempt in range(_REGISTRATION_ATTEMPTS) - ) - return next(write for write in attempts if write is not None) - - -_runtime_registered_model_cost: Final[dict[str, dict[str, object]]] = {} # mutable-ok: replayed on reload +_runtime_registered_model_cost: Final[Mapping[str, Mapping[str, object]]] = RuntimeRegistrations() def runtime_model_cost_registrations( @@ -3922,30 +3654,6 @@ def set_live_deployment_replay(replay: Callable[[], None]) -> None: _LiveDeploymentReplay.callbacks = (replay,) -def reapply_runtime_model_cost_registrations() -> None: - """Re-apply runtime model metadata on top of a freshly adopted cost map. - - Adopting a new catalog replaces ``litellm.model_cost`` wholesale, which on - its own discards everything registered at runtime: the deployment - ``model_info`` the Router registers from ``model_list``, and pricing - overrides passed to ``register_model``. Both are re-applied here so a price - data reload only updates pricing rather than erasing operator-supplied model - metadata. - - The two are restored differently, and the difference is what keeps this - bounded. Deployment metadata is re-derived from the routers that are alive - right now, so a deployment that has been deleted or repointed, and a router - that has been discarded, are simply not part of the rebuild; nothing has to - withdraw them and nothing accumulates. Only ``register_model`` calls that - have no such owner are recorded and replayed, and a registration describing - a single request opts out of even that. - """ - for replay in _LiveDeploymentReplay.callbacks: - replay() - if _runtime_registered_model_cost: - register_model(model_cost=dict(_runtime_registered_model_cost)) - - def cost_map_omits_token_price(*keys: object) -> bool: """Whether the raw ``litellm.model_cost`` entries under ``keys`` exist but none carries a per-token price. @@ -4021,28 +3729,20 @@ def register_model( return model_cost loaded_model_cost = served_rows(accepted) - if persist_across_reloads: - _registrations: Final[Mapping[str, Mapping[str, object]]] = loaded_model_cost - for _registered_key, _registered_value in _registrations.items(): - _runtime_registered_model_cost[_registered_key] = update_dictionary( - dict(_runtime_registered_model_cost.get(_registered_key, {})), - _RATE_TABLE.validate_python(_registered_value), - ) + from litellm._catalog._register import register - for key, value in loaded_model_cost.items(): - from litellm._catalog._install import LOCK - - with LOCK: - write = _register_on_current_map( - str(key), value, custom_llm_provider, warning_display_name, persist_across_reloads - ) - - # Invalidate case-insensitive lookup map since model_cost was modified - _invalidate_model_cost_lowercase_map() - - verbose_logger.debug("added/updated model=%s in litellm.model_cost: %s", write.key, write.key) + entries: Final = {str(key): _RATE_TABLE.validate_python(value) for key, value in loaded_model_cost.items()} + register( + entries, + persist=persist_across_reloads, + custom_llm_provider=custom_llm_provider, + warning_display_name=warning_display_name, + ) + _invalidate_model_cost_lowercase_map() + for key, value in entries.items(): + verbose_logger.debug("added/updated model=%s in litellm.model_cost: %s", key, key) if add_to_provider_model_sets and value.get("litellm_provider") == "openrouter": - litellm.openrouter_models.add(str(key).split("/", 1)[-1]) + litellm.openrouter_models.add(key.split("/", 1)[-1]) if add_to_provider_model_sets: litellm.add_known_models(loaded_model_cost) return model_cost diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 7edd1bf073a..d8b1af070c0 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -293,20 +293,13 @@ def isolate_litellm_globals() -> Iterator[None]: @pytest.fixture(autouse=True) def isolate_router_model_cost_state() -> Iterator[None]: original_live_routers: Final = frozenset(litellm_router_module._live_routers) - original_runtime_registered_model_cost: Final = { - model_key: dict(model_value) - for model_key, model_value in litellm_utils_module._runtime_registered_model_cost.items() - } litellm_utils_module._invalidate_model_cost_lowercase_map() litellm_router_module._live_routers.clear() - litellm_utils_module._runtime_registered_model_cost.clear() yield for router in tuple(litellm_router_module._live_routers): litellm_router_module._live_routers.discard(router) for router in original_live_routers: litellm_router_module._live_routers.add(router) - litellm_utils_module._runtime_registered_model_cost.clear() - litellm_utils_module._runtime_registered_model_cost.update(original_runtime_registered_model_cost) litellm_utils_module._invalidate_model_cost_lowercase_map() litellm.get_model_info.cache_clear() @@ -352,6 +345,7 @@ def isolate_model_catalog() -> Iterator[None]: set_fallback_generalizations(original_rules) model_cost.clear() model_cost.update(model_cost_entries) + litellm.model_cost = model_cost models_by_provider.clear() models_by_provider.update(models_by_provider_entries) for models, entries in provider_model_sets: diff --git a/tests/unit/litellm_core_utils/test_get_model_cost_map.py b/tests/unit/litellm_core_utils/test_get_model_cost_map.py index e4f87bbf0df..93f2ba35c74 100644 --- a/tests/unit/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/unit/litellm_core_utils/test_get_model_cost_map.py @@ -553,7 +553,9 @@ def test_register_url_uses_explicit_source_under_local_flag(tmp_path: Path, monk def test_failed_url_registration_is_a_single_logged_noop(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: before: Final = copy.deepcopy(litellm.model_cost) - registrations: Final = copy.deepcopy(litellm.utils._runtime_registered_model_cost) + registrations: Final = { + key: copy.deepcopy(dict(row)) for key, row in litellm.utils._runtime_registered_model_cost.items() + } with caplog.at_level(logging.WARNING, logger="LiteLLM"): url: Final = (tmp_path / "missing.json").as_uri() assert litellm.register_model(url) == url diff --git a/tests/unit/test_conftest_isolation.py b/tests/unit/test_conftest_isolation.py index 92457119a82..4b40b6ab4e0 100644 --- a/tests/unit/test_conftest_isolation.py +++ b/tests/unit/test_conftest_isolation.py @@ -159,7 +159,10 @@ def test_aws_cache_reset_sees_a_replaced_module(monkeypatch: pytest.MonkeyPatch) @pytest.fixture(scope="module", autouse=True) def registered_collection_models() -> Iterator[None]: - original_registrations: Final = dict(litellm_utils_module._runtime_registered_model_cost) + from litellm._catalog import _install + + original_layers: Final = _install.LAYERS + _install.LAYERS = _install.ModelCostLayers() original_routers: Final = frozenset(litellm_router_module._live_routers) litellm.register_model( {"collection-isolation-ledger": {"litellm_provider": "openai", "input_cost_per_token": 0.001}} @@ -173,8 +176,7 @@ def registered_collection_models() -> Iterator[None]: ] ) yield - litellm_utils_module._runtime_registered_model_cost.clear() - litellm_utils_module._runtime_registered_model_cost.update(original_registrations) + _install.LAYERS = original_layers litellm_router_module._live_routers.discard(collection_router) litellm_router_module._live_routers.update(original_routers) diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 27b25c1f98e..3008ba43dd2 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -4815,6 +4815,7 @@ def test_completion_cost_legacy_mantle_route_prices_after_router_registration(lo /v1/chat/completions and /v1/messages alike, bills $0.""" from litellm import Router + sibling: Final = dict(litellm.model_cost["anthropic.claude-sonnet-5"]) Router( model_list=[ { @@ -4826,7 +4827,10 @@ def test_completion_cost_legacy_mantle_route_prices_after_router_registration(lo } ] ) - assert "bedrock/mantle/anthropic.claude-sonnet-5" not in litellm.model_cost + registered: Final = litellm.model_cost["bedrock/mantle/anthropic.claude-sonnet-5"] + assert registered["input_cost_per_token"] == sibling["input_cost_per_token"] + assert registered["output_cost_per_token"] == sibling["output_cost_per_token"] + assert litellm.model_cost["anthropic.claude-sonnet-5"] == sibling response = litellm.ModelResponse( id="msg_x", @@ -10118,16 +10122,14 @@ def test_non_bedrock_cost_ignores_bedrock_model_id(_local_model_cost_map: None) @pytest.mark.parametrize("metadata_prefix", ("bedrock/", "bedrock/converse/")) def test_bedrock_converse_profile_preserves_registered_price_override( _local_model_cost_map: None, - monkeypatch: pytest.MonkeyPatch, rate: float, persist_across_reloads: bool, routed_response: bool, metadata_prefix: str, ) -> None: - import litellm.utils as model_utils from litellm.cost_calculator import pricing_entry_for_cost_calc + from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map - monkeypatch.setattr(model_utils, "_runtime_registered_model_cost", {}) catalog: Final = dict(litellm.model_cost) model: Final = "openai.gpt-5.6-sol" litellm.register_model( @@ -10191,9 +10193,7 @@ def test_bedrock_converse_profile_preserves_registered_price_override( optional_params={"model_id": f"us.{model}"}, ) == pytest.approx(10 * updated_rate + 5 * rate) - monkeypatch.setattr(litellm, "model_cost", catalog) - model_utils._invalidate_model_cost_lowercase_map() - model_utils.reapply_runtime_model_cost_registrations() + adopt_model_cost_map(catalog) profile_prices: Final = litellm.get_model_info(f"us.{model}", custom_llm_provider="bedrock") expected_after_reload: Final = ( 10 * updated_rate + 5 * rate @@ -10209,11 +10209,8 @@ def test_bedrock_converse_profile_preserves_registered_price_override( def test_bedrock_converse_profile_ignores_registered_capabilities_without_prices( - _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch + _local_model_cost_map: None, ) -> None: - import litellm.utils as model_utils - - monkeypatch.setattr(model_utils, "_runtime_registered_model_cost", {}) model: Final = "openai.gpt-6-sol" litellm.register_model({model: {"supports_function_calling": True}}) response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)) @@ -10245,8 +10242,8 @@ def test_bedrock_converse_profile_preserves_cache_only_registered_prices( ) -> None: import litellm.utils as model_utils from litellm.cost_calculator import pricing_entry_for_cost_calc + from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map - monkeypatch.setattr(model_utils, "_runtime_registered_model_cost", {}) model: Final = "openai.cache-overlay-test" profile: Final = f"{profile_prefix}.{model}" catalog: Final = { @@ -10317,9 +10314,7 @@ def test_bedrock_converse_profile_preserves_cache_only_registered_prices( assert entry[1][cache_field] == rate assert litellm.model_cost[profile] == profile_before - monkeypatch.setattr(litellm, "model_cost", dict(catalog)) - model_utils._invalidate_model_cost_lowercase_map() - model_utils.reapply_runtime_model_cost_registrations() + adopt_model_cost_map(catalog) expected_after_reload: Final = ( 10 * 0.02 + 3 * 0.04 + 5 * (rate if persist_across_reloads else catalog[profile][cache_field]) ) diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index bbdfd985feb..f4b54731512 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -245,39 +245,12 @@ def test_register_model_strips_none_litellm_provider(): litellm.model_cost.pop(model_key, None) -def test_register_model_strips_none_litellm_provider_from_get_model_info(monkeypatch): - """Directly exercise the strip in ``register_model``. - - The companion test above hits the ``except Exception`` branch where - ``existing_model`` is an empty dict, so the ``pop`` is a no-op. This - test patches ``get_model_info`` to return the failure mode the strip - was added to handle, namely a populated dict whose ``litellm_provider`` - is ``None``. Without the strip, the merged entry in - ``litellm.model_cost`` would carry ``litellm_provider: None`` and - ``_check_provider_match`` would drop custom pricing. - - Regression test for https://github.com/BerriAI/litellm/issues/28336. - """ - from litellm import utils as litellm_utils +def test_register_model_strips_none_provider_from_the_served_row(monkeypatch: pytest.MonkeyPatch) -> None: from litellm.utils import _check_provider_match - model_key = "test-strip-none-provider-from-get-model-info-28336" - litellm.model_cost.pop(model_key, None) - - def _fake_get_model_info(model, *args, **kwargs): - assert model == model_key - return { - "key": model_key, - "litellm_provider": None, - "mode": "chat", - "max_tokens": 4096, - } - - # ``register_model`` calls ``get_model_info.cache_clear`` via - # ``_invalidate_model_cost_lowercase_map``, so the replacement must - # expose a no-op ``cache_clear`` attribute. - _fake_get_model_info.cache_clear = lambda: None - monkeypatch.setattr(litellm_utils, "get_model_info", _fake_get_model_info) + model_key: Final = "test-strip-none-provider-28336" + row: Final = {"litellm_provider": None, "mode": "chat", "max_tokens": 4096} + monkeypatch.setitem(litellm.model_cost, model_key, row) try: litellm.register_model( @@ -289,24 +262,13 @@ def test_register_model_strips_none_litellm_provider_from_get_model_info(monkeyp } ) - registered = litellm.model_cost.get(model_key) + registered: Final = litellm.model_cost.get(model_key) assert registered is not None, f"{model_key} should be in model_cost" - # The strip must have removed the None-valued provider that - # ``get_model_info`` returned. The key may be absent entirely, but - # it must never be present with value ``None``. - assert "litellm_provider" not in registered or (registered["litellm_provider"] is not None), ( - "register_model failed to strip litellm_provider=None returned " - f"by get_model_info, got {registered.get('litellm_provider')!r}" - ) - # Metadata from the patched ``get_model_info`` must still flow - # through, so we know the strip did not nuke the rest of the entry. + assert "litellm_provider" not in registered assert registered.get("mode") == "chat" assert registered.get("max_tokens") == 4096 - # And custom pricing from the registration call must be preserved. assert registered.get("input_cost_per_token") == 0.001 assert registered.get("output_cost_per_token") == 0.002 - # Downstream _check_provider_match must accept any provider for - # this entry, mirroring the cost calculator path. assert _check_provider_match(registered, "openai") is True assert _check_provider_match(registered, "anthropic") is True finally: @@ -847,7 +809,7 @@ def test_register_model_keeps_a_catalog_rows_own_prices_when_the_lookup_fails(mo @pytest.mark.parametrize("route", ["", "converse/", "invoke/"]) -def test_register_model_leaves_no_shadow_key_when_the_lookup_trips_once( +def test_register_model_keeps_caller_pricing_independent_when_the_lookup_trips_once( monkeypatch: pytest.MonkeyPatch, route: str ) -> None: catalog_key: Final = "au.anthropic.lit-5853-geo-model" @@ -879,11 +841,13 @@ def test_register_model_leaves_no_shadow_key_when_the_lookup_trips_once( with pytest.MonkeyPatch.context() as lookup_patch: lookup_patch.setattr(litellm.utils, "get_model_info", lookup_tripped_once_by_a_concurrent_registration) litellm.register_model({alias: {"litellm_provider": "bedrock", "mode": "chat"}}, persist_across_reloads=False) + assert litellm.model_cost[catalog_key]["input_cost_per_token"] == 3.3e-06 litellm.register_model({catalog_key: {"input_cost_per_token": repriced_input_cost}}, persist_across_reloads=False) try: - assert litellm.get_model_info(alias)["key"] == catalog_key - assert litellm.get_model_info(alias)["input_cost_per_token"] == repriced_input_cost + assert litellm.get_model_info(alias)["key"] == alias + assert litellm.get_model_info(alias)["input_cost_per_token"] == 3.3e-06 + assert litellm.model_cost[catalog_key]["input_cost_per_token"] == repriced_input_cost finally: _invalidate_model_cost_lowercase_map() @@ -982,7 +946,13 @@ def test_register_model_prices_a_provider_less_azure_ai_key_without_repricing_th _invalidate_model_cost_lowercase_map() -def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monkeypatch: pytest.MonkeyPatch) -> None: +@pytest.mark.parametrize("persist", [False, True]) +def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers( + monkeypatch: pytest.MonkeyPatch, persist: bool +) -> None: + from litellm._catalog import _register + from litellm.rust_bridge._native import RegistrationTarget + catalog_key: Final = "au.anthropic.lit-5853-reloaded-model" deployment_key: Final = f"bedrock/{catalog_key}" catalog_row: Final = { @@ -1001,30 +971,41 @@ def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monk **builtin_cost_map, catalog_key: {**catalog_row, "input_cost_per_token": reloaded_input_cost}, } - real_lookup: Final = litellm.utils.get_model_info + real_target: Final = _register.registration_target lookups: Final = itertools.count() - def lookup_racing_a_price_data_reload(model: str, custom_llm_provider: str | None = None) -> object: - info: Final = real_lookup(model=model, custom_llm_provider=custom_llm_provider) + def target_racing_a_price_data_reload( + rows: dict[str, object], key: str, *, provider: str | None = None + ) -> RegistrationTarget: + target: Final = real_target(rows, key, provider=provider) if next(lookups) == 0: adopt_model_cost_map(reloaded_cost_map) - return info + return target with pytest.MonkeyPatch.context() as lookup_patch: - lookup_patch.setattr(litellm.utils, "get_model_info", lookup_racing_a_price_data_reload) - litellm.register_model({deployment_key: {"litellm_provider": "bedrock"}}, persist_across_reloads=False) + lookup_patch.setattr(_register, "registration_target", target_racing_a_price_data_reload) + litellm.register_model({deployment_key: {"litellm_provider": "bedrock"}}, persist_across_reloads=persist) try: assert litellm.model_cost is live_cost_map assert litellm.model_cost[catalog_key]["input_cost_per_token"] == reloaded_input_cost assert litellm.get_model_info(deployment_key)["input_cost_per_token"] == reloaded_input_cost + adopt_model_cost_map(reloaded_cost_map) + assert (deployment_key in litellm.model_cost) is persist + if persist: + assert litellm.get_model_info(deployment_key)["input_cost_per_token"] == reloaded_input_cost finally: _invalidate_model_cost_lowercase_map() +@pytest.mark.parametrize("persist", [False, True]) def test_register_model_keeps_its_price_when_a_reload_adopts_a_map_as_it_writes( - monkeypatch: pytest.MonkeyPatch, + monkeypatch: pytest.MonkeyPatch, persist: bool, ) -> None: + from pydantic import JsonValue + + from litellm._catalog import _install + catalog_key: Final = "au.anthropic.lit-5853-reloaded-as-written-model" deployment_key: Final = f"bedrock/{catalog_key}" catalog_row: Final = { @@ -1042,33 +1023,35 @@ def test_register_model_keeps_its_price_when_a_reload_adopts_a_map_as_it_writes( } writes: Final = itertools.count() + encode: Final = _install._encoded_row - class CostMapAReloadReplacesAsItIsWritten(dict[str, dict[str, object]]): - def update(self, other: Mapping[str, dict[str, object]], /) -> None: # pyright: ignore[reportIncompatibleMethodOverride] # registration writes pass one mapping - if next(writes) == 0: - adopt_model_cost_map(reloaded_cost_map) - super().update(other) + def encode_racing_a_price_data_reload(row: Mapping[str, object]) -> dict[str, JsonValue]: + encoded: Final = encode(row) + if row.get("output_cost_per_token") == registered_output_cost and next(writes) == 0: + adopt_model_cost_map(reloaded_cost_map) + return encoded monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") adopt_model_cost_map({**builtin_cost_map, catalog_key: dict(catalog_row)}) - monkeypatch.setattr( - litellm, - "model_cost", - CostMapAReloadReplacesAsItIsWritten({**builtin_cost_map, catalog_key: dict(catalog_row)}), - ) + monkeypatch.setattr(_install, "_encoded_row", encode_racing_a_price_data_reload) _invalidate_model_cost_lowercase_map() live_cost_map: Final = litellm.model_cost litellm.register_model( {deployment_key: {"litellm_provider": "bedrock", "output_cost_per_token": registered_output_cost}}, - persist_across_reloads=False, + persist_across_reloads=persist, ) try: assert litellm.model_cost is live_cost_map assert litellm.get_model_info(deployment_key)["input_cost_per_token"] == reloaded_input_cost assert litellm.get_model_info(deployment_key)["output_cost_per_token"] == registered_output_cost + adopt_model_cost_map(reloaded_cost_map) + assert (deployment_key in litellm.model_cost) is persist + if persist: + assert litellm.get_model_info(deployment_key)["input_cost_per_token"] == reloaded_input_cost + assert litellm.get_model_info(deployment_key)["output_cost_per_token"] == registered_output_cost finally: _invalidate_model_cost_lowercase_map() @@ -1341,6 +1324,62 @@ def test_register_model_keeps_custom_flat_prices_over_inherited_rate_variants( _invalidate_model_cost_lowercase_map() +@pytest.mark.parametrize( + ("base_key", "caller", "provider", "served_provider", "keeps_variant"), + ( + ("registration-shape-model", "openai/registration-shape-model", "openai", "openai", True), + ("us.anthropic.shape-model", "bedrock/us.anthropic.shape-model", "bedrock", "bedrock", True), + ("us.anthropic.shape-model", "bedrock/bedrock/bedrock/us.anthropic.shape-model", "bedrock", None, True), + ("anthropic.shape-model", "bedrock/ap-northeast-1.anthropic.shape-model", "bedrock", None, False), + ("registration-shape-model", "azure_ai/registration-shape-model", "openai", None, True), + ), +) +def test_registration_keeps_primary_and_fallback_field_shaping_across_reload( + monkeypatch: pytest.MonkeyPatch, + base_key: str, + caller: str, + provider: str, + served_provider: str | None, + keeps_variant: bool, +) -> None: + row: Final = { + "litellm_provider": provider, + "mode": "chat", + "input_cost_per_token": 0.001, + "input_cost_per_token_above_200k_tokens": 0.004, + "cache_read_input_token_cost": 0.0001, + "supports_vision": True, + } + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setenv("AZURE_AI_API_BASE", "https://registration.openai.azure.com") + adopt_model_cost_map({base_key: row}) + litellm.register_model({caller: {"input_cost_per_token": 0.002}}) + expected: Final = { + "mode": "chat", + "input_cost_per_token": 0.002, + "cache_read_input_token_cost": row["cache_read_input_token_cost"], + "supports_vision": row["supports_vision"], + **({"litellm_provider": served_provider} if served_provider is not None else {}), + **( + {"input_cost_per_token_above_200k_tokens": row["input_cost_per_token_above_200k_tokens"]} + if keeps_variant + else {} + ), + } + assert litellm.model_cost[base_key] == row + assert litellm.model_cost[caller] == expected + expected_cost: Final = 250_000 * (0.004 if keeps_variant else 0.002) + assert litellm.cost_per_token(model=caller, prompt_tokens=250_000, completion_tokens=0) == pytest.approx( + (expected_cost, 0) + ) + adopt_model_cost_map({base_key: row}) + assert litellm.model_cost[base_key] == row + assert litellm.model_cost[caller] == expected + assert litellm.cost_per_token(model=caller, prompt_tokens=250_000, completion_tokens=0) == pytest.approx( + (expected_cost, 0) + ) + + def test_register_model_never_inherits_pricing_across_providers(monkeypatch: pytest.MonkeyPatch) -> None: openai_key: Final = "lit-5853-openai-only-model" monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index f001008ee2c..0726ec76852 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -44,7 +44,6 @@ from litellm.types.router import ( ) from litellm.utils import ( _invalidate_model_cost_lowercase_map, - reapply_runtime_model_cost_registrations, ) @@ -52,18 +51,9 @@ _COPILOT_TOKEN_EXPIRES_AT: Final = 4_102_444_800 def _simulate_price_data_reload(fetched_catalog): - """Drive what a price data reload does to this process's litellm state. + from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map - Mirrors `litellm.proxy.proxy_server._swap_in_model_cost_map`, which is the - one place both reload paths adopt a freshly fetched catalog; that wiring is - covered in the proxy's own tests, so these exercise the replay itself - without dragging the proxy in. The provider model sets that helper also - repopulates are left alone, since nothing here reads them and rebuilding - them from a two-entry catalog would outlive the test. - """ - litellm.model_cost = fetched_catalog - _invalidate_model_cost_lowercase_map() - reapply_runtime_model_cost_registrations() + adopt_model_cost_map(fetched_catalog) def _nested_container_ids(value: object) -> frozenset[int]: @@ -2172,14 +2162,6 @@ def test_price_data_reload_preserves_router_registered_model_info(monkeypatch): max_input_tokens / max_output_tokens from every custom model group and /model_group/info starts reporting nulls. """ - from litellm import utils as litellm_utils - - monkeypatch.setattr( - litellm_utils, - "_runtime_registered_model_cost", - dict(litellm_utils._runtime_registered_model_cost), - ) - router = Router( model_list=[ { @@ -2220,14 +2202,6 @@ def test_price_data_reload_preserves_custom_override_of_a_catalog_model(monkeypa the same bug: the reload does not blank the metadata, it reverts the operator's model_info override to the upstream catalog values. """ - from litellm import utils as litellm_utils - - monkeypatch.setattr( - litellm_utils, - "_runtime_registered_model_cost", - dict(litellm_utils._runtime_registered_model_cost), - ) - router = Router( model_list=[ { @@ -2272,14 +2246,6 @@ def test_deleted_deployments_are_not_replayed_onto_later_reloads(monkeypatch): key that another live deployment still points at must survive the same deletion. """ - from litellm import utils as litellm_utils - - monkeypatch.setattr( - litellm_utils, - "_runtime_registered_model_cost", - dict(litellm_utils._runtime_registered_model_cost), - ) - router = Router( model_list=[ { @@ -2326,14 +2292,6 @@ def test_deleting_a_deployment_leaves_catalog_pricing_for_its_backend_model(monk A backend key is shared with the fetched catalog, so withdrawing the entries a deleted deployment owns must not take real upstream pricing down with it. """ - from litellm import utils as litellm_utils - - monkeypatch.setattr( - litellm_utils, - "_runtime_registered_model_cost", - dict(litellm_utils._runtime_registered_model_cost), - ) - backend_model = "gemini/gemini-2.5-pro" catalog_entry = litellm.get_model_info(model=backend_model) catalog_input_cost = catalog_entry["input_cost_per_token"] @@ -2371,14 +2329,6 @@ def test_repointing_a_deployment_drops_its_previous_backend_key(monkeypatch): old backend key behind, and a replayed registry would re-assert it onto every later catalog for the life of the process. """ - from litellm import utils as litellm_utils - - monkeypatch.setattr( - litellm_utils, - "_runtime_registered_model_cost", - dict(litellm_utils._runtime_registered_model_cost), - ) - router = Router( model_list=[ { @@ -2786,7 +2736,8 @@ def test_a_github_copilot_chat_pin_opts_a_dual_endpoint_model_out_of_the_respons @pytest.mark.parametrize("registered_mode, pinned_mode", [("chat", "responses"), ("responses", "chat")]) @pytest.mark.parametrize( - "model, custom_llm_provider", [("github_copilot/routing-test-model", None), ("routing-test-model", "github_copilot")] + "model, custom_llm_provider", + [("github_copilot/routing-test-model", None), ("routing-test-model", "github_copilot")], ) def test_a_github_copilot_responses_pin_still_opts_the_model_into_the_responses_api( monkeypatch: pytest.MonkeyPatch, @@ -3130,14 +3081,6 @@ def test_strategy_router_alias_pricing_never_enters_model_cost(monkeypatch): strip must also survive a price-data reload, which rebuilds entries by walking the live routers. """ - from litellm import utils as litellm_utils - - monkeypatch.setattr( - litellm_utils, - "_runtime_registered_model_cost", - dict(litellm_utils._runtime_registered_model_cost), - ) - router = Router( model_list=[ { @@ -3827,12 +3770,7 @@ def test_every_declaring_deployment_is_named(caplog): def _simulate_price_data_reload_with_provider_sets(monkeypatch, fetched_catalog): - """Like `_simulate_price_data_reload`, plus the provider model-set refresh the proxy's - `_swap_in_model_cost_map` does before replaying, so bare names in the new catalog resolve.""" - monkeypatch.setattr(litellm, "model_cost", fetched_catalog) - _invalidate_model_cost_lowercase_map() - litellm.add_known_models(model_cost_map=fetched_catalog) - reapply_runtime_model_cost_registrations() + _simulate_price_data_reload(fetched_catalog) def test_a_config_deployment_dropped_by_a_stale_cost_map_comes_back_on_reload(monkeypatch): @@ -3926,7 +3864,7 @@ def test_price_data_reload_refreshes_the_cached_model_group_and_deployment_info( assert router.cached_deployment_model_info("dep-a", "openai/gpt-4o")["input_cost_per_token"] == old_price new_price = old_price * 10 - fresh_catalog = copy.deepcopy(litellm.model_cost) + fresh_catalog = litellm.get_model_cost_map(url="") fresh_catalog["gpt-4o"]["input_cost_per_token"] = new_price _simulate_price_data_reload_with_provider_sets(monkeypatch, fresh_catalog) @@ -3934,7 +3872,9 @@ def test_price_data_reload_refreshes_the_cached_model_group_and_deployment_info( assert router.cached_deployment_model_info("dep-a", "openai/gpt-4o")["input_cost_per_token"] == new_price -def test_price_data_reload_keeps_geo_deployments_priced_while_requests_register_pricing(monkeypatch: pytest.MonkeyPatch) -> None: +def test_price_data_reload_keeps_geo_deployments_priced_while_requests_register_pricing( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") catalog: Final = litellm.get_model_cost_map(url="") monkeypatch.setattr(litellm, "model_cost", dict(catalog)) @@ -3996,9 +3936,13 @@ def test_price_data_reload_keeps_geo_deployments_priced_while_requests_register_ registrations_landed: Final = writer_outcome.get(timeout=5) try: assert isinstance(registrations_landed, int) and registrations_landed > 0, registrations_landed - assert tuple(backend for backend in backends if backend in litellm.model_cost) == () + assert tuple(backend for backend in backends if backend in litellm.model_cost) == backends for backend in backends: assert litellm.get_model_info(backend)["input_cost_per_token"] == canonical_prices[backend] + assert ( + litellm.model_cost[backend.removeprefix("bedrock/")]["input_cost_per_token"] + == canonical_prices[backend] + ) assert tuple(router.get_model_names()) == tuple(backend.rsplit(".", 1)[-1] for backend in backends) finally: _invalidate_model_cost_lowercase_map() diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 55b7bc78a11..1df1dd427b8 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -2258,6 +2258,166 @@ class TestProxyFunctionCalling: ) +@pytest.mark.parametrize( + ("key", "base_key", "provider", "mode"), + ( + ("databricks/databricks-claude-opus-4-99", "databricks/databricks-claude-opus-4", "databricks", "chat"), + ("openai/o3-pro", "o3-pro", "openai", "responses"), + ("bedrock/anthropic.claude-sonnet-4-6-20260101", "bedrock/anthropic.claude-sonnet-4-6", "bedrock", "chat"), + ), +) +def test_register_model_preserves_caller_key_and_inherits_without_repricing_sibling( + key: str, base_key: str, provider: str, mode: str +) -> None: + from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map + from litellm.utils import model_cost_registration_row + + base_row: Final = { + "litellm_provider": provider, + "mode": mode, + "max_input_tokens": 8000, + "input_cost_per_token": 0.001, + "supports_vision": True, + } + adopt_model_cost_map({base_key: base_row}) + live: Final = litellm.model_cost + litellm.register_model( + {key: {"max_input_tokens": 32000, "input_cost_per_token": 0.005, "litellm_provider": provider}} + ) + + assert litellm.model_cost is live + assert litellm.model_cost[base_key] == base_row + assert litellm.model_cost[key] == {**base_row, "max_input_tokens": 32000, "input_cost_per_token": 0.005} + assert model_cost_registration_row(key, provider)["mode"] == mode + + refreshed_row: Final = {**base_row, "supports_vision": False, "input_cost_per_token": 0.002} + adopt_model_cost_map({base_key: refreshed_row}) + + assert litellm.model_cost is live + assert litellm.model_cost[base_key] == refreshed_row + assert litellm.model_cost[key] == {**refreshed_row, "max_input_tokens": 32000, "input_cost_per_token": 0.005} + + +def test_register_model_keeps_case_variant_fields_independent_after_reload() -> None: + from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map + + upper: Final = "hosted_vllm/Acme-Model" + lower: Final = "hosted_vllm/acme-model" + rows: Final = { + upper: {"max_input_tokens": 8000, "supports_vision": True, "litellm_provider": "hosted_vllm"}, + lower: {"max_input_tokens": 32000, "litellm_provider": "hosted_vllm"}, + } + base: Final = {"baseline-model": {"mode": "chat"}} + adopt_model_cost_map(base) + for key, row in rows.items(): + litellm.register_model({key: row}) + + assert dict(litellm.model_cost) == {**base, **rows} + adopt_model_cost_map(base) + assert dict(litellm.model_cost) == {**base, **rows} + + +@pytest.mark.parametrize("registration_last", (True, False)) +def test_register_model_and_direct_write_keep_last_writer_after_reload(registration_last: bool) -> None: + from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map + + key: Final = "gpt-4o" + row: Final = {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.001} + adopt_model_cost_map({key: row}) + litellm.model_cost[key]["output_cost_per_token"] = 0.8 + if registration_last: + litellm.model_cost[key]["input_cost_per_token"] = 0.7 + litellm.register_model({key: {"input_cost_per_token": 0.5}}) + else: + litellm.register_model({key: {"input_cost_per_token": 0.5}}) + litellm.model_cost[key]["input_cost_per_token"] = 0.7 + expected: Final = 0.5 if registration_last else 0.7 + assert litellm.model_cost[key]["input_cost_per_token"] == expected + adopt_model_cost_map({key: {**row, "input_cost_per_token": 0.003}}) + assert litellm.model_cost[key]["input_cost_per_token"] == expected + assert litellm.model_cost[key]["output_cost_per_token"] == 0.8 + + +def test_registration_provider_scope_replaces_an_exact_cross_provider_collision() -> None: + from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map + + key: Final = "opaque-deployment-id" + catalog: Final = { + key: {"litellm_provider": "bedrock", "input_cost_per_token": 0.1, "output_cost_per_token": 0.2} + } + adopt_model_cost_map(catalog) + expected: Final = {"mode": "chat", "input_cost_per_token": 0.5} + litellm.register_model({key: expected}, custom_llm_provider="openai", add_to_provider_model_sets=False) + assert litellm.model_cost[key] == expected + adopt_model_cost_map(catalog) + assert litellm.model_cost[key] == expected + + +def test_register_model_nonpersistent_write_is_not_captured_as_a_direct_write() -> None: + from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map + + row: Final = {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.001} + adopt_model_cost_map({"gpt-4o": row}) + litellm.register_model({"openai/gpt-4o": {"input_cost_per_token": 0.5}}, persist_across_reloads=False) + litellm.register_model({"opaque-request-id": {"max_tokens": 1234}}, persist_across_reloads=False) + assert litellm.model_cost["openai/gpt-4o"]["input_cost_per_token"] == 0.5 + assert litellm.model_cost["opaque-request-id"]["max_tokens"] == 1234 + + adopt_model_cost_map({"gpt-4o": row}) + + assert dict(litellm.model_cost) == {"gpt-4o": row} + + +def test_runtime_registration_view_is_sparse_detached_read_only_and_persistent() -> None: + from uuid import UUID + + from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map + from litellm.utils import _runtime_registered_model_cost + + key: Final = "openai/registration-view" + row: Final = {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 1.0} + metadata: Final = UUID(int=17) + adopt_model_cost_map({key: row}) + litellm.register_model({key: {"input_cost_per_token": 0, "supports_vision": False, "metadata": [metadata]}}) + snapshot: Final = dict(_runtime_registered_model_cost.items()) + expected: Final = {"input_cost_per_token": 0, "supports_vision": False, "metadata": [metadata]} + assert snapshot == {key: expected} + with pytest.raises(TypeError): + _runtime_registered_model_cost[key] = {} + with pytest.raises(TypeError): + snapshot[key]["input_cost_per_token"] = 7 + snapshot[key]["metadata"].append(UUID(int=18)) + assert dict(_runtime_registered_model_cost.items()) == {key: expected} + litellm.register_model({key: {"output_cost_per_token": 2}}) + litellm.model_cost["direct-only"] = {"mode": "chat"} + litellm.register_model({"ephemeral-only": {"mode": "chat"}}, persist_across_reloads=False) + assert dict(_runtime_registered_model_cost.items()) == {key: {**expected, "output_cost_per_token": 2}} + assert "output_cost_per_token" not in snapshot[key] + adopt_model_cost_map({key: {**row, "input_cost_per_token": 9}}) + assert litellm.model_cost[key] == {**row, **expected, "output_cost_per_token": 2} + assert litellm.model_cost["direct-only"] == {"mode": "chat"} + assert "ephemeral-only" not in litellm.model_cost + + +@pytest.mark.parametrize("register_after_delete", [False, True]) +def test_registration_tombstone_order_survives_repeated_reload(register_after_delete: bool) -> None: + from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map + + key: Final = "openai/deleted-registration" + row: Final = {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 1.0} + adopt_model_cost_map({key: row}) + litellm.register_model({key: {"input_cost_per_token": 2.0}}) + del litellm.model_cost[key] + if register_after_delete: + litellm.register_model({key: {"output_cost_per_token": 3.0}}) + expected: Final = {key: {"output_cost_per_token": 3.0}} if register_after_delete else {} + assert dict(litellm.model_cost) == expected + adopt_model_cost_map({key: row}) + assert dict(litellm.model_cost) == expected + adopt_model_cost_map({key: row}) + assert dict(litellm.model_cost) == expected + + @pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map") def test_register_model_price_is_used_for_completion_cost() -> None: model: Final = "migration-custom-pricing-model" @@ -2328,8 +2488,10 @@ def test_completion_registers_test_owned_model_prices() -> None: ) assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages} - assert litellm.model_cost[model]["input_cost_per_token"] == 0.001 - assert litellm.model_cost[model]["output_cost_per_token"] == 0.002 + assert litellm.model_cost[f"openai/{model}"]["input_cost_per_token"] == 0.001 + assert litellm.model_cost[f"openai/{model}"]["output_cost_per_token"] == 0.002 + assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001 + assert litellm.model_cost[model]["output_cost_per_token"] == 0.0002 assert response._hidden_params["response_cost"] == pytest.approx(0.018) @@ -2410,23 +2572,30 @@ def test_register_model_with_scientific_notation(): @pytest.mark.usefixtures("isolated_openai_model_sets") -@respx.mock -def test_register_model_url_fetch_uses_single_attempt(monkeypatch): +def test_register_model_url_fetch_uses_single_attempt(monkeypatch: pytest.MonkeyPatch) -> None: + from unittest.mock import Mock + + from litellm._catalog import _fetch + from litellm.rust_bridge._native import RegistryUnavailable + monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) - before = dict(litellm.model_cost) - threads_before = {thread.name for thread in threading.enumerate()} - route = respx.get("https://example.invalid/custom_pricing.json").mock(return_value=httpx.Response(503)) + before: Final = dict(litellm.model_cost) + threads_before: Final = {thread.name for thread in threading.enumerate()} + fetch_once: Final = Mock(return_value=Mock(spec=RegistryUnavailable, reason="HTTP 503", retryable=True)) + monkeypatch.setattr(_fetch, "fetch_once", fetch_once) litellm.register_model(model_cost="https://example.invalid/custom_pricing.json") - threads_after = {thread.name for thread in threading.enumerate()} - assert route.call_count == 1 + threads_after: Final = {thread.name for thread in threading.enumerate()} + fetch_once.assert_called_once_with( + "https://example.invalid/custom_pricing.json", if_none_match=None, timeout_seconds=5.0 + ) assert not (threads_after - threads_before) & {"litellm-model-cost-map-retry"} assert not any( thread.name == "litellm-model-cost-map-retry" and thread.is_alive() for thread in threading.enumerate() ) - assert litellm.model_cost.keys() >= before.keys() + assert litellm.model_cost == before def test_register_model_openrouter_without_slash(): @@ -4929,7 +5098,7 @@ async def test_builtin_string_callback_registers_when_subclass_already_active( @pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map") @pytest.mark.parametrize("rate", (0.0, 0.000123)) @pytest.mark.parametrize("updated_rate", (None, 0.0, 0.000456)) -def test_reapply_runtime_registrations_replays_register_model_overrides( +def test_install_preserves_register_model_overrides( monkeypatch: pytest.MonkeyPatch, rate: float, updated_rate: float | None ) -> None: """ @@ -4938,22 +5107,8 @@ def test_reapply_runtime_registrations_replays_register_model_overrides( so without replaying those registrations the override is silently lost and the model reverts to upstream pricing. """ - from litellm import utils as litellm_utils - from litellm.utils import ( - _invalidate_model_cost_lowercase_map, - reapply_runtime_model_cost_registrations, - ) - - monkeypatch.setattr( - litellm_utils, - "_runtime_registered_model_cost", - dict(litellm_utils._runtime_registered_model_cost), - ) - # Only the recorded half is under test here; the live-router rebuild is covered - # in test_router_model_cost_isolation.py. Routers built by earlier tests in this - # process stay in the weak set until they are collected, so leaving the callback - # installed would make this depend on when that happens. - monkeypatch.setattr(litellm_utils._LiveDeploymentReplay, "callback", None) + from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map + from litellm.utils import _invalidate_model_cost_lowercase_map saved_model_cost: Final = litellm.model_cost monkeypatch.setattr(litellm, "model_cost", dict(saved_model_cost)) @@ -4974,16 +5129,16 @@ def test_reapply_runtime_registrations_replays_register_model_overrides( ) litellm.register_model({"openai/gpt-4o": {"supports_function_calling": False}}) - litellm.model_cost = { - "openai/gpt-4o": { - "litellm_provider": "openai", - "mode": "chat", - "input_cost_per_token": 0.000999, - "max_input_tokens": 4242, + adopt_model_cost_map( + { + "openai/gpt-4o": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 0.000999, + "max_input_tokens": 4242, + } } - } - _invalidate_model_cost_lowercase_map() - reapply_runtime_model_cost_registrations() + ) expected_rate: Final = rate if updated_rate is None else updated_rate assert litellm.model_cost["openai/gpt-4o"]["input_cost_per_token"] == expected_rate @@ -4995,24 +5150,15 @@ def test_reapply_runtime_registrations_replays_register_model_overrides( @pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map") -def test_reapply_runtime_registrations_drops_request_scoped_registrations(monkeypatch): +def test_install_drops_request_scoped_registrations(monkeypatch): """ Per-request custom pricing describes one call, so it must not be re-asserted over every future catalog. Replaying it would let a one-off price outlive the catalog generation it was applied to and silently beat fresh upstream pricing forever, while a durable override registered alongside it survives. """ - from litellm import utils as litellm_utils - from litellm.utils import ( - _invalidate_model_cost_lowercase_map, - reapply_runtime_model_cost_registrations, - ) - - monkeypatch.setattr( - litellm_utils, - "_runtime_registered_model_cost", - dict(litellm_utils._runtime_registered_model_cost), - ) + from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map + from litellm.utils import _invalidate_model_cost_lowercase_map saved_model_cost = litellm.model_cost try: @@ -5025,12 +5171,12 @@ def test_reapply_runtime_registrations_drops_request_scoped_registrations(monkey persist_across_reloads=False, ) - litellm.model_cost = { - "openai/gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 0.000999}, - "openai/gpt-4o-mini": {"litellm_provider": "openai", "input_cost_per_token": 0.000888}, - } - _invalidate_model_cost_lowercase_map() - reapply_runtime_model_cost_registrations() + adopt_model_cost_map( + { + "openai/gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 0.000999}, + "openai/gpt-4o-mini": {"litellm_provider": "openai", "input_cost_per_token": 0.000888}, + } + ) assert litellm.model_cost["openai/gpt-4o"]["input_cost_per_token"] == 0.000111 assert litellm.model_cost["openai/gpt-4o-mini"]["input_cost_per_token"] == 0.000888 @@ -7377,9 +7523,7 @@ async def test_router_resolves_an_oidc_api_key_only_on_a_proxy_level_deployment( try: with respx.mock(assert_all_called=True) as upstream: upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) - await router.acompletion( - model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"} - ) + await router.acompletion(model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"}) sent: Final = upstream.calls.last.request.headers["Authorization"] finally: router.discard() @@ -7447,7 +7591,9 @@ async def test_an_inline_oidc_api_key_reaches_the_endpoint_its_deployment_config with respx.mock(assert_all_called=False) as upstream: upstream.route().respond(200, json=_BEDROCK_REPLY) await router.acompletion( - model="claude", messages=_PING, aws_bedrock_runtime_endpoint="https://BEDROCK-runtime.us-east-1.amazonaws.com" + model="claude", + messages=_PING, + aws_bedrock_runtime_endpoint="https://BEDROCK-runtime.us-east-1.amazonaws.com", ) sent: Final = tuple( (call.request.url.host, call.request.headers["Authorization"]) for call in upstream.calls @@ -7656,7 +7802,9 @@ async def test_a_team_row_naming_an_oidc_credential_reaches_only_the_credentials ( lambda reference: LiteLLM_Params( **litellm.Router( - model_list=[{"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": reference}}] + model_list=[ + {"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": reference}} + ] ).get_model_list(model_name="openai/*")[0]["litellm_params"] ), lambda reference: "Bearer listing-token", @@ -8137,10 +8285,9 @@ def _vcr_outcome_gate(request, vcr): @pytest.fixture(scope="function") def setup_and_teardown(): - """ - This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. - """ - importlib.reload(litellm) + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap, adopt_model_cost_map + + adopt_model_cost_map(GetModelCostMap.load_local_model_cost_map()) loop = asyncio.get_event_loop_policy().new_event_loop() asyncio.set_event_loop(loop) print(litellm) @@ -8308,15 +8455,12 @@ _SCALAR_DEFAULTS = { @pytest.fixture(scope="module") def setup_and_teardown_local_testing(): - """ - Module-scoped setup. Reloads litellm only in single-process mode - (skipped under xdist to avoid cross-worker interference). - """ import litellm + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap, adopt_model_cost_map worker_id = os.environ.get("PYTEST_XDIST_WORKER", None) if worker_id is None: - importlib.reload(litellm) + adopt_model_cost_map(GetModelCostMap.load_local_model_cost_map()) try: if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): import litellm.proxy.proxy_server