refactor(catalog): register caller keys through native layers

This commit is contained in:
Devin AI 2026-10-26 06:11:33 +00:00
parent 430d3be2c7
commit 69aef480f4
10 changed files with 573 additions and 569 deletions

View file

@ -2,7 +2,7 @@ from __future__ import annotations
import copy
import threading
from collections.abc import Callable, Mapping
from collections.abc import Mapping
from dataclasses import dataclass, field
from datetime import datetime, timezone
from itertools import groupby
@ -101,39 +101,50 @@ def remember_written_rows(keys: frozenset[str]) -> None:
def registration_write(
target: dict[str, Mapping[str, object]], # mutable-ok: retain the live public dict and subclass write behavior
key: str,
entry: Mapping[str, object],
registration: Mapping[str, object],
base: Mapping[str, object],
fields: Mapping[str, object],
*,
persist: bool,
inherits_from: str | None,
replace: Callable[ # mutable-ok: legacy registration publishes into the live dict
[dict[str, Mapping[str, object]], str, Mapping[str, object], Mapping[str, object]], None
],
) -> None:
generation: int,
) -> bool:
from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry
with LOCK:
current: Final = dict[str, Mapping[str, object]].get(target, key)
target: Final = _live_rows()
current: Final = dict[str, object].get(target, key)
names: Final = current.names if isinstance(current, AliasedCostEntry) else ()
keys: Final = frozenset((key, *(name for name in names if target.get(name) is current)))
record_pending_writes(keys)
generation: Final = STATE.generation
fields: Final = {name: entry.get(name) for name, value in registration.items() if value is not None}
erased: Final = {name: None for name, value in entry.items() if value is None}
overrides: Final = _ROW.validate_python({**fields, **erased})
replace(target, key, entry, registration)
if STATE.generation != generation:
refreshed: Final = {
alias: AliasedCostEntry({**STATE.rows.get(alias, {}), **overrides}, tuple(keys), frozenset(fields))
for alias in keys
}
write_rows(_live_rows(), dict(refreshed), [])
encoded: Final = _encoded_row(fields)
local: Final = ModelCostLayers()
local.diff({key: ({}, None)})
local.diff({key: (None, _encoded_row(base))})
local.record_registration(key, None, encoded)
registry: Final = STATE.registry or ModelRegistry.bundled()
composed: Final = local.compose_keys(registry, frozenset({key}))
entry: Final = composed[0][1]
if entry is None:
return True
previous_fields: Final[frozenset[str]] = (
current.registered_fields if isinstance(current, AliasedCostEntry) else frozenset()
)
published: Final = AliasedCostEntry(
{name: _decoded_value(value) for name, value in entry.items()},
tuple(keys),
previous_fields | frozenset(fields),
)
snapshot: Final = copy.deepcopy(dict(published))
if generation != STATE.generation:
return False
if persist:
for alias in keys:
LAYERS.record_registration(alias, inherits_from, _encoded_row(overrides))
remember_written_rows(keys)
LAYERS.record_registration(alias, inherits_from, encoded)
write_rows(target, dict.fromkeys(keys, published), [])
for alias in keys:
STATE.rows[alias] = snapshot
return True
def accepted_catalog(catalog: LoadedCatalog) -> LoadedCatalog | None:

View file

@ -0,0 +1,173 @@
from __future__ import annotations
from collections.abc import ItemsView, Iterator, Mapping
from types import MappingProxyType
from typing import Final
from pydantic import TypeAdapter
from litellm._catalog import _install
from litellm._logging import verbose_logger
from litellm.rust_bridge._native import RegistrationTarget, registration_target
_ROW: Final = TypeAdapter(Mapping[str, object])
def _base_fields(
target: RegistrationTarget,
inherited: Mapping[str, object],
value: Mapping[str, object],
custom_llm_provider: str | None,
) -> Mapping[str, object]:
from litellm import utils
lookups: Final = (utils.lookup_model_info_for_registration(target.write, custom_llm_provider) for _ in range(3))
read: Final = next(
(lookup for lookup in lookups if not (isinstance(lookup, utils.ModelInfoLookupMiss) and lookup.tripped)),
utils.ModelInfoLookupMiss(tripped=True),
)
primary: Final = not isinstance(read, utils.ModelInfoLookupMiss) and read.get("key") == target.inherit_from
if primary:
return {name: field for name, field in inherited.items() if name != "litellm_provider" or field is not None}
rates: Final = (
inherited
if target.names_the_row
else utils._without_overridden_price_variants( # pyright: ignore[reportPrivateUsage] # shared rate policy
inherited, value
)
)
generalized: Final = _ROW.validate_python(
utils.match_capability_generalizations( # pyright: ignore[reportUnknownMemberType] # validate legacy rules
target.write
)
or {}
)
own_fields: Final = {
name: field
for name, field in rates.items()
if name != "litellm_provider" or (target.write == target.inherit_from and field is not None)
}
return {**generalized, **own_fields}
def registration_row(key: str, provider: str | None) -> Mapping[str, object]:
with _install.LOCK:
live: Final = _install._live_rows() # pyright: ignore[reportPrivateUsage] # retain public dict identity
target: Final = registration_target(live, key, provider=provider)
return _ROW.validate_python(live.get(target.inherit_from, {}) if target.inherit_from is not None else {})
def _base_fields_replaced_by(value: Mapping[str, object]) -> frozenset[str]:
return (
frozenset({"explicit_cache_read_input_token_cost"})
if value.get("cache_read_input_token_cost") is not None
and value.get("explicit_cache_read_input_token_cost") is None
else frozenset()
)
def _register_entry(
key: str,
value: Mapping[str, object],
*,
persist: bool,
custom_llm_provider: str | None,
warning_display_name: str | None,
) -> bool:
from litellm import utils
generation: Final = _install.STATE.generation
live: Final = _install._live_rows() # pyright: ignore[reportPrivateUsage] # retain public dict identity
provider: Final = value.get("litellm_provider")
target: Final = registration_target(
live, key, provider=custom_llm_provider or (provider if isinstance(provider, str) else None)
)
inherited: Final = _ROW.validate_python(
live.get(target.inherit_from, {}) if target.inherit_from is not None else {}
)
base: Final = _base_fields(target, inherited, value, custom_llm_provider)
merged: Final = utils.update_dictionary(dict(base), value)
fields: Final = {
name: field
for name, field in merged.items()
if value.get(name) is not None or name not in inherited or field != inherited[name]
}
erased: Final = _base_fields_replaced_by(value)
previous: Final = _ROW.validate_python(live.get(key, {}))
removed: Final = (frozenset(inherited) | frozenset(previous)) - frozenset(merged)
overrides: Final = {**fields, **dict.fromkeys(removed | erased)}
if generation != _install.STATE.generation:
return False
if not _install.registration_write(
target.write,
base,
overrides,
persist=persist,
inherits_from=target.inherit_from if target.inherit_from != target.write else None,
generation=generation,
):
return False
if target.inherit_from is None and (
value.get("cache_creation_input_token_cost") is None
and value.get("cache_read_input_token_cost") is None
and value.get("tiered_pricing") is None
and (value.get("input_cost_per_token") is not None or value.get("output_cost_per_token") is not None)
):
verbose_logger.warning(
"register_model: model=%s has custom pricing but not in built-in cost map and no prefix/region variant "
"matched; cache_creation_input_token_cost and cache_read_input_token_cost will default to 0 for this "
"model (input/output cost tracking is unaffected). To track cache cost, add them to model_info",
warning_display_name or key,
)
return True
def register(
entries: Mapping[str, Mapping[str, object]],
*,
persist: bool,
custom_llm_provider: str | None,
warning_display_name: str | None,
) -> None:
for key, value in entries.items():
with _install.LOCK:
while not _register_entry(
key,
value,
persist=persist,
custom_llm_provider=custom_llm_provider,
warning_display_name=warning_display_name,
):
pass
class RuntimeRegistrations(Mapping[str, Mapping[str, object]]):
@staticmethod
def _snapshot() -> Mapping[str, Mapping[str, object]]:
with _install.LOCK:
entries: Final[dict[str, Mapping[str, object]]] = {} # mutable-ok: fold ordered native sparse records
for key, _, fields in _install.LAYERS.registrations():
entries[key] = MappingProxyType(
{
**entries.get(key, {}),
**{
name: _install._decoded_value( # pyright: ignore[reportPrivateUsage] # opaque metadata
value
)
for name, value in fields.items()
},
}
)
return MappingProxyType(entries)
def __getitem__(self, key: str) -> Mapping[str, object]:
return self._snapshot()[key]
def __iter__(self) -> Iterator[str]:
return iter(self._snapshot())
def __len__(self) -> int:
return len(self._snapshot())
def items(self) -> ItemsView[str, Mapping[str, object]]:
return self._snapshot().items()

View file

@ -79,7 +79,6 @@ from litellm.constants import (
MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE,
NON_INFERENCE_CALL_TYPES,
OPENAI_EMBEDDING_PARAMS,
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
)
from litellm.litellm_core_utils.core_helpers import (
@ -3486,17 +3485,6 @@ def _convert_stringified_numbers(value):
return value
_BEDROCK_REGION_PREFIXES: Final = (
"us.",
"eu.",
"apac.",
"jp.",
"au.",
"us-gov.",
"global.",
"ap-northeast-1.",
)
_TOKEN_COST_FIELDS: Final = ("input_cost_per_token", "output_cost_per_token")
@ -3504,39 +3492,6 @@ def _builtin_cost_map() -> dict[str, Mapping[str, object]]:
return cast(dict[str, Mapping[str, object]], litellm.model_cost)
def _replace_cost_map_entry(
cost_map: dict[str, Mapping[str, object]], key: str, entry: Mapping[str, object], registration: Mapping[str, object]
) -> None:
current: Final = cost_map.get(key)
merged: Final = {**(current or {}), **entry}
names: Final = current.names if isinstance(current, AliasedCostEntry) else ()
previous_fields: Final[frozenset[str]] = (
current.registered_fields if isinstance(current, AliasedCostEntry) else frozenset()
)
registered_fields: Final = previous_fields | frozenset(
name for name, value in registration.items() if value is not None
)
sharing_names: Final = tuple(name for name in names if cost_map.get(name) is current)
cost_map.update(dict.fromkeys((key, *sharing_names), AliasedCostEntry(merged, names, registered_fields)))
def _without_synthesized_costs(
model_info: Mapping[str, object], key: str, value: Mapping[str, object]
) -> Mapping[str, object]:
"""``_get_model_info_helper`` synthesizes ``input_cost_per_token`` / ``output_cost_per_token`` = 0 when the raw
entry has neither (#30198). Persisting those zeros would turn a name-priced entry into a free one and make
``_is_cost_explicitly_configured`` disable budget enforcement on the next re-registration."""
builtin_cost_map: Final = _builtin_cost_map()
raw_fields: Final = frozenset(builtin_cost_map.get(str(model_info["key"]), builtin_cost_map.get(key, {})))
return MappingProxyType(
{
name: field_value
for name, field_value in model_info.items()
if name not in _TOKEN_COST_FIELDS or name in raw_fields or name in value
}
)
_DIRECTION_RATE_FIELDS: Final = MappingProxyType(
{
"input_cost_per_token": ("input_cost_per_token",),
@ -3617,84 +3572,11 @@ def model_info_with_pricing_overrides(model_info: ModelInfo, registration: Mappi
)
@dataclass(frozen=True, slots=True)
class _BuiltinCostMatch:
key: str
entry: Mapping[str, object]
names_the_row: bool
@dataclass(frozen=True, slots=True)
class ModelInfoLookupMiss:
tripped: bool
@dataclass(frozen=True, slots=True)
class _FallbackRegistration:
entry: Mapping[str, object]
matched_builtin: bool
def _fallback_registration(key: str, provider: str, value: Mapping[str, object]) -> _FallbackRegistration:
"""The key keeps its own entry, seeded from the capability rules an unregistered lookup resolves it to, with the
matched built-in entry on top. The matched row's provider stays behind, since the key's own lookup can reject it
(an ``azure_ai/`` key never resolves to an ``openai`` entry), and the key keeps the provider its caller gave."""
match: Final = _resolve_builtin_model_cost_entry(key=key, provider=provider)
generalized: Final[Mapping[str, object]] = match_capability_generalizations(key) or {}
if match is None:
return _FallbackRegistration(entry=generalized, matched_builtin=False)
inherited: Final = match.entry if match.names_the_row else _without_overridden_price_variants(match.entry, value)
own_fields: Final = {name: field_value for name, field_value in inherited.items() if name != "litellm_provider"}
return _FallbackRegistration(entry={**generalized, **own_fields}, matched_builtin=True)
def _resolve_builtin_model_cost_entry(key: str, provider: str) -> _BuiltinCostMatch | None:
"""Best-effort lookup of a built-in ``model_cost`` entry for a custom key
whose shape ``get_model_info`` cannot resolve (repeated provider prefixes
like ``bedrock/bedrock/bedrock/us.anthropic.claude-sonnet-4-6`` or region
aliases), or whose lookup tripped.
The key itself is tried first, then the key behind its provider prefixes
and bedrock route, so a catalog row whose lookup tripped inherits from
itself rather than from a region-stripped sibling priced at the base rate.
``names_the_row`` is set when the match is the row the key names, which
``get_model_info`` resolves it to, so the caller keeps that row's rates as
an untripped registration does. Returns ``None`` when no candidate exists
under a provider ``get_model_info`` would accept for ``provider``, or, when
the value names none, for the provider prefix the key itself carries,
which is what ``get_model_info`` derives when it reads the key back.
"""
from litellm.llms.bedrock.common_utils import strip_bedrock_routing_prefix
segments: Final = key.split("/")
match_provider: Final = provider or (segments[0] if len(segments) > 1 and segments[0] in LlmProvidersSet else "")
provider_prefix_count: Final = next(
(idx for idx, segment in enumerate(segments[:-1]) if segment not in LlmProvidersSet), len(segments) - 1
)
prefix_stripped: Final = tuple("/".join(segments[idx:]) for idx in range(1, provider_prefix_count + 1))
prefix_free: Final = prefix_stripped[-1] if prefix_stripped else key
base: Final = (
strip_bedrock_routing_prefix(prefix_free) if match_provider in ("bedrock", "bedrock_converse") else prefix_free
)
region_stripped: Final = tuple(
base[len(region_prefix) :] for region_prefix in _BEDROCK_REGION_PREFIXES if base.startswith(region_prefix)
)
stripped: Final = _strip_model_name(model=base, custom_llm_provider=match_provider) if match_provider else base
row_names: Final = (key, *prefix_stripped, base, stripped)
candidates: Final = (key, *prefix_stripped, base, *region_stripped, stripped)
builtin_cost_map: Final = _builtin_cost_map()
for candidate in candidates:
entry = builtin_cost_map.get(candidate)
if (
entry is not None
and entry.get("litellm_provider") is not None
and _check_provider_match(dict(entry), match_provider)
):
return _BuiltinCostMatch(key=candidate, entry=dict(entry), names_the_row=candidate in row_names)
return None
def is_generalized_model_info(model_info: ModelInfo) -> bool:
"""Whether ``model_info`` came from a fallback-generalization capability rule.
@ -3709,9 +3591,6 @@ def is_generalized_model_info(model_info: ModelInfo) -> bool:
return key not in litellm.model_cost and match_capability_generalizations(key) is not None
_REGISTRATION_ATTEMPTS: Final = 3
def lookup_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | ModelInfoLookupMiss:
try:
info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
@ -3722,162 +3601,15 @@ def lookup_model_info_for_registration(model: str, custom_llm_provider: str | No
return ModelInfoLookupMiss(tripped=False) if is_generalized_model_info(info) else info
def _get_builtin_model_info_for_registration(
model: str, custom_llm_provider: str | None
) -> ModelInfo | ModelInfoLookupMiss:
"""Resolve ``model`` to its built-in cost-map entry for registration merging.
Misses when the lookup cannot map the model or resolved it via a
fallback-generalization capability rule, since a rule-derived entry carries no
pricing. A lookup that raised anything else, such as a concurrent registration
changing the map mid-walk, is retried, so a transient failure registers exactly
where the untripped lookup would.
"""
lookups: Final = (
lookup_model_info_for_registration(model, custom_llm_provider) for _ in range(_REGISTRATION_ATTEMPTS)
)
return next(
(lookup for lookup in lookups if not (isinstance(lookup, ModelInfoLookupMiss) and lookup.tripped)),
ModelInfoLookupMiss(tripped=True),
)
def _skips_builtin_registration_lookup(key: str, provider: object) -> bool:
return provider in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO or any(
key.startswith(f"{p}/") for p in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO
)
@dataclass(frozen=True, slots=True)
class _RegistrationWrite:
key: str
entry: Mapping[str, object]
def model_cost_registration_row(key: str, provider: object) -> Mapping[str, object]:
"""The row a ``register_model`` write of a ``key`` entry with this ``litellm_provider`` merges onto: the
``litellm.model_cost`` row it resolves to (``openai/o3-pro`` lands on ``o3-pro``), else the row it seeds ``key``
with from the built-in entry it falls back to (an OpenAI fine-tune takes its base model's row)."""
base: Final = _registration_base(
key, {"litellm_provider": provider}, custom_llm_provider=None, warning_display_name=None
)
return _builtin_cost_map().get(base.key) or base.entry
from litellm._catalog._register import registration_row
return registration_row(key, provider if isinstance(provider, str) else None)
def _registration_base(
key: str, value: Mapping[str, object], custom_llm_provider: str | None, warning_display_name: str | None
) -> _RegistrationWrite:
provider: Final = str(value.get("litellm_provider") or "")
if _skips_builtin_registration_lookup(key, provider):
return _RegistrationWrite(key=key, entry=dict(_builtin_cost_map().get(key, {})))
builtin_model_info: Final = _get_builtin_model_info_for_registration(
model=key, custom_llm_provider=custom_llm_provider
)
if not isinstance(builtin_model_info, ModelInfoLookupMiss):
return _RegistrationWrite(
key=builtin_model_info["key"],
entry=dict(_without_synthesized_costs(builtin_model_info, key=key, value=value)),
)
fallback: Final = _fallback_registration(key=key, provider=provider, value=value)
if not fallback.matched_builtin and (
value.get("cache_creation_input_token_cost") is None
and value.get("cache_read_input_token_cost") is None
and value.get("tiered_pricing") is None
and (value.get("input_cost_per_token") is not None or value.get("output_cost_per_token") is not None)
):
verbose_logger.warning(
"register_model: model=%s has custom pricing but not in built-in cost map and no prefix/region variant matched; cache_creation_input_token_cost and cache_read_input_token_cost will default to 0 for this model (input/output cost tracking is unaffected). To track cache cost, add them to model_info",
warning_display_name or key,
)
return _RegistrationWrite(key=key, entry=dict(fallback.entry))
from litellm._catalog._register import RuntimeRegistrations
def _base_fields_replaced_by(value: Mapping[str, object]) -> frozenset[str]:
return (
frozenset({"explicit_cache_read_input_token_cost"})
if value.get("cache_read_input_token_cost") is not None
and value.get("explicit_cache_read_input_token_cost") is None
else frozenset()
)
def _registration_write(
key: str, value: Mapping[str, object], custom_llm_provider: str | None, warning_display_name: str | None
) -> _RegistrationWrite:
base: Final = _registration_base(key, value, custom_llm_provider, warning_display_name)
# ``get_model_info`` returns ``litellm_provider: None`` when the
# provider is unknown (e.g. custom deployments registered via
# ``Router.add_deployment``). Persisting that None into
# ``litellm.model_cost`` causes ``_check_provider_match`` to drop
# custom pricing on subsequent cost lookups.
existing_model: Final = {
name: field_value
for name, field_value in base.entry.items()
if name != "litellm_provider" or field_value is not None
}
return _RegistrationWrite(
key=base.key,
entry={**update_dictionary(existing_model, value), **dict.fromkeys(_base_fields_replaced_by(value))},
)
def _registration_attempt(
key: str,
value: Mapping[str, object],
custom_llm_provider: str | None,
warning_display_name: str | None,
final_attempt: bool,
persist_across_reloads: bool = True,
) -> _RegistrationWrite | None:
from litellm._catalog._install import registration_write
generation: Final = get_model_cost_mutation_generation()
source: Final = _builtin_cost_map()
write: Final = _registration_write(key, value, custom_llm_provider, warning_display_name)
if not final_attempt and get_model_cost_mutation_generation() != generation:
return None
provider: Final = value.get("litellm_provider")
match: Final = _resolve_builtin_model_cost_entry(
key=write.key, provider=provider if isinstance(provider, str) else ""
)
registration_write(
_builtin_cost_map() if final_attempt else source,
write.key,
write.entry,
value,
persist=persist_across_reloads,
inherits_from=match.key if match is not None and match.key != write.key else None,
replace=_replace_cost_map_entry,
)
return write if final_attempt or get_model_cost_mutation_generation() == generation else None
def _register_on_current_map(
key: str,
value: Mapping[str, object],
custom_llm_provider: str | None,
warning_display_name: str | None,
persist_across_reloads: bool = True,
) -> _RegistrationWrite:
"""A price data reload that swaps the map while the entry is being built or written would otherwise get the old
map's prices written over its own, or drop the entry with the old map, so each attempt writes into the map it was
built from and the entry is rebuilt when a reload replaced that map before the write finished. The last attempt
writes into the current map, so the entry is never lost."""
attempts: Final = (
_registration_attempt(
key,
value,
custom_llm_provider,
warning_display_name,
final_attempt=attempt == _REGISTRATION_ATTEMPTS - 1,
persist_across_reloads=persist_across_reloads,
)
for attempt in range(_REGISTRATION_ATTEMPTS)
)
return next(write for write in attempts if write is not None)
_runtime_registered_model_cost: Final[dict[str, dict[str, object]]] = {} # mutable-ok: replayed on reload
_runtime_registered_model_cost: Final[Mapping[str, Mapping[str, object]]] = RuntimeRegistrations()
def runtime_model_cost_registrations(
@ -3922,30 +3654,6 @@ def set_live_deployment_replay(replay: Callable[[], None]) -> None:
_LiveDeploymentReplay.callbacks = (replay,)
def reapply_runtime_model_cost_registrations() -> None:
"""Re-apply runtime model metadata on top of a freshly adopted cost map.
Adopting a new catalog replaces ``litellm.model_cost`` wholesale, which on
its own discards everything registered at runtime: the deployment
``model_info`` the Router registers from ``model_list``, and pricing
overrides passed to ``register_model``. Both are re-applied here so a price
data reload only updates pricing rather than erasing operator-supplied model
metadata.
The two are restored differently, and the difference is what keeps this
bounded. Deployment metadata is re-derived from the routers that are alive
right now, so a deployment that has been deleted or repointed, and a router
that has been discarded, are simply not part of the rebuild; nothing has to
withdraw them and nothing accumulates. Only ``register_model`` calls that
have no such owner are recorded and replayed, and a registration describing
a single request opts out of even that.
"""
for replay in _LiveDeploymentReplay.callbacks:
replay()
if _runtime_registered_model_cost:
register_model(model_cost=dict(_runtime_registered_model_cost))
def cost_map_omits_token_price(*keys: object) -> bool:
"""Whether the raw ``litellm.model_cost`` entries under ``keys`` exist but none carries a per-token price.
@ -4021,28 +3729,20 @@ def register_model(
return model_cost
loaded_model_cost = served_rows(accepted)
if persist_across_reloads:
_registrations: Final[Mapping[str, Mapping[str, object]]] = loaded_model_cost
for _registered_key, _registered_value in _registrations.items():
_runtime_registered_model_cost[_registered_key] = update_dictionary(
dict(_runtime_registered_model_cost.get(_registered_key, {})),
_RATE_TABLE.validate_python(_registered_value),
)
from litellm._catalog._register import register
for key, value in loaded_model_cost.items():
from litellm._catalog._install import LOCK
with LOCK:
write = _register_on_current_map(
str(key), value, custom_llm_provider, warning_display_name, persist_across_reloads
)
# Invalidate case-insensitive lookup map since model_cost was modified
_invalidate_model_cost_lowercase_map()
verbose_logger.debug("added/updated model=%s in litellm.model_cost: %s", write.key, write.key)
entries: Final = {str(key): _RATE_TABLE.validate_python(value) for key, value in loaded_model_cost.items()}
register(
entries,
persist=persist_across_reloads,
custom_llm_provider=custom_llm_provider,
warning_display_name=warning_display_name,
)
_invalidate_model_cost_lowercase_map()
for key, value in entries.items():
verbose_logger.debug("added/updated model=%s in litellm.model_cost: %s", key, key)
if add_to_provider_model_sets and value.get("litellm_provider") == "openrouter":
litellm.openrouter_models.add(str(key).split("/", 1)[-1])
litellm.openrouter_models.add(key.split("/", 1)[-1])
if add_to_provider_model_sets:
litellm.add_known_models(loaded_model_cost)
return model_cost

View file

@ -293,20 +293,13 @@ def isolate_litellm_globals() -> Iterator[None]:
@pytest.fixture(autouse=True)
def isolate_router_model_cost_state() -> Iterator[None]:
original_live_routers: Final = frozenset(litellm_router_module._live_routers)
original_runtime_registered_model_cost: Final = {
model_key: dict(model_value)
for model_key, model_value in litellm_utils_module._runtime_registered_model_cost.items()
}
litellm_utils_module._invalidate_model_cost_lowercase_map()
litellm_router_module._live_routers.clear()
litellm_utils_module._runtime_registered_model_cost.clear()
yield
for router in tuple(litellm_router_module._live_routers):
litellm_router_module._live_routers.discard(router)
for router in original_live_routers:
litellm_router_module._live_routers.add(router)
litellm_utils_module._runtime_registered_model_cost.clear()
litellm_utils_module._runtime_registered_model_cost.update(original_runtime_registered_model_cost)
litellm_utils_module._invalidate_model_cost_lowercase_map()
litellm.get_model_info.cache_clear()
@ -352,6 +345,7 @@ def isolate_model_catalog() -> Iterator[None]:
set_fallback_generalizations(original_rules)
model_cost.clear()
model_cost.update(model_cost_entries)
litellm.model_cost = model_cost
models_by_provider.clear()
models_by_provider.update(models_by_provider_entries)
for models, entries in provider_model_sets:

View file

@ -553,7 +553,9 @@ def test_register_url_uses_explicit_source_under_local_flag(tmp_path: Path, monk
def test_failed_url_registration_is_a_single_logged_noop(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None:
before: Final = copy.deepcopy(litellm.model_cost)
registrations: Final = copy.deepcopy(litellm.utils._runtime_registered_model_cost)
registrations: Final = {
key: copy.deepcopy(dict(row)) for key, row in litellm.utils._runtime_registered_model_cost.items()
}
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
url: Final = (tmp_path / "missing.json").as_uri()
assert litellm.register_model(url) == url

View file

@ -159,7 +159,10 @@ def test_aws_cache_reset_sees_a_replaced_module(monkeypatch: pytest.MonkeyPatch)
@pytest.fixture(scope="module", autouse=True)
def registered_collection_models() -> Iterator[None]:
original_registrations: Final = dict(litellm_utils_module._runtime_registered_model_cost)
from litellm._catalog import _install
original_layers: Final = _install.LAYERS
_install.LAYERS = _install.ModelCostLayers()
original_routers: Final = frozenset(litellm_router_module._live_routers)
litellm.register_model(
{"collection-isolation-ledger": {"litellm_provider": "openai", "input_cost_per_token": 0.001}}
@ -173,8 +176,7 @@ def registered_collection_models() -> Iterator[None]:
]
)
yield
litellm_utils_module._runtime_registered_model_cost.clear()
litellm_utils_module._runtime_registered_model_cost.update(original_registrations)
_install.LAYERS = original_layers
litellm_router_module._live_routers.discard(collection_router)
litellm_router_module._live_routers.update(original_routers)

View file

@ -4815,6 +4815,7 @@ def test_completion_cost_legacy_mantle_route_prices_after_router_registration(lo
/v1/chat/completions and /v1/messages alike, bills $0."""
from litellm import Router
sibling: Final = dict(litellm.model_cost["anthropic.claude-sonnet-5"])
Router(
model_list=[
{
@ -4826,7 +4827,10 @@ def test_completion_cost_legacy_mantle_route_prices_after_router_registration(lo
}
]
)
assert "bedrock/mantle/anthropic.claude-sonnet-5" not in litellm.model_cost
registered: Final = litellm.model_cost["bedrock/mantle/anthropic.claude-sonnet-5"]
assert registered["input_cost_per_token"] == sibling["input_cost_per_token"]
assert registered["output_cost_per_token"] == sibling["output_cost_per_token"]
assert litellm.model_cost["anthropic.claude-sonnet-5"] == sibling
response = litellm.ModelResponse(
id="msg_x",
@ -10118,16 +10122,14 @@ def test_non_bedrock_cost_ignores_bedrock_model_id(_local_model_cost_map: None)
@pytest.mark.parametrize("metadata_prefix", ("bedrock/", "bedrock/converse/"))
def test_bedrock_converse_profile_preserves_registered_price_override(
_local_model_cost_map: None,
monkeypatch: pytest.MonkeyPatch,
rate: float,
persist_across_reloads: bool,
routed_response: bool,
metadata_prefix: str,
) -> None:
import litellm.utils as model_utils
from litellm.cost_calculator import pricing_entry_for_cost_calc
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
monkeypatch.setattr(model_utils, "_runtime_registered_model_cost", {})
catalog: Final = dict(litellm.model_cost)
model: Final = "openai.gpt-5.6-sol"
litellm.register_model(
@ -10191,9 +10193,7 @@ def test_bedrock_converse_profile_preserves_registered_price_override(
optional_params={"model_id": f"us.{model}"},
) == pytest.approx(10 * updated_rate + 5 * rate)
monkeypatch.setattr(litellm, "model_cost", catalog)
model_utils._invalidate_model_cost_lowercase_map()
model_utils.reapply_runtime_model_cost_registrations()
adopt_model_cost_map(catalog)
profile_prices: Final = litellm.get_model_info(f"us.{model}", custom_llm_provider="bedrock")
expected_after_reload: Final = (
10 * updated_rate + 5 * rate
@ -10209,11 +10209,8 @@ def test_bedrock_converse_profile_preserves_registered_price_override(
def test_bedrock_converse_profile_ignores_registered_capabilities_without_prices(
_local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
_local_model_cost_map: None,
) -> None:
import litellm.utils as model_utils
monkeypatch.setattr(model_utils, "_runtime_registered_model_cost", {})
model: Final = "openai.gpt-6-sol"
litellm.register_model({model: {"supports_function_calling": True}})
response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
@ -10245,8 +10242,8 @@ def test_bedrock_converse_profile_preserves_cache_only_registered_prices(
) -> None:
import litellm.utils as model_utils
from litellm.cost_calculator import pricing_entry_for_cost_calc
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
monkeypatch.setattr(model_utils, "_runtime_registered_model_cost", {})
model: Final = "openai.cache-overlay-test"
profile: Final = f"{profile_prefix}.{model}"
catalog: Final = {
@ -10317,9 +10314,7 @@ def test_bedrock_converse_profile_preserves_cache_only_registered_prices(
assert entry[1][cache_field] == rate
assert litellm.model_cost[profile] == profile_before
monkeypatch.setattr(litellm, "model_cost", dict(catalog))
model_utils._invalidate_model_cost_lowercase_map()
model_utils.reapply_runtime_model_cost_registrations()
adopt_model_cost_map(catalog)
expected_after_reload: Final = (
10 * 0.02 + 3 * 0.04 + 5 * (rate if persist_across_reloads else catalog[profile][cache_field])
)

View file

@ -245,39 +245,12 @@ def test_register_model_strips_none_litellm_provider():
litellm.model_cost.pop(model_key, None)
def test_register_model_strips_none_litellm_provider_from_get_model_info(monkeypatch):
"""Directly exercise the strip in ``register_model``.
The companion test above hits the ``except Exception`` branch where
``existing_model`` is an empty dict, so the ``pop`` is a no-op. This
test patches ``get_model_info`` to return the failure mode the strip
was added to handle, namely a populated dict whose ``litellm_provider``
is ``None``. Without the strip, the merged entry in
``litellm.model_cost`` would carry ``litellm_provider: None`` and
``_check_provider_match`` would drop custom pricing.
Regression test for https://github.com/BerriAI/litellm/issues/28336.
"""
from litellm import utils as litellm_utils
def test_register_model_strips_none_provider_from_the_served_row(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.utils import _check_provider_match
model_key = "test-strip-none-provider-from-get-model-info-28336"
litellm.model_cost.pop(model_key, None)
def _fake_get_model_info(model, *args, **kwargs):
assert model == model_key
return {
"key": model_key,
"litellm_provider": None,
"mode": "chat",
"max_tokens": 4096,
}
# ``register_model`` calls ``get_model_info.cache_clear`` via
# ``_invalidate_model_cost_lowercase_map``, so the replacement must
# expose a no-op ``cache_clear`` attribute.
_fake_get_model_info.cache_clear = lambda: None
monkeypatch.setattr(litellm_utils, "get_model_info", _fake_get_model_info)
model_key: Final = "test-strip-none-provider-28336"
row: Final = {"litellm_provider": None, "mode": "chat", "max_tokens": 4096}
monkeypatch.setitem(litellm.model_cost, model_key, row)
try:
litellm.register_model(
@ -289,24 +262,13 @@ def test_register_model_strips_none_litellm_provider_from_get_model_info(monkeyp
}
)
registered = litellm.model_cost.get(model_key)
registered: Final = litellm.model_cost.get(model_key)
assert registered is not None, f"{model_key} should be in model_cost"
# The strip must have removed the None-valued provider that
# ``get_model_info`` returned. The key may be absent entirely, but
# it must never be present with value ``None``.
assert "litellm_provider" not in registered or (registered["litellm_provider"] is not None), (
"register_model failed to strip litellm_provider=None returned "
f"by get_model_info, got {registered.get('litellm_provider')!r}"
)
# Metadata from the patched ``get_model_info`` must still flow
# through, so we know the strip did not nuke the rest of the entry.
assert "litellm_provider" not in registered
assert registered.get("mode") == "chat"
assert registered.get("max_tokens") == 4096
# And custom pricing from the registration call must be preserved.
assert registered.get("input_cost_per_token") == 0.001
assert registered.get("output_cost_per_token") == 0.002
# Downstream _check_provider_match must accept any provider for
# this entry, mirroring the cost calculator path.
assert _check_provider_match(registered, "openai") is True
assert _check_provider_match(registered, "anthropic") is True
finally:
@ -847,7 +809,7 @@ def test_register_model_keeps_a_catalog_rows_own_prices_when_the_lookup_fails(mo
@pytest.mark.parametrize("route", ["", "converse/", "invoke/"])
def test_register_model_leaves_no_shadow_key_when_the_lookup_trips_once(
def test_register_model_keeps_caller_pricing_independent_when_the_lookup_trips_once(
monkeypatch: pytest.MonkeyPatch, route: str
) -> None:
catalog_key: Final = "au.anthropic.lit-5853-geo-model"
@ -879,11 +841,13 @@ def test_register_model_leaves_no_shadow_key_when_the_lookup_trips_once(
with pytest.MonkeyPatch.context() as lookup_patch:
lookup_patch.setattr(litellm.utils, "get_model_info", lookup_tripped_once_by_a_concurrent_registration)
litellm.register_model({alias: {"litellm_provider": "bedrock", "mode": "chat"}}, persist_across_reloads=False)
assert litellm.model_cost[catalog_key]["input_cost_per_token"] == 3.3e-06
litellm.register_model({catalog_key: {"input_cost_per_token": repriced_input_cost}}, persist_across_reloads=False)
try:
assert litellm.get_model_info(alias)["key"] == catalog_key
assert litellm.get_model_info(alias)["input_cost_per_token"] == repriced_input_cost
assert litellm.get_model_info(alias)["key"] == alias
assert litellm.get_model_info(alias)["input_cost_per_token"] == 3.3e-06
assert litellm.model_cost[catalog_key]["input_cost_per_token"] == repriced_input_cost
finally:
_invalidate_model_cost_lowercase_map()
@ -982,7 +946,13 @@ def test_register_model_prices_a_provider_less_azure_ai_key_without_repricing_th
_invalidate_model_cost_lowercase_map()
def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monkeypatch: pytest.MonkeyPatch) -> None:
@pytest.mark.parametrize("persist", [False, True])
def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(
monkeypatch: pytest.MonkeyPatch, persist: bool
) -> None:
from litellm._catalog import _register
from litellm.rust_bridge._native import RegistrationTarget
catalog_key: Final = "au.anthropic.lit-5853-reloaded-model"
deployment_key: Final = f"bedrock/{catalog_key}"
catalog_row: Final = {
@ -1001,30 +971,41 @@ def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monk
**builtin_cost_map,
catalog_key: {**catalog_row, "input_cost_per_token": reloaded_input_cost},
}
real_lookup: Final = litellm.utils.get_model_info
real_target: Final = _register.registration_target
lookups: Final = itertools.count()
def lookup_racing_a_price_data_reload(model: str, custom_llm_provider: str | None = None) -> object:
info: Final = real_lookup(model=model, custom_llm_provider=custom_llm_provider)
def target_racing_a_price_data_reload(
rows: dict[str, object], key: str, *, provider: str | None = None
) -> RegistrationTarget:
target: Final = real_target(rows, key, provider=provider)
if next(lookups) == 0:
adopt_model_cost_map(reloaded_cost_map)
return info
return target
with pytest.MonkeyPatch.context() as lookup_patch:
lookup_patch.setattr(litellm.utils, "get_model_info", lookup_racing_a_price_data_reload)
litellm.register_model({deployment_key: {"litellm_provider": "bedrock"}}, persist_across_reloads=False)
lookup_patch.setattr(_register, "registration_target", target_racing_a_price_data_reload)
litellm.register_model({deployment_key: {"litellm_provider": "bedrock"}}, persist_across_reloads=persist)
try:
assert litellm.model_cost is live_cost_map
assert litellm.model_cost[catalog_key]["input_cost_per_token"] == reloaded_input_cost
assert litellm.get_model_info(deployment_key)["input_cost_per_token"] == reloaded_input_cost
adopt_model_cost_map(reloaded_cost_map)
assert (deployment_key in litellm.model_cost) is persist
if persist:
assert litellm.get_model_info(deployment_key)["input_cost_per_token"] == reloaded_input_cost
finally:
_invalidate_model_cost_lowercase_map()
@pytest.mark.parametrize("persist", [False, True])
def test_register_model_keeps_its_price_when_a_reload_adopts_a_map_as_it_writes(
monkeypatch: pytest.MonkeyPatch,
monkeypatch: pytest.MonkeyPatch, persist: bool,
) -> None:
from pydantic import JsonValue
from litellm._catalog import _install
catalog_key: Final = "au.anthropic.lit-5853-reloaded-as-written-model"
deployment_key: Final = f"bedrock/{catalog_key}"
catalog_row: Final = {
@ -1042,33 +1023,35 @@ def test_register_model_keeps_its_price_when_a_reload_adopts_a_map_as_it_writes(
}
writes: Final = itertools.count()
encode: Final = _install._encoded_row
class CostMapAReloadReplacesAsItIsWritten(dict[str, dict[str, object]]):
def update(self, other: Mapping[str, dict[str, object]], /) -> None: # pyright: ignore[reportIncompatibleMethodOverride] # registration writes pass one mapping
if next(writes) == 0:
adopt_model_cost_map(reloaded_cost_map)
super().update(other)
def encode_racing_a_price_data_reload(row: Mapping[str, object]) -> dict[str, JsonValue]:
encoded: Final = encode(row)
if row.get("output_cost_per_token") == registered_output_cost and next(writes) == 0:
adopt_model_cost_map(reloaded_cost_map)
return encoded
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
adopt_model_cost_map({**builtin_cost_map, catalog_key: dict(catalog_row)})
monkeypatch.setattr(
litellm,
"model_cost",
CostMapAReloadReplacesAsItIsWritten({**builtin_cost_map, catalog_key: dict(catalog_row)}),
)
monkeypatch.setattr(_install, "_encoded_row", encode_racing_a_price_data_reload)
_invalidate_model_cost_lowercase_map()
live_cost_map: Final = litellm.model_cost
litellm.register_model(
{deployment_key: {"litellm_provider": "bedrock", "output_cost_per_token": registered_output_cost}},
persist_across_reloads=False,
persist_across_reloads=persist,
)
try:
assert litellm.model_cost is live_cost_map
assert litellm.get_model_info(deployment_key)["input_cost_per_token"] == reloaded_input_cost
assert litellm.get_model_info(deployment_key)["output_cost_per_token"] == registered_output_cost
adopt_model_cost_map(reloaded_cost_map)
assert (deployment_key in litellm.model_cost) is persist
if persist:
assert litellm.get_model_info(deployment_key)["input_cost_per_token"] == reloaded_input_cost
assert litellm.get_model_info(deployment_key)["output_cost_per_token"] == registered_output_cost
finally:
_invalidate_model_cost_lowercase_map()
@ -1341,6 +1324,62 @@ def test_register_model_keeps_custom_flat_prices_over_inherited_rate_variants(
_invalidate_model_cost_lowercase_map()
@pytest.mark.parametrize(
("base_key", "caller", "provider", "served_provider", "keeps_variant"),
(
("registration-shape-model", "openai/registration-shape-model", "openai", "openai", True),
("us.anthropic.shape-model", "bedrock/us.anthropic.shape-model", "bedrock", "bedrock", True),
("us.anthropic.shape-model", "bedrock/bedrock/bedrock/us.anthropic.shape-model", "bedrock", None, True),
("anthropic.shape-model", "bedrock/ap-northeast-1.anthropic.shape-model", "bedrock", None, False),
("registration-shape-model", "azure_ai/registration-shape-model", "openai", None, True),
),
)
def test_registration_keeps_primary_and_fallback_field_shaping_across_reload(
monkeypatch: pytest.MonkeyPatch,
base_key: str,
caller: str,
provider: str,
served_provider: str | None,
keeps_variant: bool,
) -> None:
row: Final = {
"litellm_provider": provider,
"mode": "chat",
"input_cost_per_token": 0.001,
"input_cost_per_token_above_200k_tokens": 0.004,
"cache_read_input_token_cost": 0.0001,
"supports_vision": True,
}
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setenv("AZURE_AI_API_BASE", "https://registration.openai.azure.com")
adopt_model_cost_map({base_key: row})
litellm.register_model({caller: {"input_cost_per_token": 0.002}})
expected: Final = {
"mode": "chat",
"input_cost_per_token": 0.002,
"cache_read_input_token_cost": row["cache_read_input_token_cost"],
"supports_vision": row["supports_vision"],
**({"litellm_provider": served_provider} if served_provider is not None else {}),
**(
{"input_cost_per_token_above_200k_tokens": row["input_cost_per_token_above_200k_tokens"]}
if keeps_variant
else {}
),
}
assert litellm.model_cost[base_key] == row
assert litellm.model_cost[caller] == expected
expected_cost: Final = 250_000 * (0.004 if keeps_variant else 0.002)
assert litellm.cost_per_token(model=caller, prompt_tokens=250_000, completion_tokens=0) == pytest.approx(
(expected_cost, 0)
)
adopt_model_cost_map({base_key: row})
assert litellm.model_cost[base_key] == row
assert litellm.model_cost[caller] == expected
assert litellm.cost_per_token(model=caller, prompt_tokens=250_000, completion_tokens=0) == pytest.approx(
(expected_cost, 0)
)
def test_register_model_never_inherits_pricing_across_providers(monkeypatch: pytest.MonkeyPatch) -> None:
openai_key: Final = "lit-5853-openai-only-model"
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")

View file

@ -44,7 +44,6 @@ from litellm.types.router import (
)
from litellm.utils import (
_invalidate_model_cost_lowercase_map,
reapply_runtime_model_cost_registrations,
)
@ -52,18 +51,9 @@ _COPILOT_TOKEN_EXPIRES_AT: Final = 4_102_444_800
def _simulate_price_data_reload(fetched_catalog):
"""Drive what a price data reload does to this process's litellm state.
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
Mirrors `litellm.proxy.proxy_server._swap_in_model_cost_map`, which is the
one place both reload paths adopt a freshly fetched catalog; that wiring is
covered in the proxy's own tests, so these exercise the replay itself
without dragging the proxy in. The provider model sets that helper also
repopulates are left alone, since nothing here reads them and rebuilding
them from a two-entry catalog would outlive the test.
"""
litellm.model_cost = fetched_catalog
_invalidate_model_cost_lowercase_map()
reapply_runtime_model_cost_registrations()
adopt_model_cost_map(fetched_catalog)
def _nested_container_ids(value: object) -> frozenset[int]:
@ -2172,14 +2162,6 @@ def test_price_data_reload_preserves_router_registered_model_info(monkeypatch):
max_input_tokens / max_output_tokens from every custom model group and
/model_group/info starts reporting nulls.
"""
from litellm import utils as litellm_utils
monkeypatch.setattr(
litellm_utils,
"_runtime_registered_model_cost",
dict(litellm_utils._runtime_registered_model_cost),
)
router = Router(
model_list=[
{
@ -2220,14 +2202,6 @@ def test_price_data_reload_preserves_custom_override_of_a_catalog_model(monkeypa
the same bug: the reload does not blank the metadata, it reverts the
operator's model_info override to the upstream catalog values.
"""
from litellm import utils as litellm_utils
monkeypatch.setattr(
litellm_utils,
"_runtime_registered_model_cost",
dict(litellm_utils._runtime_registered_model_cost),
)
router = Router(
model_list=[
{
@ -2272,14 +2246,6 @@ def test_deleted_deployments_are_not_replayed_onto_later_reloads(monkeypatch):
key that another live deployment still points at must survive the same
deletion.
"""
from litellm import utils as litellm_utils
monkeypatch.setattr(
litellm_utils,
"_runtime_registered_model_cost",
dict(litellm_utils._runtime_registered_model_cost),
)
router = Router(
model_list=[
{
@ -2326,14 +2292,6 @@ def test_deleting_a_deployment_leaves_catalog_pricing_for_its_backend_model(monk
A backend key is shared with the fetched catalog, so withdrawing the entries
a deleted deployment owns must not take real upstream pricing down with it.
"""
from litellm import utils as litellm_utils
monkeypatch.setattr(
litellm_utils,
"_runtime_registered_model_cost",
dict(litellm_utils._runtime_registered_model_cost),
)
backend_model = "gemini/gemini-2.5-pro"
catalog_entry = litellm.get_model_info(model=backend_model)
catalog_input_cost = catalog_entry["input_cost_per_token"]
@ -2371,14 +2329,6 @@ def test_repointing_a_deployment_drops_its_previous_backend_key(monkeypatch):
old backend key behind, and a replayed registry would re-assert it onto every
later catalog for the life of the process.
"""
from litellm import utils as litellm_utils
monkeypatch.setattr(
litellm_utils,
"_runtime_registered_model_cost",
dict(litellm_utils._runtime_registered_model_cost),
)
router = Router(
model_list=[
{
@ -2786,7 +2736,8 @@ def test_a_github_copilot_chat_pin_opts_a_dual_endpoint_model_out_of_the_respons
@pytest.mark.parametrize("registered_mode, pinned_mode", [("chat", "responses"), ("responses", "chat")])
@pytest.mark.parametrize(
"model, custom_llm_provider", [("github_copilot/routing-test-model", None), ("routing-test-model", "github_copilot")]
"model, custom_llm_provider",
[("github_copilot/routing-test-model", None), ("routing-test-model", "github_copilot")],
)
def test_a_github_copilot_responses_pin_still_opts_the_model_into_the_responses_api(
monkeypatch: pytest.MonkeyPatch,
@ -3130,14 +3081,6 @@ def test_strategy_router_alias_pricing_never_enters_model_cost(monkeypatch):
strip must also survive a price-data reload, which rebuilds entries by
walking the live routers.
"""
from litellm import utils as litellm_utils
monkeypatch.setattr(
litellm_utils,
"_runtime_registered_model_cost",
dict(litellm_utils._runtime_registered_model_cost),
)
router = Router(
model_list=[
{
@ -3827,12 +3770,7 @@ def test_every_declaring_deployment_is_named(caplog):
def _simulate_price_data_reload_with_provider_sets(monkeypatch, fetched_catalog):
"""Like `_simulate_price_data_reload`, plus the provider model-set refresh the proxy's
`_swap_in_model_cost_map` does before replaying, so bare names in the new catalog resolve."""
monkeypatch.setattr(litellm, "model_cost", fetched_catalog)
_invalidate_model_cost_lowercase_map()
litellm.add_known_models(model_cost_map=fetched_catalog)
reapply_runtime_model_cost_registrations()
_simulate_price_data_reload(fetched_catalog)
def test_a_config_deployment_dropped_by_a_stale_cost_map_comes_back_on_reload(monkeypatch):
@ -3926,7 +3864,7 @@ def test_price_data_reload_refreshes_the_cached_model_group_and_deployment_info(
assert router.cached_deployment_model_info("dep-a", "openai/gpt-4o")["input_cost_per_token"] == old_price
new_price = old_price * 10
fresh_catalog = copy.deepcopy(litellm.model_cost)
fresh_catalog = litellm.get_model_cost_map(url="")
fresh_catalog["gpt-4o"]["input_cost_per_token"] = new_price
_simulate_price_data_reload_with_provider_sets(monkeypatch, fresh_catalog)
@ -3934,7 +3872,9 @@ def test_price_data_reload_refreshes_the_cached_model_group_and_deployment_info(
assert router.cached_deployment_model_info("dep-a", "openai/gpt-4o")["input_cost_per_token"] == new_price
def test_price_data_reload_keeps_geo_deployments_priced_while_requests_register_pricing(monkeypatch: pytest.MonkeyPatch) -> None:
def test_price_data_reload_keeps_geo_deployments_priced_while_requests_register_pricing(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
catalog: Final = litellm.get_model_cost_map(url="")
monkeypatch.setattr(litellm, "model_cost", dict(catalog))
@ -3996,9 +3936,13 @@ def test_price_data_reload_keeps_geo_deployments_priced_while_requests_register_
registrations_landed: Final = writer_outcome.get(timeout=5)
try:
assert isinstance(registrations_landed, int) and registrations_landed > 0, registrations_landed
assert tuple(backend for backend in backends if backend in litellm.model_cost) == ()
assert tuple(backend for backend in backends if backend in litellm.model_cost) == backends
for backend in backends:
assert litellm.get_model_info(backend)["input_cost_per_token"] == canonical_prices[backend]
assert (
litellm.model_cost[backend.removeprefix("bedrock/")]["input_cost_per_token"]
== canonical_prices[backend]
)
assert tuple(router.get_model_names()) == tuple(backend.rsplit(".", 1)[-1] for backend in backends)
finally:
_invalidate_model_cost_lowercase_map()

View file

@ -2258,6 +2258,166 @@ class TestProxyFunctionCalling:
)
@pytest.mark.parametrize(
("key", "base_key", "provider", "mode"),
(
("databricks/databricks-claude-opus-4-99", "databricks/databricks-claude-opus-4", "databricks", "chat"),
("openai/o3-pro", "o3-pro", "openai", "responses"),
("bedrock/anthropic.claude-sonnet-4-6-20260101", "bedrock/anthropic.claude-sonnet-4-6", "bedrock", "chat"),
),
)
def test_register_model_preserves_caller_key_and_inherits_without_repricing_sibling(
key: str, base_key: str, provider: str, mode: str
) -> None:
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
from litellm.utils import model_cost_registration_row
base_row: Final = {
"litellm_provider": provider,
"mode": mode,
"max_input_tokens": 8000,
"input_cost_per_token": 0.001,
"supports_vision": True,
}
adopt_model_cost_map({base_key: base_row})
live: Final = litellm.model_cost
litellm.register_model(
{key: {"max_input_tokens": 32000, "input_cost_per_token": 0.005, "litellm_provider": provider}}
)
assert litellm.model_cost is live
assert litellm.model_cost[base_key] == base_row
assert litellm.model_cost[key] == {**base_row, "max_input_tokens": 32000, "input_cost_per_token": 0.005}
assert model_cost_registration_row(key, provider)["mode"] == mode
refreshed_row: Final = {**base_row, "supports_vision": False, "input_cost_per_token": 0.002}
adopt_model_cost_map({base_key: refreshed_row})
assert litellm.model_cost is live
assert litellm.model_cost[base_key] == refreshed_row
assert litellm.model_cost[key] == {**refreshed_row, "max_input_tokens": 32000, "input_cost_per_token": 0.005}
def test_register_model_keeps_case_variant_fields_independent_after_reload() -> None:
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
upper: Final = "hosted_vllm/Acme-Model"
lower: Final = "hosted_vllm/acme-model"
rows: Final = {
upper: {"max_input_tokens": 8000, "supports_vision": True, "litellm_provider": "hosted_vllm"},
lower: {"max_input_tokens": 32000, "litellm_provider": "hosted_vllm"},
}
base: Final = {"baseline-model": {"mode": "chat"}}
adopt_model_cost_map(base)
for key, row in rows.items():
litellm.register_model({key: row})
assert dict(litellm.model_cost) == {**base, **rows}
adopt_model_cost_map(base)
assert dict(litellm.model_cost) == {**base, **rows}
@pytest.mark.parametrize("registration_last", (True, False))
def test_register_model_and_direct_write_keep_last_writer_after_reload(registration_last: bool) -> None:
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
key: Final = "gpt-4o"
row: Final = {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.001}
adopt_model_cost_map({key: row})
litellm.model_cost[key]["output_cost_per_token"] = 0.8
if registration_last:
litellm.model_cost[key]["input_cost_per_token"] = 0.7
litellm.register_model({key: {"input_cost_per_token": 0.5}})
else:
litellm.register_model({key: {"input_cost_per_token": 0.5}})
litellm.model_cost[key]["input_cost_per_token"] = 0.7
expected: Final = 0.5 if registration_last else 0.7
assert litellm.model_cost[key]["input_cost_per_token"] == expected
adopt_model_cost_map({key: {**row, "input_cost_per_token": 0.003}})
assert litellm.model_cost[key]["input_cost_per_token"] == expected
assert litellm.model_cost[key]["output_cost_per_token"] == 0.8
def test_registration_provider_scope_replaces_an_exact_cross_provider_collision() -> None:
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
key: Final = "opaque-deployment-id"
catalog: Final = {
key: {"litellm_provider": "bedrock", "input_cost_per_token": 0.1, "output_cost_per_token": 0.2}
}
adopt_model_cost_map(catalog)
expected: Final = {"mode": "chat", "input_cost_per_token": 0.5}
litellm.register_model({key: expected}, custom_llm_provider="openai", add_to_provider_model_sets=False)
assert litellm.model_cost[key] == expected
adopt_model_cost_map(catalog)
assert litellm.model_cost[key] == expected
def test_register_model_nonpersistent_write_is_not_captured_as_a_direct_write() -> None:
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
row: Final = {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.001}
adopt_model_cost_map({"gpt-4o": row})
litellm.register_model({"openai/gpt-4o": {"input_cost_per_token": 0.5}}, persist_across_reloads=False)
litellm.register_model({"opaque-request-id": {"max_tokens": 1234}}, persist_across_reloads=False)
assert litellm.model_cost["openai/gpt-4o"]["input_cost_per_token"] == 0.5
assert litellm.model_cost["opaque-request-id"]["max_tokens"] == 1234
adopt_model_cost_map({"gpt-4o": row})
assert dict(litellm.model_cost) == {"gpt-4o": row}
def test_runtime_registration_view_is_sparse_detached_read_only_and_persistent() -> None:
from uuid import UUID
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
from litellm.utils import _runtime_registered_model_cost
key: Final = "openai/registration-view"
row: Final = {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 1.0}
metadata: Final = UUID(int=17)
adopt_model_cost_map({key: row})
litellm.register_model({key: {"input_cost_per_token": 0, "supports_vision": False, "metadata": [metadata]}})
snapshot: Final = dict(_runtime_registered_model_cost.items())
expected: Final = {"input_cost_per_token": 0, "supports_vision": False, "metadata": [metadata]}
assert snapshot == {key: expected}
with pytest.raises(TypeError):
_runtime_registered_model_cost[key] = {}
with pytest.raises(TypeError):
snapshot[key]["input_cost_per_token"] = 7
snapshot[key]["metadata"].append(UUID(int=18))
assert dict(_runtime_registered_model_cost.items()) == {key: expected}
litellm.register_model({key: {"output_cost_per_token": 2}})
litellm.model_cost["direct-only"] = {"mode": "chat"}
litellm.register_model({"ephemeral-only": {"mode": "chat"}}, persist_across_reloads=False)
assert dict(_runtime_registered_model_cost.items()) == {key: {**expected, "output_cost_per_token": 2}}
assert "output_cost_per_token" not in snapshot[key]
adopt_model_cost_map({key: {**row, "input_cost_per_token": 9}})
assert litellm.model_cost[key] == {**row, **expected, "output_cost_per_token": 2}
assert litellm.model_cost["direct-only"] == {"mode": "chat"}
assert "ephemeral-only" not in litellm.model_cost
@pytest.mark.parametrize("register_after_delete", [False, True])
def test_registration_tombstone_order_survives_repeated_reload(register_after_delete: bool) -> None:
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
key: Final = "openai/deleted-registration"
row: Final = {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 1.0}
adopt_model_cost_map({key: row})
litellm.register_model({key: {"input_cost_per_token": 2.0}})
del litellm.model_cost[key]
if register_after_delete:
litellm.register_model({key: {"output_cost_per_token": 3.0}})
expected: Final = {key: {"output_cost_per_token": 3.0}} if register_after_delete else {}
assert dict(litellm.model_cost) == expected
adopt_model_cost_map({key: row})
assert dict(litellm.model_cost) == expected
adopt_model_cost_map({key: row})
assert dict(litellm.model_cost) == expected
@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map")
def test_register_model_price_is_used_for_completion_cost() -> None:
model: Final = "migration-custom-pricing-model"
@ -2328,8 +2488,10 @@ def test_completion_registers_test_owned_model_prices() -> None:
)
assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages}
assert litellm.model_cost[model]["input_cost_per_token"] == 0.001
assert litellm.model_cost[model]["output_cost_per_token"] == 0.002
assert litellm.model_cost[f"openai/{model}"]["input_cost_per_token"] == 0.001
assert litellm.model_cost[f"openai/{model}"]["output_cost_per_token"] == 0.002
assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001
assert litellm.model_cost[model]["output_cost_per_token"] == 0.0002
assert response._hidden_params["response_cost"] == pytest.approx(0.018)
@ -2410,23 +2572,30 @@ def test_register_model_with_scientific_notation():
@pytest.mark.usefixtures("isolated_openai_model_sets")
@respx.mock
def test_register_model_url_fetch_uses_single_attempt(monkeypatch):
def test_register_model_url_fetch_uses_single_attempt(monkeypatch: pytest.MonkeyPatch) -> None:
from unittest.mock import Mock
from litellm._catalog import _fetch
from litellm.rust_bridge._native import RegistryUnavailable
monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False)
monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost))
before = dict(litellm.model_cost)
threads_before = {thread.name for thread in threading.enumerate()}
route = respx.get("https://example.invalid/custom_pricing.json").mock(return_value=httpx.Response(503))
before: Final = dict(litellm.model_cost)
threads_before: Final = {thread.name for thread in threading.enumerate()}
fetch_once: Final = Mock(return_value=Mock(spec=RegistryUnavailable, reason="HTTP 503", retryable=True))
monkeypatch.setattr(_fetch, "fetch_once", fetch_once)
litellm.register_model(model_cost="https://example.invalid/custom_pricing.json")
threads_after = {thread.name for thread in threading.enumerate()}
assert route.call_count == 1
threads_after: Final = {thread.name for thread in threading.enumerate()}
fetch_once.assert_called_once_with(
"https://example.invalid/custom_pricing.json", if_none_match=None, timeout_seconds=5.0
)
assert not (threads_after - threads_before) & {"litellm-model-cost-map-retry"}
assert not any(
thread.name == "litellm-model-cost-map-retry" and thread.is_alive() for thread in threading.enumerate()
)
assert litellm.model_cost.keys() >= before.keys()
assert litellm.model_cost == before
def test_register_model_openrouter_without_slash():
@ -4929,7 +5098,7 @@ async def test_builtin_string_callback_registers_when_subclass_already_active(
@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map")
@pytest.mark.parametrize("rate", (0.0, 0.000123))
@pytest.mark.parametrize("updated_rate", (None, 0.0, 0.000456))
def test_reapply_runtime_registrations_replays_register_model_overrides(
def test_install_preserves_register_model_overrides(
monkeypatch: pytest.MonkeyPatch, rate: float, updated_rate: float | None
) -> None:
"""
@ -4938,22 +5107,8 @@ def test_reapply_runtime_registrations_replays_register_model_overrides(
so without replaying those registrations the override is silently lost and
the model reverts to upstream pricing.
"""
from litellm import utils as litellm_utils
from litellm.utils import (
_invalidate_model_cost_lowercase_map,
reapply_runtime_model_cost_registrations,
)
monkeypatch.setattr(
litellm_utils,
"_runtime_registered_model_cost",
dict(litellm_utils._runtime_registered_model_cost),
)
# Only the recorded half is under test here; the live-router rebuild is covered
# in test_router_model_cost_isolation.py. Routers built by earlier tests in this
# process stay in the weak set until they are collected, so leaving the callback
# installed would make this depend on when that happens.
monkeypatch.setattr(litellm_utils._LiveDeploymentReplay, "callback", None)
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
from litellm.utils import _invalidate_model_cost_lowercase_map
saved_model_cost: Final = litellm.model_cost
monkeypatch.setattr(litellm, "model_cost", dict(saved_model_cost))
@ -4974,16 +5129,16 @@ def test_reapply_runtime_registrations_replays_register_model_overrides(
)
litellm.register_model({"openai/gpt-4o": {"supports_function_calling": False}})
litellm.model_cost = {
"openai/gpt-4o": {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 0.000999,
"max_input_tokens": 4242,
adopt_model_cost_map(
{
"openai/gpt-4o": {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 0.000999,
"max_input_tokens": 4242,
}
}
}
_invalidate_model_cost_lowercase_map()
reapply_runtime_model_cost_registrations()
)
expected_rate: Final = rate if updated_rate is None else updated_rate
assert litellm.model_cost["openai/gpt-4o"]["input_cost_per_token"] == expected_rate
@ -4995,24 +5150,15 @@ def test_reapply_runtime_registrations_replays_register_model_overrides(
@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map")
def test_reapply_runtime_registrations_drops_request_scoped_registrations(monkeypatch):
def test_install_drops_request_scoped_registrations(monkeypatch):
"""
Per-request custom pricing describes one call, so it must not be re-asserted
over every future catalog. Replaying it would let a one-off price outlive
the catalog generation it was applied to and silently beat fresh upstream
pricing forever, while a durable override registered alongside it survives.
"""
from litellm import utils as litellm_utils
from litellm.utils import (
_invalidate_model_cost_lowercase_map,
reapply_runtime_model_cost_registrations,
)
monkeypatch.setattr(
litellm_utils,
"_runtime_registered_model_cost",
dict(litellm_utils._runtime_registered_model_cost),
)
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
from litellm.utils import _invalidate_model_cost_lowercase_map
saved_model_cost = litellm.model_cost
try:
@ -5025,12 +5171,12 @@ def test_reapply_runtime_registrations_drops_request_scoped_registrations(monkey
persist_across_reloads=False,
)
litellm.model_cost = {
"openai/gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 0.000999},
"openai/gpt-4o-mini": {"litellm_provider": "openai", "input_cost_per_token": 0.000888},
}
_invalidate_model_cost_lowercase_map()
reapply_runtime_model_cost_registrations()
adopt_model_cost_map(
{
"openai/gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 0.000999},
"openai/gpt-4o-mini": {"litellm_provider": "openai", "input_cost_per_token": 0.000888},
}
)
assert litellm.model_cost["openai/gpt-4o"]["input_cost_per_token"] == 0.000111
assert litellm.model_cost["openai/gpt-4o-mini"]["input_cost_per_token"] == 0.000888
@ -7377,9 +7523,7 @@ async def test_router_resolves_an_oidc_api_key_only_on_a_proxy_level_deployment(
try:
with respx.mock(assert_all_called=True) as upstream:
upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY)
await router.acompletion(
model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"}
)
await router.acompletion(model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"})
sent: Final = upstream.calls.last.request.headers["Authorization"]
finally:
router.discard()
@ -7447,7 +7591,9 @@ async def test_an_inline_oidc_api_key_reaches_the_endpoint_its_deployment_config
with respx.mock(assert_all_called=False) as upstream:
upstream.route().respond(200, json=_BEDROCK_REPLY)
await router.acompletion(
model="claude", messages=_PING, aws_bedrock_runtime_endpoint="https://BEDROCK-runtime.us-east-1.amazonaws.com"
model="claude",
messages=_PING,
aws_bedrock_runtime_endpoint="https://BEDROCK-runtime.us-east-1.amazonaws.com",
)
sent: Final = tuple(
(call.request.url.host, call.request.headers["Authorization"]) for call in upstream.calls
@ -7656,7 +7802,9 @@ async def test_a_team_row_naming_an_oidc_credential_reaches_only_the_credentials
(
lambda reference: LiteLLM_Params(
**litellm.Router(
model_list=[{"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": reference}}]
model_list=[
{"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": reference}}
]
).get_model_list(model_name="openai/*")[0]["litellm_params"]
),
lambda reference: "Bearer listing-token",
@ -8137,10 +8285,9 @@ def _vcr_outcome_gate(request, vcr):
@pytest.fixture(scope="function")
def setup_and_teardown():
"""
This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained.
"""
importlib.reload(litellm)
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap, adopt_model_cost_map
adopt_model_cost_map(GetModelCostMap.load_local_model_cost_map())
loop = asyncio.get_event_loop_policy().new_event_loop()
asyncio.set_event_loop(loop)
print(litellm)
@ -8308,15 +8455,12 @@ _SCALAR_DEFAULTS = {
@pytest.fixture(scope="module")
def setup_and_teardown_local_testing():
"""
Module-scoped setup. Reloads litellm only in single-process mode
(skipped under xdist to avoid cross-worker interference).
"""
import litellm
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap, adopt_model_cost_map
worker_id = os.environ.get("PYTEST_XDIST_WORKER", None)
if worker_id is None:
importlib.reload(litellm)
adopt_model_cost_map(GetModelCostMap.load_local_model_cost_map())
try:
if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"):
import litellm.proxy.proxy_server