mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
refactor(catalog): register caller keys through native layers
This commit is contained in:
parent
430d3be2c7
commit
69aef480f4
10 changed files with 573 additions and 569 deletions
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import copy
|
||||
import threading
|
||||
from collections.abc import Callable, Mapping
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from itertools import groupby
|
||||
|
|
@ -101,39 +101,50 @@ def remember_written_rows(keys: frozenset[str]) -> None:
|
|||
|
||||
|
||||
def registration_write(
|
||||
target: dict[str, Mapping[str, object]], # mutable-ok: retain the live public dict and subclass write behavior
|
||||
key: str,
|
||||
entry: Mapping[str, object],
|
||||
registration: Mapping[str, object],
|
||||
base: Mapping[str, object],
|
||||
fields: Mapping[str, object],
|
||||
*,
|
||||
persist: bool,
|
||||
inherits_from: str | None,
|
||||
replace: Callable[ # mutable-ok: legacy registration publishes into the live dict
|
||||
[dict[str, Mapping[str, object]], str, Mapping[str, object], Mapping[str, object]], None
|
||||
],
|
||||
) -> None:
|
||||
generation: int,
|
||||
) -> bool:
|
||||
from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry
|
||||
|
||||
with LOCK:
|
||||
current: Final = dict[str, Mapping[str, object]].get(target, key)
|
||||
target: Final = _live_rows()
|
||||
current: Final = dict[str, object].get(target, key)
|
||||
names: Final = current.names if isinstance(current, AliasedCostEntry) else ()
|
||||
keys: Final = frozenset((key, *(name for name in names if target.get(name) is current)))
|
||||
record_pending_writes(keys)
|
||||
generation: Final = STATE.generation
|
||||
fields: Final = {name: entry.get(name) for name, value in registration.items() if value is not None}
|
||||
erased: Final = {name: None for name, value in entry.items() if value is None}
|
||||
overrides: Final = _ROW.validate_python({**fields, **erased})
|
||||
replace(target, key, entry, registration)
|
||||
if STATE.generation != generation:
|
||||
refreshed: Final = {
|
||||
alias: AliasedCostEntry({**STATE.rows.get(alias, {}), **overrides}, tuple(keys), frozenset(fields))
|
||||
for alias in keys
|
||||
}
|
||||
write_rows(_live_rows(), dict(refreshed), [])
|
||||
encoded: Final = _encoded_row(fields)
|
||||
local: Final = ModelCostLayers()
|
||||
local.diff({key: ({}, None)})
|
||||
local.diff({key: (None, _encoded_row(base))})
|
||||
local.record_registration(key, None, encoded)
|
||||
registry: Final = STATE.registry or ModelRegistry.bundled()
|
||||
composed: Final = local.compose_keys(registry, frozenset({key}))
|
||||
entry: Final = composed[0][1]
|
||||
if entry is None:
|
||||
return True
|
||||
previous_fields: Final[frozenset[str]] = (
|
||||
current.registered_fields if isinstance(current, AliasedCostEntry) else frozenset()
|
||||
)
|
||||
published: Final = AliasedCostEntry(
|
||||
{name: _decoded_value(value) for name, value in entry.items()},
|
||||
tuple(keys),
|
||||
previous_fields | frozenset(fields),
|
||||
)
|
||||
snapshot: Final = copy.deepcopy(dict(published))
|
||||
if generation != STATE.generation:
|
||||
return False
|
||||
if persist:
|
||||
for alias in keys:
|
||||
LAYERS.record_registration(alias, inherits_from, _encoded_row(overrides))
|
||||
remember_written_rows(keys)
|
||||
LAYERS.record_registration(alias, inherits_from, encoded)
|
||||
write_rows(target, dict.fromkeys(keys, published), [])
|
||||
for alias in keys:
|
||||
STATE.rows[alias] = snapshot
|
||||
return True
|
||||
|
||||
|
||||
def accepted_catalog(catalog: LoadedCatalog) -> LoadedCatalog | None:
|
||||
|
|
|
|||
173
litellm/_catalog/_register.py
Normal file
173
litellm/_catalog/_register.py
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import ItemsView, Iterator, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._catalog import _install
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.rust_bridge._native import RegistrationTarget, registration_target
|
||||
|
||||
_ROW: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _base_fields(
|
||||
target: RegistrationTarget,
|
||||
inherited: Mapping[str, object],
|
||||
value: Mapping[str, object],
|
||||
custom_llm_provider: str | None,
|
||||
) -> Mapping[str, object]:
|
||||
from litellm import utils
|
||||
|
||||
lookups: Final = (utils.lookup_model_info_for_registration(target.write, custom_llm_provider) for _ in range(3))
|
||||
read: Final = next(
|
||||
(lookup for lookup in lookups if not (isinstance(lookup, utils.ModelInfoLookupMiss) and lookup.tripped)),
|
||||
utils.ModelInfoLookupMiss(tripped=True),
|
||||
)
|
||||
primary: Final = not isinstance(read, utils.ModelInfoLookupMiss) and read.get("key") == target.inherit_from
|
||||
if primary:
|
||||
return {name: field for name, field in inherited.items() if name != "litellm_provider" or field is not None}
|
||||
rates: Final = (
|
||||
inherited
|
||||
if target.names_the_row
|
||||
else utils._without_overridden_price_variants( # pyright: ignore[reportPrivateUsage] # shared rate policy
|
||||
inherited, value
|
||||
)
|
||||
)
|
||||
generalized: Final = _ROW.validate_python(
|
||||
utils.match_capability_generalizations( # pyright: ignore[reportUnknownMemberType] # validate legacy rules
|
||||
target.write
|
||||
)
|
||||
or {}
|
||||
)
|
||||
own_fields: Final = {
|
||||
name: field
|
||||
for name, field in rates.items()
|
||||
if name != "litellm_provider" or (target.write == target.inherit_from and field is not None)
|
||||
}
|
||||
return {**generalized, **own_fields}
|
||||
|
||||
|
||||
def registration_row(key: str, provider: str | None) -> Mapping[str, object]:
|
||||
with _install.LOCK:
|
||||
live: Final = _install._live_rows() # pyright: ignore[reportPrivateUsage] # retain public dict identity
|
||||
target: Final = registration_target(live, key, provider=provider)
|
||||
return _ROW.validate_python(live.get(target.inherit_from, {}) if target.inherit_from is not None else {})
|
||||
|
||||
|
||||
def _base_fields_replaced_by(value: Mapping[str, object]) -> frozenset[str]:
|
||||
return (
|
||||
frozenset({"explicit_cache_read_input_token_cost"})
|
||||
if value.get("cache_read_input_token_cost") is not None
|
||||
and value.get("explicit_cache_read_input_token_cost") is None
|
||||
else frozenset()
|
||||
)
|
||||
|
||||
|
||||
def _register_entry(
|
||||
key: str,
|
||||
value: Mapping[str, object],
|
||||
*,
|
||||
persist: bool,
|
||||
custom_llm_provider: str | None,
|
||||
warning_display_name: str | None,
|
||||
) -> bool:
|
||||
from litellm import utils
|
||||
|
||||
generation: Final = _install.STATE.generation
|
||||
live: Final = _install._live_rows() # pyright: ignore[reportPrivateUsage] # retain public dict identity
|
||||
provider: Final = value.get("litellm_provider")
|
||||
target: Final = registration_target(
|
||||
live, key, provider=custom_llm_provider or (provider if isinstance(provider, str) else None)
|
||||
)
|
||||
inherited: Final = _ROW.validate_python(
|
||||
live.get(target.inherit_from, {}) if target.inherit_from is not None else {}
|
||||
)
|
||||
base: Final = _base_fields(target, inherited, value, custom_llm_provider)
|
||||
merged: Final = utils.update_dictionary(dict(base), value)
|
||||
fields: Final = {
|
||||
name: field
|
||||
for name, field in merged.items()
|
||||
if value.get(name) is not None or name not in inherited or field != inherited[name]
|
||||
}
|
||||
erased: Final = _base_fields_replaced_by(value)
|
||||
previous: Final = _ROW.validate_python(live.get(key, {}))
|
||||
removed: Final = (frozenset(inherited) | frozenset(previous)) - frozenset(merged)
|
||||
overrides: Final = {**fields, **dict.fromkeys(removed | erased)}
|
||||
if generation != _install.STATE.generation:
|
||||
return False
|
||||
if not _install.registration_write(
|
||||
target.write,
|
||||
base,
|
||||
overrides,
|
||||
persist=persist,
|
||||
inherits_from=target.inherit_from if target.inherit_from != target.write else None,
|
||||
generation=generation,
|
||||
):
|
||||
return False
|
||||
if target.inherit_from is None and (
|
||||
value.get("cache_creation_input_token_cost") is None
|
||||
and value.get("cache_read_input_token_cost") is None
|
||||
and value.get("tiered_pricing") is None
|
||||
and (value.get("input_cost_per_token") is not None or value.get("output_cost_per_token") is not None)
|
||||
):
|
||||
verbose_logger.warning(
|
||||
"register_model: model=%s has custom pricing but not in built-in cost map and no prefix/region variant "
|
||||
"matched; cache_creation_input_token_cost and cache_read_input_token_cost will default to 0 for this "
|
||||
"model (input/output cost tracking is unaffected). To track cache cost, add them to model_info",
|
||||
warning_display_name or key,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def register(
|
||||
entries: Mapping[str, Mapping[str, object]],
|
||||
*,
|
||||
persist: bool,
|
||||
custom_llm_provider: str | None,
|
||||
warning_display_name: str | None,
|
||||
) -> None:
|
||||
for key, value in entries.items():
|
||||
with _install.LOCK:
|
||||
while not _register_entry(
|
||||
key,
|
||||
value,
|
||||
persist=persist,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
warning_display_name=warning_display_name,
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
class RuntimeRegistrations(Mapping[str, Mapping[str, object]]):
|
||||
@staticmethod
|
||||
def _snapshot() -> Mapping[str, Mapping[str, object]]:
|
||||
with _install.LOCK:
|
||||
entries: Final[dict[str, Mapping[str, object]]] = {} # mutable-ok: fold ordered native sparse records
|
||||
for key, _, fields in _install.LAYERS.registrations():
|
||||
entries[key] = MappingProxyType(
|
||||
{
|
||||
**entries.get(key, {}),
|
||||
**{
|
||||
name: _install._decoded_value( # pyright: ignore[reportPrivateUsage] # opaque metadata
|
||||
value
|
||||
)
|
||||
for name, value in fields.items()
|
||||
},
|
||||
}
|
||||
)
|
||||
return MappingProxyType(entries)
|
||||
|
||||
def __getitem__(self, key: str) -> Mapping[str, object]:
|
||||
return self._snapshot()[key]
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
return iter(self._snapshot())
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._snapshot())
|
||||
|
||||
def items(self) -> ItemsView[str, Mapping[str, object]]:
|
||||
return self._snapshot().items()
|
||||
334
litellm/utils.py
334
litellm/utils.py
|
|
@ -79,7 +79,6 @@ from litellm.constants import (
|
|||
MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE,
|
||||
NON_INFERENCE_CALL_TYPES,
|
||||
OPENAI_EMBEDDING_PARAMS,
|
||||
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
|
||||
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
|
|
@ -3486,17 +3485,6 @@ def _convert_stringified_numbers(value):
|
|||
return value
|
||||
|
||||
|
||||
_BEDROCK_REGION_PREFIXES: Final = (
|
||||
"us.",
|
||||
"eu.",
|
||||
"apac.",
|
||||
"jp.",
|
||||
"au.",
|
||||
"us-gov.",
|
||||
"global.",
|
||||
"ap-northeast-1.",
|
||||
)
|
||||
|
||||
_TOKEN_COST_FIELDS: Final = ("input_cost_per_token", "output_cost_per_token")
|
||||
|
||||
|
||||
|
|
@ -3504,39 +3492,6 @@ def _builtin_cost_map() -> dict[str, Mapping[str, object]]:
|
|||
return cast(dict[str, Mapping[str, object]], litellm.model_cost)
|
||||
|
||||
|
||||
def _replace_cost_map_entry(
|
||||
cost_map: dict[str, Mapping[str, object]], key: str, entry: Mapping[str, object], registration: Mapping[str, object]
|
||||
) -> None:
|
||||
current: Final = cost_map.get(key)
|
||||
merged: Final = {**(current or {}), **entry}
|
||||
names: Final = current.names if isinstance(current, AliasedCostEntry) else ()
|
||||
previous_fields: Final[frozenset[str]] = (
|
||||
current.registered_fields if isinstance(current, AliasedCostEntry) else frozenset()
|
||||
)
|
||||
registered_fields: Final = previous_fields | frozenset(
|
||||
name for name, value in registration.items() if value is not None
|
||||
)
|
||||
sharing_names: Final = tuple(name for name in names if cost_map.get(name) is current)
|
||||
cost_map.update(dict.fromkeys((key, *sharing_names), AliasedCostEntry(merged, names, registered_fields)))
|
||||
|
||||
|
||||
def _without_synthesized_costs(
|
||||
model_info: Mapping[str, object], key: str, value: Mapping[str, object]
|
||||
) -> Mapping[str, object]:
|
||||
"""``_get_model_info_helper`` synthesizes ``input_cost_per_token`` / ``output_cost_per_token`` = 0 when the raw
|
||||
entry has neither (#30198). Persisting those zeros would turn a name-priced entry into a free one and make
|
||||
``_is_cost_explicitly_configured`` disable budget enforcement on the next re-registration."""
|
||||
builtin_cost_map: Final = _builtin_cost_map()
|
||||
raw_fields: Final = frozenset(builtin_cost_map.get(str(model_info["key"]), builtin_cost_map.get(key, {})))
|
||||
return MappingProxyType(
|
||||
{
|
||||
name: field_value
|
||||
for name, field_value in model_info.items()
|
||||
if name not in _TOKEN_COST_FIELDS or name in raw_fields or name in value
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_DIRECTION_RATE_FIELDS: Final = MappingProxyType(
|
||||
{
|
||||
"input_cost_per_token": ("input_cost_per_token",),
|
||||
|
|
@ -3617,84 +3572,11 @@ def model_info_with_pricing_overrides(model_info: ModelInfo, registration: Mappi
|
|||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _BuiltinCostMatch:
|
||||
key: str
|
||||
entry: Mapping[str, object]
|
||||
names_the_row: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModelInfoLookupMiss:
|
||||
tripped: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _FallbackRegistration:
|
||||
entry: Mapping[str, object]
|
||||
matched_builtin: bool
|
||||
|
||||
|
||||
def _fallback_registration(key: str, provider: str, value: Mapping[str, object]) -> _FallbackRegistration:
|
||||
"""The key keeps its own entry, seeded from the capability rules an unregistered lookup resolves it to, with the
|
||||
matched built-in entry on top. The matched row's provider stays behind, since the key's own lookup can reject it
|
||||
(an ``azure_ai/`` key never resolves to an ``openai`` entry), and the key keeps the provider its caller gave."""
|
||||
match: Final = _resolve_builtin_model_cost_entry(key=key, provider=provider)
|
||||
generalized: Final[Mapping[str, object]] = match_capability_generalizations(key) or {}
|
||||
if match is None:
|
||||
return _FallbackRegistration(entry=generalized, matched_builtin=False)
|
||||
inherited: Final = match.entry if match.names_the_row else _without_overridden_price_variants(match.entry, value)
|
||||
own_fields: Final = {name: field_value for name, field_value in inherited.items() if name != "litellm_provider"}
|
||||
return _FallbackRegistration(entry={**generalized, **own_fields}, matched_builtin=True)
|
||||
|
||||
|
||||
def _resolve_builtin_model_cost_entry(key: str, provider: str) -> _BuiltinCostMatch | None:
|
||||
"""Best-effort lookup of a built-in ``model_cost`` entry for a custom key
|
||||
whose shape ``get_model_info`` cannot resolve (repeated provider prefixes
|
||||
like ``bedrock/bedrock/bedrock/us.anthropic.claude-sonnet-4-6`` or region
|
||||
aliases), or whose lookup tripped.
|
||||
|
||||
The key itself is tried first, then the key behind its provider prefixes
|
||||
and bedrock route, so a catalog row whose lookup tripped inherits from
|
||||
itself rather than from a region-stripped sibling priced at the base rate.
|
||||
``names_the_row`` is set when the match is the row the key names, which
|
||||
``get_model_info`` resolves it to, so the caller keeps that row's rates as
|
||||
an untripped registration does. Returns ``None`` when no candidate exists
|
||||
under a provider ``get_model_info`` would accept for ``provider``, or, when
|
||||
the value names none, for the provider prefix the key itself carries,
|
||||
which is what ``get_model_info`` derives when it reads the key back.
|
||||
"""
|
||||
from litellm.llms.bedrock.common_utils import strip_bedrock_routing_prefix
|
||||
|
||||
segments: Final = key.split("/")
|
||||
match_provider: Final = provider or (segments[0] if len(segments) > 1 and segments[0] in LlmProvidersSet else "")
|
||||
provider_prefix_count: Final = next(
|
||||
(idx for idx, segment in enumerate(segments[:-1]) if segment not in LlmProvidersSet), len(segments) - 1
|
||||
)
|
||||
prefix_stripped: Final = tuple("/".join(segments[idx:]) for idx in range(1, provider_prefix_count + 1))
|
||||
prefix_free: Final = prefix_stripped[-1] if prefix_stripped else key
|
||||
base: Final = (
|
||||
strip_bedrock_routing_prefix(prefix_free) if match_provider in ("bedrock", "bedrock_converse") else prefix_free
|
||||
)
|
||||
region_stripped: Final = tuple(
|
||||
base[len(region_prefix) :] for region_prefix in _BEDROCK_REGION_PREFIXES if base.startswith(region_prefix)
|
||||
)
|
||||
stripped: Final = _strip_model_name(model=base, custom_llm_provider=match_provider) if match_provider else base
|
||||
row_names: Final = (key, *prefix_stripped, base, stripped)
|
||||
candidates: Final = (key, *prefix_stripped, base, *region_stripped, stripped)
|
||||
|
||||
builtin_cost_map: Final = _builtin_cost_map()
|
||||
for candidate in candidates:
|
||||
entry = builtin_cost_map.get(candidate)
|
||||
if (
|
||||
entry is not None
|
||||
and entry.get("litellm_provider") is not None
|
||||
and _check_provider_match(dict(entry), match_provider)
|
||||
):
|
||||
return _BuiltinCostMatch(key=candidate, entry=dict(entry), names_the_row=candidate in row_names)
|
||||
return None
|
||||
|
||||
|
||||
def is_generalized_model_info(model_info: ModelInfo) -> bool:
|
||||
"""Whether ``model_info`` came from a fallback-generalization capability rule.
|
||||
|
||||
|
|
@ -3709,9 +3591,6 @@ def is_generalized_model_info(model_info: ModelInfo) -> bool:
|
|||
return key not in litellm.model_cost and match_capability_generalizations(key) is not None
|
||||
|
||||
|
||||
_REGISTRATION_ATTEMPTS: Final = 3
|
||||
|
||||
|
||||
def lookup_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | ModelInfoLookupMiss:
|
||||
try:
|
||||
info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
|
@ -3722,162 +3601,15 @@ def lookup_model_info_for_registration(model: str, custom_llm_provider: str | No
|
|||
return ModelInfoLookupMiss(tripped=False) if is_generalized_model_info(info) else info
|
||||
|
||||
|
||||
def _get_builtin_model_info_for_registration(
|
||||
model: str, custom_llm_provider: str | None
|
||||
) -> ModelInfo | ModelInfoLookupMiss:
|
||||
"""Resolve ``model`` to its built-in cost-map entry for registration merging.
|
||||
|
||||
Misses when the lookup cannot map the model or resolved it via a
|
||||
fallback-generalization capability rule, since a rule-derived entry carries no
|
||||
pricing. A lookup that raised anything else, such as a concurrent registration
|
||||
changing the map mid-walk, is retried, so a transient failure registers exactly
|
||||
where the untripped lookup would.
|
||||
"""
|
||||
lookups: Final = (
|
||||
lookup_model_info_for_registration(model, custom_llm_provider) for _ in range(_REGISTRATION_ATTEMPTS)
|
||||
)
|
||||
return next(
|
||||
(lookup for lookup in lookups if not (isinstance(lookup, ModelInfoLookupMiss) and lookup.tripped)),
|
||||
ModelInfoLookupMiss(tripped=True),
|
||||
)
|
||||
|
||||
|
||||
def _skips_builtin_registration_lookup(key: str, provider: object) -> bool:
|
||||
return provider in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO or any(
|
||||
key.startswith(f"{p}/") for p in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _RegistrationWrite:
|
||||
key: str
|
||||
entry: Mapping[str, object]
|
||||
|
||||
|
||||
def model_cost_registration_row(key: str, provider: object) -> Mapping[str, object]:
|
||||
"""The row a ``register_model`` write of a ``key`` entry with this ``litellm_provider`` merges onto: the
|
||||
``litellm.model_cost`` row it resolves to (``openai/o3-pro`` lands on ``o3-pro``), else the row it seeds ``key``
|
||||
with from the built-in entry it falls back to (an OpenAI fine-tune takes its base model's row)."""
|
||||
base: Final = _registration_base(
|
||||
key, {"litellm_provider": provider}, custom_llm_provider=None, warning_display_name=None
|
||||
)
|
||||
return _builtin_cost_map().get(base.key) or base.entry
|
||||
from litellm._catalog._register import registration_row
|
||||
|
||||
return registration_row(key, provider if isinstance(provider, str) else None)
|
||||
|
||||
|
||||
def _registration_base(
|
||||
key: str, value: Mapping[str, object], custom_llm_provider: str | None, warning_display_name: str | None
|
||||
) -> _RegistrationWrite:
|
||||
provider: Final = str(value.get("litellm_provider") or "")
|
||||
if _skips_builtin_registration_lookup(key, provider):
|
||||
return _RegistrationWrite(key=key, entry=dict(_builtin_cost_map().get(key, {})))
|
||||
builtin_model_info: Final = _get_builtin_model_info_for_registration(
|
||||
model=key, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
if not isinstance(builtin_model_info, ModelInfoLookupMiss):
|
||||
return _RegistrationWrite(
|
||||
key=builtin_model_info["key"],
|
||||
entry=dict(_without_synthesized_costs(builtin_model_info, key=key, value=value)),
|
||||
)
|
||||
fallback: Final = _fallback_registration(key=key, provider=provider, value=value)
|
||||
if not fallback.matched_builtin and (
|
||||
value.get("cache_creation_input_token_cost") is None
|
||||
and value.get("cache_read_input_token_cost") is None
|
||||
and value.get("tiered_pricing") is None
|
||||
and (value.get("input_cost_per_token") is not None or value.get("output_cost_per_token") is not None)
|
||||
):
|
||||
verbose_logger.warning(
|
||||
"register_model: model=%s has custom pricing but not in built-in cost map and no prefix/region variant matched; cache_creation_input_token_cost and cache_read_input_token_cost will default to 0 for this model (input/output cost tracking is unaffected). To track cache cost, add them to model_info",
|
||||
warning_display_name or key,
|
||||
)
|
||||
return _RegistrationWrite(key=key, entry=dict(fallback.entry))
|
||||
from litellm._catalog._register import RuntimeRegistrations
|
||||
|
||||
|
||||
def _base_fields_replaced_by(value: Mapping[str, object]) -> frozenset[str]:
|
||||
return (
|
||||
frozenset({"explicit_cache_read_input_token_cost"})
|
||||
if value.get("cache_read_input_token_cost") is not None
|
||||
and value.get("explicit_cache_read_input_token_cost") is None
|
||||
else frozenset()
|
||||
)
|
||||
|
||||
|
||||
def _registration_write(
|
||||
key: str, value: Mapping[str, object], custom_llm_provider: str | None, warning_display_name: str | None
|
||||
) -> _RegistrationWrite:
|
||||
base: Final = _registration_base(key, value, custom_llm_provider, warning_display_name)
|
||||
# ``get_model_info`` returns ``litellm_provider: None`` when the
|
||||
# provider is unknown (e.g. custom deployments registered via
|
||||
# ``Router.add_deployment``). Persisting that None into
|
||||
# ``litellm.model_cost`` causes ``_check_provider_match`` to drop
|
||||
# custom pricing on subsequent cost lookups.
|
||||
existing_model: Final = {
|
||||
name: field_value
|
||||
for name, field_value in base.entry.items()
|
||||
if name != "litellm_provider" or field_value is not None
|
||||
}
|
||||
return _RegistrationWrite(
|
||||
key=base.key,
|
||||
entry={**update_dictionary(existing_model, value), **dict.fromkeys(_base_fields_replaced_by(value))},
|
||||
)
|
||||
|
||||
|
||||
def _registration_attempt(
|
||||
key: str,
|
||||
value: Mapping[str, object],
|
||||
custom_llm_provider: str | None,
|
||||
warning_display_name: str | None,
|
||||
final_attempt: bool,
|
||||
persist_across_reloads: bool = True,
|
||||
) -> _RegistrationWrite | None:
|
||||
from litellm._catalog._install import registration_write
|
||||
|
||||
generation: Final = get_model_cost_mutation_generation()
|
||||
source: Final = _builtin_cost_map()
|
||||
write: Final = _registration_write(key, value, custom_llm_provider, warning_display_name)
|
||||
if not final_attempt and get_model_cost_mutation_generation() != generation:
|
||||
return None
|
||||
provider: Final = value.get("litellm_provider")
|
||||
match: Final = _resolve_builtin_model_cost_entry(
|
||||
key=write.key, provider=provider if isinstance(provider, str) else ""
|
||||
)
|
||||
registration_write(
|
||||
_builtin_cost_map() if final_attempt else source,
|
||||
write.key,
|
||||
write.entry,
|
||||
value,
|
||||
persist=persist_across_reloads,
|
||||
inherits_from=match.key if match is not None and match.key != write.key else None,
|
||||
replace=_replace_cost_map_entry,
|
||||
)
|
||||
return write if final_attempt or get_model_cost_mutation_generation() == generation else None
|
||||
|
||||
|
||||
def _register_on_current_map(
|
||||
key: str,
|
||||
value: Mapping[str, object],
|
||||
custom_llm_provider: str | None,
|
||||
warning_display_name: str | None,
|
||||
persist_across_reloads: bool = True,
|
||||
) -> _RegistrationWrite:
|
||||
"""A price data reload that swaps the map while the entry is being built or written would otherwise get the old
|
||||
map's prices written over its own, or drop the entry with the old map, so each attempt writes into the map it was
|
||||
built from and the entry is rebuilt when a reload replaced that map before the write finished. The last attempt
|
||||
writes into the current map, so the entry is never lost."""
|
||||
attempts: Final = (
|
||||
_registration_attempt(
|
||||
key,
|
||||
value,
|
||||
custom_llm_provider,
|
||||
warning_display_name,
|
||||
final_attempt=attempt == _REGISTRATION_ATTEMPTS - 1,
|
||||
persist_across_reloads=persist_across_reloads,
|
||||
)
|
||||
for attempt in range(_REGISTRATION_ATTEMPTS)
|
||||
)
|
||||
return next(write for write in attempts if write is not None)
|
||||
|
||||
|
||||
_runtime_registered_model_cost: Final[dict[str, dict[str, object]]] = {} # mutable-ok: replayed on reload
|
||||
_runtime_registered_model_cost: Final[Mapping[str, Mapping[str, object]]] = RuntimeRegistrations()
|
||||
|
||||
|
||||
def runtime_model_cost_registrations(
|
||||
|
|
@ -3922,30 +3654,6 @@ def set_live_deployment_replay(replay: Callable[[], None]) -> None:
|
|||
_LiveDeploymentReplay.callbacks = (replay,)
|
||||
|
||||
|
||||
def reapply_runtime_model_cost_registrations() -> None:
|
||||
"""Re-apply runtime model metadata on top of a freshly adopted cost map.
|
||||
|
||||
Adopting a new catalog replaces ``litellm.model_cost`` wholesale, which on
|
||||
its own discards everything registered at runtime: the deployment
|
||||
``model_info`` the Router registers from ``model_list``, and pricing
|
||||
overrides passed to ``register_model``. Both are re-applied here so a price
|
||||
data reload only updates pricing rather than erasing operator-supplied model
|
||||
metadata.
|
||||
|
||||
The two are restored differently, and the difference is what keeps this
|
||||
bounded. Deployment metadata is re-derived from the routers that are alive
|
||||
right now, so a deployment that has been deleted or repointed, and a router
|
||||
that has been discarded, are simply not part of the rebuild; nothing has to
|
||||
withdraw them and nothing accumulates. Only ``register_model`` calls that
|
||||
have no such owner are recorded and replayed, and a registration describing
|
||||
a single request opts out of even that.
|
||||
"""
|
||||
for replay in _LiveDeploymentReplay.callbacks:
|
||||
replay()
|
||||
if _runtime_registered_model_cost:
|
||||
register_model(model_cost=dict(_runtime_registered_model_cost))
|
||||
|
||||
|
||||
def cost_map_omits_token_price(*keys: object) -> bool:
|
||||
"""Whether the raw ``litellm.model_cost`` entries under ``keys`` exist but none carries a per-token price.
|
||||
|
||||
|
|
@ -4021,28 +3729,20 @@ def register_model(
|
|||
return model_cost
|
||||
loaded_model_cost = served_rows(accepted)
|
||||
|
||||
if persist_across_reloads:
|
||||
_registrations: Final[Mapping[str, Mapping[str, object]]] = loaded_model_cost
|
||||
for _registered_key, _registered_value in _registrations.items():
|
||||
_runtime_registered_model_cost[_registered_key] = update_dictionary(
|
||||
dict(_runtime_registered_model_cost.get(_registered_key, {})),
|
||||
_RATE_TABLE.validate_python(_registered_value),
|
||||
)
|
||||
from litellm._catalog._register import register
|
||||
|
||||
for key, value in loaded_model_cost.items():
|
||||
from litellm._catalog._install import LOCK
|
||||
|
||||
with LOCK:
|
||||
write = _register_on_current_map(
|
||||
str(key), value, custom_llm_provider, warning_display_name, persist_across_reloads
|
||||
)
|
||||
|
||||
# Invalidate case-insensitive lookup map since model_cost was modified
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
verbose_logger.debug("added/updated model=%s in litellm.model_cost: %s", write.key, write.key)
|
||||
entries: Final = {str(key): _RATE_TABLE.validate_python(value) for key, value in loaded_model_cost.items()}
|
||||
register(
|
||||
entries,
|
||||
persist=persist_across_reloads,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
warning_display_name=warning_display_name,
|
||||
)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
for key, value in entries.items():
|
||||
verbose_logger.debug("added/updated model=%s in litellm.model_cost: %s", key, key)
|
||||
if add_to_provider_model_sets and value.get("litellm_provider") == "openrouter":
|
||||
litellm.openrouter_models.add(str(key).split("/", 1)[-1])
|
||||
litellm.openrouter_models.add(key.split("/", 1)[-1])
|
||||
if add_to_provider_model_sets:
|
||||
litellm.add_known_models(loaded_model_cost)
|
||||
return model_cost
|
||||
|
|
|
|||
|
|
@ -293,20 +293,13 @@ def isolate_litellm_globals() -> Iterator[None]:
|
|||
@pytest.fixture(autouse=True)
|
||||
def isolate_router_model_cost_state() -> Iterator[None]:
|
||||
original_live_routers: Final = frozenset(litellm_router_module._live_routers)
|
||||
original_runtime_registered_model_cost: Final = {
|
||||
model_key: dict(model_value)
|
||||
for model_key, model_value in litellm_utils_module._runtime_registered_model_cost.items()
|
||||
}
|
||||
litellm_utils_module._invalidate_model_cost_lowercase_map()
|
||||
litellm_router_module._live_routers.clear()
|
||||
litellm_utils_module._runtime_registered_model_cost.clear()
|
||||
yield
|
||||
for router in tuple(litellm_router_module._live_routers):
|
||||
litellm_router_module._live_routers.discard(router)
|
||||
for router in original_live_routers:
|
||||
litellm_router_module._live_routers.add(router)
|
||||
litellm_utils_module._runtime_registered_model_cost.clear()
|
||||
litellm_utils_module._runtime_registered_model_cost.update(original_runtime_registered_model_cost)
|
||||
litellm_utils_module._invalidate_model_cost_lowercase_map()
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
|
@ -352,6 +345,7 @@ def isolate_model_catalog() -> Iterator[None]:
|
|||
set_fallback_generalizations(original_rules)
|
||||
model_cost.clear()
|
||||
model_cost.update(model_cost_entries)
|
||||
litellm.model_cost = model_cost
|
||||
models_by_provider.clear()
|
||||
models_by_provider.update(models_by_provider_entries)
|
||||
for models, entries in provider_model_sets:
|
||||
|
|
|
|||
|
|
@ -553,7 +553,9 @@ def test_register_url_uses_explicit_source_under_local_flag(tmp_path: Path, monk
|
|||
|
||||
def test_failed_url_registration_is_a_single_logged_noop(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None:
|
||||
before: Final = copy.deepcopy(litellm.model_cost)
|
||||
registrations: Final = copy.deepcopy(litellm.utils._runtime_registered_model_cost)
|
||||
registrations: Final = {
|
||||
key: copy.deepcopy(dict(row)) for key, row in litellm.utils._runtime_registered_model_cost.items()
|
||||
}
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
|
||||
url: Final = (tmp_path / "missing.json").as_uri()
|
||||
assert litellm.register_model(url) == url
|
||||
|
|
|
|||
|
|
@ -159,7 +159,10 @@ def test_aws_cache_reset_sees_a_replaced_module(monkeypatch: pytest.MonkeyPatch)
|
|||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def registered_collection_models() -> Iterator[None]:
|
||||
original_registrations: Final = dict(litellm_utils_module._runtime_registered_model_cost)
|
||||
from litellm._catalog import _install
|
||||
|
||||
original_layers: Final = _install.LAYERS
|
||||
_install.LAYERS = _install.ModelCostLayers()
|
||||
original_routers: Final = frozenset(litellm_router_module._live_routers)
|
||||
litellm.register_model(
|
||||
{"collection-isolation-ledger": {"litellm_provider": "openai", "input_cost_per_token": 0.001}}
|
||||
|
|
@ -173,8 +176,7 @@ def registered_collection_models() -> Iterator[None]:
|
|||
]
|
||||
)
|
||||
yield
|
||||
litellm_utils_module._runtime_registered_model_cost.clear()
|
||||
litellm_utils_module._runtime_registered_model_cost.update(original_registrations)
|
||||
_install.LAYERS = original_layers
|
||||
litellm_router_module._live_routers.discard(collection_router)
|
||||
litellm_router_module._live_routers.update(original_routers)
|
||||
|
||||
|
|
|
|||
|
|
@ -4815,6 +4815,7 @@ def test_completion_cost_legacy_mantle_route_prices_after_router_registration(lo
|
|||
/v1/chat/completions and /v1/messages alike, bills $0."""
|
||||
from litellm import Router
|
||||
|
||||
sibling: Final = dict(litellm.model_cost["anthropic.claude-sonnet-5"])
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
|
|
@ -4826,7 +4827,10 @@ def test_completion_cost_legacy_mantle_route_prices_after_router_registration(lo
|
|||
}
|
||||
]
|
||||
)
|
||||
assert "bedrock/mantle/anthropic.claude-sonnet-5" not in litellm.model_cost
|
||||
registered: Final = litellm.model_cost["bedrock/mantle/anthropic.claude-sonnet-5"]
|
||||
assert registered["input_cost_per_token"] == sibling["input_cost_per_token"]
|
||||
assert registered["output_cost_per_token"] == sibling["output_cost_per_token"]
|
||||
assert litellm.model_cost["anthropic.claude-sonnet-5"] == sibling
|
||||
|
||||
response = litellm.ModelResponse(
|
||||
id="msg_x",
|
||||
|
|
@ -10118,16 +10122,14 @@ def test_non_bedrock_cost_ignores_bedrock_model_id(_local_model_cost_map: None)
|
|||
@pytest.mark.parametrize("metadata_prefix", ("bedrock/", "bedrock/converse/"))
|
||||
def test_bedrock_converse_profile_preserves_registered_price_override(
|
||||
_local_model_cost_map: None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
rate: float,
|
||||
persist_across_reloads: bool,
|
||||
routed_response: bool,
|
||||
metadata_prefix: str,
|
||||
) -> None:
|
||||
import litellm.utils as model_utils
|
||||
from litellm.cost_calculator import pricing_entry_for_cost_calc
|
||||
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
|
||||
|
||||
monkeypatch.setattr(model_utils, "_runtime_registered_model_cost", {})
|
||||
catalog: Final = dict(litellm.model_cost)
|
||||
model: Final = "openai.gpt-5.6-sol"
|
||||
litellm.register_model(
|
||||
|
|
@ -10191,9 +10193,7 @@ def test_bedrock_converse_profile_preserves_registered_price_override(
|
|||
optional_params={"model_id": f"us.{model}"},
|
||||
) == pytest.approx(10 * updated_rate + 5 * rate)
|
||||
|
||||
monkeypatch.setattr(litellm, "model_cost", catalog)
|
||||
model_utils._invalidate_model_cost_lowercase_map()
|
||||
model_utils.reapply_runtime_model_cost_registrations()
|
||||
adopt_model_cost_map(catalog)
|
||||
profile_prices: Final = litellm.get_model_info(f"us.{model}", custom_llm_provider="bedrock")
|
||||
expected_after_reload: Final = (
|
||||
10 * updated_rate + 5 * rate
|
||||
|
|
@ -10209,11 +10209,8 @@ def test_bedrock_converse_profile_preserves_registered_price_override(
|
|||
|
||||
|
||||
def test_bedrock_converse_profile_ignores_registered_capabilities_without_prices(
|
||||
_local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
|
||||
_local_model_cost_map: None,
|
||||
) -> None:
|
||||
import litellm.utils as model_utils
|
||||
|
||||
monkeypatch.setattr(model_utils, "_runtime_registered_model_cost", {})
|
||||
model: Final = "openai.gpt-6-sol"
|
||||
litellm.register_model({model: {"supports_function_calling": True}})
|
||||
response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15))
|
||||
|
|
@ -10245,8 +10242,8 @@ def test_bedrock_converse_profile_preserves_cache_only_registered_prices(
|
|||
) -> None:
|
||||
import litellm.utils as model_utils
|
||||
from litellm.cost_calculator import pricing_entry_for_cost_calc
|
||||
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
|
||||
|
||||
monkeypatch.setattr(model_utils, "_runtime_registered_model_cost", {})
|
||||
model: Final = "openai.cache-overlay-test"
|
||||
profile: Final = f"{profile_prefix}.{model}"
|
||||
catalog: Final = {
|
||||
|
|
@ -10317,9 +10314,7 @@ def test_bedrock_converse_profile_preserves_cache_only_registered_prices(
|
|||
assert entry[1][cache_field] == rate
|
||||
assert litellm.model_cost[profile] == profile_before
|
||||
|
||||
monkeypatch.setattr(litellm, "model_cost", dict(catalog))
|
||||
model_utils._invalidate_model_cost_lowercase_map()
|
||||
model_utils.reapply_runtime_model_cost_registrations()
|
||||
adopt_model_cost_map(catalog)
|
||||
expected_after_reload: Final = (
|
||||
10 * 0.02 + 3 * 0.04 + 5 * (rate if persist_across_reloads else catalog[profile][cache_field])
|
||||
)
|
||||
|
|
|
|||
|
|
@ -245,39 +245,12 @@ def test_register_model_strips_none_litellm_provider():
|
|||
litellm.model_cost.pop(model_key, None)
|
||||
|
||||
|
||||
def test_register_model_strips_none_litellm_provider_from_get_model_info(monkeypatch):
|
||||
"""Directly exercise the strip in ``register_model``.
|
||||
|
||||
The companion test above hits the ``except Exception`` branch where
|
||||
``existing_model`` is an empty dict, so the ``pop`` is a no-op. This
|
||||
test patches ``get_model_info`` to return the failure mode the strip
|
||||
was added to handle, namely a populated dict whose ``litellm_provider``
|
||||
is ``None``. Without the strip, the merged entry in
|
||||
``litellm.model_cost`` would carry ``litellm_provider: None`` and
|
||||
``_check_provider_match`` would drop custom pricing.
|
||||
|
||||
Regression test for https://github.com/BerriAI/litellm/issues/28336.
|
||||
"""
|
||||
from litellm import utils as litellm_utils
|
||||
def test_register_model_strips_none_provider_from_the_served_row(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.utils import _check_provider_match
|
||||
|
||||
model_key = "test-strip-none-provider-from-get-model-info-28336"
|
||||
litellm.model_cost.pop(model_key, None)
|
||||
|
||||
def _fake_get_model_info(model, *args, **kwargs):
|
||||
assert model == model_key
|
||||
return {
|
||||
"key": model_key,
|
||||
"litellm_provider": None,
|
||||
"mode": "chat",
|
||||
"max_tokens": 4096,
|
||||
}
|
||||
|
||||
# ``register_model`` calls ``get_model_info.cache_clear`` via
|
||||
# ``_invalidate_model_cost_lowercase_map``, so the replacement must
|
||||
# expose a no-op ``cache_clear`` attribute.
|
||||
_fake_get_model_info.cache_clear = lambda: None
|
||||
monkeypatch.setattr(litellm_utils, "get_model_info", _fake_get_model_info)
|
||||
model_key: Final = "test-strip-none-provider-28336"
|
||||
row: Final = {"litellm_provider": None, "mode": "chat", "max_tokens": 4096}
|
||||
monkeypatch.setitem(litellm.model_cost, model_key, row)
|
||||
|
||||
try:
|
||||
litellm.register_model(
|
||||
|
|
@ -289,24 +262,13 @@ def test_register_model_strips_none_litellm_provider_from_get_model_info(monkeyp
|
|||
}
|
||||
)
|
||||
|
||||
registered = litellm.model_cost.get(model_key)
|
||||
registered: Final = litellm.model_cost.get(model_key)
|
||||
assert registered is not None, f"{model_key} should be in model_cost"
|
||||
# The strip must have removed the None-valued provider that
|
||||
# ``get_model_info`` returned. The key may be absent entirely, but
|
||||
# it must never be present with value ``None``.
|
||||
assert "litellm_provider" not in registered or (registered["litellm_provider"] is not None), (
|
||||
"register_model failed to strip litellm_provider=None returned "
|
||||
f"by get_model_info, got {registered.get('litellm_provider')!r}"
|
||||
)
|
||||
# Metadata from the patched ``get_model_info`` must still flow
|
||||
# through, so we know the strip did not nuke the rest of the entry.
|
||||
assert "litellm_provider" not in registered
|
||||
assert registered.get("mode") == "chat"
|
||||
assert registered.get("max_tokens") == 4096
|
||||
# And custom pricing from the registration call must be preserved.
|
||||
assert registered.get("input_cost_per_token") == 0.001
|
||||
assert registered.get("output_cost_per_token") == 0.002
|
||||
# Downstream _check_provider_match must accept any provider for
|
||||
# this entry, mirroring the cost calculator path.
|
||||
assert _check_provider_match(registered, "openai") is True
|
||||
assert _check_provider_match(registered, "anthropic") is True
|
||||
finally:
|
||||
|
|
@ -847,7 +809,7 @@ def test_register_model_keeps_a_catalog_rows_own_prices_when_the_lookup_fails(mo
|
|||
|
||||
|
||||
@pytest.mark.parametrize("route", ["", "converse/", "invoke/"])
|
||||
def test_register_model_leaves_no_shadow_key_when_the_lookup_trips_once(
|
||||
def test_register_model_keeps_caller_pricing_independent_when_the_lookup_trips_once(
|
||||
monkeypatch: pytest.MonkeyPatch, route: str
|
||||
) -> None:
|
||||
catalog_key: Final = "au.anthropic.lit-5853-geo-model"
|
||||
|
|
@ -879,11 +841,13 @@ def test_register_model_leaves_no_shadow_key_when_the_lookup_trips_once(
|
|||
with pytest.MonkeyPatch.context() as lookup_patch:
|
||||
lookup_patch.setattr(litellm.utils, "get_model_info", lookup_tripped_once_by_a_concurrent_registration)
|
||||
litellm.register_model({alias: {"litellm_provider": "bedrock", "mode": "chat"}}, persist_across_reloads=False)
|
||||
assert litellm.model_cost[catalog_key]["input_cost_per_token"] == 3.3e-06
|
||||
litellm.register_model({catalog_key: {"input_cost_per_token": repriced_input_cost}}, persist_across_reloads=False)
|
||||
|
||||
try:
|
||||
assert litellm.get_model_info(alias)["key"] == catalog_key
|
||||
assert litellm.get_model_info(alias)["input_cost_per_token"] == repriced_input_cost
|
||||
assert litellm.get_model_info(alias)["key"] == alias
|
||||
assert litellm.get_model_info(alias)["input_cost_per_token"] == 3.3e-06
|
||||
assert litellm.model_cost[catalog_key]["input_cost_per_token"] == repriced_input_cost
|
||||
finally:
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
|
@ -982,7 +946,13 @@ def test_register_model_prices_a_provider_less_azure_ai_key_without_repricing_th
|
|||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
@pytest.mark.parametrize("persist", [False, True])
|
||||
def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(
|
||||
monkeypatch: pytest.MonkeyPatch, persist: bool
|
||||
) -> None:
|
||||
from litellm._catalog import _register
|
||||
from litellm.rust_bridge._native import RegistrationTarget
|
||||
|
||||
catalog_key: Final = "au.anthropic.lit-5853-reloaded-model"
|
||||
deployment_key: Final = f"bedrock/{catalog_key}"
|
||||
catalog_row: Final = {
|
||||
|
|
@ -1001,30 +971,41 @@ def test_register_model_keeps_the_prices_a_reload_adopts_while_it_registers(monk
|
|||
**builtin_cost_map,
|
||||
catalog_key: {**catalog_row, "input_cost_per_token": reloaded_input_cost},
|
||||
}
|
||||
real_lookup: Final = litellm.utils.get_model_info
|
||||
real_target: Final = _register.registration_target
|
||||
lookups: Final = itertools.count()
|
||||
|
||||
def lookup_racing_a_price_data_reload(model: str, custom_llm_provider: str | None = None) -> object:
|
||||
info: Final = real_lookup(model=model, custom_llm_provider=custom_llm_provider)
|
||||
def target_racing_a_price_data_reload(
|
||||
rows: dict[str, object], key: str, *, provider: str | None = None
|
||||
) -> RegistrationTarget:
|
||||
target: Final = real_target(rows, key, provider=provider)
|
||||
if next(lookups) == 0:
|
||||
adopt_model_cost_map(reloaded_cost_map)
|
||||
return info
|
||||
return target
|
||||
|
||||
with pytest.MonkeyPatch.context() as lookup_patch:
|
||||
lookup_patch.setattr(litellm.utils, "get_model_info", lookup_racing_a_price_data_reload)
|
||||
litellm.register_model({deployment_key: {"litellm_provider": "bedrock"}}, persist_across_reloads=False)
|
||||
lookup_patch.setattr(_register, "registration_target", target_racing_a_price_data_reload)
|
||||
litellm.register_model({deployment_key: {"litellm_provider": "bedrock"}}, persist_across_reloads=persist)
|
||||
|
||||
try:
|
||||
assert litellm.model_cost is live_cost_map
|
||||
assert litellm.model_cost[catalog_key]["input_cost_per_token"] == reloaded_input_cost
|
||||
assert litellm.get_model_info(deployment_key)["input_cost_per_token"] == reloaded_input_cost
|
||||
adopt_model_cost_map(reloaded_cost_map)
|
||||
assert (deployment_key in litellm.model_cost) is persist
|
||||
if persist:
|
||||
assert litellm.get_model_info(deployment_key)["input_cost_per_token"] == reloaded_input_cost
|
||||
finally:
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("persist", [False, True])
|
||||
def test_register_model_keeps_its_price_when_a_reload_adopts_a_map_as_it_writes(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
monkeypatch: pytest.MonkeyPatch, persist: bool,
|
||||
) -> None:
|
||||
from pydantic import JsonValue
|
||||
|
||||
from litellm._catalog import _install
|
||||
|
||||
catalog_key: Final = "au.anthropic.lit-5853-reloaded-as-written-model"
|
||||
deployment_key: Final = f"bedrock/{catalog_key}"
|
||||
catalog_row: Final = {
|
||||
|
|
@ -1042,33 +1023,35 @@ def test_register_model_keeps_its_price_when_a_reload_adopts_a_map_as_it_writes(
|
|||
}
|
||||
|
||||
writes: Final = itertools.count()
|
||||
encode: Final = _install._encoded_row
|
||||
|
||||
class CostMapAReloadReplacesAsItIsWritten(dict[str, dict[str, object]]):
|
||||
def update(self, other: Mapping[str, dict[str, object]], /) -> None: # pyright: ignore[reportIncompatibleMethodOverride] # registration writes pass one mapping
|
||||
if next(writes) == 0:
|
||||
adopt_model_cost_map(reloaded_cost_map)
|
||||
super().update(other)
|
||||
def encode_racing_a_price_data_reload(row: Mapping[str, object]) -> dict[str, JsonValue]:
|
||||
encoded: Final = encode(row)
|
||||
if row.get("output_cost_per_token") == registered_output_cost and next(writes) == 0:
|
||||
adopt_model_cost_map(reloaded_cost_map)
|
||||
return encoded
|
||||
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
adopt_model_cost_map({**builtin_cost_map, catalog_key: dict(catalog_row)})
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"model_cost",
|
||||
CostMapAReloadReplacesAsItIsWritten({**builtin_cost_map, catalog_key: dict(catalog_row)}),
|
||||
)
|
||||
monkeypatch.setattr(_install, "_encoded_row", encode_racing_a_price_data_reload)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
live_cost_map: Final = litellm.model_cost
|
||||
|
||||
litellm.register_model(
|
||||
{deployment_key: {"litellm_provider": "bedrock", "output_cost_per_token": registered_output_cost}},
|
||||
persist_across_reloads=False,
|
||||
persist_across_reloads=persist,
|
||||
)
|
||||
|
||||
try:
|
||||
assert litellm.model_cost is live_cost_map
|
||||
assert litellm.get_model_info(deployment_key)["input_cost_per_token"] == reloaded_input_cost
|
||||
assert litellm.get_model_info(deployment_key)["output_cost_per_token"] == registered_output_cost
|
||||
adopt_model_cost_map(reloaded_cost_map)
|
||||
assert (deployment_key in litellm.model_cost) is persist
|
||||
if persist:
|
||||
assert litellm.get_model_info(deployment_key)["input_cost_per_token"] == reloaded_input_cost
|
||||
assert litellm.get_model_info(deployment_key)["output_cost_per_token"] == registered_output_cost
|
||||
finally:
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
|
@ -1341,6 +1324,62 @@ def test_register_model_keeps_custom_flat_prices_over_inherited_rate_variants(
|
|||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("base_key", "caller", "provider", "served_provider", "keeps_variant"),
|
||||
(
|
||||
("registration-shape-model", "openai/registration-shape-model", "openai", "openai", True),
|
||||
("us.anthropic.shape-model", "bedrock/us.anthropic.shape-model", "bedrock", "bedrock", True),
|
||||
("us.anthropic.shape-model", "bedrock/bedrock/bedrock/us.anthropic.shape-model", "bedrock", None, True),
|
||||
("anthropic.shape-model", "bedrock/ap-northeast-1.anthropic.shape-model", "bedrock", None, False),
|
||||
("registration-shape-model", "azure_ai/registration-shape-model", "openai", None, True),
|
||||
),
|
||||
)
|
||||
def test_registration_keeps_primary_and_fallback_field_shaping_across_reload(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
base_key: str,
|
||||
caller: str,
|
||||
provider: str,
|
||||
served_provider: str | None,
|
||||
keeps_variant: bool,
|
||||
) -> None:
|
||||
row: Final = {
|
||||
"litellm_provider": provider,
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 0.001,
|
||||
"input_cost_per_token_above_200k_tokens": 0.004,
|
||||
"cache_read_input_token_cost": 0.0001,
|
||||
"supports_vision": True,
|
||||
}
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setenv("AZURE_AI_API_BASE", "https://registration.openai.azure.com")
|
||||
adopt_model_cost_map({base_key: row})
|
||||
litellm.register_model({caller: {"input_cost_per_token": 0.002}})
|
||||
expected: Final = {
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 0.002,
|
||||
"cache_read_input_token_cost": row["cache_read_input_token_cost"],
|
||||
"supports_vision": row["supports_vision"],
|
||||
**({"litellm_provider": served_provider} if served_provider is not None else {}),
|
||||
**(
|
||||
{"input_cost_per_token_above_200k_tokens": row["input_cost_per_token_above_200k_tokens"]}
|
||||
if keeps_variant
|
||||
else {}
|
||||
),
|
||||
}
|
||||
assert litellm.model_cost[base_key] == row
|
||||
assert litellm.model_cost[caller] == expected
|
||||
expected_cost: Final = 250_000 * (0.004 if keeps_variant else 0.002)
|
||||
assert litellm.cost_per_token(model=caller, prompt_tokens=250_000, completion_tokens=0) == pytest.approx(
|
||||
(expected_cost, 0)
|
||||
)
|
||||
adopt_model_cost_map({base_key: row})
|
||||
assert litellm.model_cost[base_key] == row
|
||||
assert litellm.model_cost[caller] == expected
|
||||
assert litellm.cost_per_token(model=caller, prompt_tokens=250_000, completion_tokens=0) == pytest.approx(
|
||||
(expected_cost, 0)
|
||||
)
|
||||
|
||||
|
||||
def test_register_model_never_inherits_pricing_across_providers(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
openai_key: Final = "lit-5853-openai-only-model"
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
|
|
|
|||
|
|
@ -44,7 +44,6 @@ from litellm.types.router import (
|
|||
)
|
||||
from litellm.utils import (
|
||||
_invalidate_model_cost_lowercase_map,
|
||||
reapply_runtime_model_cost_registrations,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -52,18 +51,9 @@ _COPILOT_TOKEN_EXPIRES_AT: Final = 4_102_444_800
|
|||
|
||||
|
||||
def _simulate_price_data_reload(fetched_catalog):
|
||||
"""Drive what a price data reload does to this process's litellm state.
|
||||
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
|
||||
|
||||
Mirrors `litellm.proxy.proxy_server._swap_in_model_cost_map`, which is the
|
||||
one place both reload paths adopt a freshly fetched catalog; that wiring is
|
||||
covered in the proxy's own tests, so these exercise the replay itself
|
||||
without dragging the proxy in. The provider model sets that helper also
|
||||
repopulates are left alone, since nothing here reads them and rebuilding
|
||||
them from a two-entry catalog would outlive the test.
|
||||
"""
|
||||
litellm.model_cost = fetched_catalog
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
reapply_runtime_model_cost_registrations()
|
||||
adopt_model_cost_map(fetched_catalog)
|
||||
|
||||
|
||||
def _nested_container_ids(value: object) -> frozenset[int]:
|
||||
|
|
@ -2172,14 +2162,6 @@ def test_price_data_reload_preserves_router_registered_model_info(monkeypatch):
|
|||
max_input_tokens / max_output_tokens from every custom model group and
|
||||
/model_group/info starts reporting nulls.
|
||||
"""
|
||||
from litellm import utils as litellm_utils
|
||||
|
||||
monkeypatch.setattr(
|
||||
litellm_utils,
|
||||
"_runtime_registered_model_cost",
|
||||
dict(litellm_utils._runtime_registered_model_cost),
|
||||
)
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
|
|
@ -2220,14 +2202,6 @@ def test_price_data_reload_preserves_custom_override_of_a_catalog_model(monkeypa
|
|||
the same bug: the reload does not blank the metadata, it reverts the
|
||||
operator's model_info override to the upstream catalog values.
|
||||
"""
|
||||
from litellm import utils as litellm_utils
|
||||
|
||||
monkeypatch.setattr(
|
||||
litellm_utils,
|
||||
"_runtime_registered_model_cost",
|
||||
dict(litellm_utils._runtime_registered_model_cost),
|
||||
)
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
|
|
@ -2272,14 +2246,6 @@ def test_deleted_deployments_are_not_replayed_onto_later_reloads(monkeypatch):
|
|||
key that another live deployment still points at must survive the same
|
||||
deletion.
|
||||
"""
|
||||
from litellm import utils as litellm_utils
|
||||
|
||||
monkeypatch.setattr(
|
||||
litellm_utils,
|
||||
"_runtime_registered_model_cost",
|
||||
dict(litellm_utils._runtime_registered_model_cost),
|
||||
)
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
|
|
@ -2326,14 +2292,6 @@ def test_deleting_a_deployment_leaves_catalog_pricing_for_its_backend_model(monk
|
|||
A backend key is shared with the fetched catalog, so withdrawing the entries
|
||||
a deleted deployment owns must not take real upstream pricing down with it.
|
||||
"""
|
||||
from litellm import utils as litellm_utils
|
||||
|
||||
monkeypatch.setattr(
|
||||
litellm_utils,
|
||||
"_runtime_registered_model_cost",
|
||||
dict(litellm_utils._runtime_registered_model_cost),
|
||||
)
|
||||
|
||||
backend_model = "gemini/gemini-2.5-pro"
|
||||
catalog_entry = litellm.get_model_info(model=backend_model)
|
||||
catalog_input_cost = catalog_entry["input_cost_per_token"]
|
||||
|
|
@ -2371,14 +2329,6 @@ def test_repointing_a_deployment_drops_its_previous_backend_key(monkeypatch):
|
|||
old backend key behind, and a replayed registry would re-assert it onto every
|
||||
later catalog for the life of the process.
|
||||
"""
|
||||
from litellm import utils as litellm_utils
|
||||
|
||||
monkeypatch.setattr(
|
||||
litellm_utils,
|
||||
"_runtime_registered_model_cost",
|
||||
dict(litellm_utils._runtime_registered_model_cost),
|
||||
)
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
|
|
@ -2786,7 +2736,8 @@ def test_a_github_copilot_chat_pin_opts_a_dual_endpoint_model_out_of_the_respons
|
|||
|
||||
@pytest.mark.parametrize("registered_mode, pinned_mode", [("chat", "responses"), ("responses", "chat")])
|
||||
@pytest.mark.parametrize(
|
||||
"model, custom_llm_provider", [("github_copilot/routing-test-model", None), ("routing-test-model", "github_copilot")]
|
||||
"model, custom_llm_provider",
|
||||
[("github_copilot/routing-test-model", None), ("routing-test-model", "github_copilot")],
|
||||
)
|
||||
def test_a_github_copilot_responses_pin_still_opts_the_model_into_the_responses_api(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
@ -3130,14 +3081,6 @@ def test_strategy_router_alias_pricing_never_enters_model_cost(monkeypatch):
|
|||
strip must also survive a price-data reload, which rebuilds entries by
|
||||
walking the live routers.
|
||||
"""
|
||||
from litellm import utils as litellm_utils
|
||||
|
||||
monkeypatch.setattr(
|
||||
litellm_utils,
|
||||
"_runtime_registered_model_cost",
|
||||
dict(litellm_utils._runtime_registered_model_cost),
|
||||
)
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
|
|
@ -3827,12 +3770,7 @@ def test_every_declaring_deployment_is_named(caplog):
|
|||
|
||||
|
||||
def _simulate_price_data_reload_with_provider_sets(monkeypatch, fetched_catalog):
|
||||
"""Like `_simulate_price_data_reload`, plus the provider model-set refresh the proxy's
|
||||
`_swap_in_model_cost_map` does before replaying, so bare names in the new catalog resolve."""
|
||||
monkeypatch.setattr(litellm, "model_cost", fetched_catalog)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
litellm.add_known_models(model_cost_map=fetched_catalog)
|
||||
reapply_runtime_model_cost_registrations()
|
||||
_simulate_price_data_reload(fetched_catalog)
|
||||
|
||||
|
||||
def test_a_config_deployment_dropped_by_a_stale_cost_map_comes_back_on_reload(monkeypatch):
|
||||
|
|
@ -3926,7 +3864,7 @@ def test_price_data_reload_refreshes_the_cached_model_group_and_deployment_info(
|
|||
assert router.cached_deployment_model_info("dep-a", "openai/gpt-4o")["input_cost_per_token"] == old_price
|
||||
|
||||
new_price = old_price * 10
|
||||
fresh_catalog = copy.deepcopy(litellm.model_cost)
|
||||
fresh_catalog = litellm.get_model_cost_map(url="")
|
||||
fresh_catalog["gpt-4o"]["input_cost_per_token"] = new_price
|
||||
_simulate_price_data_reload_with_provider_sets(monkeypatch, fresh_catalog)
|
||||
|
||||
|
|
@ -3934,7 +3872,9 @@ def test_price_data_reload_refreshes_the_cached_model_group_and_deployment_info(
|
|||
assert router.cached_deployment_model_info("dep-a", "openai/gpt-4o")["input_cost_per_token"] == new_price
|
||||
|
||||
|
||||
def test_price_data_reload_keeps_geo_deployments_priced_while_requests_register_pricing(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def test_price_data_reload_keeps_geo_deployments_priced_while_requests_register_pricing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
catalog: Final = litellm.get_model_cost_map(url="")
|
||||
monkeypatch.setattr(litellm, "model_cost", dict(catalog))
|
||||
|
|
@ -3996,9 +3936,13 @@ def test_price_data_reload_keeps_geo_deployments_priced_while_requests_register_
|
|||
registrations_landed: Final = writer_outcome.get(timeout=5)
|
||||
try:
|
||||
assert isinstance(registrations_landed, int) and registrations_landed > 0, registrations_landed
|
||||
assert tuple(backend for backend in backends if backend in litellm.model_cost) == ()
|
||||
assert tuple(backend for backend in backends if backend in litellm.model_cost) == backends
|
||||
for backend in backends:
|
||||
assert litellm.get_model_info(backend)["input_cost_per_token"] == canonical_prices[backend]
|
||||
assert (
|
||||
litellm.model_cost[backend.removeprefix("bedrock/")]["input_cost_per_token"]
|
||||
== canonical_prices[backend]
|
||||
)
|
||||
assert tuple(router.get_model_names()) == tuple(backend.rsplit(".", 1)[-1] for backend in backends)
|
||||
finally:
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
|
|
|||
|
|
@ -2258,6 +2258,166 @@ class TestProxyFunctionCalling:
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("key", "base_key", "provider", "mode"),
|
||||
(
|
||||
("databricks/databricks-claude-opus-4-99", "databricks/databricks-claude-opus-4", "databricks", "chat"),
|
||||
("openai/o3-pro", "o3-pro", "openai", "responses"),
|
||||
("bedrock/anthropic.claude-sonnet-4-6-20260101", "bedrock/anthropic.claude-sonnet-4-6", "bedrock", "chat"),
|
||||
),
|
||||
)
|
||||
def test_register_model_preserves_caller_key_and_inherits_without_repricing_sibling(
|
||||
key: str, base_key: str, provider: str, mode: str
|
||||
) -> None:
|
||||
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
|
||||
from litellm.utils import model_cost_registration_row
|
||||
|
||||
base_row: Final = {
|
||||
"litellm_provider": provider,
|
||||
"mode": mode,
|
||||
"max_input_tokens": 8000,
|
||||
"input_cost_per_token": 0.001,
|
||||
"supports_vision": True,
|
||||
}
|
||||
adopt_model_cost_map({base_key: base_row})
|
||||
live: Final = litellm.model_cost
|
||||
litellm.register_model(
|
||||
{key: {"max_input_tokens": 32000, "input_cost_per_token": 0.005, "litellm_provider": provider}}
|
||||
)
|
||||
|
||||
assert litellm.model_cost is live
|
||||
assert litellm.model_cost[base_key] == base_row
|
||||
assert litellm.model_cost[key] == {**base_row, "max_input_tokens": 32000, "input_cost_per_token": 0.005}
|
||||
assert model_cost_registration_row(key, provider)["mode"] == mode
|
||||
|
||||
refreshed_row: Final = {**base_row, "supports_vision": False, "input_cost_per_token": 0.002}
|
||||
adopt_model_cost_map({base_key: refreshed_row})
|
||||
|
||||
assert litellm.model_cost is live
|
||||
assert litellm.model_cost[base_key] == refreshed_row
|
||||
assert litellm.model_cost[key] == {**refreshed_row, "max_input_tokens": 32000, "input_cost_per_token": 0.005}
|
||||
|
||||
|
||||
def test_register_model_keeps_case_variant_fields_independent_after_reload() -> None:
|
||||
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
|
||||
|
||||
upper: Final = "hosted_vllm/Acme-Model"
|
||||
lower: Final = "hosted_vllm/acme-model"
|
||||
rows: Final = {
|
||||
upper: {"max_input_tokens": 8000, "supports_vision": True, "litellm_provider": "hosted_vllm"},
|
||||
lower: {"max_input_tokens": 32000, "litellm_provider": "hosted_vllm"},
|
||||
}
|
||||
base: Final = {"baseline-model": {"mode": "chat"}}
|
||||
adopt_model_cost_map(base)
|
||||
for key, row in rows.items():
|
||||
litellm.register_model({key: row})
|
||||
|
||||
assert dict(litellm.model_cost) == {**base, **rows}
|
||||
adopt_model_cost_map(base)
|
||||
assert dict(litellm.model_cost) == {**base, **rows}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("registration_last", (True, False))
|
||||
def test_register_model_and_direct_write_keep_last_writer_after_reload(registration_last: bool) -> None:
|
||||
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
|
||||
|
||||
key: Final = "gpt-4o"
|
||||
row: Final = {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.001}
|
||||
adopt_model_cost_map({key: row})
|
||||
litellm.model_cost[key]["output_cost_per_token"] = 0.8
|
||||
if registration_last:
|
||||
litellm.model_cost[key]["input_cost_per_token"] = 0.7
|
||||
litellm.register_model({key: {"input_cost_per_token": 0.5}})
|
||||
else:
|
||||
litellm.register_model({key: {"input_cost_per_token": 0.5}})
|
||||
litellm.model_cost[key]["input_cost_per_token"] = 0.7
|
||||
expected: Final = 0.5 if registration_last else 0.7
|
||||
assert litellm.model_cost[key]["input_cost_per_token"] == expected
|
||||
adopt_model_cost_map({key: {**row, "input_cost_per_token": 0.003}})
|
||||
assert litellm.model_cost[key]["input_cost_per_token"] == expected
|
||||
assert litellm.model_cost[key]["output_cost_per_token"] == 0.8
|
||||
|
||||
|
||||
def test_registration_provider_scope_replaces_an_exact_cross_provider_collision() -> None:
|
||||
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
|
||||
|
||||
key: Final = "opaque-deployment-id"
|
||||
catalog: Final = {
|
||||
key: {"litellm_provider": "bedrock", "input_cost_per_token": 0.1, "output_cost_per_token": 0.2}
|
||||
}
|
||||
adopt_model_cost_map(catalog)
|
||||
expected: Final = {"mode": "chat", "input_cost_per_token": 0.5}
|
||||
litellm.register_model({key: expected}, custom_llm_provider="openai", add_to_provider_model_sets=False)
|
||||
assert litellm.model_cost[key] == expected
|
||||
adopt_model_cost_map(catalog)
|
||||
assert litellm.model_cost[key] == expected
|
||||
|
||||
|
||||
def test_register_model_nonpersistent_write_is_not_captured_as_a_direct_write() -> None:
|
||||
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
|
||||
|
||||
row: Final = {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.001}
|
||||
adopt_model_cost_map({"gpt-4o": row})
|
||||
litellm.register_model({"openai/gpt-4o": {"input_cost_per_token": 0.5}}, persist_across_reloads=False)
|
||||
litellm.register_model({"opaque-request-id": {"max_tokens": 1234}}, persist_across_reloads=False)
|
||||
assert litellm.model_cost["openai/gpt-4o"]["input_cost_per_token"] == 0.5
|
||||
assert litellm.model_cost["opaque-request-id"]["max_tokens"] == 1234
|
||||
|
||||
adopt_model_cost_map({"gpt-4o": row})
|
||||
|
||||
assert dict(litellm.model_cost) == {"gpt-4o": row}
|
||||
|
||||
|
||||
def test_runtime_registration_view_is_sparse_detached_read_only_and_persistent() -> None:
|
||||
from uuid import UUID
|
||||
|
||||
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
|
||||
from litellm.utils import _runtime_registered_model_cost
|
||||
|
||||
key: Final = "openai/registration-view"
|
||||
row: Final = {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 1.0}
|
||||
metadata: Final = UUID(int=17)
|
||||
adopt_model_cost_map({key: row})
|
||||
litellm.register_model({key: {"input_cost_per_token": 0, "supports_vision": False, "metadata": [metadata]}})
|
||||
snapshot: Final = dict(_runtime_registered_model_cost.items())
|
||||
expected: Final = {"input_cost_per_token": 0, "supports_vision": False, "metadata": [metadata]}
|
||||
assert snapshot == {key: expected}
|
||||
with pytest.raises(TypeError):
|
||||
_runtime_registered_model_cost[key] = {}
|
||||
with pytest.raises(TypeError):
|
||||
snapshot[key]["input_cost_per_token"] = 7
|
||||
snapshot[key]["metadata"].append(UUID(int=18))
|
||||
assert dict(_runtime_registered_model_cost.items()) == {key: expected}
|
||||
litellm.register_model({key: {"output_cost_per_token": 2}})
|
||||
litellm.model_cost["direct-only"] = {"mode": "chat"}
|
||||
litellm.register_model({"ephemeral-only": {"mode": "chat"}}, persist_across_reloads=False)
|
||||
assert dict(_runtime_registered_model_cost.items()) == {key: {**expected, "output_cost_per_token": 2}}
|
||||
assert "output_cost_per_token" not in snapshot[key]
|
||||
adopt_model_cost_map({key: {**row, "input_cost_per_token": 9}})
|
||||
assert litellm.model_cost[key] == {**row, **expected, "output_cost_per_token": 2}
|
||||
assert litellm.model_cost["direct-only"] == {"mode": "chat"}
|
||||
assert "ephemeral-only" not in litellm.model_cost
|
||||
|
||||
|
||||
@pytest.mark.parametrize("register_after_delete", [False, True])
|
||||
def test_registration_tombstone_order_survives_repeated_reload(register_after_delete: bool) -> None:
|
||||
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
|
||||
|
||||
key: Final = "openai/deleted-registration"
|
||||
row: Final = {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 1.0}
|
||||
adopt_model_cost_map({key: row})
|
||||
litellm.register_model({key: {"input_cost_per_token": 2.0}})
|
||||
del litellm.model_cost[key]
|
||||
if register_after_delete:
|
||||
litellm.register_model({key: {"output_cost_per_token": 3.0}})
|
||||
expected: Final = {key: {"output_cost_per_token": 3.0}} if register_after_delete else {}
|
||||
assert dict(litellm.model_cost) == expected
|
||||
adopt_model_cost_map({key: row})
|
||||
assert dict(litellm.model_cost) == expected
|
||||
adopt_model_cost_map({key: row})
|
||||
assert dict(litellm.model_cost) == expected
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map")
|
||||
def test_register_model_price_is_used_for_completion_cost() -> None:
|
||||
model: Final = "migration-custom-pricing-model"
|
||||
|
|
@ -2328,8 +2488,10 @@ def test_completion_registers_test_owned_model_prices() -> None:
|
|||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages}
|
||||
assert litellm.model_cost[model]["input_cost_per_token"] == 0.001
|
||||
assert litellm.model_cost[model]["output_cost_per_token"] == 0.002
|
||||
assert litellm.model_cost[f"openai/{model}"]["input_cost_per_token"] == 0.001
|
||||
assert litellm.model_cost[f"openai/{model}"]["output_cost_per_token"] == 0.002
|
||||
assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001
|
||||
assert litellm.model_cost[model]["output_cost_per_token"] == 0.0002
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(0.018)
|
||||
|
||||
|
||||
|
|
@ -2410,23 +2572,30 @@ def test_register_model_with_scientific_notation():
|
|||
|
||||
|
||||
@pytest.mark.usefixtures("isolated_openai_model_sets")
|
||||
@respx.mock
|
||||
def test_register_model_url_fetch_uses_single_attempt(monkeypatch):
|
||||
def test_register_model_url_fetch_uses_single_attempt(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from unittest.mock import Mock
|
||||
|
||||
from litellm._catalog import _fetch
|
||||
from litellm.rust_bridge._native import RegistryUnavailable
|
||||
|
||||
monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False)
|
||||
monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost))
|
||||
before = dict(litellm.model_cost)
|
||||
threads_before = {thread.name for thread in threading.enumerate()}
|
||||
route = respx.get("https://example.invalid/custom_pricing.json").mock(return_value=httpx.Response(503))
|
||||
before: Final = dict(litellm.model_cost)
|
||||
threads_before: Final = {thread.name for thread in threading.enumerate()}
|
||||
fetch_once: Final = Mock(return_value=Mock(spec=RegistryUnavailable, reason="HTTP 503", retryable=True))
|
||||
monkeypatch.setattr(_fetch, "fetch_once", fetch_once)
|
||||
|
||||
litellm.register_model(model_cost="https://example.invalid/custom_pricing.json")
|
||||
|
||||
threads_after = {thread.name for thread in threading.enumerate()}
|
||||
assert route.call_count == 1
|
||||
threads_after: Final = {thread.name for thread in threading.enumerate()}
|
||||
fetch_once.assert_called_once_with(
|
||||
"https://example.invalid/custom_pricing.json", if_none_match=None, timeout_seconds=5.0
|
||||
)
|
||||
assert not (threads_after - threads_before) & {"litellm-model-cost-map-retry"}
|
||||
assert not any(
|
||||
thread.name == "litellm-model-cost-map-retry" and thread.is_alive() for thread in threading.enumerate()
|
||||
)
|
||||
assert litellm.model_cost.keys() >= before.keys()
|
||||
assert litellm.model_cost == before
|
||||
|
||||
|
||||
def test_register_model_openrouter_without_slash():
|
||||
|
|
@ -4929,7 +5098,7 @@ async def test_builtin_string_callback_registers_when_subclass_already_active(
|
|||
@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map")
|
||||
@pytest.mark.parametrize("rate", (0.0, 0.000123))
|
||||
@pytest.mark.parametrize("updated_rate", (None, 0.0, 0.000456))
|
||||
def test_reapply_runtime_registrations_replays_register_model_overrides(
|
||||
def test_install_preserves_register_model_overrides(
|
||||
monkeypatch: pytest.MonkeyPatch, rate: float, updated_rate: float | None
|
||||
) -> None:
|
||||
"""
|
||||
|
|
@ -4938,22 +5107,8 @@ def test_reapply_runtime_registrations_replays_register_model_overrides(
|
|||
so without replaying those registrations the override is silently lost and
|
||||
the model reverts to upstream pricing.
|
||||
"""
|
||||
from litellm import utils as litellm_utils
|
||||
from litellm.utils import (
|
||||
_invalidate_model_cost_lowercase_map,
|
||||
reapply_runtime_model_cost_registrations,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
litellm_utils,
|
||||
"_runtime_registered_model_cost",
|
||||
dict(litellm_utils._runtime_registered_model_cost),
|
||||
)
|
||||
# Only the recorded half is under test here; the live-router rebuild is covered
|
||||
# in test_router_model_cost_isolation.py. Routers built by earlier tests in this
|
||||
# process stay in the weak set until they are collected, so leaving the callback
|
||||
# installed would make this depend on when that happens.
|
||||
monkeypatch.setattr(litellm_utils._LiveDeploymentReplay, "callback", None)
|
||||
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
|
||||
saved_model_cost: Final = litellm.model_cost
|
||||
monkeypatch.setattr(litellm, "model_cost", dict(saved_model_cost))
|
||||
|
|
@ -4974,16 +5129,16 @@ def test_reapply_runtime_registrations_replays_register_model_overrides(
|
|||
)
|
||||
litellm.register_model({"openai/gpt-4o": {"supports_function_calling": False}})
|
||||
|
||||
litellm.model_cost = {
|
||||
"openai/gpt-4o": {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 0.000999,
|
||||
"max_input_tokens": 4242,
|
||||
adopt_model_cost_map(
|
||||
{
|
||||
"openai/gpt-4o": {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 0.000999,
|
||||
"max_input_tokens": 4242,
|
||||
}
|
||||
}
|
||||
}
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
reapply_runtime_model_cost_registrations()
|
||||
)
|
||||
|
||||
expected_rate: Final = rate if updated_rate is None else updated_rate
|
||||
assert litellm.model_cost["openai/gpt-4o"]["input_cost_per_token"] == expected_rate
|
||||
|
|
@ -4995,24 +5150,15 @@ def test_reapply_runtime_registrations_replays_register_model_overrides(
|
|||
|
||||
|
||||
@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map")
|
||||
def test_reapply_runtime_registrations_drops_request_scoped_registrations(monkeypatch):
|
||||
def test_install_drops_request_scoped_registrations(monkeypatch):
|
||||
"""
|
||||
Per-request custom pricing describes one call, so it must not be re-asserted
|
||||
over every future catalog. Replaying it would let a one-off price outlive
|
||||
the catalog generation it was applied to and silently beat fresh upstream
|
||||
pricing forever, while a durable override registered alongside it survives.
|
||||
"""
|
||||
from litellm import utils as litellm_utils
|
||||
from litellm.utils import (
|
||||
_invalidate_model_cost_lowercase_map,
|
||||
reapply_runtime_model_cost_registrations,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
litellm_utils,
|
||||
"_runtime_registered_model_cost",
|
||||
dict(litellm_utils._runtime_registered_model_cost),
|
||||
)
|
||||
from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
|
||||
saved_model_cost = litellm.model_cost
|
||||
try:
|
||||
|
|
@ -5025,12 +5171,12 @@ def test_reapply_runtime_registrations_drops_request_scoped_registrations(monkey
|
|||
persist_across_reloads=False,
|
||||
)
|
||||
|
||||
litellm.model_cost = {
|
||||
"openai/gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 0.000999},
|
||||
"openai/gpt-4o-mini": {"litellm_provider": "openai", "input_cost_per_token": 0.000888},
|
||||
}
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
reapply_runtime_model_cost_registrations()
|
||||
adopt_model_cost_map(
|
||||
{
|
||||
"openai/gpt-4o": {"litellm_provider": "openai", "input_cost_per_token": 0.000999},
|
||||
"openai/gpt-4o-mini": {"litellm_provider": "openai", "input_cost_per_token": 0.000888},
|
||||
}
|
||||
)
|
||||
|
||||
assert litellm.model_cost["openai/gpt-4o"]["input_cost_per_token"] == 0.000111
|
||||
assert litellm.model_cost["openai/gpt-4o-mini"]["input_cost_per_token"] == 0.000888
|
||||
|
|
@ -7377,9 +7523,7 @@ async def test_router_resolves_an_oidc_api_key_only_on_a_proxy_level_deployment(
|
|||
try:
|
||||
with respx.mock(assert_all_called=True) as upstream:
|
||||
upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY)
|
||||
await router.acompletion(
|
||||
model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"}
|
||||
)
|
||||
await router.acompletion(model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"})
|
||||
sent: Final = upstream.calls.last.request.headers["Authorization"]
|
||||
finally:
|
||||
router.discard()
|
||||
|
|
@ -7447,7 +7591,9 @@ async def test_an_inline_oidc_api_key_reaches_the_endpoint_its_deployment_config
|
|||
with respx.mock(assert_all_called=False) as upstream:
|
||||
upstream.route().respond(200, json=_BEDROCK_REPLY)
|
||||
await router.acompletion(
|
||||
model="claude", messages=_PING, aws_bedrock_runtime_endpoint="https://BEDROCK-runtime.us-east-1.amazonaws.com"
|
||||
model="claude",
|
||||
messages=_PING,
|
||||
aws_bedrock_runtime_endpoint="https://BEDROCK-runtime.us-east-1.amazonaws.com",
|
||||
)
|
||||
sent: Final = tuple(
|
||||
(call.request.url.host, call.request.headers["Authorization"]) for call in upstream.calls
|
||||
|
|
@ -7656,7 +7802,9 @@ async def test_a_team_row_naming_an_oidc_credential_reaches_only_the_credentials
|
|||
(
|
||||
lambda reference: LiteLLM_Params(
|
||||
**litellm.Router(
|
||||
model_list=[{"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": reference}}]
|
||||
model_list=[
|
||||
{"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": reference}}
|
||||
]
|
||||
).get_model_list(model_name="openai/*")[0]["litellm_params"]
|
||||
),
|
||||
lambda reference: "Bearer listing-token",
|
||||
|
|
@ -8137,10 +8285,9 @@ def _vcr_outcome_gate(request, vcr):
|
|||
|
||||
@pytest.fixture(scope="function")
|
||||
def setup_and_teardown():
|
||||
"""
|
||||
This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained.
|
||||
"""
|
||||
importlib.reload(litellm)
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap, adopt_model_cost_map
|
||||
|
||||
adopt_model_cost_map(GetModelCostMap.load_local_model_cost_map())
|
||||
loop = asyncio.get_event_loop_policy().new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
print(litellm)
|
||||
|
|
@ -8308,15 +8455,12 @@ _SCALAR_DEFAULTS = {
|
|||
|
||||
@pytest.fixture(scope="module")
|
||||
def setup_and_teardown_local_testing():
|
||||
"""
|
||||
Module-scoped setup. Reloads litellm only in single-process mode
|
||||
(skipped under xdist to avoid cross-worker interference).
|
||||
"""
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap, adopt_model_cost_map
|
||||
|
||||
worker_id = os.environ.get("PYTEST_XDIST_WORKER", None)
|
||||
if worker_id is None:
|
||||
importlib.reload(litellm)
|
||||
adopt_model_cost_map(GetModelCostMap.load_local_model_cost_map())
|
||||
try:
|
||||
if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"):
|
||||
import litellm.proxy.proxy_server
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue