diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 91a22144805..5471fe50d5f 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -17,14 +17,16 @@ import random import sys import threading import time -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass, replace from datetime import datetime, timezone from importlib.resources import files from pathlib import Path +from types import MappingProxyType from typing import Final, Protocol import httpx +from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict from litellm import verbose_logger @@ -37,6 +39,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( ) FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations" +_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) _CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"}) @@ -88,6 +91,18 @@ class GetModelCostMap: """Load the local backup model cost map bundled with the package.""" return GetModelCostMap.load_local_model_cost_map_with_revision().model_cost_map + _loaded_catalog: Mapping[str, Mapping[str, object]] = MappingProxyType({}) + + @classmethod + def loaded_model_cost_map(cls) -> Mapping[str, Mapping[str, object]]: + """The catalog as last loaded (bundled or remote), untouched by ``register_model`` or router registrations.""" + return cls._loaded_catalog + + @classmethod + def _snapshot_loaded_catalog(cls, model_cost: Mapping[str, object]) -> None: + raw: Final = _CATALOG_ADAPTER.validate_python(model_cost) + cls._loaded_catalog = MappingProxyType({key: MappingProxyType(entry) for key, entry in raw.items()}) + @classmethod def _get_backup_model_count(cls) -> int: """Return the number of models in the local backup (cached int).""" @@ -533,7 +548,9 @@ def _finalize_model_cost_map(model_cost: dict) -> dict: def _finalize_loaded_model_cost_map(loaded: ModelCostMapReloaded) -> ModelCostMapReloaded: _cost_map_source_info.source_revision = loaded.revision _cost_map_source_info.etag = loaded.etag - return replace(loaded, model_cost_map=_finalize_model_cost_map(loaded.model_cost_map)) + finalized: Final = _finalize_model_cost_map(loaded.model_cost_map) + GetModelCostMap._snapshot_loaded_catalog(finalized) # pyright: ignore[reportPrivateUsage] # same module + return replace(loaded, model_cost_map=finalized) def adopt_model_cost_map( diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 10a0a2f3104..fcadcfe2cae 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -14,12 +14,12 @@ import asyncio import datetime import json from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence -from contextlib import AbstractAsyncContextManager, asynccontextmanager +from contextlib import AbstractAsyncContextManager, asynccontextmanager, suppress from dataclasses import dataclass from fnmatch import fnmatchcase from json import JSONDecodeError from types import MappingProxyType -from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeVar, cast, runtime_checkable +from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable from fastapi import APIRouter, Depends, Header, HTTPException, Request, status from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError, field_validator @@ -29,6 +29,7 @@ from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.constants import LITELLM_PROXY_ADMIN_NAME from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.ptu_pricing import ( CUSTOM_PRICING_FIELDS, PTU_EMPTIED_PRICING_FIELDS, @@ -139,7 +140,12 @@ from litellm.types.router import ( updateDeployment, updateLiteLLMParams, ) -from litellm.types.utils import echoed_cost_map_pricing_fields, without_server_derived_pricing +from litellm.types.utils import ( + COST_MAP_LOOKUP_KEY, + echoed_cost_map_fields, + echoed_cost_map_pricing_fields, + without_server_derived_pricing, +) from litellm.utils import get_utc_datetime if TYPE_CHECKING: @@ -928,7 +934,33 @@ def _ptu_priced_deployment(model_params: Deployment) -> Deployment: ) -def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> PrismaCompatibleUpdateDBModel: +def _cost_map_entry(db_model: Deployment, incoming_model_info: Mapping[str, object]) -> Mapping[str, object]: + base_model: Final = incoming_model_info.get("base_model") + lookup: Final = base_model if isinstance(base_model, str) else _decrypted_model(db_model.litellm_params.model) + if lookup is None: + return MappingProxyType({}) + with suppress(Exception): + return MappingProxyType(dict(litellm.get_model_info(model=lookup))) + return MappingProxyType({}) + + +LoadedCatalog: TypeAlias = Callable[[], Mapping[str, Mapping[str, object]]] # mutable-ok: Callable parameter syntax + + +def _loaded_catalog_entry( + incoming_model_info: Mapping[str, object], loaded_catalog: LoadedCatalog +) -> Mapping[str, object]: + catalog_key: Final = incoming_model_info.get(COST_MAP_LOOKUP_KEY) + if not isinstance(catalog_key, str): + return MappingProxyType({}) + return loaded_catalog().get(catalog_key, MappingProxyType({})) + + +def update_db_model( + db_model: Deployment, + updated_patch: updateDeployment, + loaded_catalog: LoadedCatalog = GetModelCostMap.loaded_model_cost_map, +) -> PrismaCompatibleUpdateDBModel: if updated_patch.model_info is not None: _raise_if_ptu_cost_attribution_disabled(updated_patch.model_info.model_dump(exclude_none=True)) merged_model_name: Final = updated_patch.model_name or db_model.model_name @@ -955,7 +987,24 @@ def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> Pr # update model info if updated_patch.model_info: - merged_model_info.update(without_server_derived_pricing(updated_patch.model_info.model_dump(exclude_none=True))) + incoming_model_info: Final = updated_patch.model_info.model_dump(exclude_none=True) + echoed_fields: Final = echoed_cost_map_fields( + incoming_model_info, + _cost_map_entry(db_model, incoming_model_info), + _loaded_catalog_entry(incoming_model_info, loaded_catalog), + ) + merged_model_info.update( + MappingProxyType( + dict( + (k, v) + for k, v in without_server_derived_pricing(incoming_model_info).items() + if k not in echoed_fields + ) + ) + ) + for k in echoed_fields: + if k in merged_model_info and merged_model_info[k] != incoming_model_info[k]: + del merged_model_info[k] # Honor explicit-null clears LAST, after both merges, so a model_info blob a client # passes through cannot silently undo a litellm_params clear via .update(). diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3cb2193661e..6cc2637357d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3842,6 +3842,24 @@ def echoed_cost_map_pricing_fields(model_info: Mapping[str, Any]) -> tuple[str, return tuple(sorted(k for k in model_info if is_server_derived_pricing_key(k))) +def echoed_cost_map_fields( + model_info: Mapping[str, object], *cost_map_entries: Mapping[str, object] +) -> tuple[str, ...]: + """Fields a ``/model/info`` echo copied from the cost map unchanged. + + Only ``litellm.get_model_info`` emits ``key``, so a blob carrying it is an echo of that + response. Anything in it that still equals a resolved cost-map entry is a display value + nobody typed; a value the operator edited differs from every entry and stays a real override. + Callers pass both the live entry, which the router rewrites with each deployment's own + overrides, and the catalog entry as loaded, so a reset to the catalog value reads as an echo either way. + """ + if COST_MAP_LOOKUP_KEY not in model_info: + return () + return tuple( + sorted(k for k, v in model_info.items() if any(k in entry and entry[k] == v for entry in cost_map_entries)) + ) + + def pricing_override_fields(*sources: Mapping[str, Any]) -> tuple[str, ...]: return tuple( sorted( diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index 53fee36b3a8..262dabb7c1b 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -448,6 +448,23 @@ async def test_refetch_records_the_blob_id_of_the_bytes_served_and_the_fetch_eta assert get_model_cost_map_provenance() == {"source_revision": git_blob_id(body), "etag": 'W/"abc123"'} +@pytest.mark.asyncio +async def test_loaded_catalog_snapshot_follows_the_fetched_map_and_ignores_later_registrations(monkeypatch): + import litellm + + edited = json.loads(_real_map_bytes()) + edited["gpt-5.4-mini"]["max_input_tokens"] = 777 + client, _ = _mock_client([httpx.Response(200, content=json.dumps(edited).encode())]) + + result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) + + assert isinstance(result, ModelCostMapReloaded) + monkeypatch.setattr(litellm, "model_cost", result.model_cost_map) + litellm.register_model({"gpt-5.4-mini": {"max_input_tokens": 2048}}, persist_across_reloads=False) + assert litellm.model_cost["gpt-5.4-mini"]["max_input_tokens"] == 2048 + assert GetModelCostMap.loaded_model_cost_map()["gpt-5.4-mini"]["max_input_tokens"] == 777 + + @pytest.mark.asyncio async def test_refetch_revision_follows_the_bytes_not_the_url(): edited = json.loads(_real_map_bytes()) diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 5e6c37c41dd..bd252169131 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -4063,6 +4063,294 @@ class TestModelInfoServerDerivedPricingFilter: assert written["access_groups"] == ["prod"] +class TestModelInfoCostMapEchoFilter: + """LIT-5534. ``/model/info`` fills a deployment's ``model_info`` from the cost map (context + limits, mode, provider, supported params, capability flags), and the Admin UI edit form sends + that whole blob back on any save. Only values that still equal the cost-map entry are the + echo; a value the operator changed is a real override and stays.""" + + def test_echoed_cost_map_metadata_is_not_persisted(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + entry = litellm.get_model_info("openai/gpt-5.6") + echo = {**entry, "id": "dep-echo-0", "db_model": True, "access_groups": ["prod"]} + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-0"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert info["access_groups"] == ["prod"] + assert set(info).isdisjoint(entry) + assert "max_input_tokens" not in info and "mode" not in info and "supports_vision" not in info, ( + "cost-map metadata must not be persisted from an unchanged /model/info echo" + ) + + def test_an_edited_value_survives_the_echo_filter(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + entry = litellm.get_model_info("openai/gpt-5.6") + echo = { + **entry, + "id": "dep-echo-1", + "db_model": True, + "access_groups": ["prod"], + "max_input_tokens": entry["max_input_tokens"] + 1, + "mode": "completion" if entry["mode"] != "completion" else "chat", + } + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-1"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert info["max_input_tokens"] == echo["max_input_tokens"] + assert info["mode"] == echo["mode"] + assert "litellm_provider" not in info + assert "supported_openai_params" not in info + + def test_metadata_without_a_cost_map_key_is_persisted(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + from litellm.types.utils import echoed_cost_map_fields + + entry = litellm.get_model_info("openai/gpt-5.6") + assert echoed_cost_map_fields({"max_input_tokens": entry["max_input_tokens"]}, entry) == () + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-2"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment( + model_info=ModelInfo( + id="dep-echo-2", + max_input_tokens=entry["max_input_tokens"], + mode=entry["mode"], + ) + ), + ) + + info = json.loads(result["model_info"]) + assert info["max_input_tokens"] == entry["max_input_tokens"] + assert info["mode"] == entry["mode"] + + def test_a_stored_mode_survives_an_echoed_save(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + entry = litellm.get_model_info("openai/gpt-5.6") + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-3", mode=entry["mode"]), + ) + echo = {**entry, "id": "dep-echo-3", "db_model": True, "access_groups": ["prod"]} + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert info["mode"] == entry["mode"] + assert "max_input_tokens" not in info + + def test_resetting_an_override_to_the_cost_map_value_removes_it(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + entry = litellm.get_model_info("openai/gpt-5.6") + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-4", mode="chat", max_input_tokens=2048), + ) + echo = {**entry, "id": "dep-echo-4", "db_model": True, "access_groups": ["staging"]} + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert "max_input_tokens" not in info + assert info["mode"] == "chat" + assert info["access_groups"] == ["staging"] + + def test_reset_is_recognised_after_the_router_registered_the_override(self, monkeypatch: pytest.MonkeyPatch): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + pristine = litellm.get_model_info("openai/gpt-5.6") + polluted = {**pristine, "max_input_tokens": 2048} + monkeypatch.setattr(litellm, "get_model_info", lambda model, **_: polluted) + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-8", max_input_tokens=2048), + ) + echo = {**pristine, "id": "dep-echo-8", "db_model": True} + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert "max_input_tokens" not in info, info + + def test_reset_to_a_remote_catalog_value_that_differs_from_the_bundled_one(self, monkeypatch: pytest.MonkeyPatch): + from types import MappingProxyType + + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + bundled = litellm.get_model_info("openai/gpt-5.6") + remote = {**bundled, "max_input_tokens": bundled["max_input_tokens"] + 1} + remote_catalog = MappingProxyType({remote["key"]: MappingProxyType(remote)}) + monkeypatch.setattr(litellm, "get_model_info", lambda model, **_: {**remote, "max_input_tokens": 2048}) + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model="openai/gpt-5.6"), + model_info=ModelInfo(id="dep-echo-9", max_input_tokens=2048), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**{**remote, "id": "dep-echo-9", "db_model": True})), + loaded_catalog=lambda: remote_catalog, + ) + + info = json.loads(result["model_info"]) + assert "max_input_tokens" not in info, info + + def test_echo_is_compared_against_the_deployments_lookup_not_the_key(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + lookup_pairs: Final = ( + ("openai/gpt-5.6", "gpt-5.6"), + ("openai/gpt-4.1-mini", "gpt-4.1-mini"), + ) + lookup_data: Final = tuple( + (deployment_model, deployment_entry, differing_fields) + for deployment_model, key_model in lookup_pairs + for deployment_entry in (litellm.get_model_info(deployment_model),) + for key_entry in (litellm.get_model_info(key_model),) + for differing_fields in ( + frozenset( + k for k in deployment_entry if k in key_entry and deployment_entry[k] != key_entry[k] + ), + ) + if differing_fields + ) + if not lookup_data: + pytest.skip("No deployment/key cost-map lookup differences are available") + + deployment_model, entry, differing_fields = lookup_data[0] + assert differing_fields + db_model = Deployment( + model_name=deployment_model, + litellm_params=LiteLLM_Params(model=deployment_model), + model_info=ModelInfo(id="dep-echo-5"), + ) + echo = {**entry, "id": "dep-echo-5", "db_model": True, "access_groups": ["prod"]} + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert not frozenset(info).intersection(frozenset(entry) - frozenset(("mode",))) + + def test_base_model_wins_over_litellm_params_model_for_the_lookup(self): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + entry = litellm.get_model_info("azure/gpt-5.6") + db_model = Deployment( + model_name="azure/my-deploy", + litellm_params=LiteLLM_Params(model="azure/my-deploy"), + model_info=ModelInfo(id="dep-echo-6", base_model="azure/gpt-5.6"), + ) + echo = { + **entry, + "id": "dep-echo-6", + "base_model": "azure/gpt-5.6", + "db_model": True, + } + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert not frozenset(info).intersection(frozenset(entry) - frozenset(("mode",))) + assert info["base_model"] == "azure/gpt-5.6" + + def test_encrypted_stored_model_is_decrypted_for_the_lookup(self, monkeypatch): + import litellm + + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-1234") + entry = litellm.get_model_info("openai/gpt-5.6") + db_model = Deployment( + model_name="gpt-5.6", + litellm_params=LiteLLM_Params(model=encrypt_value_helper(value="openai/gpt-5.6")), + model_info=ModelInfo(id="dep-echo-7", mode="chat"), + ) + echo = {**entry, "id": "dep-echo-7", "db_model": True, "access_groups": ["prod"]} + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment(model_info=ModelInfo(**echo)), + ) + + info = json.loads(result["model_info"]) + assert not frozenset(info).intersection(frozenset(entry) - frozenset(("mode",))) + assert info["mode"] == "chat" + assert info["access_groups"] == ["prod"] + + class TestUpdateDBModelClearCacheControlInjectionPoints: def test_explicit_null_removes_stored_injection_points(self): from litellm.proxy.management_endpoints.model_management_endpoints import (