From 76cb0fec1c3c7e16507117119dc826b0bdb61dd2 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 12 Sep 2026 17:29:35 -0700 Subject: [PATCH] fix(model_management): stop persisting cost map pricing as a deployment override /model/info fills a deployment's missing pricing in from the model cost map so the Admin UI has a rate to display. Clients echo that whole model_info blob back on save, and update_db_model merged it into the row, so editing an unrelated setting turned that day's catalog price into a real per-deployment override. After that the deployment ignored the cost map and Reload Price Data could no longer move it, because the reload replays each deployment's stored pricing over the fresh catalog. Drop the derived pricing from incoming model_info on the two write paths. The drop-set is read off the same objects the read path uses, CustomPricingLiteLLMParams plus the tiered *_above_N_tokens pattern that get_model_info passes through and no model declares, so it cannot drift as new rates are added. output_vector_size is exempt: it lives on the pricing model but is an embedding dimension, not a rate. A deployment's own pricing still rides litellm_params, which is untouched, as is the explicit-null clear, which reads the incoming model rather than the filtered dict. The filter sits in the endpoint bodies rather than _add_model_to_db, which master-key rotation reuses to re-serialize every stored deployment. --- .../model_management_endpoints.py | 12 +- litellm/types/utils.py | 33 +++ litellm/utils.py | 3 +- .../test_model_management_endpoints.py | 198 ++++++++++++++++++ 4 files changed, 241 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 8484279c69a..2234e825090 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -123,6 +123,7 @@ from litellm.types.router import ( ModelInfo, updateDeployment, ) +from litellm.types.utils import without_server_derived_pricing from litellm.utils import get_utc_datetime if TYPE_CHECKING: @@ -747,11 +748,10 @@ def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> Pr # update model info if updated_patch.model_info: - merged_model_info.update(updated_patch.model_info.model_dump(exclude_none=True)) + merged_model_info.update(without_server_derived_pricing(updated_patch.model_info.model_dump(exclude_none=True))) - # Honor explicit-null clears LAST, after both merges, so a model_info blob the UI - # passes through (which today re-sends the OLD pricing on every save) cannot - # silently undo a litellm_params clear via .update(). + # Honor explicit-null clears LAST, after both merges, so a model_info blob a client + # passes through cannot silently undo a litellm_params clear via .update(). # # Restricted to SPECIAL_MODEL_INFO_PARAMS (input/output cost per token/character # and cache read/write costs) so this path cannot be used to null out privileged @@ -2094,6 +2094,10 @@ async def add_new_model( enforced=bool(general_settings.get(ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING, False)), ) + model_params.model_info = ModelInfo( # rebind-ok: downstream team-model handling mutates this same object + **without_server_derived_pricing(model_params.model_info.model_dump(exclude_none=True)) + ) + model_response: prisma_models.LiteLLM_ProxyModelTable | LiteLLM_ProxyModelTable | None = None # update DB incoming_model_info: Final = model_params.model_info.model_dump(exclude_none=True) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index e3ea37dc0c8..07c173213d3 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1,4 +1,5 @@ import json +import re import time from collections.abc import Mapping, Sequence from enum import Enum @@ -3645,6 +3646,38 @@ def shared_backend_model_info(model_info: dict[str, Any]) -> dict[str, Any]: return {k: v for k, v in model_info.items() if k in SHARED_BACKEND_MODEL_INFO_FIELDS} +ABOVE_THRESHOLD_COST_KEY_PATTERN: Final = re.compile(r"_above_\d+k?_tokens$") + +_PRICING_FIELD_EXEMPTIONS: Final[frozenset[str]] = frozenset({"output_vector_size"}) + +SERVER_DERIVED_PRICING_FIELDS: Final[frozenset[str]] = ( + frozenset(CustomPricingLiteLLMParams.model_fields) - _PRICING_FIELD_EXEMPTIONS +) + + +def is_server_derived_pricing_key(key: str) -> bool: + """Whether ``/model/info`` can fill ``key`` into ``model_info`` from the cost map. + + Two sources, because ``get_model_info`` emits two: the declared pricing fields, and + the tiered ``*_above__tokens`` rates that ride through on a pattern match and are + declared nowhere. Both are read here from the same objects the read path uses, so the + set cannot drift as new rates are added. + """ + return key in SERVER_DERIVED_PRICING_FIELDS or ABOVE_THRESHOLD_COST_KEY_PATTERN.search(key) is not None + + +def without_server_derived_pricing(model_info: Mapping[str, Any]) -> Mapping[str, Any]: + """Drop the pricing ``/model/info`` derives for display, keeping everything else. + + ``/model/info`` fills a deployment's missing pricing in from the cost map so the + Admin UI has a rate to show. Clients that echo that response back on save would + otherwise persist the display value as a real per-deployment override, freezing the + deployment at that day's price where no cost map refresh can reach it. A deployment's + own pricing belongs on ``litellm_params``, which is unaffected. + """ + return MappingProxyType({k: v for k, v in model_info.items() if not is_server_derived_pricing_key(k)}) + + # Server-controlled fields that bound or drive an interceptor's agentic loop # (depth, cycle fingerprints, ceiling, code-interpreter sandbox state). Listed # in all_litellm_params so they are treated as LiteLLM-level and excluded from diff --git a/litellm/utils.py b/litellm/utils.py index 394ab4b4094..27e97ab5fb2 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -216,6 +216,7 @@ from litellm.types.llms.openai import ( OpenAIWebSearchOptions, ) from litellm.types.utils import ( + ABOVE_THRESHOLD_COST_KEY_PATTERN, OPENAI_RESPONSE_HEADERS, CallTypes, ChatCompletionDeltaToolCall, @@ -5637,7 +5638,7 @@ def _is_potential_model_name_in_model_cost( ) -_ABOVE_THRESHOLD_COST_KEY: Final = re.compile(r"_above_\d+k?_tokens$") +_ABOVE_THRESHOLD_COST_KEY: Final = ABOVE_THRESHOLD_COST_KEY_PATTERN def _get_model_info_helper( diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index c66095dc1fd..c3ad66397ea 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -3666,6 +3666,204 @@ class TestUpdateDBModelClearPricing: assert info["cache_creation_input_token_cost"] == 0.000003 +class TestModelInfoServerDerivedPricingFilter: + """LIT-5292. `/model/info` fills a deployment's missing pricing in from the cost map + so the Admin UI has a rate to display. Clients echo that whole blob back on save, so + without a write-path filter an unrelated edit persists the display value as a real + per-deployment override and no cost map refresh can move the deployment again. + + A deployment's own pricing rides `litellm_params`, which stays writable. + """ + + def test_echoed_cost_map_pricing_is_not_persisted(self): + """The ticket's repro: a deployment with no override, edited for an unrelated + reason, must not gain one from the pricing the form was displaying.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + db_model = Deployment( + model_name="haiku", + litellm_params=LiteLLM_Params(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"), + model_info=ModelInfo(id="dep-unpriced-0"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment( + model_info=ModelInfo( + id="dep-unpriced-0", + access_groups=["prod"], + input_cost_per_token=0.0000008, + output_cost_per_token=0.000004, + cache_read_input_token_cost=0.00000008, + ) + ), + ) + + info = json.loads(result["model_info"]) + params = json.loads(result["litellm_params"]) + assert info["access_groups"] == ["prod"] + for field in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost"): + assert field not in info, f"{field} was persisted as a per-deployment override" + assert field not in params + + def test_tiered_above_threshold_pricing_is_dropped(self): + """Tiered rates ride `get_model_info` on a pattern match and are declared on no + model, so a filter built only from the declared pricing fields would miss them.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + db_model = Deployment( + model_name="sonnet", + litellm_params=LiteLLM_Params(model="claude-sonnet-4-5"), + model_info=ModelInfo(id="dep-tiered-0"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment( + model_info=ModelInfo( + id="dep-tiered-0", + input_cost_per_token_above_200k_tokens=0.000006, + cache_creation_input_token_cost_above_1hr_above_200k_tokens=0.000012, + ) + ), + ) + + info = json.loads(result["model_info"]) + assert "input_cost_per_token_above_200k_tokens" not in info + assert "cache_creation_input_token_cost_above_1hr_above_200k_tokens" not in info + + def test_output_vector_size_and_client_owned_fields_survive(self): + """`output_vector_size` sits on the pricing model but is an embedding dimension, + not a rate. It and the operator-owned keys stay writable.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + db_model = Deployment( + model_name="embed", + litellm_params=LiteLLM_Params(model="openai/text-embedding-3-large"), + model_info=ModelInfo(id="dep-embed-0"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment( + model_info=ModelInfo( + id="dep-embed-0", + output_vector_size=3072, + base_model="azure/text-embedding-3-large", + tier="paid", + team_id="team-1", + access_groups=["research"], + my_custom_key="my_custom_value", + ) + ), + ) + + info = json.loads(result["model_info"]) + assert info["output_vector_size"] == 3072 + assert info["base_model"] == "azure/text-embedding-3-large" + assert info["tier"] == "paid" + assert info["team_id"] == "team-1" + assert info["access_groups"] == ["research"] + assert info["my_custom_key"] == "my_custom_value" + + def test_litellm_params_pricing_still_persists(self): + """The supported way to set a deployment override is untouched.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + from litellm.types.router import ( + Deployment, + LiteLLM_Params, + ModelInfo, + updateLiteLLMParams, + ) + + db_model = Deployment( + model_name="haiku", + litellm_params=LiteLLM_Params(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"), + model_info=ModelInfo(id="dep-priced-0"), + ) + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment( + litellm_params=updateLiteLLMParams(input_cost_per_token=0.00000123), + model_info=ModelInfo(id="dep-priced-0", input_cost_per_token=0.0000008), + ), + ) + + params = json.loads(result["litellm_params"]) + assert params["input_cost_per_token"] == 0.00000123 + + @pytest.mark.asyncio + async def test_add_new_model_drops_echoed_pricing_and_keeps_identity(self): + """The create path filters too, and rebuilding the blob must not mint a fresh id + or flip `db_model`, which would detach the row from its router deployment.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + add_new_model, + ) + + model_id = "dep-create-0" + db_row = LiteLLM_ProxyModelTable( + model_id=model_id, + model_name="haiku", + litellm_params={"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"}, + model_info={"id": model_id}, + created_by="test-admin", + updated_by="test-admin", + ) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_proxymodeltable = AsyncMock() + mock_prisma.db.litellm_proxymodeltable.create = AsyncMock(return_value=db_row) + + mock_proxy_config = MagicMock() + mock_proxy_config.add_deployment = AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)) + + mock_router = MagicMock() + mock_router.get_model_ids.return_value = [model_id] + + _PS = "litellm.proxy.proxy_server" + _ENCRYPT = "litellm.proxy.management_endpoints.model_management_endpoints.encrypt_value_helper" + with ( + patch(f"{_PS}.prisma_client", mock_prisma), + patch(f"{_PS}.store_model_in_db", True), + patch(f"{_PS}.proxy_config", mock_proxy_config), + patch(f"{_PS}.proxy_logging_obj", MagicMock()), + patch(f"{_PS}.general_settings", {}), + patch(f"{_PS}.premium_user", True), + patch(f"{_PS}.llm_router", mock_router), + patch(_ENCRYPT, side_effect=lambda value, **kwargs: value), + ): + await add_new_model( + model_params=Deployment( + model_name="haiku", + litellm_params=LiteLLM_Params(model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"), + model_info={ + "id": model_id, + "access_groups": ["prod"], + "input_cost_per_token": 0.0000008, + }, + ), + user_api_key_dict=UserAPIKeyAuth(user_id="test-admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + written = json.loads(mock_prisma.db.litellm_proxymodeltable.create.call_args.kwargs["data"]["model_info"]) + assert "input_cost_per_token" not in written + assert written["id"] == model_id, "filtering must not mint a fresh deployment id" + assert written["access_groups"] == ["prod"] + + class TestGetModelInfoWithIdBlocked: """`ProxyConfig.get_model_info_with_id` must propagate the DB-level `blocked` column into the in-memory `model_info` dict so the router filter can read it."""