From a627a1a363328be0bdf5a7a74ca5f246d6499114 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:54:17 -0700 Subject: [PATCH] fix(proxy): treat only keyed /model/update bodies as echoes and write the stored model_info as saved A typed model_info value that matched the cost map, or the previous base model's entry, was dropped and could erase a stored override. Echo detection now follows PATCH, only a body carrying the /model/info key counts, compared against the base model the row has after the write. The locked write patches the stored JSON instead of dumping the hydrated Deployment, so litellm_params pricing is no longer copied into model_info. Tests run the real Router with discovery over a mock upstream --- .../model_management_endpoints.py | 56 +++++---- terraform/provider/CHANGELOG.md | 2 +- .../provider/litellm/resource_model_crud.go | 2 +- .../test_model_management_endpoints.py | 107 ++++++++++-------- 4 files changed, 99 insertions(+), 68 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 04369e6aa0d..544793a559c 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -1128,26 +1128,23 @@ def legacy_update_model_info_changes( Only fields the caller sent with a value count, so ``ModelInfo`` defaults and the blank strings typed clients send for unset fields never overwrite stored values. Server-owned, - display-only, team, PTU and pricing fields keep their dedicated write paths, and values - echoed from a ``/model/info`` response stay display-only whether or not the echo kept its - ``key``: a value equal to the old or new base model's cost map entry (live or as loaded), - the entry ``key`` names, or a limit discovered from the upstream. An echoed value that - differs from the stored override resets that override (``None``), the way ``update_db_model`` does. + display-only, team, PTU and pricing fields keep their dedicated write paths. As on + ``update_db_model``, only a body carrying the ``key`` that ``/model/info`` emits is an echo: + its values equal to the cost map entry of the base model the row has after the write, the + entry ``key`` names, or a limit discovered from the upstream stay display-only, and an echoed + value that differs from the stored override resets that override (``None``). """ stored: Final = db_model.model_info.model_dump(exclude_none=True) - sent: Final = { - k: v - for k, v in incoming_model_info.model_dump(exclude_unset=True, exclude_none=True).items() - if k not in _LEGACY_UPDATE_IGNORED_MODEL_INFO_FIELDS and v != "" - } - live_entries: Final = (_cost_map_entry(db_model, sent), _cost_map_entry(db_model, stored)) - echo_sources: Final = ( - *live_entries, - *(_loaded_catalog_entry(entry, loaded_catalog) for entry in live_entries), - _loaded_catalog_entry(incoming_model_info.model_dump(exclude_none=True), loaded_catalog), - discovered_model_info, + requested: Final = incoming_model_info.model_dump(exclude_unset=True, exclude_none=True) + sent: Final = {k: v for k, v in requested.items() if k not in _LEGACY_UPDATE_IGNORED_MODEL_INFO_FIELDS and v != ""} + echoed: Final = frozenset( + echoed_cost_map_fields( + requested, + _cost_map_entry(db_model, {**stored, **sent}), + _loaded_catalog_entry(requested, loaded_catalog), + discovered_model_info, + ) ) - echoed: Final = frozenset(k for k, v in sent.items() if any(source.get(k) == v for source in echo_sources)) kept: Final = without_server_derived_pricing(sent) return MappingProxyType( { @@ -1180,9 +1177,28 @@ async def _locked_legacy_update_model_info_json( row: Final = await table.find_unique(where={"model_id": model_id}) if row is None: return None - current: Final = Deployment(**row.model_dump()) - writes: Final = _legacy_update_model_info_writes(current, incoming_model_info, discovered_model_info, member_marker) - return current.model_info.model_copy(update=writes).model_dump_json(exclude_none=True) if writes else None + payload: Final = row.model_dump() + writes: Final = _legacy_update_model_info_writes( + Deployment(**payload), incoming_model_info, discovered_model_info, member_marker + ) + if not writes: + return None + stored_model_info: Final[object] = payload.get("model_info") + stored: Final = ( + TypeAdapter(dict[str, object]).validate_json(stored_model_info) + if isinstance(stored_model_info, str) + else TypeAdapter(dict[str, object]).validate_python(stored_model_info or {}) + ) + return ( + TypeAdapter(dict[str, object]) + .dump_json( + { + **{k: v for k, v in stored.items() if k not in writes}, + **{k: v for k, v in writes.items() if v is not None}, + } + ) + .decode() + ) def _param_keys_to_remove(patch: updateDeployment) -> tuple[str, ...]: diff --git a/terraform/provider/CHANGELOG.md b/terraform/provider/CHANGELOG.md index 597e0c6d4d8..1ee88a17a3e 100644 --- a/terraform/provider/CHANGELOG.md +++ b/terraform/provider/CHANGELOG.md @@ -18,7 +18,7 @@ longer signal it. - **credential, model, mcp_server**: Write-only secret attributes so secrets never land in Terraform state (Terraform 1.11 or later). `litellm_credential` gains `credential_values_wo` (a JSON object, `jsonencode({...})`, since the SDK cannot mark a map write-only) with `credential_values_wo_version`, and `credential_values` becomes optional with exactly one of the two required. `litellm_model` gains `model_api_key_wo`, `aws_secret_access_key_wo` and `vertex_credentials_wo`, each with a `_wo_version` companion and each conflicting with its in-state twin. `litellm_mcp_server` gains `client_id`, `client_secret_wo` and `auth_value_wo` (with `_wo_version` companions), sent to the proxy as `credentials`, and `auth_type` now accepts every value the proxy does (`oauth2`, `api_key`, `bearer_token`, ...). Every `_wo` value is sent on each apply from the configuration and produces no diff, so bump its `_wo_version` to push a rotated secret. `litellm_vector_store.litellm_params` warns at plan time when it carries a secret-looking key such as `api_key` - **team**: Optional `team_member_model_max_budget` argument on `litellm_team`, a JSON string of per-model budgets applied to every team member and sent as `team_member_model_max_budget` on `/team/new` and `/team/update`; read back from `team_member_budget_table.model_max_budget` on `/team/info` so out-of-band changes show as drift, with `budget_limit`/`time_period` treated as equal to the `max_budget`/`budget_duration` spelling the proxy stores; removing it sends `{}`, which clears every per-model member budget; `tpm_limit`/`rpm_limit` are rejected at plan time because the proxy rejects them on team budgets. Needs a proxy that includes [#43887](https://github.com/BerriAI/litellm/pull/43887): v1.105.0 and v1.106.0-rc.1 accept the field but silently drop it, so every plan shows a diff -- **model**: Optional `display_name` argument on `litellm_model`, sent as `model_info.display_name` and returned as `display_name` by `/v1/models`, so client model pickers show a readable name; changes are persisted through `/model/{id}/update` since `/model/update` ignores `model_info`; also exported by the `litellm_model` and `litellm_models` data sources +- **model**: Optional `display_name` argument on `litellm_model`, sent as `model_info.display_name` and returned as `display_name` by `/v1/models`, so client model pickers show a readable name; changes are also persisted through `/model/{id}/update`, since proxies older than this provider drop the `model_info` sent to `/model/update`; also exported by the `litellm_model` and `litellm_models` data sources - **key**: Computed `server_metadata` attribute on `litellm_key` exposing every metadata entry the proxy stores, so metadata created outside Terraform is visible in state and drift on it shows on refresh, while `metadata` keeps tracking only the declared entries and updates keep preserving undeclared ones - **team_member_add**: `tpm_limit`, `rpm_limit`, `budget_duration`, and `allowed_models` attributes on `litellm_team_member_add`, applied to every member of the resource; `budget_duration` and `allowed_models` ride on `/team/member_add`, while the limits are sent through `/team/member_update`, which is where the proxy accepts them - **team**: Optional `team_id` argument on `litellm_team`, so teams can be created with a stable, human-readable ID instead of a provider-generated UUID; changing it forces replacement diff --git a/terraform/provider/litellm/resource_model_crud.go b/terraform/provider/litellm/resource_model_crud.go index 3db6f68760d..1f5aa62560f 100644 --- a/terraform/provider/litellm/resource_model_crud.go +++ b/terraform/provider/litellm/resource_model_crud.go @@ -357,7 +357,7 @@ func createOrUpdateModel(d *schema.ResourceData, m interface{}, isUpdate bool) e return retryModelRead(d, m, 5) } -// /model/update only merges litellm_params, so model_info changes go through the PATCH endpoint. +// Older proxies drop the model_info sent to /model/update, so display_name also goes through the PATCH endpoint. func patchModelDisplayName(client *Client, modelID, displayName string) error { resp, err := MakeRequest(client, "PATCH", fmt.Sprintf(endpointModelPatch, url.PathEscape(modelID)), ModelInfoPatch{ ModelInfo: ModelInfoPatchFields{ID: modelID, DisplayName: displayName}, diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 1bb2bb8909b..933ce6555a7 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -4,17 +4,18 @@ import contextlib import json import socket from collections.abc import AsyncIterator, Iterator, Mapping -from dataclasses import dataclass from types import MappingProxyType, SimpleNamespace from typing import Dict, Final, Optional from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest from fastapi import HTTPException from fastapi.encoders import jsonable_encoder from fastapi.testclient import TestClient from litellm._uuid import uuid +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.models.credentials import CredentialItem from litellm.proxy._types import ( @@ -1668,19 +1669,20 @@ class _ContendedModelDb: self._other_writers = self._other_writers[1:] -@dataclass(frozen=True, slots=True) -class _ServingRouter: - model_id: str - discovered: Mapping[str, int] +_AZURE_DEPLOYMENT_PARAMS: Final = MappingProxyType({"model": "azure/my-deployment", "api_key": "sk-existing"}) +_LOCAL_DEPLOYMENT_PARAMS: Final = MappingProxyType( + {"model": "hosted_vllm/local-model", "api_base": "https://local.test/v1", "api_key": "local-key"} +) - def get_model_ids(self) -> tuple[str, ...]: - return (self.model_id,) - def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]: - return self.discovered - - def config_deployments(self) -> tuple[()]: - return () +async def _discover_from_upstream(router: Router, model_card: Mapping[str, object]) -> None: + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": [dict(model_card)]})) + ) as client: + handler.client = client + await router.arefresh_model_info(client=handler) class TestUpdateModel: @@ -1692,22 +1694,30 @@ class TestUpdateModel: async def _update_model( stored_model_info: Mapping[str, object], model_info: ModelInfo, - discovered: Mapping[str, int] = MappingProxyType({}), + litellm_params: Mapping[str, object] = _AZURE_DEPLOYMENT_PARAMS, + upstream_model_card: Mapping[str, object] | None = None, other_writers: tuple[Mapping[str, object], ...] = (), ) -> "_ContendedModelDb": db: Final = _ContendedModelDb( LiteLLM_ProxyModelTable( model_id="m1", model_name="my-model", - litellm_params={"model": "azure/my-deployment", "api_key": "sk-existing"}, + litellm_params=dict(litellm_params), model_info=dict(stored_model_info), ), other_writers, ) + router: Final = Router( + model_list=[ + {"model_name": "my-model", "litellm_params": dict(litellm_params), "model_info": {"id": "m1"}} + ] + ) + if upstream_model_card is not None: + await _discover_from_upstream(router, upstream_model_card) with ( patch("litellm.proxy.proxy_server.prisma_client", db), - patch("litellm.proxy.proxy_server.llm_router", _ServingRouter(model_id="m1", discovered=discovered)), + patch("litellm.proxy.proxy_server.llm_router", router), patch("litellm.proxy.proxy_server.store_model_in_db", True), patch("litellm.proxy.proxy_server.premium_user", True), patch( # test-quality-ok: [TQ008] the route reloads the proxy-global router with no seam to inject @@ -1793,29 +1803,24 @@ class TestUpdateModel: assert not {"key", "max_tokens", "input_cost_per_token"} & persisted.keys() @pytest.mark.asyncio - async def test_update_model_does_not_pin_catalog_values_echoed_without_a_key(self): + async def test_update_model_saves_typed_values_that_equal_the_catalog(self): import litellm catalog_entry: Final = litellm.get_model_info(model="azure/gpt-4o") - deliberate_max_output_tokens: Final = catalog_entry["max_output_tokens"] // 2 + typed: Final = { + "max_tokens": catalog_entry["max_tokens"], + "max_input_tokens": catalog_entry["max_input_tokens"], + "supports_vision": catalog_entry["supports_vision"], + } db: Final = await self._update_model( - {"id": "m1"}, - ModelInfo( - id="m1", - base_model="azure/gpt-4o", - max_tokens=catalog_entry["max_tokens"], - max_input_tokens=catalog_entry["max_input_tokens"], - supports_vision=catalog_entry["supports_vision"], - max_output_tokens=deliberate_max_output_tokens, - ), + {"id": "m1", "max_input_tokens": catalog_entry["max_input_tokens"] // 2}, + ModelInfo(id="m1", base_model="azure/gpt-4o", **typed), ) persisted: Final = db.row.model_info assert persisted is not None - assert persisted["base_model"] == "azure/gpt-4o" - assert persisted["max_output_tokens"] == deliberate_max_output_tokens - assert not {"max_tokens", "max_input_tokens", "supports_vision"} & persisted.keys() + assert {k: persisted.get(k) for k in typed} == typed @pytest.mark.asyncio async def test_update_model_keeps_stored_value_when_a_field_is_sent_blank(self): @@ -1868,40 +1873,39 @@ class TestUpdateModel: assert dict(changes) == {"base_model": "test-provider/next"} - def test_update_model_does_not_pin_the_previous_base_models_loaded_catalog_value_echoed_without_a_key(self): + @pytest.mark.asyncio + async def test_update_model_saves_a_value_the_previous_base_model_has_in_the_catalog(self): import litellm - from litellm.proxy.management_endpoints.model_management_endpoints import legacy_update_model_info_changes previous: Final = litellm.get_model_info(model="azure/gpt-4o") - loaded_max_tokens: Final = previous["max_tokens"] + 1 - catalog: Final = MappingProxyType({previous["key"]: MappingProxyType({"max_tokens": loaded_max_tokens})}) - stored: Final = Deployment( - model_name="my-model", - litellm_params=LiteLLM_Params(model="test-provider/my-deployment"), - model_info=ModelInfo(id="m1", base_model="azure/gpt-4o"), + + db: Final = await self._update_model( + {"id": "m1", "base_model": "azure/gpt-4o", "max_input_tokens": previous["max_input_tokens"] // 2}, + ModelInfo(id="m1", base_model="azure/gpt-4.1", max_input_tokens=previous["max_input_tokens"]), ) - changes: Final = legacy_update_model_info_changes( - stored, - ModelInfo(id="m1", base_model="test-provider/next", max_tokens=loaded_max_tokens), - loaded_catalog=lambda: catalog, - ) - - assert dict(changes) == {"base_model": "test-provider/next"} + persisted: Final = db.row.model_info + assert persisted is not None + assert {k: persisted.get(k) for k in ("base_model", "max_input_tokens")} == { + "base_model": "azure/gpt-4.1", + "max_input_tokens": previous["max_input_tokens"], + } @pytest.mark.asyncio - async def test_update_model_does_not_pin_discovered_limits_or_team_access_echoed_without_a_key(self): + async def test_update_model_does_not_pin_discovered_limits_or_team_access_echoed_from_model_info(self): db: Final = await self._update_model( {"id": "m1"}, ModelInfo( id="m1", base_model="test-provider/test-model", + key="test-provider/test-model", max_input_tokens=128000, max_output_tokens=64000, direct_access=True, access_via_team_ids=["team-a"], ), - discovered=MappingProxyType({"max_input_tokens": 128000, "max_output_tokens": 32000}), + litellm_params=_LOCAL_DEPLOYMENT_PARAMS, + upstream_model_card=MappingProxyType({"id": "local-model", "max_model_len": 128000, "max_output_tokens": 32000}), ) persisted: Final = db.row.model_info @@ -1910,6 +1914,17 @@ class TestUpdateModel: assert persisted["max_output_tokens"] == 64000 assert not {"max_input_tokens", "direct_access", "access_via_team_ids"} & persisted.keys() + @pytest.mark.asyncio + async def test_update_model_does_not_copy_litellm_params_pricing_into_model_info(self): + db: Final = await self._update_model( + {"id": "m1"}, + ModelInfo(id="m1", tier="paid"), + litellm_params={**_AZURE_DEPLOYMENT_PARAMS, "input_cost_per_token": 0.000003}, + ) + + persisted: Final = db.row.model_info + assert persisted == {"id": "m1", "tier": "paid"} + @pytest.mark.asyncio async def test_update_model_resets_a_stored_override_echoed_back_at_its_catalog_value(self): import litellm