fix(proxy): treat only keyed /model/update bodies as echoes and write the stored model_info as saved

A typed model_info value that matched the cost map, or the previous base model's entry,
was dropped and could erase a stored override. Echo detection now follows PATCH, only a
body carrying the /model/info key counts, compared against the base model the row has
after the write. The locked write patches the stored JSON instead of dumping the hydrated
Deployment, so litellm_params pricing is no longer copied into model_info. Tests run the
real Router with discovery over a mock upstream
This commit is contained in:
mateo-berri 2026-10-28 12:54:17 -07:00
parent ce02463ad2
commit a627a1a363
4 changed files with 99 additions and 68 deletions

View file

@ -1128,26 +1128,23 @@ def legacy_update_model_info_changes(
Only fields the caller sent with a value count, so ``ModelInfo`` defaults and the blank
strings typed clients send for unset fields never overwrite stored values. Server-owned,
display-only, team, PTU and pricing fields keep their dedicated write paths, and values
echoed from a ``/model/info`` response stay display-only whether or not the echo kept its
``key``: a value equal to the old or new base model's cost map entry (live or as loaded),
the entry ``key`` names, or a limit discovered from the upstream. An echoed value that
differs from the stored override resets that override (``None``), the way ``update_db_model`` does.
display-only, team, PTU and pricing fields keep their dedicated write paths. As on
``update_db_model``, only a body carrying the ``key`` that ``/model/info`` emits is an echo:
its values equal to the cost map entry of the base model the row has after the write, the
entry ``key`` names, or a limit discovered from the upstream stay display-only, and an echoed
value that differs from the stored override resets that override (``None``).
"""
stored: Final = db_model.model_info.model_dump(exclude_none=True)
sent: Final = {
k: v
for k, v in incoming_model_info.model_dump(exclude_unset=True, exclude_none=True).items()
if k not in _LEGACY_UPDATE_IGNORED_MODEL_INFO_FIELDS and v != ""
}
live_entries: Final = (_cost_map_entry(db_model, sent), _cost_map_entry(db_model, stored))
echo_sources: Final = (
*live_entries,
*(_loaded_catalog_entry(entry, loaded_catalog) for entry in live_entries),
_loaded_catalog_entry(incoming_model_info.model_dump(exclude_none=True), loaded_catalog),
discovered_model_info,
requested: Final = incoming_model_info.model_dump(exclude_unset=True, exclude_none=True)
sent: Final = {k: v for k, v in requested.items() if k not in _LEGACY_UPDATE_IGNORED_MODEL_INFO_FIELDS and v != ""}
echoed: Final = frozenset(
echoed_cost_map_fields(
requested,
_cost_map_entry(db_model, {**stored, **sent}),
_loaded_catalog_entry(requested, loaded_catalog),
discovered_model_info,
)
)
echoed: Final = frozenset(k for k, v in sent.items() if any(source.get(k) == v for source in echo_sources))
kept: Final = without_server_derived_pricing(sent)
return MappingProxyType(
{
@ -1180,9 +1177,28 @@ async def _locked_legacy_update_model_info_json(
row: Final = await table.find_unique(where={"model_id": model_id})
if row is None:
return None
current: Final = Deployment(**row.model_dump())
writes: Final = _legacy_update_model_info_writes(current, incoming_model_info, discovered_model_info, member_marker)
return current.model_info.model_copy(update=writes).model_dump_json(exclude_none=True) if writes else None
payload: Final = row.model_dump()
writes: Final = _legacy_update_model_info_writes(
Deployment(**payload), incoming_model_info, discovered_model_info, member_marker
)
if not writes:
return None
stored_model_info: Final[object] = payload.get("model_info")
stored: Final = (
TypeAdapter(dict[str, object]).validate_json(stored_model_info)
if isinstance(stored_model_info, str)
else TypeAdapter(dict[str, object]).validate_python(stored_model_info or {})
)
return (
TypeAdapter(dict[str, object])
.dump_json(
{
**{k: v for k, v in stored.items() if k not in writes},
**{k: v for k, v in writes.items() if v is not None},
}
)
.decode()
)
def _param_keys_to_remove(patch: updateDeployment) -> tuple[str, ...]:

View file

@ -18,7 +18,7 @@ longer signal it.
- **credential, model, mcp_server**: Write-only secret attributes so secrets never land in Terraform state (Terraform 1.11 or later). `litellm_credential` gains `credential_values_wo` (a JSON object, `jsonencode({...})`, since the SDK cannot mark a map write-only) with `credential_values_wo_version`, and `credential_values` becomes optional with exactly one of the two required. `litellm_model` gains `model_api_key_wo`, `aws_secret_access_key_wo` and `vertex_credentials_wo`, each with a `_wo_version` companion and each conflicting with its in-state twin. `litellm_mcp_server` gains `client_id`, `client_secret_wo` and `auth_value_wo` (with `_wo_version` companions), sent to the proxy as `credentials`, and `auth_type` now accepts every value the proxy does (`oauth2`, `api_key`, `bearer_token`, ...). Every `_wo` value is sent on each apply from the configuration and produces no diff, so bump its `_wo_version` to push a rotated secret. `litellm_vector_store.litellm_params` warns at plan time when it carries a secret-looking key such as `api_key`
- **team**: Optional `team_member_model_max_budget` argument on `litellm_team`, a JSON string of per-model budgets applied to every team member and sent as `team_member_model_max_budget` on `/team/new` and `/team/update`; read back from `team_member_budget_table.model_max_budget` on `/team/info` so out-of-band changes show as drift, with `budget_limit`/`time_period` treated as equal to the `max_budget`/`budget_duration` spelling the proxy stores; removing it sends `{}`, which clears every per-model member budget; `tpm_limit`/`rpm_limit` are rejected at plan time because the proxy rejects them on team budgets. Needs a proxy that includes [#43887](https://github.com/BerriAI/litellm/pull/43887): v1.105.0 and v1.106.0-rc.1 accept the field but silently drop it, so every plan shows a diff
- **model**: Optional `display_name` argument on `litellm_model`, sent as `model_info.display_name` and returned as `display_name` by `/v1/models`, so client model pickers show a readable name; changes are persisted through `/model/{id}/update` since `/model/update` ignores `model_info`; also exported by the `litellm_model` and `litellm_models` data sources
- **model**: Optional `display_name` argument on `litellm_model`, sent as `model_info.display_name` and returned as `display_name` by `/v1/models`, so client model pickers show a readable name; changes are also persisted through `/model/{id}/update`, since proxies older than this provider drop the `model_info` sent to `/model/update`; also exported by the `litellm_model` and `litellm_models` data sources
- **key**: Computed `server_metadata` attribute on `litellm_key` exposing every metadata entry the proxy stores, so metadata created outside Terraform is visible in state and drift on it shows on refresh, while `metadata` keeps tracking only the declared entries and updates keep preserving undeclared ones
- **team_member_add**: `tpm_limit`, `rpm_limit`, `budget_duration`, and `allowed_models` attributes on `litellm_team_member_add`, applied to every member of the resource; `budget_duration` and `allowed_models` ride on `/team/member_add`, while the limits are sent through `/team/member_update`, which is where the proxy accepts them
- **team**: Optional `team_id` argument on `litellm_team`, so teams can be created with a stable, human-readable ID instead of a provider-generated UUID; changing it forces replacement

View file

@ -357,7 +357,7 @@ func createOrUpdateModel(d *schema.ResourceData, m interface{}, isUpdate bool) e
return retryModelRead(d, m, 5)
}
// /model/update only merges litellm_params, so model_info changes go through the PATCH endpoint.
// Older proxies drop the model_info sent to /model/update, so display_name also goes through the PATCH endpoint.
func patchModelDisplayName(client *Client, modelID, displayName string) error {
resp, err := MakeRequest(client, "PATCH", fmt.Sprintf(endpointModelPatch, url.PathEscape(modelID)), ModelInfoPatch{
ModelInfo: ModelInfoPatchFields{ID: modelID, DisplayName: displayName},

View file

@ -4,17 +4,18 @@ import contextlib
import json
import socket
from collections.abc import AsyncIterator, Iterator, Mapping
from dataclasses import dataclass
from types import MappingProxyType, SimpleNamespace
from typing import Dict, Final, Optional
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from fastapi import HTTPException
from fastapi.encoders import jsonable_encoder
from fastapi.testclient import TestClient
from litellm._uuid import uuid
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.models.credentials import CredentialItem
from litellm.proxy._types import (
@ -1668,19 +1669,20 @@ class _ContendedModelDb:
self._other_writers = self._other_writers[1:]
@dataclass(frozen=True, slots=True)
class _ServingRouter:
model_id: str
discovered: Mapping[str, int]
_AZURE_DEPLOYMENT_PARAMS: Final = MappingProxyType({"model": "azure/my-deployment", "api_key": "sk-existing"})
_LOCAL_DEPLOYMENT_PARAMS: Final = MappingProxyType(
{"model": "hosted_vllm/local-model", "api_base": "https://local.test/v1", "api_key": "local-key"}
)
def get_model_ids(self) -> tuple[str, ...]:
return (self.model_id,)
def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]:
return self.discovered
def config_deployments(self) -> tuple[()]:
return ()
async def _discover_from_upstream(router: Router, model_card: Mapping[str, object]) -> None:
handler: Final = AsyncHTTPHandler()
await handler.client.aclose()
async with httpx.AsyncClient(
transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": [dict(model_card)]}))
) as client:
handler.client = client
await router.arefresh_model_info(client=handler)
class TestUpdateModel:
@ -1692,22 +1694,30 @@ class TestUpdateModel:
async def _update_model(
stored_model_info: Mapping[str, object],
model_info: ModelInfo,
discovered: Mapping[str, int] = MappingProxyType({}),
litellm_params: Mapping[str, object] = _AZURE_DEPLOYMENT_PARAMS,
upstream_model_card: Mapping[str, object] | None = None,
other_writers: tuple[Mapping[str, object], ...] = (),
) -> "_ContendedModelDb":
db: Final = _ContendedModelDb(
LiteLLM_ProxyModelTable(
model_id="m1",
model_name="my-model",
litellm_params={"model": "azure/my-deployment", "api_key": "sk-existing"},
litellm_params=dict(litellm_params),
model_info=dict(stored_model_info),
),
other_writers,
)
router: Final = Router(
model_list=[
{"model_name": "my-model", "litellm_params": dict(litellm_params), "model_info": {"id": "m1"}}
]
)
if upstream_model_card is not None:
await _discover_from_upstream(router, upstream_model_card)
with (
patch("litellm.proxy.proxy_server.prisma_client", db),
patch("litellm.proxy.proxy_server.llm_router", _ServingRouter(model_id="m1", discovered=discovered)),
patch("litellm.proxy.proxy_server.llm_router", router),
patch("litellm.proxy.proxy_server.store_model_in_db", True),
patch("litellm.proxy.proxy_server.premium_user", True),
patch( # test-quality-ok: [TQ008] the route reloads the proxy-global router with no seam to inject
@ -1793,29 +1803,24 @@ class TestUpdateModel:
assert not {"key", "max_tokens", "input_cost_per_token"} & persisted.keys()
@pytest.mark.asyncio
async def test_update_model_does_not_pin_catalog_values_echoed_without_a_key(self):
async def test_update_model_saves_typed_values_that_equal_the_catalog(self):
import litellm
catalog_entry: Final = litellm.get_model_info(model="azure/gpt-4o")
deliberate_max_output_tokens: Final = catalog_entry["max_output_tokens"] // 2
typed: Final = {
"max_tokens": catalog_entry["max_tokens"],
"max_input_tokens": catalog_entry["max_input_tokens"],
"supports_vision": catalog_entry["supports_vision"],
}
db: Final = await self._update_model(
{"id": "m1"},
ModelInfo(
id="m1",
base_model="azure/gpt-4o",
max_tokens=catalog_entry["max_tokens"],
max_input_tokens=catalog_entry["max_input_tokens"],
supports_vision=catalog_entry["supports_vision"],
max_output_tokens=deliberate_max_output_tokens,
),
{"id": "m1", "max_input_tokens": catalog_entry["max_input_tokens"] // 2},
ModelInfo(id="m1", base_model="azure/gpt-4o", **typed),
)
persisted: Final = db.row.model_info
assert persisted is not None
assert persisted["base_model"] == "azure/gpt-4o"
assert persisted["max_output_tokens"] == deliberate_max_output_tokens
assert not {"max_tokens", "max_input_tokens", "supports_vision"} & persisted.keys()
assert {k: persisted.get(k) for k in typed} == typed
@pytest.mark.asyncio
async def test_update_model_keeps_stored_value_when_a_field_is_sent_blank(self):
@ -1868,40 +1873,39 @@ class TestUpdateModel:
assert dict(changes) == {"base_model": "test-provider/next"}
def test_update_model_does_not_pin_the_previous_base_models_loaded_catalog_value_echoed_without_a_key(self):
@pytest.mark.asyncio
async def test_update_model_saves_a_value_the_previous_base_model_has_in_the_catalog(self):
import litellm
from litellm.proxy.management_endpoints.model_management_endpoints import legacy_update_model_info_changes
previous: Final = litellm.get_model_info(model="azure/gpt-4o")
loaded_max_tokens: Final = previous["max_tokens"] + 1
catalog: Final = MappingProxyType({previous["key"]: MappingProxyType({"max_tokens": loaded_max_tokens})})
stored: Final = Deployment(
model_name="my-model",
litellm_params=LiteLLM_Params(model="test-provider/my-deployment"),
model_info=ModelInfo(id="m1", base_model="azure/gpt-4o"),
db: Final = await self._update_model(
{"id": "m1", "base_model": "azure/gpt-4o", "max_input_tokens": previous["max_input_tokens"] // 2},
ModelInfo(id="m1", base_model="azure/gpt-4.1", max_input_tokens=previous["max_input_tokens"]),
)
changes: Final = legacy_update_model_info_changes(
stored,
ModelInfo(id="m1", base_model="test-provider/next", max_tokens=loaded_max_tokens),
loaded_catalog=lambda: catalog,
)
assert dict(changes) == {"base_model": "test-provider/next"}
persisted: Final = db.row.model_info
assert persisted is not None
assert {k: persisted.get(k) for k in ("base_model", "max_input_tokens")} == {
"base_model": "azure/gpt-4.1",
"max_input_tokens": previous["max_input_tokens"],
}
@pytest.mark.asyncio
async def test_update_model_does_not_pin_discovered_limits_or_team_access_echoed_without_a_key(self):
async def test_update_model_does_not_pin_discovered_limits_or_team_access_echoed_from_model_info(self):
db: Final = await self._update_model(
{"id": "m1"},
ModelInfo(
id="m1",
base_model="test-provider/test-model",
key="test-provider/test-model",
max_input_tokens=128000,
max_output_tokens=64000,
direct_access=True,
access_via_team_ids=["team-a"],
),
discovered=MappingProxyType({"max_input_tokens": 128000, "max_output_tokens": 32000}),
litellm_params=_LOCAL_DEPLOYMENT_PARAMS,
upstream_model_card=MappingProxyType({"id": "local-model", "max_model_len": 128000, "max_output_tokens": 32000}),
)
persisted: Final = db.row.model_info
@ -1910,6 +1914,17 @@ class TestUpdateModel:
assert persisted["max_output_tokens"] == 64000
assert not {"max_input_tokens", "direct_access", "access_via_team_ids"} & persisted.keys()
@pytest.mark.asyncio
async def test_update_model_does_not_copy_litellm_params_pricing_into_model_info(self):
db: Final = await self._update_model(
{"id": "m1"},
ModelInfo(id="m1", tier="paid"),
litellm_params={**_AZURE_DEPLOYMENT_PARAMS, "input_cost_per_token": 0.000003},
)
persisted: Final = db.row.model_info
assert persisted == {"id": "m1", "tier": "paid"}
@pytest.mark.asyncio
async def test_update_model_resets_a_stored_override_echoed_back_at_its_catalog_value(self):
import litellm