mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): treat only keyed /model/update bodies as echoes and write the stored model_info as saved
A typed model_info value that matched the cost map, or the previous base model's entry, was dropped and could erase a stored override. Echo detection now follows PATCH, only a body carrying the /model/info key counts, compared against the base model the row has after the write. The locked write patches the stored JSON instead of dumping the hydrated Deployment, so litellm_params pricing is no longer copied into model_info. Tests run the real Router with discovery over a mock upstream
This commit is contained in:
parent
ce02463ad2
commit
a627a1a363
4 changed files with 99 additions and 68 deletions
|
|
@ -1128,26 +1128,23 @@ def legacy_update_model_info_changes(
|
|||
|
||||
Only fields the caller sent with a value count, so ``ModelInfo`` defaults and the blank
|
||||
strings typed clients send for unset fields never overwrite stored values. Server-owned,
|
||||
display-only, team, PTU and pricing fields keep their dedicated write paths, and values
|
||||
echoed from a ``/model/info`` response stay display-only whether or not the echo kept its
|
||||
``key``: a value equal to the old or new base model's cost map entry (live or as loaded),
|
||||
the entry ``key`` names, or a limit discovered from the upstream. An echoed value that
|
||||
differs from the stored override resets that override (``None``), the way ``update_db_model`` does.
|
||||
display-only, team, PTU and pricing fields keep their dedicated write paths. As on
|
||||
``update_db_model``, only a body carrying the ``key`` that ``/model/info`` emits is an echo:
|
||||
its values equal to the cost map entry of the base model the row has after the write, the
|
||||
entry ``key`` names, or a limit discovered from the upstream stay display-only, and an echoed
|
||||
value that differs from the stored override resets that override (``None``).
|
||||
"""
|
||||
stored: Final = db_model.model_info.model_dump(exclude_none=True)
|
||||
sent: Final = {
|
||||
k: v
|
||||
for k, v in incoming_model_info.model_dump(exclude_unset=True, exclude_none=True).items()
|
||||
if k not in _LEGACY_UPDATE_IGNORED_MODEL_INFO_FIELDS and v != ""
|
||||
}
|
||||
live_entries: Final = (_cost_map_entry(db_model, sent), _cost_map_entry(db_model, stored))
|
||||
echo_sources: Final = (
|
||||
*live_entries,
|
||||
*(_loaded_catalog_entry(entry, loaded_catalog) for entry in live_entries),
|
||||
_loaded_catalog_entry(incoming_model_info.model_dump(exclude_none=True), loaded_catalog),
|
||||
discovered_model_info,
|
||||
requested: Final = incoming_model_info.model_dump(exclude_unset=True, exclude_none=True)
|
||||
sent: Final = {k: v for k, v in requested.items() if k not in _LEGACY_UPDATE_IGNORED_MODEL_INFO_FIELDS and v != ""}
|
||||
echoed: Final = frozenset(
|
||||
echoed_cost_map_fields(
|
||||
requested,
|
||||
_cost_map_entry(db_model, {**stored, **sent}),
|
||||
_loaded_catalog_entry(requested, loaded_catalog),
|
||||
discovered_model_info,
|
||||
)
|
||||
)
|
||||
echoed: Final = frozenset(k for k, v in sent.items() if any(source.get(k) == v for source in echo_sources))
|
||||
kept: Final = without_server_derived_pricing(sent)
|
||||
return MappingProxyType(
|
||||
{
|
||||
|
|
@ -1180,9 +1177,28 @@ async def _locked_legacy_update_model_info_json(
|
|||
row: Final = await table.find_unique(where={"model_id": model_id})
|
||||
if row is None:
|
||||
return None
|
||||
current: Final = Deployment(**row.model_dump())
|
||||
writes: Final = _legacy_update_model_info_writes(current, incoming_model_info, discovered_model_info, member_marker)
|
||||
return current.model_info.model_copy(update=writes).model_dump_json(exclude_none=True) if writes else None
|
||||
payload: Final = row.model_dump()
|
||||
writes: Final = _legacy_update_model_info_writes(
|
||||
Deployment(**payload), incoming_model_info, discovered_model_info, member_marker
|
||||
)
|
||||
if not writes:
|
||||
return None
|
||||
stored_model_info: Final[object] = payload.get("model_info")
|
||||
stored: Final = (
|
||||
TypeAdapter(dict[str, object]).validate_json(stored_model_info)
|
||||
if isinstance(stored_model_info, str)
|
||||
else TypeAdapter(dict[str, object]).validate_python(stored_model_info or {})
|
||||
)
|
||||
return (
|
||||
TypeAdapter(dict[str, object])
|
||||
.dump_json(
|
||||
{
|
||||
**{k: v for k, v in stored.items() if k not in writes},
|
||||
**{k: v for k, v in writes.items() if v is not None},
|
||||
}
|
||||
)
|
||||
.decode()
|
||||
)
|
||||
|
||||
|
||||
def _param_keys_to_remove(patch: updateDeployment) -> tuple[str, ...]:
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ longer signal it.
|
|||
|
||||
- **credential, model, mcp_server**: Write-only secret attributes so secrets never land in Terraform state (Terraform 1.11 or later). `litellm_credential` gains `credential_values_wo` (a JSON object, `jsonencode({...})`, since the SDK cannot mark a map write-only) with `credential_values_wo_version`, and `credential_values` becomes optional with exactly one of the two required. `litellm_model` gains `model_api_key_wo`, `aws_secret_access_key_wo` and `vertex_credentials_wo`, each with a `_wo_version` companion and each conflicting with its in-state twin. `litellm_mcp_server` gains `client_id`, `client_secret_wo` and `auth_value_wo` (with `_wo_version` companions), sent to the proxy as `credentials`, and `auth_type` now accepts every value the proxy does (`oauth2`, `api_key`, `bearer_token`, ...). Every `_wo` value is sent on each apply from the configuration and produces no diff, so bump its `_wo_version` to push a rotated secret. `litellm_vector_store.litellm_params` warns at plan time when it carries a secret-looking key such as `api_key`
|
||||
- **team**: Optional `team_member_model_max_budget` argument on `litellm_team`, a JSON string of per-model budgets applied to every team member and sent as `team_member_model_max_budget` on `/team/new` and `/team/update`; read back from `team_member_budget_table.model_max_budget` on `/team/info` so out-of-band changes show as drift, with `budget_limit`/`time_period` treated as equal to the `max_budget`/`budget_duration` spelling the proxy stores; removing it sends `{}`, which clears every per-model member budget; `tpm_limit`/`rpm_limit` are rejected at plan time because the proxy rejects them on team budgets. Needs a proxy that includes [#43887](https://github.com/BerriAI/litellm/pull/43887): v1.105.0 and v1.106.0-rc.1 accept the field but silently drop it, so every plan shows a diff
|
||||
- **model**: Optional `display_name` argument on `litellm_model`, sent as `model_info.display_name` and returned as `display_name` by `/v1/models`, so client model pickers show a readable name; changes are persisted through `/model/{id}/update` since `/model/update` ignores `model_info`; also exported by the `litellm_model` and `litellm_models` data sources
|
||||
- **model**: Optional `display_name` argument on `litellm_model`, sent as `model_info.display_name` and returned as `display_name` by `/v1/models`, so client model pickers show a readable name; changes are also persisted through `/model/{id}/update`, since proxies older than this provider drop the `model_info` sent to `/model/update`; also exported by the `litellm_model` and `litellm_models` data sources
|
||||
- **key**: Computed `server_metadata` attribute on `litellm_key` exposing every metadata entry the proxy stores, so metadata created outside Terraform is visible in state and drift on it shows on refresh, while `metadata` keeps tracking only the declared entries and updates keep preserving undeclared ones
|
||||
- **team_member_add**: `tpm_limit`, `rpm_limit`, `budget_duration`, and `allowed_models` attributes on `litellm_team_member_add`, applied to every member of the resource; `budget_duration` and `allowed_models` ride on `/team/member_add`, while the limits are sent through `/team/member_update`, which is where the proxy accepts them
|
||||
- **team**: Optional `team_id` argument on `litellm_team`, so teams can be created with a stable, human-readable ID instead of a provider-generated UUID; changing it forces replacement
|
||||
|
|
|
|||
|
|
@ -357,7 +357,7 @@ func createOrUpdateModel(d *schema.ResourceData, m interface{}, isUpdate bool) e
|
|||
return retryModelRead(d, m, 5)
|
||||
}
|
||||
|
||||
// /model/update only merges litellm_params, so model_info changes go through the PATCH endpoint.
|
||||
// Older proxies drop the model_info sent to /model/update, so display_name also goes through the PATCH endpoint.
|
||||
func patchModelDisplayName(client *Client, modelID, displayName string) error {
|
||||
resp, err := MakeRequest(client, "PATCH", fmt.Sprintf(endpointModelPatch, url.PathEscape(modelID)), ModelInfoPatch{
|
||||
ModelInfo: ModelInfoPatchFields{ID: modelID, DisplayName: displayName},
|
||||
|
|
|
|||
|
|
@ -4,17 +4,18 @@ import contextlib
|
|||
import json
|
||||
import socket
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import Dict, Final, Optional
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from litellm._uuid import uuid
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.models.credentials import CredentialItem
|
||||
|
||||
from litellm.proxy._types import (
|
||||
|
|
@ -1668,19 +1669,20 @@ class _ContendedModelDb:
|
|||
self._other_writers = self._other_writers[1:]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ServingRouter:
|
||||
model_id: str
|
||||
discovered: Mapping[str, int]
|
||||
_AZURE_DEPLOYMENT_PARAMS: Final = MappingProxyType({"model": "azure/my-deployment", "api_key": "sk-existing"})
|
||||
_LOCAL_DEPLOYMENT_PARAMS: Final = MappingProxyType(
|
||||
{"model": "hosted_vllm/local-model", "api_base": "https://local.test/v1", "api_key": "local-key"}
|
||||
)
|
||||
|
||||
def get_model_ids(self) -> tuple[str, ...]:
|
||||
return (self.model_id,)
|
||||
|
||||
def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]:
|
||||
return self.discovered
|
||||
|
||||
def config_deployments(self) -> tuple[()]:
|
||||
return ()
|
||||
async def _discover_from_upstream(router: Router, model_card: Mapping[str, object]) -> None:
|
||||
handler: Final = AsyncHTTPHandler()
|
||||
await handler.client.aclose()
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": [dict(model_card)]}))
|
||||
) as client:
|
||||
handler.client = client
|
||||
await router.arefresh_model_info(client=handler)
|
||||
|
||||
|
||||
class TestUpdateModel:
|
||||
|
|
@ -1692,22 +1694,30 @@ class TestUpdateModel:
|
|||
async def _update_model(
|
||||
stored_model_info: Mapping[str, object],
|
||||
model_info: ModelInfo,
|
||||
discovered: Mapping[str, int] = MappingProxyType({}),
|
||||
litellm_params: Mapping[str, object] = _AZURE_DEPLOYMENT_PARAMS,
|
||||
upstream_model_card: Mapping[str, object] | None = None,
|
||||
other_writers: tuple[Mapping[str, object], ...] = (),
|
||||
) -> "_ContendedModelDb":
|
||||
db: Final = _ContendedModelDb(
|
||||
LiteLLM_ProxyModelTable(
|
||||
model_id="m1",
|
||||
model_name="my-model",
|
||||
litellm_params={"model": "azure/my-deployment", "api_key": "sk-existing"},
|
||||
litellm_params=dict(litellm_params),
|
||||
model_info=dict(stored_model_info),
|
||||
),
|
||||
other_writers,
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "my-model", "litellm_params": dict(litellm_params), "model_info": {"id": "m1"}}
|
||||
]
|
||||
)
|
||||
if upstream_model_card is not None:
|
||||
await _discover_from_upstream(router, upstream_model_card)
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", db),
|
||||
patch("litellm.proxy.proxy_server.llm_router", _ServingRouter(model_id="m1", discovered=discovered)),
|
||||
patch("litellm.proxy.proxy_server.llm_router", router),
|
||||
patch("litellm.proxy.proxy_server.store_model_in_db", True),
|
||||
patch("litellm.proxy.proxy_server.premium_user", True),
|
||||
patch( # test-quality-ok: [TQ008] the route reloads the proxy-global router with no seam to inject
|
||||
|
|
@ -1793,29 +1803,24 @@ class TestUpdateModel:
|
|||
assert not {"key", "max_tokens", "input_cost_per_token"} & persisted.keys()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_does_not_pin_catalog_values_echoed_without_a_key(self):
|
||||
async def test_update_model_saves_typed_values_that_equal_the_catalog(self):
|
||||
import litellm
|
||||
|
||||
catalog_entry: Final = litellm.get_model_info(model="azure/gpt-4o")
|
||||
deliberate_max_output_tokens: Final = catalog_entry["max_output_tokens"] // 2
|
||||
typed: Final = {
|
||||
"max_tokens": catalog_entry["max_tokens"],
|
||||
"max_input_tokens": catalog_entry["max_input_tokens"],
|
||||
"supports_vision": catalog_entry["supports_vision"],
|
||||
}
|
||||
|
||||
db: Final = await self._update_model(
|
||||
{"id": "m1"},
|
||||
ModelInfo(
|
||||
id="m1",
|
||||
base_model="azure/gpt-4o",
|
||||
max_tokens=catalog_entry["max_tokens"],
|
||||
max_input_tokens=catalog_entry["max_input_tokens"],
|
||||
supports_vision=catalog_entry["supports_vision"],
|
||||
max_output_tokens=deliberate_max_output_tokens,
|
||||
),
|
||||
{"id": "m1", "max_input_tokens": catalog_entry["max_input_tokens"] // 2},
|
||||
ModelInfo(id="m1", base_model="azure/gpt-4o", **typed),
|
||||
)
|
||||
|
||||
persisted: Final = db.row.model_info
|
||||
assert persisted is not None
|
||||
assert persisted["base_model"] == "azure/gpt-4o"
|
||||
assert persisted["max_output_tokens"] == deliberate_max_output_tokens
|
||||
assert not {"max_tokens", "max_input_tokens", "supports_vision"} & persisted.keys()
|
||||
assert {k: persisted.get(k) for k in typed} == typed
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_keeps_stored_value_when_a_field_is_sent_blank(self):
|
||||
|
|
@ -1868,40 +1873,39 @@ class TestUpdateModel:
|
|||
|
||||
assert dict(changes) == {"base_model": "test-provider/next"}
|
||||
|
||||
def test_update_model_does_not_pin_the_previous_base_models_loaded_catalog_value_echoed_without_a_key(self):
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_saves_a_value_the_previous_base_model_has_in_the_catalog(self):
|
||||
import litellm
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import legacy_update_model_info_changes
|
||||
|
||||
previous: Final = litellm.get_model_info(model="azure/gpt-4o")
|
||||
loaded_max_tokens: Final = previous["max_tokens"] + 1
|
||||
catalog: Final = MappingProxyType({previous["key"]: MappingProxyType({"max_tokens": loaded_max_tokens})})
|
||||
stored: Final = Deployment(
|
||||
model_name="my-model",
|
||||
litellm_params=LiteLLM_Params(model="test-provider/my-deployment"),
|
||||
model_info=ModelInfo(id="m1", base_model="azure/gpt-4o"),
|
||||
|
||||
db: Final = await self._update_model(
|
||||
{"id": "m1", "base_model": "azure/gpt-4o", "max_input_tokens": previous["max_input_tokens"] // 2},
|
||||
ModelInfo(id="m1", base_model="azure/gpt-4.1", max_input_tokens=previous["max_input_tokens"]),
|
||||
)
|
||||
|
||||
changes: Final = legacy_update_model_info_changes(
|
||||
stored,
|
||||
ModelInfo(id="m1", base_model="test-provider/next", max_tokens=loaded_max_tokens),
|
||||
loaded_catalog=lambda: catalog,
|
||||
)
|
||||
|
||||
assert dict(changes) == {"base_model": "test-provider/next"}
|
||||
persisted: Final = db.row.model_info
|
||||
assert persisted is not None
|
||||
assert {k: persisted.get(k) for k in ("base_model", "max_input_tokens")} == {
|
||||
"base_model": "azure/gpt-4.1",
|
||||
"max_input_tokens": previous["max_input_tokens"],
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_does_not_pin_discovered_limits_or_team_access_echoed_without_a_key(self):
|
||||
async def test_update_model_does_not_pin_discovered_limits_or_team_access_echoed_from_model_info(self):
|
||||
db: Final = await self._update_model(
|
||||
{"id": "m1"},
|
||||
ModelInfo(
|
||||
id="m1",
|
||||
base_model="test-provider/test-model",
|
||||
key="test-provider/test-model",
|
||||
max_input_tokens=128000,
|
||||
max_output_tokens=64000,
|
||||
direct_access=True,
|
||||
access_via_team_ids=["team-a"],
|
||||
),
|
||||
discovered=MappingProxyType({"max_input_tokens": 128000, "max_output_tokens": 32000}),
|
||||
litellm_params=_LOCAL_DEPLOYMENT_PARAMS,
|
||||
upstream_model_card=MappingProxyType({"id": "local-model", "max_model_len": 128000, "max_output_tokens": 32000}),
|
||||
)
|
||||
|
||||
persisted: Final = db.row.model_info
|
||||
|
|
@ -1910,6 +1914,17 @@ class TestUpdateModel:
|
|||
assert persisted["max_output_tokens"] == 64000
|
||||
assert not {"max_input_tokens", "direct_access", "access_via_team_ids"} & persisted.keys()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_does_not_copy_litellm_params_pricing_into_model_info(self):
|
||||
db: Final = await self._update_model(
|
||||
{"id": "m1"},
|
||||
ModelInfo(id="m1", tier="paid"),
|
||||
litellm_params={**_AZURE_DEPLOYMENT_PARAMS, "input_cost_per_token": 0.000003},
|
||||
)
|
||||
|
||||
persisted: Final = db.row.model_info
|
||||
assert persisted == {"id": "m1", "tier": "paid"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_resets_a_stored_override_echoed_back_at_its_catalog_value(self):
|
||||
import litellm
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue