scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-01 07:03:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 129

fix(proxy)!: persist sent model_info fields on POST /model/update (#46902)

Browse source 
* fix(proxy): persist sent model_info fields on POST /model/update

POST /model/update rebuilt litellm_params from the request but dropped the
request's model_info, so changing base_model (or mode, tier, etc.) on an
existing DB model was silently ignored. Clients that only use this endpoint,
like the Terraform provider, could never fix a deployment's base_model
without deleting and recreating it.

Persist the model_info fields the caller actually sent, on top of the stored
ones. ModelInfo defaults never overwrite stored values, server-owned, team,
PTU and pricing fields stay on their dedicated write paths, and values echoed
from a /model/info response are not pinned.

* fix(proxy): keep echoed and blank model_info values out of POST /model/update writes

Blank strings from typed clients no longer overwrite stored values. Values echoed from
/model/info (the stored base model's cost map entry and discovered limits) are not
pinned, and an echoed catalog value resets a differing stored override like PATCH does.
direct_access and access_via_team_ids stay display-only. The tests stop patching the
auth check class and give the team lookup a real row instead.

* fix(proxy): lock the model row and skip unchanged model_info on POST /model/update

* fix(proxy): treat only keyed /model/update bodies as echoes and write the stored model_info as saved

A typed model_info value that matched the cost map, or the previous base model's entry,
was dropped and could erase a stored override. Echo detection now follows PATCH, only a
body carrying the /model/info key counts, compared against the base model the row has
after the write. The locked write patches the stored JSON instead of dumping the hydrated
Deployment, so litellm_params pricing is no longer copied into model_info. Tests run the
real Router with discovery over a mock upstream

* docs(terraform): note that model_info updates now take effect and overwrite values set outside Terraform

* test: expect POST /model/update to save the cache point flag, and wrap long test lines

test_x06 pinned the old contract, where POST /model/update answered 200 and ignored model_info.
This PR makes POST save model_info the way PATCH does, so x06 now asserts what its PATCH sibling
x01 asserts, the stored flag flips to true and cache points come back on.

* fix(proxy): answer 400 when POST /model/update gets an access_groups that is not a list of strings

* fix(proxy): read discovered model info through the router registry entry point

---------

Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
Maxime Boudier 2026-10-30 20:10:13 +01:00 • committed by GitHub
parent e752528ac8
commit b20acb4869
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 506 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

167
litellm/proxy/management_endpoints/model_management_endpoints.py
View file

@ -486,6 +486,7 @@ async def _raise_on_invalid_credential_name(
AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY: Final = 5_872_301
_CAPABILITY_LOCK_SQL: Final = "SELECT 1 AS locked FROM pg_advisory_xact_lock($1)"
_MODEL_ROW_LOCK_SQL: Final = 'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = $1 FOR UPDATE'
_STORED_LITELLM_PARAMS_SQL: Final = (
"(CASE jsonb_typeof(litellm_params) WHEN 'string' THEN (litellm_params #>> '{}')::jsonb ELSE litellm_params END)"
)
@ -633,6 +634,7 @@ async def _auto_router_capability_slot(
effective_params: Mapping[str, object],
model_id: str | None,
member_write: MemberAutoRouterWrite | None = None,
lock_model_row: bool = False,
) -> AsyncGenerator[_ProxyModelTable, None]:
"""Hand out the model table to write through while the row's claim on a licensed capability is settled.
@ -649,6 +651,9 @@ async def _auto_router_capability_slot(
A heuristic-v1 router whose tuning has moved off its recorded baseline is judged the same
way under the same lock, against the DB rows plus this proxy's config.yaml routers.
``lock_model_row`` runs the write in a transaction holding the row's ``FOR UPDATE`` lock,
so a caller that re-reads the row through the yielded table writes onto its committed state.
"""
from litellm.proxy.proxy_server import (
_license_check, # pyright: ignore[reportPrivateUsage] # existing capability slot reads the proxy license singleton
@ -662,7 +667,8 @@ async def _auto_router_capability_slot(
baselines: Final = heuristic_v1_tuning_baselines
tuning_candidate: Final = _tuning_candidate(effective_params, model_id=model_id)
judges_tuning: Final = baselines is not None and is_mutable_tuned_candidate(tuning_candidate, baselines)
if member_write is None and (limit is None or (not capabilities and not judges_tuning)):
checks_license: Final = member_write is not None or (limit is not None and (bool(capabilities) or judges_tuning))
if not checks_license and not lock_model_row:
yield _proxy_model_table(prisma_client)
return
transaction_client: Final = _ModelTransactionClient.model_validate(prisma_client.db)
@ -673,14 +679,15 @@ async def _auto_router_capability_slot(
)
async with transaction as tx_ctx:
tables: Final[_TxModelTables] = tx_ctx
await tx_ctx.query_raw(_CAPABILITY_LOCK_SQL, AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY)
if checks_license:
await tx_ctx.query_raw(_CAPABILITY_LOCK_SQL, AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY)
locked_model_id: Final = (
member_write.model_id if member_write is not None else (model_id if lock_model_row else None)
)
if locked_model_id is not None:
await tx_ctx.query_raw(_MODEL_ROW_LOCK_SQL, locked_model_id)
config_rows: Final = () if llm_router is None else tuple(llm_router.registry().deployments(source="config"))
if member_write is not None:
if member_write.model_id is not None:
await tx_ctx.query_raw(
'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = $1 FOR UPDATE',
member_write.model_id,
)
pinned_client: Final = _TransactionClient(tx_ctx)
team_where: Final[prisma_types.LiteLLM_TeamTableWhereUniqueInput] = {"team_id": member_write.team_id}
team_include: Final[prisma_types.LiteLLM_TeamTableInclude] = {"litellm_model_table": True}
@ -752,7 +759,8 @@ async def _auto_router_capability_slot(
prisma_client=pinned_client,
llm_router=llm_router,
)
for capability in capabilities:
judged_capabilities: Final = capabilities if checks_license else ()
for capability in judged_capabilities:
if (
violation := await _claimed_capability_violation(
tables, capability=capability, config_rows=config_rows, model_id=model_id, limit=limit
@ -761,7 +769,7 @@ async def _auto_router_capability_slot(
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN, detail=f"{violation} {AUTO_ROUTER_LICENSE_REMEDY}"
)
if judges_tuning and baselines is not None:
if checks_license and judges_tuning and baselines is not None:
model_rows: Final = await ModelRepository(_TransactionClient(tx_ctx)).find_all_except(model_id or "")
_raise_on_tuning_quota_violation(
candidate=tuning_candidate,
@ -1049,6 +1057,16 @@ def _raise_if_ptu_cost_attribution_disabled(incoming_model_info: Mapping[str, ob
)
_ACCESS_GROUPS: Final[TypeAdapter[tuple[str, ...] | None]] = TypeAdapter(tuple[str, ...] | None)
def _raise_if_access_groups_malformed(model_info_writes: Mapping[str, object]) -> None:
try:
_ACCESS_GROUPS.validate_python(model_info_writes.get("access_groups"))
except ValidationError:
raise HTTPException(status_code=400, detail="model_info.access_groups must be a list of strings") from None
def _validate_ptu_model_info(model_info: Mapping[str, object]) -> None:
"""Enforce the PTU cross-field invariant on the effective model_info.
@ -1291,6 +1309,109 @@ def _loaded_catalog_entry(
return loaded_catalog().get(catalog_key, MappingProxyType({}))
_LEGACY_UPDATE_IGNORED_MODEL_INFO_FIELDS: Final = frozenset(
{
"id",
"db_model",
"created_at",
"created_by",
"updated_at",
"updated_by",
"blocked",
"member_auto_router",
"team_id",
"team_public_model_name",
"direct_access",
"access_via_team_ids",
COST_MAP_LOOKUP_KEY,
*PTU_MODEL_INFO_FIELDS,
}
)
def legacy_update_model_info_changes(
db_model: Deployment,
incoming_model_info: ModelInfo,
discovered_model_info: Mapping[str, object] = MappingProxyType({}),
loaded_catalog: LoadedCatalog = GetModelCostMap.loaded_model_cost_map,
) -> Mapping[str, object]:
"""The ``model_info`` fields ``POST /model/update`` persists from its request.
Only fields the caller sent with a value count, so ``ModelInfo`` defaults and the blank
strings typed clients send for unset fields never overwrite stored values. Server-owned,
display-only, team, PTU and pricing fields keep their dedicated write paths. As on
``update_db_model``, only a body carrying the ``key`` that ``/model/info`` emits is an echo:
its values equal to the cost map entry of the base model the row has after the write, the
entry ``key`` names, or a limit discovered from the upstream stay display-only, and an echoed
value that differs from the stored override resets that override (``None``).
"""
stored: Final = db_model.model_info.model_dump(exclude_none=True)
requested: Final = incoming_model_info.model_dump(exclude_unset=True, exclude_none=True)
sent: Final = {k: v for k, v in requested.items() if k not in _LEGACY_UPDATE_IGNORED_MODEL_INFO_FIELDS and v != ""}
echoed: Final = frozenset(
echoed_cost_map_fields(
requested,
_cost_map_entry(db_model, {**stored, **sent}),
_loaded_catalog_entry(requested, loaded_catalog),
discovered_model_info,
)
)
kept: Final = without_server_derived_pricing(sent)
return MappingProxyType(
{
**{k: getattr(incoming_model_info, k) for k in kept if k not in echoed},
**{k: None for k, v in kept.items() if k in echoed and stored.get(k) not in (None, v)},
}
)
def _legacy_update_model_info_writes(
current: Deployment,
incoming_model_info: ModelInfo,
discovered_model_info: Mapping[str, object],
member_marker: bool | None,
) -> Mapping[str, object]:
changes: Final = {
**legacy_update_model_info_changes(current, incoming_model_info, discovered_model_info),
**({"member_auto_router": member_marker} if member_marker is not None else {}),
}
return MappingProxyType({k: v for k, v in changes.items() if getattr(current.model_info, k, None) != v})
async def _locked_legacy_update_model_info_json(
table: _ProxyModelTable,
model_id: str,
incoming_model_info: ModelInfo,
discovered_model_info: Mapping[str, object],
member_marker: bool | None,
) -> str | None:
row: Final = await table.find_unique(where={"model_id": model_id})
if row is None:
return None
payload: Final = row.model_dump()
writes: Final = _legacy_update_model_info_writes(
Deployment(**payload), incoming_model_info, discovered_model_info, member_marker
)
if not writes:
return None
stored_model_info: Final[object] = payload.get("model_info")
stored: Final = (
TypeAdapter(dict[str, object]).validate_json(stored_model_info)
if isinstance(stored_model_info, str)
else TypeAdapter(dict[str, object]).validate_python(stored_model_info or {})
)
return (
TypeAdapter(dict[str, object])
.dump_json(
{
**{k: v for k, v in stored.items() if k not in writes},
**{k: v for k, v in writes.items() if v is not None},
}
)
.decode()
)
def _param_keys_to_remove(patch: updateDeployment) -> tuple[str, ...]:
return patch.litellm_params_to_remove if isinstance(patch, PatchModelRequest) else ()
@ -3279,22 +3400,32 @@ async def update_model(
renamed_update: Final[PrismaCompatibleUpdateDBModel] = (
{**base_update, "model_name": renamed_to} if renamed_to is not None else base_update
)
_data: Final[PrismaCompatibleUpdateDBModel] = (
{
**renamed_update,
"model_info": deployment.model_info.model_copy(
update=MappingProxyType({"member_auto_router": member_marker})
).model_dump_json(exclude_none=True),
}
if member_marker is not None
else renamed_update
discovered_model_info: Final[Mapping[str, int]] = (
llm_router.registry().discovered_model_info(_model_id)
if llm_router is not None
else MappingProxyType({})
)
requested_model_info: Final = _legacy_update_model_info_writes(
deployment, _model_info, discovered_model_info, member_marker
)
_raise_if_access_groups_malformed(requested_model_info)
async with _auto_router_capability_slot(
prisma_client,
effective_params=effective_params,
model_id=_model_id,
member_write=member_write,
lock_model_row=bool(requested_model_info),
) as table:
model_info_json: Final = (
await _locked_legacy_update_model_info_json(
table, _model_id, _model_info, discovered_model_info, member_marker
)
if requested_model_info
else None
)
_data: Final[PrismaCompatibleUpdateDBModel] = (
{**renamed_update, "model_info": model_info_json} if model_info_json is not None else renamed_update
)
model_response: Final = await table.update(
where={"model_id": _model_id},
data=_data,

3
terraform/provider/CHANGELOG.md
View file

@ -18,7 +18,7 @@ longer signal it.
- **credential, model, mcp_server**: Write-only secret attributes so secrets never land in Terraform state (Terraform 1.11 or later). `litellm_credential` gains `credential_values_wo` (a JSON object, `jsonencode({...})`, since the SDK cannot mark a map write-only) with `credential_values_wo_version`, and `credential_values` becomes optional with exactly one of the two required. `litellm_model` gains `model_api_key_wo`, `aws_secret_access_key_wo` and `vertex_credentials_wo`, each with a `_wo_version` companion and each conflicting with its in-state twin. `litellm_mcp_server` gains `client_id`, `client_secret_wo` and `auth_value_wo` (with `_wo_version` companions), sent to the proxy as `credentials`, and `auth_type` now accepts every value the proxy does (`oauth2`, `api_key`, `bearer_token`, ...). Every `_wo` value is sent on each apply from the configuration and produces no diff, so bump its `_wo_version` to push a rotated secret. `litellm_vector_store.litellm_params` warns at plan time when it carries a secret-looking key such as `api_key`
- **team**: Optional `team_member_model_max_budget` argument on `litellm_team`, a JSON string of per-model budgets applied to every team member and sent as `team_member_model_max_budget` on `/team/new` and `/team/update`; read back from `team_member_budget_table.model_max_budget` on `/team/info` so out-of-band changes show as drift, with `budget_limit`/`time_period` treated as equal to the `max_budget`/`budget_duration` spelling the proxy stores; removing it sends `{}`, which clears every per-model member budget; `tpm_limit`/`rpm_limit` are rejected at plan time because the proxy rejects them on team budgets. Needs a proxy that includes [#43887](https://github.com/BerriAI/litellm/pull/43887): v1.105.0 and v1.106.0-rc.1 accept the field but silently drop it, so every plan shows a diff
- **model**: Optional `display_name` argument on `litellm_model`, sent as `model_info.display_name` and returned as `display_name` by `/v1/models`, so client model pickers show a readable name; changes are persisted through `/model/{id}/update` since `/model/update` ignores `model_info`; also exported by the `litellm_model` and `litellm_models` data sources
- **model**: Optional `display_name` argument on `litellm_model`, sent as `model_info.display_name` and returned as `display_name` by `/v1/models`, so client model pickers show a readable name; changes are also persisted through `/model/{id}/update`, since proxies older than this provider drop the `model_info` sent to `/model/update`; also exported by the `litellm_model` and `litellm_models` data sources
- **key**: Computed `server_metadata` attribute on `litellm_key` exposing every metadata entry the proxy stores, so metadata created outside Terraform is visible in state and drift on it shows on refresh, while `metadata` keeps tracking only the declared entries and updates keep preserving undeclared ones
- **team_member_add**: `tpm_limit`, `rpm_limit`, `budget_duration`, and `allowed_models` attributes on `litellm_team_member_add`, applied to every member of the resource; `budget_duration` and `allowed_models` ride on `/team/member_add`, while the limits are sent through `/team/member_update`, which is where the proxy accepts them
- **team**: Optional `team_id` argument on `litellm_team`, so teams can be created with a stable, human-readable ID instead of a provider-generated UUID; changing it forces replacement
@ -70,6 +70,7 @@ longer signal it.
### Changed
- **model** (breaking): on a proxy that includes [#46902](https://github.com/BerriAI/litellm/pull/46902), `litellm_model` updates now save the `model_info` the provider sends to `/model/update`, so `pricing_base_model`, `tier` and `mode` changes take effect. The first apply after upgrading the proxy overwrites values set outside Terraform with the configuration's, including `tier`, which defaults to `free`, and `base_model`, which falls back to the deployment name when `pricing_base_model` is unset. Run `terraform plan` before that apply and copy any value to keep into the configuration
- **model** (breaking): a key named in `additional_litellm_params.additional_drop_params` is no longer deleted from the deployment's `litellm_params`. A configuration that also sets that key, such as `vertex_location`, stores it on the deployment at its next apply, and the proxy applies the drop list to requests instead. For `vertex_location` this changes routing. `/v1/messages` calls now go to the configured region, while chat completions drop a listed `vertex_location` and use the default region. To keep such a key off a deployment, remove its attribute and replace the model with `terraform apply -replace`, which gives it a new model id, since an update keeps a stored value the provider leaves out. `model`, `custom_llm_provider` and `merge_reasoning_content_in_choices` are always sent. A value that is not a JSON list of names, such as a bare `"store"` or an empty string, now fails the plan, and with it `terraform destroy`, instead of being stored as a string the proxy ignores. Rewrite it as `jsonencode([...])`, or remove the model without fixing it through `terraform destroy -refresh=false`
- **key** (breaking): `model_max_budget` on `litellm_key` is now a JSON string of per-model budget objects (`jsonencode({"gpt-4o-mini" = {budget_limit = 50, time_period = "30d"}})`), matching `litellm_user`, `litellm_budget` and `litellm_tag`. The old `map(number)` form sent bare numbers to `/key/generate`, which the proxy rejects with a 500 (`'int' object is not iterable`), so every key with a non-empty `model_max_budget` failed to apply. Existing state upgrades automatically (schema version 1) and the attribute is refilled from the proxy on the next read; configurations still using the map form must be rewritten
- **Versioning**: the provider is now published at the LiteLLM version, from the same commit as the proxy, on every LiteLLM release (dev, rc, stable). The `0.x` line ends at `0.4.0`; a `~> 0.4` constraint will not receive further releases, so re-pin to the LiteLLM version your proxy runs (for example `~> 1.99.0`). Existing `0.x` versions remain in the registry and keep verifying

2
terraform/provider/litellm/resource_model_crud.go
View file

@ -357,7 +357,7 @@ func createOrUpdateModel(d *schema.ResourceData, m interface{}, isUpdate bool) e
return retryModelRead(d, m, 5)
}
// /model/update only merges litellm_params, so model_info changes go through the PATCH endpoint.
// Older proxies drop the model_info sent to /model/update, so display_name also goes through the PATCH endpoint.
func patchModelDisplayName(client *Client, modelID, displayName string) error {
resp, err := MakeRequest(client, "PATCH", fmt.Sprintf(endpointModelPatch, url.PathEscape(modelID)), ModelInfoPatch{
ModelInfo: ModelInfoPatchFields{ID: modelID, DisplayName: displayName},

18
tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py
View file

@ -1325,7 +1325,9 @@ def test_x01_flipping_the_flag_to_true_through_a_patch_update_turns_cache_points
@pytest.mark.timeout(600)
def test_x06_the_legacy_post_update_answers_200_and_leaves_the_stored_flag_alone(gateway: Gateway) -> None:
def test_x06_flipping_the_flag_to_true_through_the_legacy_post_update_turns_cache_points_back_on(
gateway: Gateway,
) -> None:
arn: Final = _arn("legacy")
with wire_server(_peer()) as wire, gateway.scenario() as scenario:
name: Final = _deployment(
@ -1333,6 +1335,7 @@ def test_x06_the_legacy_post_update_answers_200_and_leaves_the_stored_flag_alone
)
before, before_received = _observe_at(gateway, wire, "chat", _chat_body(name, _prompt(), _SYSTEM_AND_USER))
_assert_answered(before)
assert before_received.cache_points == 0, before_received.body
identity: Final = _model_id(name)
updated: Final = gateway.request(
"POST",
@ -1344,13 +1347,14 @@ def test_x06_the_legacy_post_update_answers_200_and_leaves_the_stored_flag_alone
},
)
assert updated.status_code == 200, updated.text
assert _stored_flag(identity) is False
_settled(gateway, name, wire)
after, after_received = _observe_at(gateway, wire, "chat", _chat_body(name, _prompt(), _SYSTEM_AND_USER))
_assert_answered(after)
assert after_received.cache_points == before_received.cache_points, (before_received.body, after_received.body)
assert _stored_flag(identity) is True
points: Final = eventually(
lambda: _six_cache_point_counts(gateway, wire, name),
lambda counts: len(counts) == 6 and all(count == 2 for count in counts),
seconds=60,
)
assert points == (2,) * 6, points
_success_row(before.response_id)
_success_row(after.response_id)
@dataclass(frozen=True, slots=True)

345
tests/unit/proxy/management_endpoints/test_model_management_endpoints.py
View file

@ -3,17 +3,19 @@ import contextlib
import inspect
import json
import socket
from collections.abc import Iterator, Mapping
from types import SimpleNamespace
from collections.abc import AsyncIterator, Iterator, Mapping
from types import MappingProxyType, SimpleNamespace
from typing import Dict, Final, Optional
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from fastapi import HTTPException
from fastapi.encoders import jsonable_encoder
from fastapi.testclient import TestClient
from litellm._uuid import uuid
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.models.credentials import CredentialItem
from litellm.models.project import LiteLLM_ProjectTable
@ -1639,11 +1641,350 @@ class TestDeletedAutoRouterAvailability:
}
class _ContendedModelDb:
def __init__(self, row: LiteLLM_ProxyModelTable, other_writers: tuple[Mapping[str, object], ...]) -> None:
self.row = row
self.writes: tuple[Mapping[str, object], ...] = ()
self._other_writers = other_writers
self._row_locked = False
self.db = self
self.litellm_proxymodeltable = self
self.litellm_teamtable = SimpleNamespace(find_unique=self._find_team)
async def _find_team(self, *, where: Mapping[str, str]) -> LiteLLM_TeamTable:
return LiteLLM_TeamTable(team_id=where["team_id"])
async def find_unique(self, *, where: Mapping[str, object]) -> LiteLLM_ProxyModelTable:
read: Final = self.row
if not self._row_locked and self._other_writers:
self._commit_next_other_writer()
return read
async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> LiteLLM_ProxyModelTable:
self.writes = (*self.writes, data)
self.row = self.row.model_copy(
update={k: json.loads(v) if k in ("litellm_params", "model_info") else v for k, v in data.items()}
)
return self.row
async def query_raw(self, query: str, *args: object) -> tuple[()]:
self._row_locked = self._row_locked or "FOR UPDATE" in query
return ()
@contextlib.asynccontextmanager
async def tx(self, timeout: object = None) -> AsyncIterator["_ContendedModelDb"]:
yield self
self._row_locked = False
while self._other_writers:
self._commit_next_other_writer()
def _commit_next_other_writer(self) -> None:
self.row = self.row.model_copy(
update={"model_info": {**(self.row.model_info or {}), **self._other_writers[0]}}
)
self._other_writers = self._other_writers[1:]
_AZURE_DEPLOYMENT_PARAMS: Final = MappingProxyType({"model": "azure/my-deployment", "api_key": "sk-existing"})
_LOCAL_DEPLOYMENT_PARAMS: Final = MappingProxyType(
{"model": "hosted_vllm/local-model", "api_base": "https://local.test/v1", "api_key": "local-key"}
)
async def _discover_from_upstream(router: Router, model_card: Mapping[str, object]) -> None:
handler: Final = AsyncHTTPHandler()
await handler.client.aclose()
async with httpx.AsyncClient(
transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": [dict(model_card)]}))
) as client:
handler.client = client
await router.arefresh_model_info(client=handler)
class TestUpdateModel:
"""
Tests for the update_model (POST /model/update) handler.
"""
@staticmethod
async def _update_model(
stored_model_info: Mapping[str, object],
model_info: ModelInfo,
litellm_params: Mapping[str, object] = _AZURE_DEPLOYMENT_PARAMS,
upstream_model_card: Mapping[str, object] | None = None,
other_writers: tuple[Mapping[str, object], ...] = (),
) -> "_ContendedModelDb":
db: Final = _ContendedModelDb(
LiteLLM_ProxyModelTable(
model_id="m1",
model_name="my-model",
litellm_params=dict(litellm_params),
model_info=dict(stored_model_info),
),
other_writers,
)
router: Final = Router(
model_list=[
{"model_name": "my-model", "litellm_params": dict(litellm_params), "model_info": {"id": "m1"}}
]
)
if upstream_model_card is not None:
await _discover_from_upstream(router, upstream_model_card)
with (
patch("litellm.proxy.proxy_server.prisma_client", db),
patch("litellm.proxy.proxy_server.llm_router", router),
patch("litellm.proxy.proxy_server.store_model_in_db", True),
patch("litellm.proxy.proxy_server.premium_user", True),
patch( # test-quality-ok: [TQ008] the route reloads the proxy-global router with no seam to inject
"litellm.proxy.management_endpoints.model_management_endpoints.clear_cache",
new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
),
):
await update_model(
model_params=updateDeployment(litellm_params=updateLiteLLMParams(), model_info=model_info),
user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
)
return db
@pytest.mark.asyncio
async def test_update_model_persists_sent_model_info_over_stored_fields(self):
stored: Final = {
"id": "m1",
"db_model": True,
"base_model": "my-deployment",
"tier": "paid",
"team_id": "team-a",
}
db: Final = await self._update_model(
stored,
ModelInfo(id="m1", base_model="test-provider/test-model", mode="chat", db_model=False, team_id="team-b"),
)
persisted: Final = db.row.model_info
assert persisted is not None
assert persisted["base_model"] == "test-provider/test-model"
assert persisted["mode"] == "chat"
assert {k: persisted[k] for k in ("db_model", "tier", "team_id")} == {
"db_model": True,
"tier": "paid",
"team_id": "team-a",
}
@pytest.mark.asyncio
async def test_update_model_with_only_an_id_leaves_model_info_untouched(self):
db: Final = await self._update_model({"id": "m1", "base_model": "my-deployment"}, ModelInfo(id="m1"))
assert tuple("model_info" in write for write in db.writes) == (False,)
@pytest.mark.asyncio
async def test_update_model_resending_the_stored_model_info_does_not_rewrite_it(self):
db: Final = await self._update_model(
{"id": "m1", "db_model": True, "tier": "paid", "mode": "chat"},
ModelInfo(id="m1", tier="paid", mode="chat"),
)
assert tuple("model_info" in write for write in db.writes) == (False,)
@pytest.mark.asyncio
async def test_update_model_keeps_model_info_fields_a_concurrent_writer_stored(self):
db: Final = await self._update_model(
{"id": "m1", "db_model": True, "mode": "embedding"},
ModelInfo(id="m1", mode="chat"),
other_writers=(MappingProxyType({"access_groups": ["group-a"]}), MappingProxyType({"tier": "paid"})),
)
persisted: Final = db.row.model_info
assert persisted is not None
assert {k: persisted.get(k) for k in ("mode", "access_groups", "tier")} == {
"mode": "chat",
"access_groups": ["group-a"],
"tier": "paid",
}
@pytest.mark.asyncio
async def test_update_model_does_not_pin_values_echoed_from_model_info(self):
import litellm
catalog_entry: Final = litellm.get_model_info(model="azure/gpt-4o")
echoed: Final = ModelInfo(
id="m1",
base_model="azure/gpt-4o",
key=catalog_entry["key"],
max_tokens=catalog_entry["max_tokens"],
input_cost_per_token=catalog_entry["input_cost_per_token"],
)
db: Final = await self._update_model({"id": "m1"}, echoed)
persisted: Final = db.row.model_info
assert persisted is not None
assert persisted["base_model"] == "azure/gpt-4o"
assert not {"key", "max_tokens", "input_cost_per_token"} & persisted.keys()
@pytest.mark.asyncio
async def test_update_model_saves_typed_values_that_equal_the_catalog(self):
import litellm
catalog_entry: Final = litellm.get_model_info(model="azure/gpt-4o")
typed: Final = {
"max_tokens": catalog_entry["max_tokens"],
"max_input_tokens": catalog_entry["max_input_tokens"],
"supports_vision": catalog_entry["supports_vision"],
}
db: Final = await self._update_model(
{"id": "m1", "max_input_tokens": catalog_entry["max_input_tokens"] // 2},
ModelInfo(id="m1", base_model="azure/gpt-4o", **typed),
)
persisted: Final = db.row.model_info
assert persisted is not None
assert {k: persisted.get(k) for k in typed} == typed
@pytest.mark.asyncio
async def test_update_model_keeps_stored_value_when_a_field_is_sent_blank(self):
db: Final = await self._update_model(
{"id": "m1", "base_model": "my-deployment", "mode": "chat"},
ModelInfo(id="m1", base_model="test-provider/test-model", mode=""),
)
persisted: Final = db.row.model_info
assert persisted is not None
assert persisted["base_model"] == "test-provider/test-model"
assert persisted["mode"] == "chat"
@pytest.mark.asyncio
async def test_update_model_ignores_ptu_blocked_and_pricing_fields(self):
db: Final = await self._update_model(
{"id": "m1"},
ModelInfo(
id="m1", base_model="test-provider/test-model", ptu_count=5, blocked=True, input_cost_per_token=0.5
),
)
persisted: Final = db.row.model_info
assert persisted is not None
assert persisted["base_model"] == "test-provider/test-model"
assert not {"ptu_count", "blocked", "input_cost_per_token"} & persisted.keys()
@pytest.mark.asyncio
@pytest.mark.parametrize("access_groups", [5, "group-a", ["group-a", 5], {"group": "group-a"}])
async def test_update_model_rejects_access_groups_that_are_not_a_list_of_strings(self, access_groups: object):
with pytest.raises(ProxyException) as error:
await self._update_model({"id": "m1"}, ModelInfo.model_validate({"id": "m1", "access_groups": access_groups}))
assert error.value.code == "400"
@pytest.mark.asyncio
async def test_update_model_saves_access_groups_sent_as_a_list_of_strings(self):
db: Final = await self._update_model(
{"id": "m1"}, ModelInfo.model_validate({"id": "m1", "access_groups": ["group-a", "group-b"]})
)
persisted: Final = db.row.model_info
assert persisted is not None
assert persisted["access_groups"] == ["group-a", "group-b"]
def test_update_model_does_not_pin_the_previous_base_models_echoed_values(self):
from litellm.proxy.management_endpoints.model_management_endpoints import legacy_update_model_info_changes
catalog: Final = MappingProxyType(
{"test-provider/previous": MappingProxyType({"supported_openai_params": ["previous-only-param"]})}
)
stored: Final = Deployment(
model_name="my-model",
litellm_params=LiteLLM_Params(model="test-provider/my-deployment"),
model_info=ModelInfo(id="m1", base_model="test-provider/previous"),
)
changes: Final = legacy_update_model_info_changes(
stored,
ModelInfo(
id="m1",
base_model="test-provider/next",
key="test-provider/previous",
supported_openai_params=["previous-only-param"],
),
loaded_catalog=lambda: catalog,
)
assert dict(changes) == {"base_model": "test-provider/next"}
@pytest.mark.asyncio
async def test_update_model_saves_a_value_the_previous_base_model_has_in_the_catalog(self):
import litellm
previous: Final = litellm.get_model_info(model="azure/gpt-4o")
db: Final = await self._update_model(
{"id": "m1", "base_model": "azure/gpt-4o", "max_input_tokens": previous["max_input_tokens"] // 2},
ModelInfo(id="m1", base_model="azure/gpt-4.1", max_input_tokens=previous["max_input_tokens"]),
)
persisted: Final = db.row.model_info
assert persisted is not None
assert {k: persisted.get(k) for k in ("base_model", "max_input_tokens")} == {
"base_model": "azure/gpt-4.1",
"max_input_tokens": previous["max_input_tokens"],
}
@pytest.mark.asyncio
async def test_update_model_does_not_pin_discovered_limits_or_team_access_echoed_from_model_info(self):
db: Final = await self._update_model(
{"id": "m1"},
ModelInfo(
id="m1",
base_model="test-provider/test-model",
key="test-provider/test-model",
max_input_tokens=128000,
max_output_tokens=64000,
direct_access=True,
access_via_team_ids=["team-a"],
),
litellm_params=_LOCAL_DEPLOYMENT_PARAMS,
upstream_model_card=MappingProxyType(
{"id": "local-model", "max_model_len": 128000, "max_output_tokens": 32000}
),
)
persisted: Final = db.row.model_info
assert persisted is not None
assert persisted["base_model"] == "test-provider/test-model"
assert persisted["max_output_tokens"] == 64000
assert not {"max_input_tokens", "direct_access", "access_via_team_ids"} & persisted.keys()
@pytest.mark.asyncio
async def test_update_model_does_not_copy_litellm_params_pricing_into_model_info(self):
db: Final = await self._update_model(
{"id": "m1"},
ModelInfo(id="m1", tier="paid"),
litellm_params={**_AZURE_DEPLOYMENT_PARAMS, "input_cost_per_token": 0.000003},
)
persisted: Final = db.row.model_info
assert persisted == {"id": "m1", "tier": "paid"}
@pytest.mark.asyncio
async def test_update_model_resets_a_stored_override_echoed_back_at_its_catalog_value(self):
import litellm
catalog_entry: Final = litellm.get_model_info(model="azure/gpt-4o")
stored_override: Final = catalog_entry["max_tokens"] // 2
db: Final = await self._update_model(
{"id": "m1", "base_model": "azure/gpt-4o", "max_tokens": stored_override},
ModelInfo(
id="m1", base_model="azure/gpt-4o", key=catalog_entry["key"], max_tokens=catalog_entry["max_tokens"]
),
)
persisted: Final = db.row.model_info
assert persisted is not None
assert "max_tokens" not in persisted
@pytest.mark.asyncio
async def test_update_model_clears_cache_after_db_write(self):
"""