mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): lock the model row and skip unchanged model_info on POST /model/update
This commit is contained in:
parent
56e35e8f6d
commit
ce02463ad2
2 changed files with 287 additions and 133 deletions
|
|
@ -463,6 +463,7 @@ async def _raise_on_invalid_credential_name(
|
|||
|
||||
AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY: Final = 5_872_301
|
||||
_CAPABILITY_LOCK_SQL: Final = "SELECT 1 AS locked FROM pg_advisory_xact_lock($1)"
|
||||
_MODEL_ROW_LOCK_SQL: Final = 'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = $1 FOR UPDATE'
|
||||
_STORED_LITELLM_PARAMS_SQL: Final = (
|
||||
"(CASE jsonb_typeof(litellm_params) WHEN 'string' THEN (litellm_params #>> '{}')::jsonb ELSE litellm_params END)"
|
||||
)
|
||||
|
|
@ -610,6 +611,7 @@ async def _auto_router_capability_slot(
|
|||
effective_params: Mapping[str, object],
|
||||
model_id: str | None,
|
||||
member_write: MemberAutoRouterWrite | None = None,
|
||||
lock_model_row: bool = False,
|
||||
) -> AsyncGenerator[_ProxyModelTable, None]:
|
||||
"""Hand out the model table to write through while the row's claim on a licensed capability is settled.
|
||||
|
||||
|
|
@ -626,6 +628,9 @@ async def _auto_router_capability_slot(
|
|||
|
||||
A heuristic-v1 router whose tuning has moved off its recorded baseline is judged the same
|
||||
way under the same lock, against the DB rows plus this proxy's config.yaml routers.
|
||||
|
||||
``lock_model_row`` runs the write in a transaction holding the row's ``FOR UPDATE`` lock,
|
||||
so a caller that re-reads the row through the yielded table writes onto its committed state.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import (
|
||||
_license_check, # pyright: ignore[reportPrivateUsage] # existing capability slot reads the proxy license singleton
|
||||
|
|
@ -639,7 +644,8 @@ async def _auto_router_capability_slot(
|
|||
baselines: Final = heuristic_v1_tuning_baselines
|
||||
tuning_candidate: Final = _tuning_candidate(effective_params, model_id=model_id)
|
||||
judges_tuning: Final = baselines is not None and is_mutable_tuned_candidate(tuning_candidate, baselines)
|
||||
if member_write is None and (limit is None or (not capabilities and not judges_tuning)):
|
||||
checks_license: Final = member_write is not None or (limit is not None and (bool(capabilities) or judges_tuning))
|
||||
if not checks_license and not lock_model_row:
|
||||
yield _proxy_model_table(prisma_client)
|
||||
return
|
||||
transaction_client: Final = _ModelTransactionClient.model_validate(prisma_client.db)
|
||||
|
|
@ -650,14 +656,15 @@ async def _auto_router_capability_slot(
|
|||
)
|
||||
async with transaction as tx_ctx:
|
||||
tables: Final[_TxModelTables] = tx_ctx
|
||||
await tx_ctx.query_raw(_CAPABILITY_LOCK_SQL, AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY)
|
||||
if checks_license:
|
||||
await tx_ctx.query_raw(_CAPABILITY_LOCK_SQL, AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY)
|
||||
locked_model_id: Final = (
|
||||
member_write.model_id if member_write is not None else (model_id if lock_model_row else None)
|
||||
)
|
||||
if locked_model_id is not None:
|
||||
await tx_ctx.query_raw(_MODEL_ROW_LOCK_SQL, locked_model_id)
|
||||
config_rows: Final = () if llm_router is None else tuple(llm_router.config_deployments())
|
||||
if member_write is not None:
|
||||
if member_write.model_id is not None:
|
||||
await tx_ctx.query_raw(
|
||||
'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = $1 FOR UPDATE',
|
||||
member_write.model_id,
|
||||
)
|
||||
pinned_client: Final = _TransactionClient(tx_ctx)
|
||||
team_where: Final[prisma_types.LiteLLM_TeamTableWhereUniqueInput] = {"team_id": member_write.team_id}
|
||||
team_include: Final[prisma_types.LiteLLM_TeamTableInclude] = {"litellm_model_table": True}
|
||||
|
|
@ -729,7 +736,8 @@ async def _auto_router_capability_slot(
|
|||
prisma_client=pinned_client,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
for capability in capabilities:
|
||||
judged_capabilities: Final = capabilities if checks_license else ()
|
||||
for capability in judged_capabilities:
|
||||
if (
|
||||
violation := await _claimed_capability_violation(
|
||||
tables, capability=capability, config_rows=config_rows, model_id=model_id, limit=limit
|
||||
|
|
@ -738,7 +746,7 @@ async def _auto_router_capability_slot(
|
|||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN, detail=f"{violation} {AUTO_ROUTER_LICENSE_REMEDY}"
|
||||
)
|
||||
if judges_tuning and baselines is not None:
|
||||
if checks_license and judges_tuning and baselines is not None:
|
||||
model_rows: Final = await ModelRepository(_TransactionClient(tx_ctx)).find_all_except(model_id or "")
|
||||
_raise_on_tuning_quota_violation(
|
||||
candidate=tuning_candidate,
|
||||
|
|
@ -1121,28 +1129,25 @@ def legacy_update_model_info_changes(
|
|||
Only fields the caller sent with a value count, so ``ModelInfo`` defaults and the blank
|
||||
strings typed clients send for unset fields never overwrite stored values. Server-owned,
|
||||
display-only, team, PTU and pricing fields keep their dedicated write paths, and values
|
||||
echoed from a ``/model/info`` response (the old or new base model's cost map entry, or
|
||||
limits discovered from the upstream) stay display-only. An echoed value that differs from
|
||||
the stored override resets that override (``None``), the way ``update_db_model`` does.
|
||||
echoed from a ``/model/info`` response stay display-only whether or not the echo kept its
|
||||
``key``: a value equal to the old or new base model's cost map entry (live or as loaded),
|
||||
the entry ``key`` names, or a limit discovered from the upstream. An echoed value that
|
||||
differs from the stored override resets that override (``None``), the way ``update_db_model`` does.
|
||||
"""
|
||||
incoming: Final = incoming_model_info.model_dump(exclude_none=True)
|
||||
stored: Final = db_model.model_info.model_dump(exclude_none=True)
|
||||
sent: Final = {
|
||||
k: v
|
||||
for k, v in incoming_model_info.model_dump(exclude_unset=True, exclude_none=True).items()
|
||||
if k not in _LEGACY_UPDATE_IGNORED_MODEL_INFO_FIELDS and v != ""
|
||||
}
|
||||
echoed: Final = frozenset(
|
||||
(
|
||||
*echoed_cost_map_fields(
|
||||
incoming,
|
||||
_cost_map_entry(db_model, sent),
|
||||
_cost_map_entry(db_model, stored),
|
||||
_loaded_catalog_entry(incoming, loaded_catalog),
|
||||
),
|
||||
*(k for k, v in sent.items() if discovered_model_info.get(k) == v),
|
||||
)
|
||||
live_entries: Final = (_cost_map_entry(db_model, sent), _cost_map_entry(db_model, stored))
|
||||
echo_sources: Final = (
|
||||
*live_entries,
|
||||
*(_loaded_catalog_entry(entry, loaded_catalog) for entry in live_entries),
|
||||
_loaded_catalog_entry(incoming_model_info.model_dump(exclude_none=True), loaded_catalog),
|
||||
discovered_model_info,
|
||||
)
|
||||
echoed: Final = frozenset(k for k, v in sent.items() if any(source.get(k) == v for source in echo_sources))
|
||||
kept: Final = without_server_derived_pricing(sent)
|
||||
return MappingProxyType(
|
||||
{
|
||||
|
|
@ -1152,6 +1157,34 @@ def legacy_update_model_info_changes(
|
|||
)
|
||||
|
||||
|
||||
def _legacy_update_model_info_writes(
|
||||
current: Deployment,
|
||||
incoming_model_info: ModelInfo,
|
||||
discovered_model_info: Mapping[str, object],
|
||||
member_marker: bool | None,
|
||||
) -> Mapping[str, object]:
|
||||
changes: Final = {
|
||||
**legacy_update_model_info_changes(current, incoming_model_info, discovered_model_info),
|
||||
**({"member_auto_router": member_marker} if member_marker is not None else {}),
|
||||
}
|
||||
return MappingProxyType({k: v for k, v in changes.items() if getattr(current.model_info, k, None) != v})
|
||||
|
||||
|
||||
async def _locked_legacy_update_model_info_json(
|
||||
table: _ProxyModelTable,
|
||||
model_id: str,
|
||||
incoming_model_info: ModelInfo,
|
||||
discovered_model_info: Mapping[str, object],
|
||||
member_marker: bool | None,
|
||||
) -> str | None:
|
||||
row: Final = await table.find_unique(where={"model_id": model_id})
|
||||
if row is None:
|
||||
return None
|
||||
current: Final = Deployment(**row.model_dump())
|
||||
writes: Final = _legacy_update_model_info_writes(current, incoming_model_info, discovered_model_info, member_marker)
|
||||
return current.model_info.model_copy(update=writes).model_dump_json(exclude_none=True) if writes else None
|
||||
|
||||
|
||||
def _param_keys_to_remove(patch: updateDeployment) -> tuple[str, ...]:
|
||||
return patch.litellm_params_to_remove if isinstance(patch, PatchModelRequest) else ()
|
||||
|
||||
|
|
@ -3084,34 +3117,29 @@ async def update_model(
|
|||
renamed_update: Final[PrismaCompatibleUpdateDBModel] = (
|
||||
{**base_update, "model_name": renamed_to} if renamed_to is not None else base_update
|
||||
)
|
||||
model_info_changes: Final = MappingProxyType(
|
||||
{
|
||||
**legacy_update_model_info_changes(
|
||||
deployment,
|
||||
_model_info,
|
||||
llm_router.get_discovered_model_info(_model_id)
|
||||
if llm_router is not None
|
||||
else MappingProxyType({}),
|
||||
),
|
||||
**({"member_auto_router": member_marker} if member_marker is not None else {}),
|
||||
}
|
||||
discovered_model_info: Final[Mapping[str, int]] = (
|
||||
llm_router.get_discovered_model_info(_model_id) if llm_router is not None else MappingProxyType({})
|
||||
)
|
||||
_data: Final[PrismaCompatibleUpdateDBModel] = (
|
||||
{
|
||||
**renamed_update,
|
||||
"model_info": deployment.model_info.model_copy(update=model_info_changes).model_dump_json(
|
||||
exclude_none=True
|
||||
),
|
||||
}
|
||||
if model_info_changes
|
||||
else renamed_update
|
||||
requested_model_info: Final = _legacy_update_model_info_writes(
|
||||
deployment, _model_info, discovered_model_info, member_marker
|
||||
)
|
||||
async with _auto_router_capability_slot(
|
||||
prisma_client,
|
||||
effective_params=effective_params,
|
||||
model_id=_model_id,
|
||||
member_write=member_write,
|
||||
lock_model_row=bool(requested_model_info),
|
||||
) as table:
|
||||
model_info_json: Final = (
|
||||
await _locked_legacy_update_model_info_json(
|
||||
table, _model_id, _model_info, discovered_model_info, member_marker
|
||||
)
|
||||
if requested_model_info
|
||||
else None
|
||||
)
|
||||
_data: Final[PrismaCompatibleUpdateDBModel] = (
|
||||
{**renamed_update, "model_info": model_info_json} if model_info_json is not None else renamed_update
|
||||
)
|
||||
model_response: Final = await table.update(
|
||||
where={"model_id": _model_id},
|
||||
data=_data,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@ import asyncio
|
|||
import contextlib
|
||||
import json
|
||||
import socket
|
||||
from collections.abc import Iterator, Mapping
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import Dict, Final, Optional
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
|
@ -1623,50 +1624,93 @@ class TestDeletedAutoRouterAvailability:
|
|||
}
|
||||
|
||||
|
||||
class _ContendedModelDb:
|
||||
def __init__(self, row: LiteLLM_ProxyModelTable, other_writers: tuple[Mapping[str, object], ...]) -> None:
|
||||
self.row = row
|
||||
self.writes: tuple[Mapping[str, object], ...] = ()
|
||||
self._other_writers = other_writers
|
||||
self._row_locked = False
|
||||
self.db = self
|
||||
self.litellm_proxymodeltable = self
|
||||
self.litellm_teamtable = SimpleNamespace(find_unique=self._find_team)
|
||||
|
||||
async def _find_team(self, *, where: Mapping[str, str]) -> LiteLLM_TeamTable:
|
||||
return LiteLLM_TeamTable(team_id=where["team_id"])
|
||||
|
||||
async def find_unique(self, *, where: Mapping[str, object]) -> LiteLLM_ProxyModelTable:
|
||||
read: Final = self.row
|
||||
if not self._row_locked and self._other_writers:
|
||||
self._commit_next_other_writer()
|
||||
return read
|
||||
|
||||
async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> LiteLLM_ProxyModelTable:
|
||||
self.writes = (*self.writes, data)
|
||||
self.row = self.row.model_copy(
|
||||
update={k: json.loads(v) if k in ("litellm_params", "model_info") else v for k, v in data.items()}
|
||||
)
|
||||
return self.row
|
||||
|
||||
async def query_raw(self, query: str, *args: object) -> tuple[()]:
|
||||
self._row_locked = self._row_locked or "FOR UPDATE" in query
|
||||
return ()
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def tx(self, timeout: object = None) -> AsyncIterator["_ContendedModelDb"]:
|
||||
yield self
|
||||
self._row_locked = False
|
||||
while self._other_writers:
|
||||
self._commit_next_other_writer()
|
||||
|
||||
def _commit_next_other_writer(self) -> None:
|
||||
self.row = self.row.model_copy(
|
||||
update={"model_info": {**(self.row.model_info or {}), **self._other_writers[0]}}
|
||||
)
|
||||
self._other_writers = self._other_writers[1:]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ServingRouter:
|
||||
model_id: str
|
||||
discovered: Mapping[str, int]
|
||||
|
||||
def get_model_ids(self) -> tuple[str, ...]:
|
||||
return (self.model_id,)
|
||||
|
||||
def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]:
|
||||
return self.discovered
|
||||
|
||||
def config_deployments(self) -> tuple[()]:
|
||||
return ()
|
||||
|
||||
|
||||
class TestUpdateModel:
|
||||
"""
|
||||
Tests for the update_model (POST /model/update) handler.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
async def _persisted_model_info(
|
||||
async def _update_model(
|
||||
stored_model_info: Mapping[str, object],
|
||||
model_info: ModelInfo,
|
||||
discovered: Mapping[str, int] = MappingProxyType({}),
|
||||
) -> object:
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_model
|
||||
|
||||
existing_row = MagicMock()
|
||||
existing_row.litellm_params = {"model": "azure/my-deployment", "api_key": "sk-existing"}
|
||||
existing_row.model_dump.return_value = {
|
||||
"model_name": "my-model",
|
||||
"litellm_params": existing_row.litellm_params,
|
||||
"model_info": dict(stored_model_info),
|
||||
}
|
||||
existing_row.model_dump_json.return_value = "{}"
|
||||
updated_row = MagicMock()
|
||||
updated_row.model_dump_json.return_value = "{}"
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=existing_row)
|
||||
mock_prisma.db.litellm_proxymodeltable.update = AsyncMock(return_value=updated_row)
|
||||
mock_router = MagicMock()
|
||||
mock_router.get_model_ids.return_value = [stored_model_info["id"]]
|
||||
mock_router.get_discovered_model_info.return_value = discovered
|
||||
stored_team_id: Final = stored_model_info.get("team_id")
|
||||
mock_prisma.db.litellm_teamtable.find_unique = AsyncMock(
|
||||
return_value=LiteLLM_TeamTable(team_id=stored_team_id) if isinstance(stored_team_id, str) else None
|
||||
other_writers: tuple[Mapping[str, object], ...] = (),
|
||||
) -> "_ContendedModelDb":
|
||||
db: Final = _ContendedModelDb(
|
||||
LiteLLM_ProxyModelTable(
|
||||
model_id="m1",
|
||||
model_name="my-model",
|
||||
litellm_params={"model": "azure/my-deployment", "api_key": "sk-existing"},
|
||||
model_info=dict(stored_model_info),
|
||||
),
|
||||
other_writers,
|
||||
)
|
||||
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
|
||||
patch("litellm.proxy.proxy_server.llm_router", mock_router),
|
||||
patch("litellm.proxy.proxy_server.prisma_client", db),
|
||||
patch("litellm.proxy.proxy_server.llm_router", _ServingRouter(model_id="m1", discovered=discovered)),
|
||||
patch("litellm.proxy.proxy_server.store_model_in_db", True),
|
||||
patch("litellm.proxy.proxy_server.premium_user", True),
|
||||
patch( # test-quality-ok: [TQ008] isolate persistence from encryption implementation
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.encrypt_value_helper",
|
||||
side_effect=lambda value: value,
|
||||
),
|
||||
patch( # test-quality-ok: [TQ008] isolate persistence from router reload implementation
|
||||
patch( # test-quality-ok: [TQ008] the route reloads the proxy-global router with no seam to inject
|
||||
"litellm.proxy.management_endpoints.model_management_endpoints.clear_cache",
|
||||
new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
|
||||
),
|
||||
|
|
@ -1676,19 +1720,20 @@ class TestUpdateModel:
|
|||
user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
written = mock_prisma.db.litellm_proxymodeltable.update.await_args.kwargs["data"]
|
||||
return json.loads(written["model_info"]) if "model_info" in written else None
|
||||
return db
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_persists_sent_model_info_over_stored_fields(self):
|
||||
stored = {"id": "m1", "db_model": True, "base_model": "my-deployment", "tier": "paid", "team_id": "team-a"}
|
||||
stored: Final = {"id": "m1", "db_model": True, "base_model": "my-deployment", "tier": "paid", "team_id": "team-a"}
|
||||
|
||||
persisted = await self._persisted_model_info(
|
||||
stored, ModelInfo(id="m1", base_model="azure/gpt-4o", mode="chat", db_model=False, team_id="team-b")
|
||||
db: Final = await self._update_model(
|
||||
stored,
|
||||
ModelInfo(id="m1", base_model="test-provider/test-model", mode="chat", db_model=False, team_id="team-b"),
|
||||
)
|
||||
|
||||
persisted: Final = db.row.model_info
|
||||
assert persisted is not None
|
||||
assert persisted["base_model"] == "azure/gpt-4o"
|
||||
assert persisted["base_model"] == "test-provider/test-model"
|
||||
assert persisted["mode"] == "chat"
|
||||
assert {k: persisted[k] for k in ("db_model", "tier", "team_id")} == {
|
||||
"db_model": True,
|
||||
|
|
@ -1698,14 +1743,41 @@ class TestUpdateModel:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_with_only_an_id_leaves_model_info_untouched(self):
|
||||
assert await self._persisted_model_info({"id": "m1", "base_model": "my-deployment"}, ModelInfo(id="m1")) is None
|
||||
db: Final = await self._update_model({"id": "m1", "base_model": "my-deployment"}, ModelInfo(id="m1"))
|
||||
|
||||
assert tuple("model_info" in write for write in db.writes) == (False,)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_resending_the_stored_model_info_does_not_rewrite_it(self):
|
||||
db: Final = await self._update_model(
|
||||
{"id": "m1", "db_model": True, "tier": "paid", "mode": "chat"},
|
||||
ModelInfo(id="m1", tier="paid", mode="chat"),
|
||||
)
|
||||
|
||||
assert tuple("model_info" in write for write in db.writes) == (False,)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_keeps_model_info_fields_a_concurrent_writer_stored(self):
|
||||
db: Final = await self._update_model(
|
||||
{"id": "m1", "db_model": True, "mode": "embedding"},
|
||||
ModelInfo(id="m1", mode="chat"),
|
||||
other_writers=(MappingProxyType({"access_groups": ["group-a"]}), MappingProxyType({"tier": "paid"})),
|
||||
)
|
||||
|
||||
persisted: Final = db.row.model_info
|
||||
assert persisted is not None
|
||||
assert {k: persisted.get(k) for k in ("mode", "access_groups", "tier")} == {
|
||||
"mode": "chat",
|
||||
"access_groups": ["group-a"],
|
||||
"tier": "paid",
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_does_not_pin_values_echoed_from_model_info(self):
|
||||
import litellm
|
||||
|
||||
catalog_entry = litellm.get_model_info(model="azure/gpt-4o")
|
||||
echoed = ModelInfo(
|
||||
catalog_entry: Final = litellm.get_model_info(model="azure/gpt-4o")
|
||||
echoed: Final = ModelInfo(
|
||||
id="m1",
|
||||
base_model="azure/gpt-4o",
|
||||
key=catalog_entry["key"],
|
||||
|
|
@ -1713,65 +1785,117 @@ class TestUpdateModel:
|
|||
input_cost_per_token=catalog_entry["input_cost_per_token"],
|
||||
)
|
||||
|
||||
persisted = await self._persisted_model_info({"id": "m1"}, echoed)
|
||||
db: Final = await self._update_model({"id": "m1"}, echoed)
|
||||
|
||||
persisted: Final = db.row.model_info
|
||||
assert persisted is not None
|
||||
assert persisted["base_model"] == "azure/gpt-4o"
|
||||
assert not {"key", "max_tokens", "input_cost_per_token"} & persisted.keys()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_keeps_stored_value_when_a_field_is_sent_blank(self):
|
||||
persisted = await self._persisted_model_info(
|
||||
{"id": "m1", "base_model": "my-deployment", "mode": "chat"},
|
||||
ModelInfo(id="m1", base_model="azure/gpt-4o", mode=""),
|
||||
)
|
||||
|
||||
assert persisted is not None
|
||||
assert persisted["base_model"] == "azure/gpt-4o"
|
||||
assert persisted["mode"] == "chat"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_ignores_ptu_blocked_and_pricing_fields(self):
|
||||
persisted = await self._persisted_model_info(
|
||||
{"id": "m1"},
|
||||
ModelInfo(id="m1", base_model="azure/gpt-4o", ptu_count=5, blocked=True, input_cost_per_token=0.5),
|
||||
)
|
||||
|
||||
assert persisted is not None
|
||||
assert persisted["base_model"] == "azure/gpt-4o"
|
||||
assert not {"ptu_count", "blocked", "input_cost_per_token"} & persisted.keys()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_does_not_pin_the_previous_base_models_echoed_values(self):
|
||||
async def test_update_model_does_not_pin_catalog_values_echoed_without_a_key(self):
|
||||
import litellm
|
||||
|
||||
previous = litellm.get_model_info(model="azure/gpt-4o")
|
||||
assert (
|
||||
previous["supported_openai_params"]
|
||||
!= litellm.get_model_info(model="azure/o3-mini")["supported_openai_params"]
|
||||
)
|
||||
catalog_entry: Final = litellm.get_model_info(model="azure/gpt-4o")
|
||||
deliberate_max_output_tokens: Final = catalog_entry["max_output_tokens"] // 2
|
||||
|
||||
persisted = await self._persisted_model_info(
|
||||
{"id": "m1", "base_model": "azure/gpt-4o"},
|
||||
ModelInfo(
|
||||
id="m1",
|
||||
base_model="azure/o3-mini",
|
||||
key=previous["key"],
|
||||
supported_openai_params=previous["supported_openai_params"],
|
||||
),
|
||||
)
|
||||
|
||||
assert persisted is not None
|
||||
assert persisted["base_model"] == "azure/o3-mini"
|
||||
assert "supported_openai_params" not in persisted
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_does_not_pin_discovered_limits_or_team_access_echoed_without_a_key(self):
|
||||
persisted = await self._persisted_model_info(
|
||||
db: Final = await self._update_model(
|
||||
{"id": "m1"},
|
||||
ModelInfo(
|
||||
id="m1",
|
||||
base_model="azure/gpt-4o",
|
||||
max_tokens=catalog_entry["max_tokens"],
|
||||
max_input_tokens=catalog_entry["max_input_tokens"],
|
||||
supports_vision=catalog_entry["supports_vision"],
|
||||
max_output_tokens=deliberate_max_output_tokens,
|
||||
),
|
||||
)
|
||||
|
||||
persisted: Final = db.row.model_info
|
||||
assert persisted is not None
|
||||
assert persisted["base_model"] == "azure/gpt-4o"
|
||||
assert persisted["max_output_tokens"] == deliberate_max_output_tokens
|
||||
assert not {"max_tokens", "max_input_tokens", "supports_vision"} & persisted.keys()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_keeps_stored_value_when_a_field_is_sent_blank(self):
|
||||
db: Final = await self._update_model(
|
||||
{"id": "m1", "base_model": "my-deployment", "mode": "chat"},
|
||||
ModelInfo(id="m1", base_model="test-provider/test-model", mode=""),
|
||||
)
|
||||
|
||||
persisted: Final = db.row.model_info
|
||||
assert persisted is not None
|
||||
assert persisted["base_model"] == "test-provider/test-model"
|
||||
assert persisted["mode"] == "chat"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_ignores_ptu_blocked_and_pricing_fields(self):
|
||||
db: Final = await self._update_model(
|
||||
{"id": "m1"},
|
||||
ModelInfo(
|
||||
id="m1", base_model="test-provider/test-model", ptu_count=5, blocked=True, input_cost_per_token=0.5
|
||||
),
|
||||
)
|
||||
|
||||
persisted: Final = db.row.model_info
|
||||
assert persisted is not None
|
||||
assert persisted["base_model"] == "test-provider/test-model"
|
||||
assert not {"ptu_count", "blocked", "input_cost_per_token"} & persisted.keys()
|
||||
|
||||
def test_update_model_does_not_pin_the_previous_base_models_echoed_values(self):
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import legacy_update_model_info_changes
|
||||
|
||||
catalog: Final = MappingProxyType(
|
||||
{"test-provider/previous": MappingProxyType({"supported_openai_params": ["previous-only-param"]})}
|
||||
)
|
||||
stored: Final = Deployment(
|
||||
model_name="my-model",
|
||||
litellm_params=LiteLLM_Params(model="test-provider/my-deployment"),
|
||||
model_info=ModelInfo(id="m1", base_model="test-provider/previous"),
|
||||
)
|
||||
|
||||
changes: Final = legacy_update_model_info_changes(
|
||||
stored,
|
||||
ModelInfo(
|
||||
id="m1",
|
||||
base_model="test-provider/next",
|
||||
key="test-provider/previous",
|
||||
supported_openai_params=["previous-only-param"],
|
||||
),
|
||||
loaded_catalog=lambda: catalog,
|
||||
)
|
||||
|
||||
assert dict(changes) == {"base_model": "test-provider/next"}
|
||||
|
||||
def test_update_model_does_not_pin_the_previous_base_models_loaded_catalog_value_echoed_without_a_key(self):
|
||||
import litellm
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import legacy_update_model_info_changes
|
||||
|
||||
previous: Final = litellm.get_model_info(model="azure/gpt-4o")
|
||||
loaded_max_tokens: Final = previous["max_tokens"] + 1
|
||||
catalog: Final = MappingProxyType({previous["key"]: MappingProxyType({"max_tokens": loaded_max_tokens})})
|
||||
stored: Final = Deployment(
|
||||
model_name="my-model",
|
||||
litellm_params=LiteLLM_Params(model="test-provider/my-deployment"),
|
||||
model_info=ModelInfo(id="m1", base_model="azure/gpt-4o"),
|
||||
)
|
||||
|
||||
changes: Final = legacy_update_model_info_changes(
|
||||
stored,
|
||||
ModelInfo(id="m1", base_model="test-provider/next", max_tokens=loaded_max_tokens),
|
||||
loaded_catalog=lambda: catalog,
|
||||
)
|
||||
|
||||
assert dict(changes) == {"base_model": "test-provider/next"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_does_not_pin_discovered_limits_or_team_access_echoed_without_a_key(self):
|
||||
db: Final = await self._update_model(
|
||||
{"id": "m1"},
|
||||
ModelInfo(
|
||||
id="m1",
|
||||
base_model="test-provider/test-model",
|
||||
max_input_tokens=128000,
|
||||
max_output_tokens=64000,
|
||||
direct_access=True,
|
||||
|
|
@ -1780,8 +1904,9 @@ class TestUpdateModel:
|
|||
discovered=MappingProxyType({"max_input_tokens": 128000, "max_output_tokens": 32000}),
|
||||
)
|
||||
|
||||
persisted: Final = db.row.model_info
|
||||
assert persisted is not None
|
||||
assert persisted["base_model"] == "azure/gpt-4o"
|
||||
assert persisted["base_model"] == "test-provider/test-model"
|
||||
assert persisted["max_output_tokens"] == 64000
|
||||
assert not {"max_input_tokens", "direct_access", "access_via_team_ids"} & persisted.keys()
|
||||
|
||||
|
|
@ -1789,16 +1914,17 @@ class TestUpdateModel:
|
|||
async def test_update_model_resets_a_stored_override_echoed_back_at_its_catalog_value(self):
|
||||
import litellm
|
||||
|
||||
catalog_entry = litellm.get_model_info(model="azure/gpt-4o")
|
||||
stored_override = catalog_entry["max_tokens"] // 2
|
||||
catalog_entry: Final = litellm.get_model_info(model="azure/gpt-4o")
|
||||
stored_override: Final = catalog_entry["max_tokens"] // 2
|
||||
|
||||
persisted = await self._persisted_model_info(
|
||||
db: Final = await self._update_model(
|
||||
{"id": "m1", "base_model": "azure/gpt-4o", "max_tokens": stored_override},
|
||||
ModelInfo(
|
||||
id="m1", base_model="azure/gpt-4o", key=catalog_entry["key"], max_tokens=catalog_entry["max_tokens"]
|
||||
),
|
||||
)
|
||||
|
||||
persisted: Final = db.row.model_info
|
||||
assert persisted is not None
|
||||
assert "max_tokens" not in persisted
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue