fix(proxy): lock the model row and skip unchanged model_info on POST /model/update

This commit is contained in:
mateo-berri 2026-10-28 11:22:01 -07:00
parent 56e35e8f6d
commit ce02463ad2
2 changed files with 287 additions and 133 deletions

View file

@ -463,6 +463,7 @@ async def _raise_on_invalid_credential_name(
AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY: Final = 5_872_301
_CAPABILITY_LOCK_SQL: Final = "SELECT 1 AS locked FROM pg_advisory_xact_lock($1)"
_MODEL_ROW_LOCK_SQL: Final = 'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = $1 FOR UPDATE'
_STORED_LITELLM_PARAMS_SQL: Final = (
"(CASE jsonb_typeof(litellm_params) WHEN 'string' THEN (litellm_params #>> '{}')::jsonb ELSE litellm_params END)"
)
@ -610,6 +611,7 @@ async def _auto_router_capability_slot(
effective_params: Mapping[str, object],
model_id: str | None,
member_write: MemberAutoRouterWrite | None = None,
lock_model_row: bool = False,
) -> AsyncGenerator[_ProxyModelTable, None]:
"""Hand out the model table to write through while the row's claim on a licensed capability is settled.
@ -626,6 +628,9 @@ async def _auto_router_capability_slot(
A heuristic-v1 router whose tuning has moved off its recorded baseline is judged the same
way under the same lock, against the DB rows plus this proxy's config.yaml routers.
``lock_model_row`` runs the write in a transaction holding the row's ``FOR UPDATE`` lock,
so a caller that re-reads the row through the yielded table writes onto its committed state.
"""
from litellm.proxy.proxy_server import (
_license_check, # pyright: ignore[reportPrivateUsage] # existing capability slot reads the proxy license singleton
@ -639,7 +644,8 @@ async def _auto_router_capability_slot(
baselines: Final = heuristic_v1_tuning_baselines
tuning_candidate: Final = _tuning_candidate(effective_params, model_id=model_id)
judges_tuning: Final = baselines is not None and is_mutable_tuned_candidate(tuning_candidate, baselines)
if member_write is None and (limit is None or (not capabilities and not judges_tuning)):
checks_license: Final = member_write is not None or (limit is not None and (bool(capabilities) or judges_tuning))
if not checks_license and not lock_model_row:
yield _proxy_model_table(prisma_client)
return
transaction_client: Final = _ModelTransactionClient.model_validate(prisma_client.db)
@ -650,14 +656,15 @@ async def _auto_router_capability_slot(
)
async with transaction as tx_ctx:
tables: Final[_TxModelTables] = tx_ctx
await tx_ctx.query_raw(_CAPABILITY_LOCK_SQL, AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY)
if checks_license:
await tx_ctx.query_raw(_CAPABILITY_LOCK_SQL, AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY)
locked_model_id: Final = (
member_write.model_id if member_write is not None else (model_id if lock_model_row else None)
)
if locked_model_id is not None:
await tx_ctx.query_raw(_MODEL_ROW_LOCK_SQL, locked_model_id)
config_rows: Final = () if llm_router is None else tuple(llm_router.config_deployments())
if member_write is not None:
if member_write.model_id is not None:
await tx_ctx.query_raw(
'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = $1 FOR UPDATE',
member_write.model_id,
)
pinned_client: Final = _TransactionClient(tx_ctx)
team_where: Final[prisma_types.LiteLLM_TeamTableWhereUniqueInput] = {"team_id": member_write.team_id}
team_include: Final[prisma_types.LiteLLM_TeamTableInclude] = {"litellm_model_table": True}
@ -729,7 +736,8 @@ async def _auto_router_capability_slot(
prisma_client=pinned_client,
llm_router=llm_router,
)
for capability in capabilities:
judged_capabilities: Final = capabilities if checks_license else ()
for capability in judged_capabilities:
if (
violation := await _claimed_capability_violation(
tables, capability=capability, config_rows=config_rows, model_id=model_id, limit=limit
@ -738,7 +746,7 @@ async def _auto_router_capability_slot(
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN, detail=f"{violation} {AUTO_ROUTER_LICENSE_REMEDY}"
)
if judges_tuning and baselines is not None:
if checks_license and judges_tuning and baselines is not None:
model_rows: Final = await ModelRepository(_TransactionClient(tx_ctx)).find_all_except(model_id or "")
_raise_on_tuning_quota_violation(
candidate=tuning_candidate,
@ -1121,28 +1129,25 @@ def legacy_update_model_info_changes(
Only fields the caller sent with a value count, so ``ModelInfo`` defaults and the blank
strings typed clients send for unset fields never overwrite stored values. Server-owned,
display-only, team, PTU and pricing fields keep their dedicated write paths, and values
echoed from a ``/model/info`` response (the old or new base model's cost map entry, or
limits discovered from the upstream) stay display-only. An echoed value that differs from
the stored override resets that override (``None``), the way ``update_db_model`` does.
echoed from a ``/model/info`` response stay display-only whether or not the echo kept its
``key``: a value equal to the old or new base model's cost map entry (live or as loaded),
the entry ``key`` names, or a limit discovered from the upstream. An echoed value that
differs from the stored override resets that override (``None``), the way ``update_db_model`` does.
"""
incoming: Final = incoming_model_info.model_dump(exclude_none=True)
stored: Final = db_model.model_info.model_dump(exclude_none=True)
sent: Final = {
k: v
for k, v in incoming_model_info.model_dump(exclude_unset=True, exclude_none=True).items()
if k not in _LEGACY_UPDATE_IGNORED_MODEL_INFO_FIELDS and v != ""
}
echoed: Final = frozenset(
(
*echoed_cost_map_fields(
incoming,
_cost_map_entry(db_model, sent),
_cost_map_entry(db_model, stored),
_loaded_catalog_entry(incoming, loaded_catalog),
),
*(k for k, v in sent.items() if discovered_model_info.get(k) == v),
)
live_entries: Final = (_cost_map_entry(db_model, sent), _cost_map_entry(db_model, stored))
echo_sources: Final = (
*live_entries,
*(_loaded_catalog_entry(entry, loaded_catalog) for entry in live_entries),
_loaded_catalog_entry(incoming_model_info.model_dump(exclude_none=True), loaded_catalog),
discovered_model_info,
)
echoed: Final = frozenset(k for k, v in sent.items() if any(source.get(k) == v for source in echo_sources))
kept: Final = without_server_derived_pricing(sent)
return MappingProxyType(
{
@ -1152,6 +1157,34 @@ def legacy_update_model_info_changes(
)
def _legacy_update_model_info_writes(
current: Deployment,
incoming_model_info: ModelInfo,
discovered_model_info: Mapping[str, object],
member_marker: bool | None,
) -> Mapping[str, object]:
changes: Final = {
**legacy_update_model_info_changes(current, incoming_model_info, discovered_model_info),
**({"member_auto_router": member_marker} if member_marker is not None else {}),
}
return MappingProxyType({k: v for k, v in changes.items() if getattr(current.model_info, k, None) != v})
async def _locked_legacy_update_model_info_json(
table: _ProxyModelTable,
model_id: str,
incoming_model_info: ModelInfo,
discovered_model_info: Mapping[str, object],
member_marker: bool | None,
) -> str | None:
row: Final = await table.find_unique(where={"model_id": model_id})
if row is None:
return None
current: Final = Deployment(**row.model_dump())
writes: Final = _legacy_update_model_info_writes(current, incoming_model_info, discovered_model_info, member_marker)
return current.model_info.model_copy(update=writes).model_dump_json(exclude_none=True) if writes else None
def _param_keys_to_remove(patch: updateDeployment) -> tuple[str, ...]:
return patch.litellm_params_to_remove if isinstance(patch, PatchModelRequest) else ()
@ -3084,34 +3117,29 @@ async def update_model(
renamed_update: Final[PrismaCompatibleUpdateDBModel] = (
{**base_update, "model_name": renamed_to} if renamed_to is not None else base_update
)
model_info_changes: Final = MappingProxyType(
{
**legacy_update_model_info_changes(
deployment,
_model_info,
llm_router.get_discovered_model_info(_model_id)
if llm_router is not None
else MappingProxyType({}),
),
**({"member_auto_router": member_marker} if member_marker is not None else {}),
}
discovered_model_info: Final[Mapping[str, int]] = (
llm_router.get_discovered_model_info(_model_id) if llm_router is not None else MappingProxyType({})
)
_data: Final[PrismaCompatibleUpdateDBModel] = (
{
**renamed_update,
"model_info": deployment.model_info.model_copy(update=model_info_changes).model_dump_json(
exclude_none=True
),
}
if model_info_changes
else renamed_update
requested_model_info: Final = _legacy_update_model_info_writes(
deployment, _model_info, discovered_model_info, member_marker
)
async with _auto_router_capability_slot(
prisma_client,
effective_params=effective_params,
model_id=_model_id,
member_write=member_write,
lock_model_row=bool(requested_model_info),
) as table:
model_info_json: Final = (
await _locked_legacy_update_model_info_json(
table, _model_id, _model_info, discovered_model_info, member_marker
)
if requested_model_info
else None
)
_data: Final[PrismaCompatibleUpdateDBModel] = (
{**renamed_update, "model_info": model_info_json} if model_info_json is not None else renamed_update
)
model_response: Final = await table.update(
where={"model_id": _model_id},
data=_data,

View file

@ -3,7 +3,8 @@ import asyncio
import contextlib
import json
import socket
from collections.abc import Iterator, Mapping
from collections.abc import AsyncIterator, Iterator, Mapping
from dataclasses import dataclass
from types import MappingProxyType, SimpleNamespace
from typing import Dict, Final, Optional
from unittest.mock import AsyncMock, MagicMock, patch
@ -1623,50 +1624,93 @@ class TestDeletedAutoRouterAvailability:
}
class _ContendedModelDb:
def __init__(self, row: LiteLLM_ProxyModelTable, other_writers: tuple[Mapping[str, object], ...]) -> None:
self.row = row
self.writes: tuple[Mapping[str, object], ...] = ()
self._other_writers = other_writers
self._row_locked = False
self.db = self
self.litellm_proxymodeltable = self
self.litellm_teamtable = SimpleNamespace(find_unique=self._find_team)
async def _find_team(self, *, where: Mapping[str, str]) -> LiteLLM_TeamTable:
return LiteLLM_TeamTable(team_id=where["team_id"])
async def find_unique(self, *, where: Mapping[str, object]) -> LiteLLM_ProxyModelTable:
read: Final = self.row
if not self._row_locked and self._other_writers:
self._commit_next_other_writer()
return read
async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> LiteLLM_ProxyModelTable:
self.writes = (*self.writes, data)
self.row = self.row.model_copy(
update={k: json.loads(v) if k in ("litellm_params", "model_info") else v for k, v in data.items()}
)
return self.row
async def query_raw(self, query: str, *args: object) -> tuple[()]:
self._row_locked = self._row_locked or "FOR UPDATE" in query
return ()
@contextlib.asynccontextmanager
async def tx(self, timeout: object = None) -> AsyncIterator["_ContendedModelDb"]:
yield self
self._row_locked = False
while self._other_writers:
self._commit_next_other_writer()
def _commit_next_other_writer(self) -> None:
self.row = self.row.model_copy(
update={"model_info": {**(self.row.model_info or {}), **self._other_writers[0]}}
)
self._other_writers = self._other_writers[1:]
@dataclass(frozen=True, slots=True)
class _ServingRouter:
model_id: str
discovered: Mapping[str, int]
def get_model_ids(self) -> tuple[str, ...]:
return (self.model_id,)
def get_discovered_model_info(self, model_id: str | None) -> Mapping[str, int]:
return self.discovered
def config_deployments(self) -> tuple[()]:
return ()
class TestUpdateModel:
"""
Tests for the update_model (POST /model/update) handler.
"""
@staticmethod
async def _persisted_model_info(
async def _update_model(
stored_model_info: Mapping[str, object],
model_info: ModelInfo,
discovered: Mapping[str, int] = MappingProxyType({}),
) -> object:
from litellm.proxy.management_endpoints.model_management_endpoints import update_model
existing_row = MagicMock()
existing_row.litellm_params = {"model": "azure/my-deployment", "api_key": "sk-existing"}
existing_row.model_dump.return_value = {
"model_name": "my-model",
"litellm_params": existing_row.litellm_params,
"model_info": dict(stored_model_info),
}
existing_row.model_dump_json.return_value = "{}"
updated_row = MagicMock()
updated_row.model_dump_json.return_value = "{}"
mock_prisma = MagicMock()
mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=existing_row)
mock_prisma.db.litellm_proxymodeltable.update = AsyncMock(return_value=updated_row)
mock_router = MagicMock()
mock_router.get_model_ids.return_value = [stored_model_info["id"]]
mock_router.get_discovered_model_info.return_value = discovered
stored_team_id: Final = stored_model_info.get("team_id")
mock_prisma.db.litellm_teamtable.find_unique = AsyncMock(
return_value=LiteLLM_TeamTable(team_id=stored_team_id) if isinstance(stored_team_id, str) else None
other_writers: tuple[Mapping[str, object], ...] = (),
) -> "_ContendedModelDb":
db: Final = _ContendedModelDb(
LiteLLM_ProxyModelTable(
model_id="m1",
model_name="my-model",
litellm_params={"model": "azure/my-deployment", "api_key": "sk-existing"},
model_info=dict(stored_model_info),
),
other_writers,
)
with (
patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),
patch("litellm.proxy.proxy_server.llm_router", mock_router),
patch("litellm.proxy.proxy_server.prisma_client", db),
patch("litellm.proxy.proxy_server.llm_router", _ServingRouter(model_id="m1", discovered=discovered)),
patch("litellm.proxy.proxy_server.store_model_in_db", True),
patch("litellm.proxy.proxy_server.premium_user", True),
patch( # test-quality-ok: [TQ008] isolate persistence from encryption implementation
"litellm.proxy.management_endpoints.model_management_endpoints.encrypt_value_helper",
side_effect=lambda value: value,
),
patch( # test-quality-ok: [TQ008] isolate persistence from router reload implementation
patch( # test-quality-ok: [TQ008] the route reloads the proxy-global router with no seam to inject
"litellm.proxy.management_endpoints.model_management_endpoints.clear_cache",
new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)),
),
@ -1676,19 +1720,20 @@ class TestUpdateModel:
user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
)
written = mock_prisma.db.litellm_proxymodeltable.update.await_args.kwargs["data"]
return json.loads(written["model_info"]) if "model_info" in written else None
return db
@pytest.mark.asyncio
async def test_update_model_persists_sent_model_info_over_stored_fields(self):
stored = {"id": "m1", "db_model": True, "base_model": "my-deployment", "tier": "paid", "team_id": "team-a"}
stored: Final = {"id": "m1", "db_model": True, "base_model": "my-deployment", "tier": "paid", "team_id": "team-a"}
persisted = await self._persisted_model_info(
stored, ModelInfo(id="m1", base_model="azure/gpt-4o", mode="chat", db_model=False, team_id="team-b")
db: Final = await self._update_model(
stored,
ModelInfo(id="m1", base_model="test-provider/test-model", mode="chat", db_model=False, team_id="team-b"),
)
persisted: Final = db.row.model_info
assert persisted is not None
assert persisted["base_model"] == "azure/gpt-4o"
assert persisted["base_model"] == "test-provider/test-model"
assert persisted["mode"] == "chat"
assert {k: persisted[k] for k in ("db_model", "tier", "team_id")} == {
"db_model": True,
@ -1698,14 +1743,41 @@ class TestUpdateModel:
@pytest.mark.asyncio
async def test_update_model_with_only_an_id_leaves_model_info_untouched(self):
assert await self._persisted_model_info({"id": "m1", "base_model": "my-deployment"}, ModelInfo(id="m1")) is None
db: Final = await self._update_model({"id": "m1", "base_model": "my-deployment"}, ModelInfo(id="m1"))
assert tuple("model_info" in write for write in db.writes) == (False,)
@pytest.mark.asyncio
async def test_update_model_resending_the_stored_model_info_does_not_rewrite_it(self):
db: Final = await self._update_model(
{"id": "m1", "db_model": True, "tier": "paid", "mode": "chat"},
ModelInfo(id="m1", tier="paid", mode="chat"),
)
assert tuple("model_info" in write for write in db.writes) == (False,)
@pytest.mark.asyncio
async def test_update_model_keeps_model_info_fields_a_concurrent_writer_stored(self):
db: Final = await self._update_model(
{"id": "m1", "db_model": True, "mode": "embedding"},
ModelInfo(id="m1", mode="chat"),
other_writers=(MappingProxyType({"access_groups": ["group-a"]}), MappingProxyType({"tier": "paid"})),
)
persisted: Final = db.row.model_info
assert persisted is not None
assert {k: persisted.get(k) for k in ("mode", "access_groups", "tier")} == {
"mode": "chat",
"access_groups": ["group-a"],
"tier": "paid",
}
@pytest.mark.asyncio
async def test_update_model_does_not_pin_values_echoed_from_model_info(self):
import litellm
catalog_entry = litellm.get_model_info(model="azure/gpt-4o")
echoed = ModelInfo(
catalog_entry: Final = litellm.get_model_info(model="azure/gpt-4o")
echoed: Final = ModelInfo(
id="m1",
base_model="azure/gpt-4o",
key=catalog_entry["key"],
@ -1713,65 +1785,117 @@ class TestUpdateModel:
input_cost_per_token=catalog_entry["input_cost_per_token"],
)
persisted = await self._persisted_model_info({"id": "m1"}, echoed)
db: Final = await self._update_model({"id": "m1"}, echoed)
persisted: Final = db.row.model_info
assert persisted is not None
assert persisted["base_model"] == "azure/gpt-4o"
assert not {"key", "max_tokens", "input_cost_per_token"} & persisted.keys()
@pytest.mark.asyncio
async def test_update_model_keeps_stored_value_when_a_field_is_sent_blank(self):
persisted = await self._persisted_model_info(
{"id": "m1", "base_model": "my-deployment", "mode": "chat"},
ModelInfo(id="m1", base_model="azure/gpt-4o", mode=""),
)
assert persisted is not None
assert persisted["base_model"] == "azure/gpt-4o"
assert persisted["mode"] == "chat"
@pytest.mark.asyncio
async def test_update_model_ignores_ptu_blocked_and_pricing_fields(self):
persisted = await self._persisted_model_info(
{"id": "m1"},
ModelInfo(id="m1", base_model="azure/gpt-4o", ptu_count=5, blocked=True, input_cost_per_token=0.5),
)
assert persisted is not None
assert persisted["base_model"] == "azure/gpt-4o"
assert not {"ptu_count", "blocked", "input_cost_per_token"} & persisted.keys()
@pytest.mark.asyncio
async def test_update_model_does_not_pin_the_previous_base_models_echoed_values(self):
async def test_update_model_does_not_pin_catalog_values_echoed_without_a_key(self):
import litellm
previous = litellm.get_model_info(model="azure/gpt-4o")
assert (
previous["supported_openai_params"]
!= litellm.get_model_info(model="azure/o3-mini")["supported_openai_params"]
)
catalog_entry: Final = litellm.get_model_info(model="azure/gpt-4o")
deliberate_max_output_tokens: Final = catalog_entry["max_output_tokens"] // 2
persisted = await self._persisted_model_info(
{"id": "m1", "base_model": "azure/gpt-4o"},
ModelInfo(
id="m1",
base_model="azure/o3-mini",
key=previous["key"],
supported_openai_params=previous["supported_openai_params"],
),
)
assert persisted is not None
assert persisted["base_model"] == "azure/o3-mini"
assert "supported_openai_params" not in persisted
@pytest.mark.asyncio
async def test_update_model_does_not_pin_discovered_limits_or_team_access_echoed_without_a_key(self):
persisted = await self._persisted_model_info(
db: Final = await self._update_model(
{"id": "m1"},
ModelInfo(
id="m1",
base_model="azure/gpt-4o",
max_tokens=catalog_entry["max_tokens"],
max_input_tokens=catalog_entry["max_input_tokens"],
supports_vision=catalog_entry["supports_vision"],
max_output_tokens=deliberate_max_output_tokens,
),
)
persisted: Final = db.row.model_info
assert persisted is not None
assert persisted["base_model"] == "azure/gpt-4o"
assert persisted["max_output_tokens"] == deliberate_max_output_tokens
assert not {"max_tokens", "max_input_tokens", "supports_vision"} & persisted.keys()
@pytest.mark.asyncio
async def test_update_model_keeps_stored_value_when_a_field_is_sent_blank(self):
db: Final = await self._update_model(
{"id": "m1", "base_model": "my-deployment", "mode": "chat"},
ModelInfo(id="m1", base_model="test-provider/test-model", mode=""),
)
persisted: Final = db.row.model_info
assert persisted is not None
assert persisted["base_model"] == "test-provider/test-model"
assert persisted["mode"] == "chat"
@pytest.mark.asyncio
async def test_update_model_ignores_ptu_blocked_and_pricing_fields(self):
db: Final = await self._update_model(
{"id": "m1"},
ModelInfo(
id="m1", base_model="test-provider/test-model", ptu_count=5, blocked=True, input_cost_per_token=0.5
),
)
persisted: Final = db.row.model_info
assert persisted is not None
assert persisted["base_model"] == "test-provider/test-model"
assert not {"ptu_count", "blocked", "input_cost_per_token"} & persisted.keys()
def test_update_model_does_not_pin_the_previous_base_models_echoed_values(self):
from litellm.proxy.management_endpoints.model_management_endpoints import legacy_update_model_info_changes
catalog: Final = MappingProxyType(
{"test-provider/previous": MappingProxyType({"supported_openai_params": ["previous-only-param"]})}
)
stored: Final = Deployment(
model_name="my-model",
litellm_params=LiteLLM_Params(model="test-provider/my-deployment"),
model_info=ModelInfo(id="m1", base_model="test-provider/previous"),
)
changes: Final = legacy_update_model_info_changes(
stored,
ModelInfo(
id="m1",
base_model="test-provider/next",
key="test-provider/previous",
supported_openai_params=["previous-only-param"],
),
loaded_catalog=lambda: catalog,
)
assert dict(changes) == {"base_model": "test-provider/next"}
def test_update_model_does_not_pin_the_previous_base_models_loaded_catalog_value_echoed_without_a_key(self):
import litellm
from litellm.proxy.management_endpoints.model_management_endpoints import legacy_update_model_info_changes
previous: Final = litellm.get_model_info(model="azure/gpt-4o")
loaded_max_tokens: Final = previous["max_tokens"] + 1
catalog: Final = MappingProxyType({previous["key"]: MappingProxyType({"max_tokens": loaded_max_tokens})})
stored: Final = Deployment(
model_name="my-model",
litellm_params=LiteLLM_Params(model="test-provider/my-deployment"),
model_info=ModelInfo(id="m1", base_model="azure/gpt-4o"),
)
changes: Final = legacy_update_model_info_changes(
stored,
ModelInfo(id="m1", base_model="test-provider/next", max_tokens=loaded_max_tokens),
loaded_catalog=lambda: catalog,
)
assert dict(changes) == {"base_model": "test-provider/next"}
@pytest.mark.asyncio
async def test_update_model_does_not_pin_discovered_limits_or_team_access_echoed_without_a_key(self):
db: Final = await self._update_model(
{"id": "m1"},
ModelInfo(
id="m1",
base_model="test-provider/test-model",
max_input_tokens=128000,
max_output_tokens=64000,
direct_access=True,
@ -1780,8 +1904,9 @@ class TestUpdateModel:
discovered=MappingProxyType({"max_input_tokens": 128000, "max_output_tokens": 32000}),
)
persisted: Final = db.row.model_info
assert persisted is not None
assert persisted["base_model"] == "azure/gpt-4o"
assert persisted["base_model"] == "test-provider/test-model"
assert persisted["max_output_tokens"] == 64000
assert not {"max_input_tokens", "direct_access", "access_via_team_ids"} & persisted.keys()
@ -1789,16 +1914,17 @@ class TestUpdateModel:
async def test_update_model_resets_a_stored_override_echoed_back_at_its_catalog_value(self):
import litellm
catalog_entry = litellm.get_model_info(model="azure/gpt-4o")
stored_override = catalog_entry["max_tokens"] // 2
catalog_entry: Final = litellm.get_model_info(model="azure/gpt-4o")
stored_override: Final = catalog_entry["max_tokens"] // 2
persisted = await self._persisted_model_info(
db: Final = await self._update_model(
{"id": "m1", "base_model": "azure/gpt-4o", "max_tokens": stored_override},
ModelInfo(
id="m1", base_model="azure/gpt-4o", key=catalog_entry["key"], max_tokens=catalog_entry["max_tokens"]
),
)
persisted: Final = db.row.model_info
assert persisted is not None
assert "max_tokens" not in persisted