mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
fix(router): keep a deployment id that names a real model from evicting its catalog entry
Deployments are keyed into litellm.model_cost alongside the built-in catalog, so evicting a deployment's stale entry by id could take a real model's entry with it: registering a deployment whose model_info.id is "gpt-4o" stripped that model's pricing, context window and capability flags process-wide, for every other deployment of it, until the next price-map reload. Only evict an entry this registration owns. A colliding id keeps the previous merge, which pollutes the catalog entry rather than emptying it. Also pins the Admin UI round trip: the model edit form echoes the whole /model/info row back on save, and that read reports every key the deployment never stored as an explicit null, so the clear path has to leave those keys alone.
This commit is contained in:
parent
2bb55035bf
commit
f4e11aff71
3 changed files with 74 additions and 1 deletions
|
|
@ -9765,7 +9765,12 @@ class Router:
|
|||
}
|
||||
|
||||
if model_id is not None:
|
||||
litellm.model_cost.pop(model_id, None)
|
||||
# Deployments key into the same cost map as the built-in catalog, so evict only an
|
||||
# entry this registration owns; a deployment id that names a real model keeps the
|
||||
# old merge rather than stripping what every other deployment of it reads.
|
||||
registered: Final = litellm.model_cost.get(model_id)
|
||||
if registered is not None and registered.get("litellm_provider") is None:
|
||||
litellm.model_cost.pop(model_id, None)
|
||||
litellm.register_model(
|
||||
model_cost={model_id: model_info},
|
||||
persist_across_reloads=False,
|
||||
|
|
|
|||
|
|
@ -3470,6 +3470,45 @@ class TestUpdateDBModelNullClearsAnyKey:
|
|||
assert info[field] == _PROTECTED_MODEL_INFO_VALUES[field]
|
||||
assert info["max_input_tokens"] == 4096
|
||||
|
||||
def test_echoing_the_read_back_blob_preserves_every_stored_key(self):
|
||||
"""The Admin UI edit form submits the whole /model/info row back, and that read reports
|
||||
every key the deployment never stored as an explicit null. Those nulls have to stay
|
||||
no-ops: a write drops None before storing, so a null in the echoed blob always names a
|
||||
key the stored row does not carry.
|
||||
"""
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
update_db_model,
|
||||
)
|
||||
|
||||
db_model = _build_db_model_with_pinned_model_info()
|
||||
echoed = {
|
||||
"id": "dep-pinned-0",
|
||||
"max_input_tokens": 4096,
|
||||
"mode": "chat",
|
||||
"supports_vision": True,
|
||||
"input_cost_per_token": 0.000001,
|
||||
"team_id": "team-keep-me",
|
||||
"base_model": None,
|
||||
"tier": None,
|
||||
"max_output_tokens": None,
|
||||
"supports_function_calling": None,
|
||||
"cache_read_input_token_cost": None,
|
||||
}
|
||||
|
||||
result = update_db_model(
|
||||
db_model=db_model,
|
||||
updated_patch=updateDeployment.model_validate({"model_info": echoed}),
|
||||
)
|
||||
|
||||
info = json.loads(result["model_info"])
|
||||
assert info["max_input_tokens"] == 4096
|
||||
assert info["mode"] == "chat"
|
||||
assert info["supports_vision"] is True
|
||||
assert info["input_cost_per_token"] == 0.000001
|
||||
assert info["team_id"] == "team-keep-me"
|
||||
for never_stored in ("base_model", "tier", "max_output_tokens", "supports_function_calling"):
|
||||
assert never_stored not in info
|
||||
|
||||
def test_null_on_pricing_key_still_clears_both_blobs(self):
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
update_db_model,
|
||||
|
|
|
|||
|
|
@ -255,6 +255,35 @@ def test_should_drop_a_price_the_deployment_no_longer_carries():
|
|||
_restore_model_cost_entries(original)
|
||||
|
||||
|
||||
def test_should_not_strip_a_builtin_entry_when_a_deployment_id_collides_with_it():
|
||||
"""Deployments are keyed into the same cost map as the built-in catalog, so a deployment
|
||||
whose id happens to name a real model must not evict that model's entry.
|
||||
|
||||
Stripping it would take the pricing and capability flags every other deployment of that
|
||||
model reads, process-wide, until the next price-map reload.
|
||||
"""
|
||||
colliding_id = "gpt-4o"
|
||||
original = {colliding_id: litellm.model_cost.get(colliding_id)}
|
||||
builtin_max_tokens = litellm.model_cost[colliding_id]["max_tokens"]
|
||||
|
||||
try:
|
||||
Router._register_deployment_in_model_cost(
|
||||
model_id=colliding_id,
|
||||
model_info={"id": colliding_id, "db_model": True, "mode": "chat"},
|
||||
model="gpt-4o-mini",
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
entry = litellm.model_cost[colliding_id]
|
||||
assert entry["max_tokens"] == builtin_max_tokens, (
|
||||
"registering a deployment under a catalog model's name wiped that model's context window"
|
||||
)
|
||||
assert entry["litellm_provider"] == "openai"
|
||||
assert entry["supports_vision"] is True
|
||||
finally:
|
||||
_restore_model_cost_entries(original)
|
||||
|
||||
|
||||
def test_should_preserve_builtin_pricing_regardless_of_deployment_order():
|
||||
"""
|
||||
The built-in pricing should be preserved no matter which deployment
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue