mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
fix(router): register a router in the live set when it gains a deployment
_live_routers was only joined when a router was constructed with a model_list, but a router built empty is populated through add_deployment, and the empty branch exists for exactly that. Such a router was invisible to the live-router scan, so deleting the deployment from another router released the shared cost-map key while it was still serving that id. Joining the set where a deployment enters the list covers every path, and it also lets a price reload rebuild what a dynamically built router serves.
This commit is contained in:
parent
9724f06365
commit
256e0f8f58
2 changed files with 33 additions and 0 deletions
|
|
@ -9586,6 +9586,9 @@ class Router:
|
|||
"""
|
||||
idx: Final = len(self.model_list)
|
||||
self.model_list.append(model)
|
||||
# A router built without a model_list joins the registry here instead, so a price
|
||||
# reload rebuilds what it serves and delete_deployment can see it still holds an id.
|
||||
_live_routers.add(self)
|
||||
self._invalidate_model_group_info_cache()
|
||||
self._invalidate_access_groups_cache()
|
||||
|
||||
|
|
|
|||
|
|
@ -382,6 +382,36 @@ def test_should_keep_the_cost_map_key_while_another_router_still_serves_it():
|
|||
_restore_model_cost_entries(original)
|
||||
|
||||
|
||||
def test_should_keep_the_cost_map_key_while_a_dynamically_built_router_serves_it():
|
||||
"""A router built with no model_list still serves whatever add_deployment gives it, so it
|
||||
counts when deciding whether the shared cost-map claim can be released."""
|
||||
from litellm.router import _DEPLOYMENT_COST_MAP_KEYS
|
||||
|
||||
model_id = "deployment-added-dynamically"
|
||||
original = {model_id: litellm.model_cost.get(model_id)}
|
||||
entry = {
|
||||
"model_name": "added-dynamically",
|
||||
"litellm_params": {"model": "gpt-4o-mini", "mock_response": "ok"},
|
||||
"model_info": {"id": model_id, "input_cost_per_token": 0.005},
|
||||
}
|
||||
configured = Router(model_list=[entry])
|
||||
dynamic = Router()
|
||||
dynamic.add_deployment(deployment=Deployment(**entry))
|
||||
|
||||
try:
|
||||
assert configured.delete_deployment(id=model_id) is not None
|
||||
|
||||
assert model_id in _DEPLOYMENT_COST_MAP_KEYS, (
|
||||
"the claim was released while a dynamically built router still served the deployment"
|
||||
)
|
||||
|
||||
assert dynamic.delete_deployment(id=model_id) is not None
|
||||
assert model_id not in _DEPLOYMENT_COST_MAP_KEYS
|
||||
finally:
|
||||
_DEPLOYMENT_COST_MAP_KEYS.discard(model_id)
|
||||
_restore_model_cost_entries(original)
|
||||
|
||||
|
||||
def test_should_preserve_builtin_pricing_regardless_of_deployment_order():
|
||||
"""
|
||||
The built-in pricing should be preserved no matter which deployment
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue