fix(proxy): keep live model groups routable when a sibling deployment is dropped

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Devin AI 2026-09-15 19:25:34 +00:00
parent 1f98f908c5
commit bd33f686bb
4 changed files with 40 additions and 45 deletions

View file

@ -6511,7 +6511,7 @@ class ProxyConfig:
added_models = 0
## ADD MODEL LOGIC
for m in sorted(db_models, key=lambda m: (getattr(m, "created_at", None) or datetime.min, m.model_id)):
for m in sorted(db_models, key=lambda m: (m.created_at or datetime.min, m.model_id)):
_litellm_params = m.litellm_params
if isinstance(_litellm_params, dict):
# decrypt values
@ -15051,9 +15051,9 @@ def _model_info_json_response(
data: Sequence[Mapping[str, object]] | Mapping[str, object],
dropped_deployments: Sequence[Mapping[str, object]] | None = None,
) -> Response:
response_data: Final = {"data": data}
if dropped_deployments is not None:
response_data["dropped_deployments"] = dropped_deployments
response_data: Final = (
{"data": data} if dropped_deployments is None else {"data": data, "dropped_deployments": dropped_deployments}
)
return Response(
content=orjson.dumps(response_data, default=jsonable_encoder, option=orjson.OPT_NON_STR_KEYS),
media_type="application/json",
@ -15276,7 +15276,7 @@ async def model_info_v1(
verbose_proxy_logger.debug("all_models: %s", visible_models)
dropped_deployments: Final = (
[asdict(dropped_deployment) for dropped_deployment in llm_router.dropped_deployments.values()]
if hasattr(user_api_key_dict, "user_role") and _user_has_admin_view(user_api_key_dict)
if _user_has_admin_view(user_api_key_dict)
else None
)
return _model_info_json_response(visible_models, dropped_deployments=dropped_deployments)

View file

@ -491,14 +491,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr
data.pop("enable_tag_filtering", None)
team_id: Final = get_team_id_from_data(data)
dropped_model_names: Final = (
frozenset(dropped_deployment.model_name for dropped_deployment in llm_router.dropped_deployments.values())
if llm_router is not None
else frozenset()
)
router_model_names: Final = (
(frozenset(llm_router.model_names) - dropped_model_names) if llm_router is not None else frozenset()
)
router_model_names: Final = llm_router.model_names if llm_router is not None else []
is_proxy_admin_without_team: Final = team_id is None and _is_proxy_admin_request(data)
# Preprocess Google GenAI generate content requests
@ -656,14 +649,11 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr
data["model"] = team_model_name
return getattr(llm_router, f"{route_type}")(**data)
elif data["model"] not in dropped_model_names and (
(
is_proxy_admin_without_team
and data["model"] not in router_model_names
and data["model"] in llm_router.team_public_model_names
)
or llm_router.is_recognized_model(data["model"])
):
elif (
is_proxy_admin_without_team
and data["model"] not in router_model_names
and data["model"] in llm_router.team_public_model_names
) or llm_router.is_recognized_model(data["model"]):
return getattr(llm_router, f"{route_type}")(**data)
elif data["model"] not in router_model_names:

View file

@ -5206,11 +5206,12 @@ async def test_model_info_v1_oci_secrets_not_leaked():
from litellm.proxy.proxy_server import model_info_v1
# Mock user authentication
mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth)
mock_user_api_key_dict.user_id = "test-user"
mock_user_api_key_dict.api_key = "test-key"
mock_user_api_key_dict.team_models = []
mock_user_api_key_dict.models = ["oci-grok-test"]
mock_user_api_key_dict = UserAPIKeyAuth(
user_id="test-user",
api_key="test-key",
team_models=[],
models=["oci-grok-test"],
)
# Mock model data with OCI sensitive information
mock_model_data = {

View file

@ -1,11 +1,10 @@
import pytest
from datetime import datetime
from typing import Final
from unittest.mock import MagicMock
import pytest
from fastapi import HTTPException
from litellm.proxy.route_llm_request import ProxyModelNotFoundError, route_request
@ -54,7 +53,7 @@ async def test_route_request_distinguishes_dropped_deployments_from_unknown_mode
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake"},
},
{
"model_name": "x",
"model_name": "live",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {
@ -62,28 +61,32 @@ async def test_route_request_distinguishes_dropped_deployments_from_unknown_mode
"tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o"},
},
},
"model_info": {"id": "id-x"},
},
{
"model_name": "y",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {
"classifier_type": "heuristic_v2",
"tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o"},
},
},
"model_info": {"id": "id-y"},
"model_info": {"id": "id-live"},
},
],
auto_router_capability_limit=lambda: 1,
ignore_invalid_deployments=True,
)
assert "id-y" in router.dropped_deployments
from litellm.types.router import Deployment
assert router.upsert_deployment(
Deployment(
model_name="x",
litellm_params={
"model": "auto_router/complexity_router",
"complexity_router_config": {
"classifier_type": "heuristic_v2",
"tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o"},
},
},
model_info={"id": "id-x"},
)
) is None
assert "id-x" in router.dropped_deployments
with pytest.raises(HTTPException) as dropped_error:
await route_request(
data={"model": "y", "messages": [{"role": "user", "content": "Hello"}]},
data={"model": "x", "messages": [{"role": "user", "content": "Hello"}]},
llm_router=router,
user_model=None,
route_type="acompletion",
@ -1205,6 +1208,7 @@ def _db_model_row(model_name: str, mock_response: str):
litellm_params={"model": "openai/gpt-4o", "api_key": "fake", "mock_response": mock_response},
model_info={},
blocked=False,
created_at=datetime.min,
)