From bd33f686bbe0a5433477fd8eb8d2fa5b7924b898 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 19:25:34 +0000 Subject: [PATCH] fix(proxy): keep live model groups routable when a sibling deployment is dropped Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 10 ++--- litellm/proxy/route_llm_request.py | 22 +++------- tests/test_litellm/proxy/test_proxy_server.py | 11 ++--- .../proxy/test_route_llm_request.py | 42 ++++++++++--------- 4 files changed, 40 insertions(+), 45 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 38d6898f2df..de1a83b374a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6511,7 +6511,7 @@ class ProxyConfig: added_models = 0 ## ADD MODEL LOGIC - for m in sorted(db_models, key=lambda m: (getattr(m, "created_at", None) or datetime.min, m.model_id)): + for m in sorted(db_models, key=lambda m: (m.created_at or datetime.min, m.model_id)): _litellm_params = m.litellm_params if isinstance(_litellm_params, dict): # decrypt values @@ -15051,9 +15051,9 @@ def _model_info_json_response( data: Sequence[Mapping[str, object]] | Mapping[str, object], dropped_deployments: Sequence[Mapping[str, object]] | None = None, ) -> Response: - response_data: Final = {"data": data} - if dropped_deployments is not None: - response_data["dropped_deployments"] = dropped_deployments + response_data: Final = ( + {"data": data} if dropped_deployments is None else {"data": data, "dropped_deployments": dropped_deployments} + ) return Response( content=orjson.dumps(response_data, default=jsonable_encoder, option=orjson.OPT_NON_STR_KEYS), media_type="application/json", @@ -15276,7 +15276,7 @@ async def model_info_v1( verbose_proxy_logger.debug("all_models: %s", visible_models) dropped_deployments: Final = ( [asdict(dropped_deployment) for dropped_deployment in llm_router.dropped_deployments.values()] - if hasattr(user_api_key_dict, "user_role") and _user_has_admin_view(user_api_key_dict) + if _user_has_admin_view(user_api_key_dict) else None ) return _model_info_json_response(visible_models, dropped_deployments=dropped_deployments) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 8352fe5a04d..d2027dc163b 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -491,14 +491,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr data.pop("enable_tag_filtering", None) team_id: Final = get_team_id_from_data(data) - dropped_model_names: Final = ( - frozenset(dropped_deployment.model_name for dropped_deployment in llm_router.dropped_deployments.values()) - if llm_router is not None - else frozenset() - ) - router_model_names: Final = ( - (frozenset(llm_router.model_names) - dropped_model_names) if llm_router is not None else frozenset() - ) + router_model_names: Final = llm_router.model_names if llm_router is not None else [] is_proxy_admin_without_team: Final = team_id is None and _is_proxy_admin_request(data) # Preprocess Google GenAI generate content requests @@ -656,14 +649,11 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr data["model"] = team_model_name return getattr(llm_router, f"{route_type}")(**data) - elif data["model"] not in dropped_model_names and ( - ( - is_proxy_admin_without_team - and data["model"] not in router_model_names - and data["model"] in llm_router.team_public_model_names - ) - or llm_router.is_recognized_model(data["model"]) - ): + elif ( + is_proxy_admin_without_team + and data["model"] not in router_model_names + and data["model"] in llm_router.team_public_model_names + ) or llm_router.is_recognized_model(data["model"]): return getattr(llm_router, f"{route_type}")(**data) elif data["model"] not in router_model_names: diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 4d5acc4d417..24b3ddb62e6 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -5206,11 +5206,12 @@ async def test_model_info_v1_oci_secrets_not_leaked(): from litellm.proxy.proxy_server import model_info_v1 # Mock user authentication - mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) - mock_user_api_key_dict.user_id = "test-user" - mock_user_api_key_dict.api_key = "test-key" - mock_user_api_key_dict.team_models = [] - mock_user_api_key_dict.models = ["oci-grok-test"] + mock_user_api_key_dict = UserAPIKeyAuth( + user_id="test-user", + api_key="test-key", + team_models=[], + models=["oci-grok-test"], + ) # Mock model data with OCI sensitive information mock_model_data = { diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index 98e72e4c986..98125b14928 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -1,11 +1,10 @@ -import pytest - - - +from datetime import datetime from typing import Final from unittest.mock import MagicMock +import pytest + from fastapi import HTTPException from litellm.proxy.route_llm_request import ProxyModelNotFoundError, route_request @@ -54,7 +53,7 @@ async def test_route_request_distinguishes_dropped_deployments_from_unknown_mode "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake"}, }, { - "model_name": "x", + "model_name": "live", "litellm_params": { "model": "auto_router/complexity_router", "complexity_router_config": { @@ -62,28 +61,32 @@ async def test_route_request_distinguishes_dropped_deployments_from_unknown_mode "tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o"}, }, }, - "model_info": {"id": "id-x"}, - }, - { - "model_name": "y", - "litellm_params": { - "model": "auto_router/complexity_router", - "complexity_router_config": { - "classifier_type": "heuristic_v2", - "tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o"}, - }, - }, - "model_info": {"id": "id-y"}, + "model_info": {"id": "id-live"}, }, ], auto_router_capability_limit=lambda: 1, ignore_invalid_deployments=True, ) - assert "id-y" in router.dropped_deployments + from litellm.types.router import Deployment + + assert router.upsert_deployment( + Deployment( + model_name="x", + litellm_params={ + "model": "auto_router/complexity_router", + "complexity_router_config": { + "classifier_type": "heuristic_v2", + "tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o"}, + }, + }, + model_info={"id": "id-x"}, + ) + ) is None + assert "id-x" in router.dropped_deployments with pytest.raises(HTTPException) as dropped_error: await route_request( - data={"model": "y", "messages": [{"role": "user", "content": "Hello"}]}, + data={"model": "x", "messages": [{"role": "user", "content": "Hello"}]}, llm_router=router, user_model=None, route_type="acompletion", @@ -1205,6 +1208,7 @@ def _db_model_row(model_name: str, mock_response: str): litellm_params={"model": "openai/gpt-4o", "api_key": "fake", "mock_response": mock_response}, model_info={}, blocked=False, + created_at=datetime.min, )