diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a6413138bd5..95e8994ff32 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -8202,6 +8202,7 @@ async def model_list( include_metadata: Optional[bool] = False, fallback_type: Optional[str] = None, scope: Optional[str] = None, + healthy_only: Optional[bool] = False, ): """ Use `/model/info` - to get detailed model information, example - pricing, mode, etc. @@ -8215,6 +8216,15 @@ async def model_list( - scope: Optional scope parameter. Currently only accepts "expand". When scope=expand is passed, proxy admins, team admins, and org admins will receive all proxy models as if they are a proxy admin. + - healthy_only: When true, hide models whose backing deployments are all marked + unhealthy by background health checks. Requires + `background_health_checks: true` in general_settings; without + health state the listing is returned unfiltered (fail open). + Models expanded from wildcard routes (e.g. `openai/*`) are not + filtered, and nothing is hidden when `allowed_fails_policy` is + configured (cooldown remains the sole exclusion mechanism). + Hiding is presentation-only: a hidden model can still be + called directly. """ global llm_model_list, general_settings, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj @@ -8248,6 +8258,19 @@ async def model_list( llm_router.get_fully_blocked_model_names() if llm_router is not None else set() ) + # Opt-in: also hide models whose deployments are all unhealthy per background + # health checks. Empty when health state is unavailable or stale (fail open). + unhealthy_names: Set[str] = set() + if healthy_only and llm_router is not None: + unhealthy_names = await llm_router.async_get_fully_unhealthy_model_names() + if not unhealthy_names: + verbose_proxy_logger.debug( + "healthy_only=true but no unhealthy deployment state is available " + "(requires background_health_checks); returning unfiltered model list" + ) + + hidden_names = blocked_names | unhealthy_names + # If scope=expand and user has admin privileges, return all proxy models if should_expand_scope: # Get all proxy models as if user is a proxy admin @@ -8280,9 +8303,9 @@ async def model_list( only_model_access_groups=only_model_access_groups or False, ) - # Hide paused models from the public listing (admins manage them via /model/info) - if blocked_names: - all_models = [m for m in all_models if m not in blocked_names] + # Hide paused/unhealthy models from the public listing + if hidden_names: + all_models = [m for m in all_models if m not in hidden_names] # Build response data with all proxy models model_data = [] @@ -8317,9 +8340,9 @@ async def model_list( user_api_key_cache=user_api_key_cache, ) - # Hide paused models from the public listing (admins manage them via /model/info) - if blocked_names: - all_models = [m for m in all_models if m not in blocked_names] + # Hide paused/unhealthy models from the public listing + if hidden_names: + all_models = [m for m in all_models if m not in hidden_names] # Build response data model_data = [] diff --git a/litellm/router.py b/litellm/router.py index d0f4e5ff44d..15de3d51175 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9948,6 +9948,63 @@ class Router: name for name, fully_blocked in blocked_by_name.items() if fully_blocked } + async def async_get_fully_unhealthy_model_names(self) -> Set[str]: + """ + Returns the set of model names where every backing deployment is currently + marked unhealthy by background health checks (and the health state is not stale). + + Used by `/v1/models?healthy_only=true` to hide models that cannot serve any + request. A model with at least one healthy (or unknown-health) deployment + remains visible. Returns an empty set when no health state is available, so + callers fail open to the unfiltered listing. + + Notes: + - Mirrors `_async_filter_health_check_unhealthy_deployments`: when + `allowed_fails_policy` is set, cooldown is the sole routing exclusion + mechanism, so nothing is hidden here either. + - Team-specific public model names (`team_public_model_name`) are + aggregated alongside `model_name`, so team aliases of fully-unhealthy + deployments are hidden too (unlike `get_fully_blocked_model_names`, + which matches `model_name` only). + - Wildcard routes (e.g. `openai/*`) are matched by their literal + deployment name only; models expanded from a wildcard route are not + hidden (fail open). + - Intentionally diverges from the routing-time safety net (which + bypasses the health filter when every candidate is unhealthy and + still attempts the request): hiding here is presentation-only — + it answers "should this model be advertised?", not "should a + request for it still be attempted?". A hidden model can still be + called directly. + """ + if self.allowed_fails_policy is not None: + return set() + unhealthy_ids = ( + await self.health_state_cache.async_get_unhealthy_deployment_ids() + ) + if not unhealthy_ids: + return set() + deployments = self.get_model_list() or [] + unhealthy_by_name: Dict[str, bool] = {} + for deployment in deployments: + model_info = deployment.get("model_info") or {} + names = [deployment.get("model_name") or ""] + team_public_model_name = model_info.get("team_public_model_name") + if team_public_model_name: + names.append(team_public_model_name) + is_unhealthy = model_info.get("id") in unhealthy_ids + for name in names: + if not name: + continue + if name in unhealthy_by_name: + unhealthy_by_name[name] = unhealthy_by_name[name] and is_unhealthy + else: + unhealthy_by_name[name] = is_unhealthy + return { + name + for name, fully_unhealthy in unhealthy_by_name.items() + if fully_unhealthy + } + def _get_team_specific_model( self, deployment: DeploymentTypedDict, team_id: Optional[str] = None ) -> Optional[str]: diff --git a/tests/test_litellm/proxy/test_model_list_healthy_only.py b/tests/test_litellm/proxy/test_model_list_healthy_only.py new file mode 100644 index 00000000000..4ab33f3bf50 --- /dev/null +++ b/tests/test_litellm/proxy/test_model_list_healthy_only.py @@ -0,0 +1,92 @@ +""" +Tests for the opt-in `healthy_only` filter on GET /v1/models (`model_list`). +""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.proxy import proxy_server +from litellm.proxy._types import UserAPIKeyAuth + + +@pytest.fixture +def patched_model_list(monkeypatch): + """Stub router + utility helpers used by `model_list`.""" + from litellm.proxy import utils as proxy_utils + + router = MagicMock() + router.get_fully_blocked_model_names = MagicMock(return_value=set()) + router.async_get_fully_unhealthy_model_names = AsyncMock( + return_value={"claude-sonnet"} + ) + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "user_model", None) + + async def _fake_get_available_models_for_user(**kwargs): + return ["gpt-4", "claude-sonnet"] + + monkeypatch.setattr( + proxy_utils, + "get_available_models_for_user", + _fake_get_available_models_for_user, + ) + + def _fake_create_model_info_response(model_id, provider="openai", **kwargs): + return {"id": model_id, "object": "model", "created": 0, "owned_by": provider} + + monkeypatch.setattr( + proxy_utils, "create_model_info_response", _fake_create_model_info_response + ) + + return router + + +@pytest.mark.asyncio +async def test_model_list_healthy_only_hides_fully_unhealthy_models( + patched_model_list, +): + response = await proxy_server.model_list( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + healthy_only=True, + ) + assert [m["id"] for m in response["data"]] == ["gpt-4"] + + +@pytest.mark.asyncio +async def test_model_list_default_keeps_unhealthy_models(patched_model_list): + response = await proxy_server.model_list( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + ) + assert [m["id"] for m in response["data"]] == ["gpt-4", "claude-sonnet"] + patched_model_list.async_get_fully_unhealthy_model_names.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_model_list_healthy_only_applies_to_scope_expand( + patched_model_list, monkeypatch +): + from litellm.proxy.auth import model_checks + from litellm.proxy.management_endpoints import common_utils + + async def _fake_admin(**kwargs): + return True + + monkeypatch.setattr(common_utils, "_user_has_admin_privileges", _fake_admin) + monkeypatch.setattr( + model_checks, + "get_complete_model_list", + lambda **kwargs: ["gpt-4", "claude-sonnet"], + ) + patched_model_list.get_model_names = MagicMock( + return_value=["gpt-4", "claude-sonnet"] + ) + patched_model_list.get_model_access_groups = MagicMock(return_value={}) + + response = await proxy_server.model_list( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + scope="expand", + healthy_only=True, + ) + assert [m["id"] for m in response["data"]] == ["gpt-4"] diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index cd235d8de67..49ee871dac6 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -4218,6 +4218,82 @@ def test_get_fully_blocked_model_names_treats_missing_key_as_unblocked(): assert router.get_fully_blocked_model_names() == set() +def _seed_unhealthy_states(router, unhealthy_ids, timestamp=None): + import time + + ts = timestamp if timestamp is not None else time.time() + router.health_state_cache.set_deployment_health_states( + { + uid: {"is_healthy": False, "timestamp": ts, "reason": "test_unhealthy"} + for uid in unhealthy_ids + } + ) + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_marks_name_when_all_unhealthy(): + router = _router_with_two_deployments([False, False]) + _seed_unhealthy_states(router, {"dep-0", "dep-1"}) + assert await router.async_get_fully_unhealthy_model_names() == {"gpt-4o"} + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_keeps_name_when_partial(): + router = _router_with_two_deployments([False, False]) + _seed_unhealthy_states(router, {"dep-0"}) + assert await router.async_get_fully_unhealthy_model_names() == set() + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_empty_without_health_state(): + router = _router_with_two_deployments([False, False]) + assert await router.async_get_fully_unhealthy_model_names() == set() + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_ignores_stale_state(): + import time + + router = _router_with_two_deployments([False, False]) + stale_ts = time.time() - (router.health_state_cache.staleness_threshold + 10) + _seed_unhealthy_states(router, {"dep-0", "dep-1"}, timestamp=stale_ts) + assert await router.async_get_fully_unhealthy_model_names() == set() + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_includes_team_alias(): + import litellm + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4o", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": { + "id": "dep-0", + "team_id": "team-1", + "team_public_model_name": "team-gpt", + }, + } + ] + ) + _seed_unhealthy_states(router, {"dep-0"}) + assert await router.async_get_fully_unhealthy_model_names() == { + "gpt-4o", + "team-gpt", + } + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_noop_with_allowed_fails_policy(): + from litellm.types.router import AllowedFailsPolicy + + router = _router_with_two_deployments([False, False]) + router.allowed_fails_policy = AllowedFailsPolicy(BadRequestErrorAllowedFails=1) + _seed_unhealthy_states(router, {"dep-0", "dep-1"}) + assert await router.async_get_fully_unhealthy_model_names() == set() + + @pytest.mark.asyncio async def test_async_get_healthy_deployments_skips_blocked_deployment(): router = _router_with_two_deployments([True, False])