feat: add opt-in healthy_only filter to GET /v1/models (#30130)

* feat: add opt-in healthy_only filter to GET /v1/models

Adds an opt-in `healthy_only=true` query parameter to GET /v1/models and
GET /models that hides models whose backing deployments are all marked
unhealthy by background health checks.

- Add Router.async_get_fully_unhealthy_model_names(), mirroring the
  semantics of get_fully_blocked_model_names(): a model is hidden only
  when every backing deployment is unhealthy and the health state is
  not stale (fail open otherwise).
- Reuses the existing DeploymentHealthCache populated by
  _run_background_health_check(), so no new health state is introduced.
- No-op when allowed_fails_policy is set, mirroring
  _async_filter_health_check_unhealthy_deployments semantics.
- team_public_model_name aliases are aggregated alongside model_name.
- Hiding is presentation-only; default behavior is unchanged.

Fixes #30128

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* docs: address Greptile review notes

- Note team-alias asymmetry vs get_fully_blocked_model_names
- Debug-log when healthy_only is set but no health state is available

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: shin-berri <shin-laptop@berri.ai>
Co-authored-by: yuneng-jiang <yuneng@berri.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
(cherry picked from commit 9dd9d2322a)

litellm/proxy/proxy_server.py

@@ -8202,6 +8202,7 @@ async def model_list(
     include_metadata: Optional[bool] = False,
     fallback_type: Optional[str] = None,
     scope: Optional[str] = None,
+    healthy_only: Optional[bool] = False,
 ):
     """
     Use `/model/info` - to get detailed model information, example - pricing, mode, etc.
@@ -8215,6 +8216,15 @@ async def model_list(
     - scope: Optional scope parameter. Currently only accepts "expand".
              When scope=expand is passed, proxy admins, team admins, and org admins
              will receive all proxy models as if they are a proxy admin.
+    - healthy_only: When true, hide models whose backing deployments are all marked
+                    unhealthy by background health checks. Requires
+                    `background_health_checks: true` in general_settings; without
+                    health state the listing is returned unfiltered (fail open).
+                    Models expanded from wildcard routes (e.g. `openai/*`) are not
+                    filtered, and nothing is hidden when `allowed_fails_policy` is
+                    configured (cooldown remains the sole exclusion mechanism).
+                    Hiding is presentation-only: a hidden model can still be
+                    called directly.
     """
     global llm_model_list, general_settings, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj
 
@@ -8248,6 +8258,19 @@ async def model_list(
         llm_router.get_fully_blocked_model_names() if llm_router is not None else set()
     )
 
+    # Opt-in: also hide models whose deployments are all unhealthy per background
+    # health checks. Empty when health state is unavailable or stale (fail open).
+    unhealthy_names: Set[str] = set()
+    if healthy_only and llm_router is not None:
+        unhealthy_names = await llm_router.async_get_fully_unhealthy_model_names()
+        if not unhealthy_names:
+            verbose_proxy_logger.debug(
+                "healthy_only=true but no unhealthy deployment state is available "
+                "(requires background_health_checks); returning unfiltered model list"
+            )
+
+    hidden_names = blocked_names | unhealthy_names
+
     # If scope=expand and user has admin privileges, return all proxy models
     if should_expand_scope:
         # Get all proxy models as if user is a proxy admin
@@ -8280,9 +8303,9 @@ async def model_list(
             only_model_access_groups=only_model_access_groups or False,
         )
 
-        # Hide paused models from the public listing (admins manage them via /model/info)
-        if blocked_names:
-            all_models = [m for m in all_models if m not in blocked_names]
+        # Hide paused/unhealthy models from the public listing
+        if hidden_names:
+            all_models = [m for m in all_models if m not in hidden_names]
 
         # Build response data with all proxy models
         model_data = []
@@ -8317,9 +8340,9 @@ async def model_list(
         user_api_key_cache=user_api_key_cache,
     )
 
-    # Hide paused models from the public listing (admins manage them via /model/info)
-    if blocked_names:
-        all_models = [m for m in all_models if m not in blocked_names]
+    # Hide paused/unhealthy models from the public listing
+    if hidden_names:
+        all_models = [m for m in all_models if m not in hidden_names]
 
     # Build response data
     model_data = []

litellm/router.py

@@ -9948,6 +9948,63 @@ class Router:
             name for name, fully_blocked in blocked_by_name.items() if fully_blocked
         }
 
+    async def async_get_fully_unhealthy_model_names(self) -> Set[str]:
+        """
+        Returns the set of model names where every backing deployment is currently
+        marked unhealthy by background health checks (and the health state is not stale).
+
+        Used by `/v1/models?healthy_only=true` to hide models that cannot serve any
+        request. A model with at least one healthy (or unknown-health) deployment
+        remains visible. Returns an empty set when no health state is available, so
+        callers fail open to the unfiltered listing.
+
+        Notes:
+            - Mirrors `_async_filter_health_check_unhealthy_deployments`: when
+              `allowed_fails_policy` is set, cooldown is the sole routing exclusion
+              mechanism, so nothing is hidden here either.
+            - Team-specific public model names (`team_public_model_name`) are
+              aggregated alongside `model_name`, so team aliases of fully-unhealthy
+              deployments are hidden too (unlike `get_fully_blocked_model_names`,
+              which matches `model_name` only).
+            - Wildcard routes (e.g. `openai/*`) are matched by their literal
+              deployment name only; models expanded from a wildcard route are not
+              hidden (fail open).
+            - Intentionally diverges from the routing-time safety net (which
+              bypasses the health filter when every candidate is unhealthy and
+              still attempts the request): hiding here is presentation-only —
+              it answers "should this model be advertised?", not "should a
+              request for it still be attempted?". A hidden model can still be
+              called directly.
+        """
+        if self.allowed_fails_policy is not None:
+            return set()
+        unhealthy_ids = (
+            await self.health_state_cache.async_get_unhealthy_deployment_ids()
+        )
+        if not unhealthy_ids:
+            return set()
+        deployments = self.get_model_list() or []
+        unhealthy_by_name: Dict[str, bool] = {}
+        for deployment in deployments:
+            model_info = deployment.get("model_info") or {}
+            names = [deployment.get("model_name") or ""]
+            team_public_model_name = model_info.get("team_public_model_name")
+            if team_public_model_name:
+                names.append(team_public_model_name)
+            is_unhealthy = model_info.get("id") in unhealthy_ids
+            for name in names:
+                if not name:
+                    continue
+                if name in unhealthy_by_name:
+                    unhealthy_by_name[name] = unhealthy_by_name[name] and is_unhealthy
+                else:
+                    unhealthy_by_name[name] = is_unhealthy
+        return {
+            name
+            for name, fully_unhealthy in unhealthy_by_name.items()
+            if fully_unhealthy
+        }
+
     def _get_team_specific_model(
         self, deployment: DeploymentTypedDict, team_id: Optional[str] = None
     ) -> Optional[str]:

tests/test_litellm/proxy/test_model_list_healthy_only.py

@@ -0,0 +1,92 @@
+"""
+Tests for the opt-in `healthy_only` filter on GET /v1/models (`model_list`).
+"""
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from litellm.proxy import proxy_server
+from litellm.proxy._types import UserAPIKeyAuth
+
+
+@pytest.fixture
+def patched_model_list(monkeypatch):
+    """Stub router + utility helpers used by `model_list`."""
+    from litellm.proxy import utils as proxy_utils
+
+    router = MagicMock()
+    router.get_fully_blocked_model_names = MagicMock(return_value=set())
+    router.async_get_fully_unhealthy_model_names = AsyncMock(
+        return_value={"claude-sonnet"}
+    )
+
+    monkeypatch.setattr(proxy_server, "llm_router", router)
+    monkeypatch.setattr(proxy_server, "user_model", None)
+
+    async def _fake_get_available_models_for_user(**kwargs):
+        return ["gpt-4", "claude-sonnet"]
+
+    monkeypatch.setattr(
+        proxy_utils,
+        "get_available_models_for_user",
+        _fake_get_available_models_for_user,
+    )
+
+    def _fake_create_model_info_response(model_id, provider="openai", **kwargs):
+        return {"id": model_id, "object": "model", "created": 0, "owned_by": provider}
+
+    monkeypatch.setattr(
+        proxy_utils, "create_model_info_response", _fake_create_model_info_response
+    )
+
+    return router
+
+
+@pytest.mark.asyncio
+async def test_model_list_healthy_only_hides_fully_unhealthy_models(
+    patched_model_list,
+):
+    response = await proxy_server.model_list(
+        user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
+        healthy_only=True,
+    )
+    assert [m["id"] for m in response["data"]] == ["gpt-4"]
+
+
+@pytest.mark.asyncio
+async def test_model_list_default_keeps_unhealthy_models(patched_model_list):
+    response = await proxy_server.model_list(
+        user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
+    )
+    assert [m["id"] for m in response["data"]] == ["gpt-4", "claude-sonnet"]
+    patched_model_list.async_get_fully_unhealthy_model_names.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_model_list_healthy_only_applies_to_scope_expand(
+    patched_model_list, monkeypatch
+):
+    from litellm.proxy.auth import model_checks
+    from litellm.proxy.management_endpoints import common_utils
+
+    async def _fake_admin(**kwargs):
+        return True
+
+    monkeypatch.setattr(common_utils, "_user_has_admin_privileges", _fake_admin)
+    monkeypatch.setattr(
+        model_checks,
+        "get_complete_model_list",
+        lambda **kwargs: ["gpt-4", "claude-sonnet"],
+    )
+    patched_model_list.get_model_names = MagicMock(
+        return_value=["gpt-4", "claude-sonnet"]
+    )
+    patched_model_list.get_model_access_groups = MagicMock(return_value={})
+
+    response = await proxy_server.model_list(
+        user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
+        scope="expand",
+        healthy_only=True,
+    )
+    assert [m["id"] for m in response["data"]] == ["gpt-4"]

tests/test_litellm/test_router.py

@@ -4218,6 +4218,82 @@ def test_get_fully_blocked_model_names_treats_missing_key_as_unblocked():
     assert router.get_fully_blocked_model_names() == set()
 
 
+def _seed_unhealthy_states(router, unhealthy_ids, timestamp=None):
+    import time
+
+    ts = timestamp if timestamp is not None else time.time()
+    router.health_state_cache.set_deployment_health_states(
+        {
+            uid: {"is_healthy": False, "timestamp": ts, "reason": "test_unhealthy"}
+            for uid in unhealthy_ids
+        }
+    )
+
+
+@pytest.mark.asyncio
+async def test_async_get_fully_unhealthy_model_names_marks_name_when_all_unhealthy():
+    router = _router_with_two_deployments([False, False])
+    _seed_unhealthy_states(router, {"dep-0", "dep-1"})
+    assert await router.async_get_fully_unhealthy_model_names() == {"gpt-4o"}
+
+
+@pytest.mark.asyncio
+async def test_async_get_fully_unhealthy_model_names_keeps_name_when_partial():
+    router = _router_with_two_deployments([False, False])
+    _seed_unhealthy_states(router, {"dep-0"})
+    assert await router.async_get_fully_unhealthy_model_names() == set()
+
+
+@pytest.mark.asyncio
+async def test_async_get_fully_unhealthy_model_names_empty_without_health_state():
+    router = _router_with_two_deployments([False, False])
+    assert await router.async_get_fully_unhealthy_model_names() == set()
+
+
+@pytest.mark.asyncio
+async def test_async_get_fully_unhealthy_model_names_ignores_stale_state():
+    import time
+
+    router = _router_with_two_deployments([False, False])
+    stale_ts = time.time() - (router.health_state_cache.staleness_threshold + 10)
+    _seed_unhealthy_states(router, {"dep-0", "dep-1"}, timestamp=stale_ts)
+    assert await router.async_get_fully_unhealthy_model_names() == set()
+
+
+@pytest.mark.asyncio
+async def test_async_get_fully_unhealthy_model_names_includes_team_alias():
+    import litellm
+
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-4o",
+                "litellm_params": {"model": "openai/gpt-4o"},
+                "model_info": {
+                    "id": "dep-0",
+                    "team_id": "team-1",
+                    "team_public_model_name": "team-gpt",
+                },
+            }
+        ]
+    )
+    _seed_unhealthy_states(router, {"dep-0"})
+    assert await router.async_get_fully_unhealthy_model_names() == {
+        "gpt-4o",
+        "team-gpt",
+    }
+
+
+@pytest.mark.asyncio
+async def test_async_get_fully_unhealthy_model_names_noop_with_allowed_fails_policy():
+    from litellm.types.router import AllowedFailsPolicy
+
+    router = _router_with_two_deployments([False, False])
+    router.allowed_fails_policy = AllowedFailsPolicy(BadRequestErrorAllowedFails=1)
+    _seed_unhealthy_states(router, {"dep-0", "dep-1"})
+    assert await router.async_get_fully_unhealthy_model_names() == set()
+
+
 @pytest.mark.asyncio
 async def test_async_get_healthy_deployments_skips_blocked_deployment():
     router = _router_with_two_deployments([True, False])