fix(health-check-routing): fix P1 transient-error filter broken on cache hits

When SharedHealthCheckManager returns cached results, exceptions_by_model_id
is always {} so the transient-error filter defaulted to status 500 for all
endpoints, incorrectly marking 429/408 endpoints as unhealthy.

Fix: store integer exception_status on each unhealthy endpoint dict in
_perform_health_check. _get_endpoint_exception_status() uses the live
exception object when available (direct path) and falls back to the stored
integer (cache-hit path). The integer is JSON-serializable and survives
the shared cache round-trip.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

litellm/proxy/health_check.py

@@ -230,7 +230,11 @@ async def _perform_health_check(
             if _model_id:
                 cleaned["model_id"] = _model_id
                 if "exception" in is_healthy:
-                    exceptions_by_model_id[_model_id] = is_healthy["exception"]
+                    exc = is_healthy["exception"]
+                    exceptions_by_model_id[_model_id] = exc
+                    # Store integer status code so shared-cache readers can
+                    # reconstruct the transient-error filter without the exception object.
+                    cleaned["exception_status"] = getattr(exc, "status_code", 500)
             unhealthy_endpoints.append(cleaned)
         else:
             cleaned = _clean_endpoint_data(litellm_params, details)
@@ -238,6 +242,7 @@ async def _perform_health_check(
                 cleaned["model_id"] = _model_id
                 if isinstance(is_healthy, Exception):
                     exceptions_by_model_id[_model_id] = is_healthy
+                    cleaned["exception_status"] = getattr(is_healthy, "status_code", 500)
             unhealthy_endpoints.append(cleaned)
 
     return healthy_endpoints, unhealthy_endpoints, exceptions_by_model_id

litellm/proxy/proxy_server.py

@@ -2106,6 +2106,20 @@ def _schedule_background_health_check_db_save(
     )
 
 
+def _get_endpoint_exception_status(endpoint: dict, exceptions: dict) -> int:
+    """Return the HTTP status code for an unhealthy endpoint.
+
+    Prefers the live exception object in `exceptions` (direct health check path).
+    Falls back to the `exception_status` integer stored on the endpoint dict
+    (shared-cache path, where exception objects are not available).
+    """
+    model_id = endpoint.get("model_id")
+    exc = exceptions.get(model_id) if model_id else None
+    if exc is not None:
+        return getattr(exc, "status_code", 500)
+    return endpoint.get("exception_status", 500)
+
+
 def _write_health_state_to_router_cache(
     healthy_endpoints: list,
     unhealthy_endpoints: list,
@@ -2134,10 +2148,7 @@ def _write_health_state_to_router_cache(
             _effective_unhealthy = [
                 ep
                 for ep in unhealthy_endpoints
-                if getattr(
-                    _exceptions.get(ep.get("model_id")), "status_code", 500
-                )
-                not in (429, 408)
+                if _get_endpoint_exception_status(ep, _exceptions) not in (429, 408)
             ]
 
         states = build_deployment_health_states(

tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py

@@ -702,3 +702,63 @@ class TestHealthCheckIgnoreTransientErrors:
                     exceptions_by_model_id={"deploy-1": rate_exc},
                 )
                 mock_cooldown.assert_called_once()
+
+
+class TestSharedCacheTransientErrorFilter:
+    """
+    When SharedHealthCheckManager returns cached results, exceptions_by_model_id
+    is always {}. The filter must fall back to the 'exception_status' field stored
+    on each endpoint dict so 429/408 endpoints are still excluded correctly.
+    """
+
+    def test_cached_429_excluded_via_exception_status_field(self):
+        """Cache-hit path: endpoint with exception_status=429 is excluded from health state."""
+        import litellm.proxy.proxy_server as proxy_module
+        from litellm.proxy.proxy_server import _write_health_state_to_router_cache
+
+        router = Router(
+            model_list=[_make_model("deploy-1"), _make_model("deploy-2", "gpt-5")],
+            enable_health_check_routing=True,
+            health_check_ignore_transient_errors=True,
+        )
+
+        # Simulate a cache-hit endpoint: exception_status stored as int, no exceptions dict
+        unhealthy_endpoints = [
+            {"model_id": "deploy-1", "error": "rate limited", "exception_status": 429},
+        ]
+
+        with patch.object(proxy_module, "llm_router", router):
+            _write_health_state_to_router_cache(
+                healthy_endpoints=[],
+                unhealthy_endpoints=unhealthy_endpoints,
+                exceptions_by_model_id={},
+            )
+
+        # deploy-1 should NOT be marked unhealthy (429 was filtered)
+        unhealthy_ids = router.health_state_cache.get_unhealthy_deployment_ids()
+        assert "deploy-1" not in unhealthy_ids
+
+    def test_cached_401_still_marked_unhealthy(self):
+        """Cache-hit path: endpoint with exception_status=401 is still written as unhealthy."""
+        import litellm.proxy.proxy_server as proxy_module
+        from litellm.proxy.proxy_server import _write_health_state_to_router_cache
+
+        router = Router(
+            model_list=[_make_model("deploy-1"), _make_model("deploy-2", "gpt-5")],
+            enable_health_check_routing=True,
+            health_check_ignore_transient_errors=True,
+        )
+
+        unhealthy_endpoints = [
+            {"model_id": "deploy-1", "error": "auth failed", "exception_status": 401},
+        ]
+
+        with patch.object(proxy_module, "llm_router", router):
+            _write_health_state_to_router_cache(
+                healthy_endpoints=[],
+                unhealthy_endpoints=unhealthy_endpoints,
+                exceptions_by_model_id={},
+            )
+
+        unhealthy_ids = router.health_state_cache.get_unhealthy_deployment_ids()
+        assert "deploy-1" in unhealthy_ids