diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index e247842fe61..3e2019ac72d 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -230,7 +230,11 @@ async def _perform_health_check( if _model_id: cleaned["model_id"] = _model_id if "exception" in is_healthy: - exceptions_by_model_id[_model_id] = is_healthy["exception"] + exc = is_healthy["exception"] + exceptions_by_model_id[_model_id] = exc + # Store integer status code so shared-cache readers can + # reconstruct the transient-error filter without the exception object. + cleaned["exception_status"] = getattr(exc, "status_code", 500) unhealthy_endpoints.append(cleaned) else: cleaned = _clean_endpoint_data(litellm_params, details) @@ -238,6 +242,7 @@ async def _perform_health_check( cleaned["model_id"] = _model_id if isinstance(is_healthy, Exception): exceptions_by_model_id[_model_id] = is_healthy + cleaned["exception_status"] = getattr(is_healthy, "status_code", 500) unhealthy_endpoints.append(cleaned) return healthy_endpoints, unhealthy_endpoints, exceptions_by_model_id diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3ab8832acea..16b9ed9288e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2106,6 +2106,20 @@ def _schedule_background_health_check_db_save( ) +def _get_endpoint_exception_status(endpoint: dict, exceptions: dict) -> int: + """Return the HTTP status code for an unhealthy endpoint. + + Prefers the live exception object in `exceptions` (direct health check path). + Falls back to the `exception_status` integer stored on the endpoint dict + (shared-cache path, where exception objects are not available). + """ + model_id = endpoint.get("model_id") + exc = exceptions.get(model_id) if model_id else None + if exc is not None: + return getattr(exc, "status_code", 500) + return endpoint.get("exception_status", 500) + + def _write_health_state_to_router_cache( healthy_endpoints: list, unhealthy_endpoints: list, @@ -2134,10 +2148,7 @@ def _write_health_state_to_router_cache( _effective_unhealthy = [ ep for ep in unhealthy_endpoints - if getattr( - _exceptions.get(ep.get("model_id")), "status_code", 500 - ) - not in (429, 408) + if _get_endpoint_exception_status(ep, _exceptions) not in (429, 408) ] states = build_deployment_health_states( diff --git a/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py b/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py index 2add9604255..fcebfa88e83 100644 --- a/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py +++ b/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py @@ -702,3 +702,63 @@ class TestHealthCheckIgnoreTransientErrors: exceptions_by_model_id={"deploy-1": rate_exc}, ) mock_cooldown.assert_called_once() + + +class TestSharedCacheTransientErrorFilter: + """ + When SharedHealthCheckManager returns cached results, exceptions_by_model_id + is always {}. The filter must fall back to the 'exception_status' field stored + on each endpoint dict so 429/408 endpoints are still excluded correctly. + """ + + def test_cached_429_excluded_via_exception_status_field(self): + """Cache-hit path: endpoint with exception_status=429 is excluded from health state.""" + import litellm.proxy.proxy_server as proxy_module + from litellm.proxy.proxy_server import _write_health_state_to_router_cache + + router = Router( + model_list=[_make_model("deploy-1"), _make_model("deploy-2", "gpt-5")], + enable_health_check_routing=True, + health_check_ignore_transient_errors=True, + ) + + # Simulate a cache-hit endpoint: exception_status stored as int, no exceptions dict + unhealthy_endpoints = [ + {"model_id": "deploy-1", "error": "rate limited", "exception_status": 429}, + ] + + with patch.object(proxy_module, "llm_router", router): + _write_health_state_to_router_cache( + healthy_endpoints=[], + unhealthy_endpoints=unhealthy_endpoints, + exceptions_by_model_id={}, + ) + + # deploy-1 should NOT be marked unhealthy (429 was filtered) + unhealthy_ids = router.health_state_cache.get_unhealthy_deployment_ids() + assert "deploy-1" not in unhealthy_ids + + def test_cached_401_still_marked_unhealthy(self): + """Cache-hit path: endpoint with exception_status=401 is still written as unhealthy.""" + import litellm.proxy.proxy_server as proxy_module + from litellm.proxy.proxy_server import _write_health_state_to_router_cache + + router = Router( + model_list=[_make_model("deploy-1"), _make_model("deploy-2", "gpt-5")], + enable_health_check_routing=True, + health_check_ignore_transient_errors=True, + ) + + unhealthy_endpoints = [ + {"model_id": "deploy-1", "error": "auth failed", "exception_status": 401}, + ] + + with patch.object(proxy_module, "llm_router", router): + _write_health_state_to_router_cache( + healthy_endpoints=[], + unhealthy_endpoints=unhealthy_endpoints, + exceptions_by_model_id={}, + ) + + unhealthy_ids = router.health_state_cache.get_unhealthy_deployment_ids() + assert "deploy-1" in unhealthy_ids