From 9a605e2bc5e2566af23899d1ad7e9e5c8074c1e7 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 2 Apr 2026 17:02:48 +0530 Subject: [PATCH] fix(router): also exclude 429/408 from health state cache when ignore_transient_errors set The previous fix only skipped cooldown counter increments. The health state cache was still marking 429/408 endpoints as is_healthy=False, causing the binary health check filter to exclude them from routing. Now, when health_check_ignore_transient_errors=True, 429/408 endpoints are also excluded from the unhealthy list passed to build_deployment_health_states(), so the binary filter treats them as unaffected (not unhealthy). Co-Authored-By: Claude Opus 4.6 (1M context) --- litellm/proxy/proxy_server.py | 12 ++++++- ..._health_check_allowed_fails_integration.py | 31 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 02782f5e714..37cc38b693a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2264,9 +2264,19 @@ def _write_health_state_to_router_cache( if llm_router is None or not llm_router.enable_health_check_routing: return + # When health_check_ignore_transient_errors is set, treat 429/408 + # endpoints as healthy so they are not filtered from routing. + _effective_unhealthy = unhealthy_endpoints + if llm_router.health_check_ignore_transient_errors: + _effective_unhealthy = [ + ep + for ep in unhealthy_endpoints + if getattr(ep.get("exception"), "status_code", 500) not in (429, 408) + ] + states = build_deployment_health_states( healthy_endpoints=healthy_endpoints, - unhealthy_endpoints=unhealthy_endpoints, + unhealthy_endpoints=_effective_unhealthy, ) if states: llm_router.health_state_cache.set_deployment_health_states(states) diff --git a/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py b/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py index 5fee978216f..b2d298bc45d 100644 --- a/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py +++ b/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py @@ -634,6 +634,37 @@ class TestHealthCheckIgnoreTransientErrors: ) mock_cooldown.assert_called_once() + def test_429_not_written_to_health_state_cache_when_flag_enabled(self): + """429 endpoint is excluded from health state cache when flag is set, + so the binary health check filter does not mark it as unhealthy.""" + import litellm.proxy.proxy_server as proxy_module + from litellm.proxy.proxy_server import _write_health_state_to_router_cache + + router = Router( + model_list=[_make_model("deploy-1")], + enable_health_check_routing=True, + health_check_ignore_transient_errors=True, + ) + + rate_exc = litellm.RateLimitError( + message="Rate limited", model="gpt-4", llm_provider="openai" + ) + + unhealthy_endpoints = [ + {"model_id": "deploy-1", "error": "rate limited", "exception": rate_exc}, + ] + + with patch.object(proxy_module, "llm_router", router): + _write_health_state_to_router_cache( + healthy_endpoints=[], + unhealthy_endpoints=unhealthy_endpoints, + ) + + # Health state cache should have NO entry for deploy-1 + # (429 was ignored, not written as unhealthy) + unhealthy_ids = router.health_state_cache.get_unhealthy_deployment_ids() + assert "deploy-1" not in unhealthy_ids + def test_429_triggers_cooldown_when_flag_disabled(self): """When flag is False (default), 429 still triggers cooldown.""" import litellm.proxy.proxy_server as proxy_module