fix(router): treat a breaker-refused Redis read as a miss in the health state cache

The sync Redis read now raises while the circuit breaker is open, and the
health state merge caught that as a generic error, skipping the local write
and logging an error on every background health check cycle. Read the shared
snapshot through a helper that treats the refused read as a miss so the merge
falls back to the pod-local copy the way a swallowed connection error already did

litellm/router_utils/health_state_cache.py

@@ -12,6 +12,7 @@ from typing_extensions import TypedDict
 
 from litellm import verbose_logger
 from litellm.caching.caching import DualCache
+from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
@@ -27,6 +28,16 @@ class DeploymentHealthStateValue(TypedDict):
     reason: str
 
 
+def _read_shared_health_snapshot(cache: DualCache, key: str) -> object:
+    redis_cache: Final = cache.redis_cache
+    if redis_cache is None:
+        return None
+    try:
+        return redis_cache.get_cache(key)
+    except RedisCircuitBreakerOpenError:
+        return None
+
+
 class DeploymentHealthCache:
     """
     Cache for deployment health states produced by background health checks.
@@ -50,13 +61,12 @@ class DeploymentHealthCache:
         coexist on the one shared entry without erasing each other's results.
         The snapshot is read from Redis when available, since a pod-local read
         would only ever see this writer's own previous merge. When the Redis
-        read comes back empty (a miss, or a swallowed connection error), the
-        pod-local copy of the last merge is used so peers are not erased.
+        read comes back empty (a miss, a swallowed connection error, or a read
+        refused by the open circuit breaker), the pod-local copy of the last
+        merge is used so peers are not erased.
         """
         try:
-            redis_raw: Final = (
-                self.cache.redis_cache.get_cache(self.CACHE_KEY) if self.cache.redis_cache is not None else None
-            )
+            redis_raw: Final = _read_shared_health_snapshot(self.cache, self.CACHE_KEY)
             raw: Final = redis_raw if isinstance(redis_raw, dict) else self.cache.get_cache(key=self.CACHE_KEY)
             existing: Final = raw if isinstance(raw, dict) else {}
             expiry_seconds: Final = self.staleness_threshold * 1.5

tests/test_litellm/router_utils/test_health_state_cache.py

@@ -7,6 +7,7 @@ import time
 import pytest
 
 from litellm.caching.caching import DualCache
+from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
 from litellm.router_utils.health_state_cache import DeploymentHealthCache
 
 
@@ -145,8 +146,11 @@ class _SharedRedisFake:
     def __init__(self):
         self.store = {}
         self.fail_get = False
+        self.breaker_open = False
 
     def get_cache(self, key, parent_otel_span=None, **kwargs):
+        if self.breaker_open:
+            raise RedisCircuitBreakerOpenError("Redis circuit breaker is open - skipping get_cache")
         if self.fail_get:
             return None  # RedisCache.get_cache swallows connection errors and returns None
         return self.store.get(key)
@@ -192,3 +196,26 @@ def test_failed_redis_read_falls_back_to_local_copy():
         {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}}
     )
     assert set(redis_fake.store[DeploymentHealthCache.CACHE_KEY]) == {"prod-bad", "internal-bad"}
+
+
+def test_open_circuit_breaker_read_still_merges_into_local_copy(caplog):
+    """A read refused by the open breaker is a miss, so the merge and local write still happen quietly."""
+    redis_fake = _SharedRedisFake()
+    pod_a = DeploymentHealthCache(cache=DualCache(redis_cache=redis_fake), staleness_threshold=60.0)
+    pod_b = DeploymentHealthCache(cache=DualCache(redis_cache=redis_fake), staleness_threshold=60.0)
+    pod_a.set_deployment_health_states(
+        {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}}
+    )
+    pod_b.set_deployment_health_states(
+        {"internal-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "timeout"}}
+    )
+    pod_a.set_deployment_health_states(
+        {"prod-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}}
+    )
+    redis_fake.breaker_open = True
+    with caplog.at_level("ERROR"):
+        pod_a.set_deployment_health_states(
+            {"prod-new-bad": {"is_healthy": False, "timestamp": time.time(), "reason": "check_failed"}}
+        )
+    assert caplog.records == []
+    assert pod_a.get_unhealthy_deployment_ids() == {"prod-bad", "internal-bad", "prod-new-bad"}