From 8aa618a1b26e2bfcd1634319a0cf2b46c23eaad1 Mon Sep 17 00:00:00 2001
From: Noah Nistler <60981020+noahnistler@users.noreply.github.com>
Date: Fri, 24 Apr 2026 09:54:26 -0500
Subject: [PATCH] fix(proxy): poll for cached health check results instead of
 2s sleep
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The shared health check manager's non-leader branch only waited 2
seconds before falling back to a local health check. Since real health
checks against multiple models typically take longer, every non-leader
pod would fall through and run redundant checks — defeating the
shared coordination.

Replace the single 2-second sleep with a polling loop (5s interval,
up to lock_ttl) that gives the lock holder realistic time to finish.
Adds early exit when the lock disappears without a cache write (crash
recovery) and defensive error handling for Redis hiccups during
polling.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .../shared_health_check_manager.py            | 53 ++++++++++++++-----
 1 file changed, 40 insertions(+), 13 deletions(-)

diff --git a/litellm/proxy/health_check_utils/shared_health_check_manager.py b/litellm/proxy/health_check_utils/shared_health_check_manager.py
index 5b8370fece8..d65dd4261b5 100644
--- a/litellm/proxy/health_check_utils/shared_health_check_manager.py
+++ b/litellm/proxy/health_check_utils/shared_health_check_manager.py
@@ -253,27 +253,54 @@ class SharedHealthCheckManager:
                 # Always release the lock
                 await self.release_health_check_lock()
         else:
-            # Lock not acquired, wait briefly and try to get cached results
+            # Lock not acquired — poll for cached results until the lock
+            # holder finishes or the lock expires, rather than falling back
+            # to a redundant local health check after only 2 seconds.
             verbose_proxy_logger.debug(
                 "Pod %s waiting for other pod to complete health check", self.pod_id
             )
 
-            # Wait a bit for the other pod to complete
-            await asyncio.sleep(2)
+            poll_interval = 5  # seconds between cache checks
+            max_wait = self.lock_ttl  # wait at most as long as the lock can live
+            elapsed = 0
 
-            # Try to get cached results again
-            cached_results = await self.get_cached_health_check_results()
-            if cached_results is not None:
-                return (
-                    cached_results.get("healthy_endpoints", []),
-                    cached_results.get("unhealthy_endpoints", []),
-                    {},
-                )
+            while elapsed < max_wait:
+                await asyncio.sleep(poll_interval)
+                elapsed += poll_interval
 
-            # Still no cache, fall back to local health check
+                cached_results = await self.get_cached_health_check_results()
+                if cached_results is not None:
+                    verbose_proxy_logger.info(
+                        "Pod %s using cached health check results after waiting %ds",
+                        self.pod_id,
+                        elapsed,
+                    )
+                    return (
+                        cached_results.get("healthy_endpoints", []),
+                        cached_results.get("unhealthy_endpoints", []),
+                        {},
+                    )
+
+                # Check if the lock is still held — if it was released without
+                # caching (e.g. the holder crashed), stop waiting early.
+                try:
+                    lock_key = self.get_health_check_lock_key()
+                    current_owner = await self.redis_cache.async_get_cache(lock_key)
+                    if current_owner is None:
+                        verbose_proxy_logger.debug(
+                            "Pod %s detected lock released without cache, stopping wait",
+                            self.pod_id,
+                        )
+                        break
+                except Exception:
+                    # Redis hiccup — continue polling rather than crashing out
+                    pass
+
+            # Exhausted wait — fall back to local health check
             verbose_proxy_logger.warning(
-                "Pod %s falling back to local health check (no cache available)",
+                "Pod %s falling back to local health check after waiting %ds (no cache available)",
                 self.pod_id,
+                elapsed,
             )
 
             return await perform_health_check(