From f712c0adbe3c2ec7eabb1917f76ee66289c5d063 Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 02:01:04 +0000 Subject: [PATCH] fix(proxy): cap in-flight auth cache invalidation publishes so a wedge cannot drain the redis pool Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../auth_cache_invalidation_pubsub.py | 9 ++++-- .../test_auth_cache_invalidation_pubsub.py | 31 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py b/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py index fcc80873a4d..2bb53c7723d 100644 --- a/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py +++ b/litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py @@ -19,7 +19,9 @@ if TYPE_CHECKING: AUTH_CACHE_INVALIDATION_CHANNEL: Final = "litellm_proxy.auth_cache_invalidation" _POLL_TIMEOUT_SECONDS: Final = 1.0 _MAX_PENDING_PUBLISHES: Final = 1024 +_MAX_IN_FLIGHT_PUBLISHES: Final = 16 _pending_publishes: Final[set[asyncio.Task[None]]] = set() # mutable-ok: strong refs keep background publishes alive +_in_flight_publishes: Final = asyncio.Semaphore(_MAX_IN_FLIGHT_PUBLISHES) _BACKOFF_INITIAL_SECONDS: Final = 5.0 _BACKOFF_MAX_SECONDS: Final = 60.0 @@ -78,7 +80,8 @@ async def _publish_to_redis(redis_cache: "RedisCache", cache_key: str, message: cache_key, ) return - await client.publish(auth_cache_invalidation_channel(redis_cache), message) + async with _in_flight_publishes: + await client.publish(auth_cache_invalidation_channel(redis_cache), message) except Exception as e: # noqa: BLE001 # best-effort publish; mutations must never fail on redis errors verbose_proxy_logger.warning("auth cache invalidation publish for %s failed: %s", cache_key, e) @@ -101,7 +104,9 @@ async def publish_auth_cache_invalidation( publish has been handed to the event loop, so a Redis that accepts connections but never replies costs the caller nothing. The DB write has already committed and the local eviction already happened, so the caller - has nothing to do with the publish result. + has nothing to do with the publish result. At most 16 publishes hold a + Redis connection at once; the rest wait in the task set, so a wedge cannot + drain the shared connection pool. """ redis_cache: Final = coordination_redis_cache() if redis_cache is None: diff --git a/tests/test_litellm/proxy/common_utils/test_auth_cache_invalidation_pubsub.py b/tests/test_litellm/proxy/common_utils/test_auth_cache_invalidation_pubsub.py index f492a7b03b5..96770ee01c4 100644 --- a/tests/test_litellm/proxy/common_utils/test_auth_cache_invalidation_pubsub.py +++ b/tests/test_litellm/proxy/common_utils/test_auth_cache_invalidation_pubsub.py @@ -8,6 +8,7 @@ from unittest.mock import patch import pytest from redis.asyncio import Redis +import litellm.proxy.common_utils.auth_cache_invalidation_pubsub as pubsub_module from litellm.caching.in_memory_cache import InMemoryCache from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( @@ -31,11 +32,16 @@ class _RecordingRedisClient(Redis): class _WedgedPublishRedisClient(Redis): def __init__(self) -> None: self.attempted: list[str] = [] + self.in_flight = 0 + self.max_in_flight = 0 self.release = asyncio.Event() async def publish(self, channel: str, message: str) -> int: + self.in_flight += 1 + self.max_in_flight = max(self.max_in_flight, self.in_flight) self.attempted.append(message) await self.release.wait() + self.in_flight -= 1 return 1 @@ -256,3 +262,28 @@ async def test_evict_and_broadcast_evicts_locally_and_returns_while_redis_publis assert client.attempted == [json.dumps({"cache_key": "user-wedged"})], "publish was not handed to redis" client.release.set() await asyncio.sleep(0) + + +@pytest.mark.asyncio +async def test_publish_holds_at_most_sixteen_redis_connections_while_redis_is_wedged( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(pubsub_module, "_in_flight_publishes", asyncio.Semaphore(16)) + monkeypatch.setattr(pubsub_module, "_pending_publishes", set()) + client = _WedgedPublishRedisClient() + + with patch( + "litellm.proxy.common_utils.auth_cache_invalidation_pubsub.coordination_redis_cache", + return_value=_FakeRedisCache(client=client), + ): + for i in range(64): + await publish_auth_cache_invalidation(cache_key=f"user-{i}") + await asyncio.sleep(0) + await asyncio.sleep(0) + + assert client.max_in_flight == 16, f"publish tasks held {client.max_in_flight} redis connections at once" + assert len(client.attempted) == 16, "waiters called publish before a semaphore slot freed" + client.release.set() + await asyncio.gather(*pubsub_module._pending_publishes) # pyright: ignore[reportPrivateUsage] # drain module-level tasks + + assert len(client.attempted) == 64