fix(proxy): cap in-flight auth cache invalidation publishes so a wedge cannot drain the redis pool

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng 2026-09-23 02:01:04 +00:00
parent 15b4a07dd4
commit f712c0adbe
2 changed files with 38 additions and 2 deletions

View file

@ -19,7 +19,9 @@ if TYPE_CHECKING:
AUTH_CACHE_INVALIDATION_CHANNEL: Final = "litellm_proxy.auth_cache_invalidation"
_POLL_TIMEOUT_SECONDS: Final = 1.0
_MAX_PENDING_PUBLISHES: Final = 1024
_MAX_IN_FLIGHT_PUBLISHES: Final = 16
_pending_publishes: Final[set[asyncio.Task[None]]] = set() # mutable-ok: strong refs keep background publishes alive
_in_flight_publishes: Final = asyncio.Semaphore(_MAX_IN_FLIGHT_PUBLISHES)
_BACKOFF_INITIAL_SECONDS: Final = 5.0
_BACKOFF_MAX_SECONDS: Final = 60.0
@ -78,7 +80,8 @@ async def _publish_to_redis(redis_cache: "RedisCache", cache_key: str, message:
cache_key,
)
return
await client.publish(auth_cache_invalidation_channel(redis_cache), message)
async with _in_flight_publishes:
await client.publish(auth_cache_invalidation_channel(redis_cache), message)
except Exception as e: # noqa: BLE001 # best-effort publish; mutations must never fail on redis errors
verbose_proxy_logger.warning("auth cache invalidation publish for %s failed: %s", cache_key, e)
@ -101,7 +104,9 @@ async def publish_auth_cache_invalidation(
publish has been handed to the event loop, so a Redis that accepts
connections but never replies costs the caller nothing. The DB write has
already committed and the local eviction already happened, so the caller
has nothing to do with the publish result.
has nothing to do with the publish result. At most 16 publishes hold a
Redis connection at once; the rest wait in the task set, so a wedge cannot
drain the shared connection pool.
"""
redis_cache: Final = coordination_redis_cache()
if redis_cache is None:

View file

@ -8,6 +8,7 @@ from unittest.mock import patch
import pytest
from redis.asyncio import Redis
import litellm.proxy.common_utils.auth_cache_invalidation_pubsub as pubsub_module
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
@ -31,11 +32,16 @@ class _RecordingRedisClient(Redis):
class _WedgedPublishRedisClient(Redis):
def __init__(self) -> None:
self.attempted: list[str] = []
self.in_flight = 0
self.max_in_flight = 0
self.release = asyncio.Event()
async def publish(self, channel: str, message: str) -> int:
self.in_flight += 1
self.max_in_flight = max(self.max_in_flight, self.in_flight)
self.attempted.append(message)
await self.release.wait()
self.in_flight -= 1
return 1
@ -256,3 +262,28 @@ async def test_evict_and_broadcast_evicts_locally_and_returns_while_redis_publis
assert client.attempted == [json.dumps({"cache_key": "user-wedged"})], "publish was not handed to redis"
client.release.set()
await asyncio.sleep(0)
@pytest.mark.asyncio
async def test_publish_holds_at_most_sixteen_redis_connections_while_redis_is_wedged(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(pubsub_module, "_in_flight_publishes", asyncio.Semaphore(16))
monkeypatch.setattr(pubsub_module, "_pending_publishes", set())
client = _WedgedPublishRedisClient()
with patch(
"litellm.proxy.common_utils.auth_cache_invalidation_pubsub.coordination_redis_cache",
return_value=_FakeRedisCache(client=client),
):
for i in range(64):
await publish_auth_cache_invalidation(cache_key=f"user-{i}")
await asyncio.sleep(0)
await asyncio.sleep(0)
assert client.max_in_flight == 16, f"publish tasks held {client.max_in_flight} redis connections at once"
assert len(client.attempted) == 16, "waiters called publish before a semaphore slot freed"
client.release.set()
await asyncio.gather(*pubsub_module._pending_publishes) # pyright: ignore[reportPrivateUsage] # drain module-level tasks
assert len(client.attempted) == 64