mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(proxy): cap in-flight auth cache invalidation publishes so a wedge cannot drain the redis pool
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
15b4a07dd4
commit
f712c0adbe
2 changed files with 38 additions and 2 deletions
|
|
@ -19,7 +19,9 @@ if TYPE_CHECKING:
|
|||
AUTH_CACHE_INVALIDATION_CHANNEL: Final = "litellm_proxy.auth_cache_invalidation"
|
||||
_POLL_TIMEOUT_SECONDS: Final = 1.0
|
||||
_MAX_PENDING_PUBLISHES: Final = 1024
|
||||
_MAX_IN_FLIGHT_PUBLISHES: Final = 16
|
||||
_pending_publishes: Final[set[asyncio.Task[None]]] = set() # mutable-ok: strong refs keep background publishes alive
|
||||
_in_flight_publishes: Final = asyncio.Semaphore(_MAX_IN_FLIGHT_PUBLISHES)
|
||||
_BACKOFF_INITIAL_SECONDS: Final = 5.0
|
||||
_BACKOFF_MAX_SECONDS: Final = 60.0
|
||||
|
||||
|
|
@ -78,7 +80,8 @@ async def _publish_to_redis(redis_cache: "RedisCache", cache_key: str, message:
|
|||
cache_key,
|
||||
)
|
||||
return
|
||||
await client.publish(auth_cache_invalidation_channel(redis_cache), message)
|
||||
async with _in_flight_publishes:
|
||||
await client.publish(auth_cache_invalidation_channel(redis_cache), message)
|
||||
except Exception as e: # noqa: BLE001 # best-effort publish; mutations must never fail on redis errors
|
||||
verbose_proxy_logger.warning("auth cache invalidation publish for %s failed: %s", cache_key, e)
|
||||
|
||||
|
|
@ -101,7 +104,9 @@ async def publish_auth_cache_invalidation(
|
|||
publish has been handed to the event loop, so a Redis that accepts
|
||||
connections but never replies costs the caller nothing. The DB write has
|
||||
already committed and the local eviction already happened, so the caller
|
||||
has nothing to do with the publish result.
|
||||
has nothing to do with the publish result. At most 16 publishes hold a
|
||||
Redis connection at once; the rest wait in the task set, so a wedge cannot
|
||||
drain the shared connection pool.
|
||||
"""
|
||||
redis_cache: Final = coordination_redis_cache()
|
||||
if redis_cache is None:
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from unittest.mock import patch
|
|||
import pytest
|
||||
from redis.asyncio import Redis
|
||||
|
||||
import litellm.proxy.common_utils.auth_cache_invalidation_pubsub as pubsub_module
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
|
||||
|
|
@ -31,11 +32,16 @@ class _RecordingRedisClient(Redis):
|
|||
class _WedgedPublishRedisClient(Redis):
|
||||
def __init__(self) -> None:
|
||||
self.attempted: list[str] = []
|
||||
self.in_flight = 0
|
||||
self.max_in_flight = 0
|
||||
self.release = asyncio.Event()
|
||||
|
||||
async def publish(self, channel: str, message: str) -> int:
|
||||
self.in_flight += 1
|
||||
self.max_in_flight = max(self.max_in_flight, self.in_flight)
|
||||
self.attempted.append(message)
|
||||
await self.release.wait()
|
||||
self.in_flight -= 1
|
||||
return 1
|
||||
|
||||
|
||||
|
|
@ -256,3 +262,28 @@ async def test_evict_and_broadcast_evicts_locally_and_returns_while_redis_publis
|
|||
assert client.attempted == [json.dumps({"cache_key": "user-wedged"})], "publish was not handed to redis"
|
||||
client.release.set()
|
||||
await asyncio.sleep(0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_publish_holds_at_most_sixteen_redis_connections_while_redis_is_wedged(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(pubsub_module, "_in_flight_publishes", asyncio.Semaphore(16))
|
||||
monkeypatch.setattr(pubsub_module, "_pending_publishes", set())
|
||||
client = _WedgedPublishRedisClient()
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.common_utils.auth_cache_invalidation_pubsub.coordination_redis_cache",
|
||||
return_value=_FakeRedisCache(client=client),
|
||||
):
|
||||
for i in range(64):
|
||||
await publish_auth_cache_invalidation(cache_key=f"user-{i}")
|
||||
await asyncio.sleep(0)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert client.max_in_flight == 16, f"publish tasks held {client.max_in_flight} redis connections at once"
|
||||
assert len(client.attempted) == 16, "waiters called publish before a semaphore slot freed"
|
||||
client.release.set()
|
||||
await asyncio.gather(*pubsub_module._pending_publishes) # pyright: ignore[reportPrivateUsage] # drain module-level tasks
|
||||
|
||||
assert len(client.attempted) == 64
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue