From cb1c2b08f13c3be1154fcb91708dc23aeef4bcae Mon Sep 17 00:00:00 2001 From: Etienne Chabert Date: Tue, 20 Oct 2026 19:04:00 +0200 Subject: [PATCH 1/7] fix(proxy): evict a deleted key from every worker's auth cache /key/delete dropped the key object only from the in-memory cache of the worker that served the call. Auth serves key objects cache-first, so every other worker, and the Redis copy when enable_redis_auth_cache is on, kept authenticating the deleted key until user_api_key_cache_ttl expired. /key/update already evicts and broadcasts through evict_and_broadcast, delete now does the same for the tokens it deletes --- .../key_management_endpoints.py | 8 +- .../test_key_management_endpoints.py | 73 ++++++++++++++++++- 2 files changed, 75 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 824b310a1dd..47aa41989a2 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -5557,11 +5557,9 @@ async def delete_verification_tokens( verbose_proxy_logger.debug(traceback.format_exc()) raise e - for key in tokens: - user_api_key_cache.delete_cache(key) - # remove hash token from cache - hashed_token = hash_token(cast(str, key)) - user_api_key_cache.delete_cache(hashed_token) + # On every worker, not just this one: auth serves key objects cache-first, so a worker that + # kept its copy would go on authenticating the deleted key until user_api_key_cache_ttl expired + await evict_and_broadcast(cache_keys=tuple(str(token) for token in tokens), user_api_key_cache=user_api_key_cache) # After credential invalidation, so a failure here can never keep a deleted key alive. for deleted_key in authorized_keys: diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index d6ee9b1d29e..6bf69461e82 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -16,6 +16,7 @@ from unittest.mock import AsyncMock, MagicMock, patch from fastapi import HTTPException from pydantic import JsonValue +from redis.asyncio import Redis import inspect @@ -5678,7 +5679,7 @@ class _RecordingEvict: self.cache_keys = () async def __call__(self, cache_keys, user_api_key_cache): - self.cache_keys = tuple(cache_keys) + self.cache_keys += tuple(cache_keys) @pytest.mark.asyncio @@ -5754,9 +5755,79 @@ async def test_delete_verification_tokens_evicts_jwt_key_mapping_cache(monkeypat assert recording_evict.cache_keys == ( jwt_key_mapping_cache_key("email", "user@example.com", None), jwt_key_mapping_cache_key("email", "user@example.com", "https://issuer.example"), + "hashed-token-1", ) +class _RecordingPublishRedis(Redis): + def __init__(self) -> None: + self.cache_keys: list[str] = [] + + async def publish(self, channel, message) -> int: + self.cache_keys.append(json.loads(message)["cache_key"]) + return 1 + + +class _CoordinationRedis: + namespace = None + + def __init__(self, client) -> None: + self._client = client + + def init_pubsub_client(self): + return self._client + + +@pytest.mark.asyncio +async def test_delete_verification_tokens_tells_every_worker_to_drop_the_deleted_key(monkeypatch): + """Auth serves key objects cache-first. A worker that is not told about the delete keeps + authenticating the deleted key from its own copy until user_api_key_cache_ttl expires""" + key = LiteLLM_VerificationToken( + token="hashed-token-1", + user_id="user-123", + team_id=None, + key_alias="doomed-key", + spend=0.0, + max_budget=None, + models=[], + aliases={}, + config={}, + permissions={}, + metadata={}, + model_max_budget={}, + model_spend={}, + soft_budget_cooldown=False, + allowed_routes=[], + ) + prisma_client = AsyncMock() + prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[key]) + prisma_client.db.litellm_jwtkeymapping = _CascadingJWTMappingTable([]) + prisma_client.db.litellm_deletedverificationtoken.create_many = AsyncMock() + prisma_client.delete_data = AsyncMock(return_value=["hashed-token-1"]) + redis = _RecordingPublishRedis() + monkeypatch.setattr( + "litellm.proxy.management_endpoints.key_management_endpoints.hash_token_if_needed", lambda token: token + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + monkeypatch.setattr( + "litellm.proxy.common_utils.auth_cache_invalidation_pubsub.coordination_redis_cache", + lambda: _CoordinationRedis(redis), + ) + cache = UserApiKeyCache() + cache.set_cache(key="hashed-token-1", value={"token": "hashed-token-1"}) + + await delete_verification_tokens( + tokens=["hashed-token-1"], + user_api_key_cache=cache, + user_api_key_dict=UserAPIKeyAuth( + user_id="admin-user", api_key="sk-admin", user_role=LitellmUserRoles.PROXY_ADMIN.value + ), + ) + + assert cache.get_cache(key="hashed-token-1") is None + assert redis.cache_keys == ["hashed-token-1"] + + @pytest.mark.asyncio async def test_delete_key_fn_persists_deleted_keys(monkeypatch): from litellm.proxy._types import KeyRequest From 869707adb1503bd3e656b4b43a02fa4eb526dec1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:36:01 -0700 Subject: [PATCH 2/7] fix(proxy): evict only the deleted rows and broadcast them in one batch Scope the cross-worker eviction on /key/delete to the rows the caller was allowed to delete and send it through delete_cache_key_objects, so a caller-supplied name can no longer drop another team's cached entries on every worker --- litellm/proxy/auth/auth_checks.py | 6 +- .../key_management_endpoints.py | 12 +- .../management/test_key_delete_revocation.py | 4 - .../test_key_management_endpoints.py | 192 ++++++++++++++---- 4 files changed, 163 insertions(+), 51 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 8887d86570a..c1e73607b72 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -3421,9 +3421,9 @@ async def delete_cache_key_objects( proxy_logging_obj: ProxyLogging | None, ) -> None: """ - Evict a batch of key objects, for callers that delete keys in bulk rather than through - `/key/delete`. Auth resolves a cached key object without re-reading its team, so a key left - cached after its row is gone keeps buying access until its TTL expires. + Evict a batch of key objects, for callers that delete keys in bulk. Auth resolves a cached key + object without re-reading its team, so a key left cached after its row is gone keeps buying + access until its TTL expires. Evicting locally only reaches this worker; `_delete_cache_key_object` itself broadcasts each token, so a deleted key left in a peer worker's in-memory cache still authenticates there until diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 47aa41989a2..8fe81f6361a 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -58,6 +58,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo _delete_cache_key_object, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export can_team_access_model, delete_cache_key_object, + delete_cache_key_objects, get_jwt_key_mapping_cache_keys_for_token, get_key_end_user_budget_id, get_org_object, @@ -5473,7 +5474,7 @@ async def delete_verification_tokens( - List of keys being deleted, this contains information about the key_alias, token, and user_id being deleted, this is passed down to the KeyManagementEventHooks to delete the keys from the secret manager and handle audit logs """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj failed_tokens: list = [] try: @@ -5558,8 +5559,13 @@ async def delete_verification_tokens( raise e # On every worker, not just this one: auth serves key objects cache-first, so a worker that - # kept its copy would go on authenticating the deleted key until user_api_key_cache_ttl expired - await evict_and_broadcast(cache_keys=tuple(str(token) for token in tokens), user_api_key_cache=user_api_key_cache) + # kept its copy would go on authenticating the deleted key until user_api_key_cache_ttl expired. + # Only the rows the caller was allowed to delete, since the request list can name any cache key. + await delete_cache_key_objects( + hashed_tokens=tuple(key.token for key in authorized_keys if key.token is not None), + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) # After credential invalidation, so a failure here can never keep a deleted key alive. for deleted_key in authorized_keys: diff --git a/tests/integration/management/test_key_delete_revocation.py b/tests/integration/management/test_key_delete_revocation.py index 5b3527a52ae..e3a093b51d7 100644 --- a/tests/integration/management/test_key_delete_revocation.py +++ b/tests/integration/management/test_key_delete_revocation.py @@ -172,10 +172,6 @@ def test_deleting_a_key_revokes_it(gateway: Gateway, spelling: str) -> None: def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gateway) -> None: - pytest.skip( - "BUG: POST /key/delete does not broadcast the key eviction, so a warmed peer keeps answering 200 for about " - "60 seconds until its in-memory cache TTL expires" - ) backend: Final = "integration-keys-auth-delete" prompts: Final = ( "delete warm primary " + uuid.uuid4().hex, diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index 6bf69461e82..8032ef4659d 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -5,7 +5,7 @@ from typing import Final from types import SimpleNamespace import json from datetime import datetime, timedelta, timezone -from uuid import UUID +from uuid import UUID, uuid4 import litellm import pytest @@ -16,7 +16,6 @@ from unittest.mock import AsyncMock, MagicMock, patch from fastapi import HTTPException from pydantic import JsonValue -from redis.asyncio import Redis import inspect @@ -41,13 +40,17 @@ from litellm.proxy._types import ( RegenerateKeyRequest, ResetSpendRequest, UpdateKeyRequest, + hash_token, ) from litellm.models.object_permission import LiteLLM_ObjectPermissionTable +from litellm.caching.in_memory_cache import InMemoryCache from litellm.proxy.auth.auth_checks import ( delete_cache_key_object, + get_key_object, jwt_key_mapping_cache_key, ) from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import AuthCacheInvalidationSubscriber from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, project_cache_key from litellm.constants import PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker @@ -5679,7 +5682,7 @@ class _RecordingEvict: self.cache_keys = () async def __call__(self, cache_keys, user_api_key_cache): - self.cache_keys += tuple(cache_keys) + self.cache_keys = tuple(cache_keys) @pytest.mark.asyncio @@ -5755,35 +5758,73 @@ async def test_delete_verification_tokens_evicts_jwt_key_mapping_cache(monkeypat assert recording_evict.cache_keys == ( jwt_key_mapping_cache_key("email", "user@example.com", None), jwt_key_mapping_cache_key("email", "user@example.com", "https://issuer.example"), - "hashed-token-1", ) -class _RecordingPublishRedis(Redis): +class _ChannelSubscription: def __init__(self) -> None: - self.cache_keys: list[str] = [] + self.messages: asyncio.Queue[dict[str, object]] = asyncio.Queue() - async def publish(self, channel, message) -> int: - self.cache_keys.append(json.loads(message)["cache_key"]) - return 1 + async def subscribe(self, *channels: str) -> None: + return None + + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> dict[str, object] | None: + try: + return await asyncio.wait_for(self.messages.get(), timeout) + except asyncio.TimeoutError: + return None + + async def aclose(self) -> None: + return None -class _CoordinationRedis: +class _ChannelPipeline: + def __init__(self, redis: "_SharedCoordinationRedis") -> None: + self._redis = redis + self._messages: tuple[str, ...] = () + + def publish(self, channel: str, message: str) -> "_ChannelPipeline": + self._messages = (*self._messages, message) + return self + + async def execute(self) -> None: + for message in self._messages: + self._redis.deliver(message) + + +class _SharedCoordinationRedis: + """The coordination Redis two workers share: whatever one worker publishes reaches the other's subscriber""" + namespace = None - def __init__(self, client) -> None: - self._client = client + def __init__(self) -> None: + self.published: tuple[str, ...] = () + self._subscriptions: tuple[_ChannelSubscription, ...] = () - def init_pubsub_client(self): - return self._client + def init_pubsub_client(self) -> "_SharedCoordinationRedis": + return self + + def pubsub(self) -> _ChannelSubscription: + subscription: Final = _ChannelSubscription() + self._subscriptions = (*self._subscriptions, subscription) + return subscription + + def pipeline(self, transaction: bool = True) -> _ChannelPipeline: + return _ChannelPipeline(self) + + async def publish(self, channel: str, message: str) -> int: + self.deliver(message) + return 1 + + def deliver(self, message: str) -> None: + self.published = (*self.published, message) + for subscription in self._subscriptions: + subscription.messages.put_nowait({"type": "message", "data": message.encode()}) -@pytest.mark.asyncio -async def test_delete_verification_tokens_tells_every_worker_to_drop_the_deleted_key(monkeypatch): - """Auth serves key objects cache-first. A worker that is not told about the delete keeps - authenticating the deleted key from its own copy until user_api_key_cache_ttl expires""" - key = LiteLLM_VerificationToken( - token="hashed-token-1", +def _owned_key_row(hashed_token: str) -> LiteLLM_VerificationToken: + return LiteLLM_VerificationToken( + token=hashed_token, user_id="user-123", team_id=None, key_alias="doomed-key", @@ -5799,33 +5840,102 @@ async def test_delete_verification_tokens_tells_every_worker_to_drop_the_deleted soft_budget_cooldown=False, allowed_routes=[], ) - prisma_client = AsyncMock() - prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[key]) + + +def _prisma_with_key_row(key_row: LiteLLM_VerificationToken) -> AsyncMock: + prisma_client: Final = AsyncMock() + prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[key_row]) prisma_client.db.litellm_jwtkeymapping = _CascadingJWTMappingTable([]) prisma_client.db.litellm_deletedverificationtoken.create_many = AsyncMock() - prisma_client.delete_data = AsyncMock(return_value=["hashed-token-1"]) - redis = _RecordingPublishRedis() - monkeypatch.setattr( - "litellm.proxy.management_endpoints.key_management_endpoints.hash_token_if_needed", lambda token: token + prisma_client.delete_data = AsyncMock(return_value=[key_row.token]) + prisma_client.get_data = AsyncMock(return_value=None) + return prisma_client + + +async def _until_evicted(cache: UserApiKeyCache, hashed_token: str) -> None: + for _ in range(200): + if cache.get_cache(key=hashed_token) is None: + return + await asyncio.sleep(0.01) + + +@pytest.mark.asyncio +async def test_delete_verification_tokens_stops_the_key_on_a_peer_worker(monkeypatch): + """Auth serves key objects cache-first, so a peer worker that is not told about the delete keeps + authenticating the deleted key from its own copy until user_api_key_cache_ttl expires""" + raw_key: Final = "sk-" + uuid4().hex + hashed_token: Final = hash_token(raw_key) + deleting_worker_cache: Final = UserApiKeyCache() + peer_worker_cache: Final = UserApiKeyCache() + for cache in (deleting_worker_cache, peer_worker_cache): + cache.set_cache(key=hashed_token, value=UserAPIKeyAuth(token=hashed_token, user_id="user-123")) + coordination_redis: Final = _SharedCoordinationRedis() + peer_subscriber: Final = AuthCacheInvalidationSubscriber( + redis_cache=coordination_redis, user_api_key_cache=peer_worker_cache ) + peer_subscriber.start() + prisma_client: Final = _prisma_with_key_row(_owned_key_row(hashed_token)) + monkeypatch.setattr("litellm.proxy.proxy_server.redis_usage_cache", coordination_redis) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) - monkeypatch.setattr( - "litellm.proxy.common_utils.auth_cache_invalidation_pubsub.coordination_redis_cache", - lambda: _CoordinationRedis(redis), - ) - cache = UserApiKeyCache() - cache.set_cache(key="hashed-token-1", value={"token": "hashed-token-1"}) - await delete_verification_tokens( - tokens=["hashed-token-1"], - user_api_key_cache=cache, - user_api_key_dict=UserAPIKeyAuth( - user_id="admin-user", api_key="sk-admin", user_role=LitellmUserRoles.PROXY_ADMIN.value - ), - ) + try: + await delete_verification_tokens( + tokens=[raw_key], + user_api_key_cache=deleting_worker_cache, + user_api_key_dict=UserAPIKeyAuth( + user_id="admin-user", api_key="sk-admin", user_role=LitellmUserRoles.PROXY_ADMIN.value + ), + ) + await _until_evicted(peer_worker_cache, hashed_token) + finally: + await peer_subscriber.stop() - assert cache.get_cache(key="hashed-token-1") is None - assert redis.cache_keys == ["hashed-token-1"] + assert deleting_worker_cache.get_cache(key=hashed_token) is None + with pytest.raises(ProxyException) as denied: + await get_key_object( + hashed_token=hashed_token, prisma_client=prisma_client, user_api_key_cache=peer_worker_cache + ) + assert denied.value.code == "401" + + +@pytest.mark.asyncio +async def test_delete_verification_tokens_leaves_unowned_cache_entries_alone_on_every_worker(monkeypatch): + """/key/delete accepts any string, and a name that is not a key row passes through unhashed. A key + owner padding the request with another team's spend counter name must not wipe that counter from + every worker's memory: only the rows the caller was allowed to delete are evicted and broadcast""" + raw_key: Final = "sk-" + uuid4().hex + hashed_token: Final = hash_token(raw_key) + other_team_counter: Final = "spend:team:other-team" + deleting_worker_cache: Final = UserApiKeyCache() + peer_worker_cache: Final = UserApiKeyCache() + peer_spend_counters: Final = InMemoryCache() + peer_spend_counters.set_cache(other_team_counter, 42.0) + peer_worker_cache.set_cache(key=hashed_token, value=UserAPIKeyAuth(token=hashed_token, user_id="user-123")) + coordination_redis: Final = _SharedCoordinationRedis() + peer_subscriber: Final = AuthCacheInvalidationSubscriber( + redis_cache=coordination_redis, + user_api_key_cache=peer_worker_cache, + additional_in_memory_caches=(peer_spend_counters,), + ) + peer_subscriber.start() + monkeypatch.setattr("litellm.proxy.proxy_server.redis_usage_cache", coordination_redis) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _prisma_with_key_row(_owned_key_row(hashed_token))) + + try: + await delete_verification_tokens( + tokens=[other_team_counter, raw_key], + user_api_key_cache=deleting_worker_cache, + user_api_key_dict=UserAPIKeyAuth( + user_id="user-123", api_key="sk-owner", user_role=LitellmUserRoles.INTERNAL_USER.value + ), + ) + await _until_evicted(peer_worker_cache, hashed_token) + finally: + await peer_subscriber.stop() + + assert peer_worker_cache.get_cache(key=hashed_token) is None + assert peer_spend_counters.get_cache(other_team_counter) == 42.0 + assert coordination_redis.published == (json.dumps({"cache_key": hashed_token}),) @pytest.mark.asyncio From 6b6c07884e11a873308496838d351ea45117969a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:34:01 -0700 Subject: [PATCH 3/7] refactor(proxy): drop the comment above the delete eviction call --- litellm/proxy/management_endpoints/key_management_endpoints.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 8fe81f6361a..076810052b0 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -5558,9 +5558,6 @@ async def delete_verification_tokens( verbose_proxy_logger.debug(traceback.format_exc()) raise e - # On every worker, not just this one: auth serves key objects cache-first, so a worker that - # kept its copy would go on authenticating the deleted key until user_api_key_cache_ttl expired. - # Only the rows the caller was allowed to delete, since the request list can name any cache key. await delete_cache_key_objects( hashed_tokens=tuple(key.token for key in authorized_keys if key.token is not None), user_api_key_cache=user_api_key_cache, From b032e6928912723ecf7a5b74a5c4f57abfa7ab59 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:25:33 -0700 Subject: [PATCH 4/7] fix(proxy): run the usage cache eviction beside the auth eviction so a hung Redis costs one wait --- litellm/proxy/auth/auth_checks.py | 33 ++++++++++------- tests/unit/proxy/auth/test_auth_checks.py | 44 ++++++++++++++++++++++- 2 files changed, 63 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index c1e73607b72..359b7d23f08 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -3421,28 +3421,35 @@ async def delete_cache_key_objects( proxy_logging_obj: ProxyLogging | None, ) -> None: """ - Evict a batch of key objects, for callers that delete keys in bulk. Auth resolves a cached key + Evict a batch of deleted key objects here and on every other worker. Auth resolves a cached key object without re-reading its team, so a key left cached after its row is gone keeps buying access until its TTL expires. - Evicting locally only reaches this worker; `_delete_cache_key_object` itself broadcasts each - token, so a deleted key left in a peer worker's in-memory cache still authenticates there until - its TTL expires. - Best-effort per key: the rows are already deleted by the time this runs, so an unreachable cache backend must not abort the caller partway through its own cascade. - Every token is dropped from this worker's memory first, then from the auth cache and the - internal usage cache in one batch each, so a deleted key stops authenticating here the instant - its rows are gone instead of refilling from Redis on the next miss. Every other worker then - gets the eviction in one pipelined broadcast: a publish per key would pass the publisher's - pending cap on a bulk delete, and the dropped keys would keep authenticating there until their - TTL. + Every token is dropped from this worker's memory first, then from the auth cache's Redis layer, + so a deleted key stops authenticating here the instant its rows are gone instead of refilling + from Redis on the next miss. Every other worker then gets the eviction in one pipelined + broadcast: a publish per key would pass the publisher's pending cap on a bulk delete, and the + dropped keys would keep authenticating there until their TTL. The internal usage cache is not + an auth refill source, so its batch delete runs beside that chain and a hung Redis costs one + bounded wait there instead of another in line. """ evict_local(cache_keys=hashed_tokens, user_api_key_cache=user_api_key_cache) + usage_eviction: Final = ( + () + if proxy_logging_obj is None + else (evict_shared(cache_keys=hashed_tokens, cache=proxy_logging_obj.internal_usage_cache.dual_cache),) + ) + _ = await asyncio.gather( + _evict_auth_cache_then_broadcast(hashed_tokens=hashed_tokens, user_api_key_cache=user_api_key_cache), + *usage_eviction, + ) + + +async def _evict_auth_cache_then_broadcast(hashed_tokens: Sequence[str], user_api_key_cache: UserApiKeyCache) -> None: await evict_shared(cache_keys=hashed_tokens, cache=user_api_key_cache) - if proxy_logging_obj is not None: - await evict_shared(cache_keys=hashed_tokens, cache=proxy_logging_obj.internal_usage_cache.dual_cache) await broadcast_evictions(cache_keys=hashed_tokens) diff --git a/tests/unit/proxy/auth/test_auth_checks.py b/tests/unit/proxy/auth/test_auth_checks.py index e1afa81d372..29a036a4142 100644 --- a/tests/unit/proxy/auth/test_auth_checks.py +++ b/tests/unit/proxy/auth/test_auth_checks.py @@ -46,7 +46,7 @@ from litellm.proxy.auth.auth_checks import ( _team_soft_budget_check, common_checks, ) -from litellm.proxy.utils import ProxyLogging +from litellm.proxy.utils import InternalUsageCache, ProxyLogging from litellm.proxy.utils import CallInfo from litellm.types.passthrough_endpoints.pass_through_endpoints import LITELLM_PASS_THROUGH_ENDPOINT_MARKER @@ -2380,3 +2380,45 @@ async def test_bulk_eviction_clears_the_redis_copies_before_waiting_on_a_wedged_ assert refilled == [] assert redis.entries == {} + + +class _WedgedDeleteRedisCache(_AuthRedisCache): + def __init__(self) -> None: + super().__init__({}) + self.release = asyncio.Event() + + async def delete_cache_keys(self, keys: Sequence[str]) -> None: + await self.release.wait() + + +@pytest.mark.asyncio +async def test_bulk_eviction_broadcasts_while_the_usage_cache_redis_is_wedged( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(pubsub_module, "_in_flight_publishes", asyncio.Semaphore(16)) + monkeypatch.setattr(pubsub_module, "_pending_publishes", set()) + monkeypatch.setattr(pubsub_module, "_BULK_EVICTION_WAIT_SECONDS", 0.5) + client: Final = _SlowPublishRedisClient() + usage_redis: Final = _WedgedDeleteRedisCache() + proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=None) + proxy_logging_obj.internal_usage_cache = InternalUsageCache( + dual_cache=DualCache(redis_cache=usage_redis) # pyright: ignore[reportArgumentType] # a dict-backed fake of the Redis layer + ) + hashed_tokens: Final = ("token-a", "token-b") + + with patch( + "litellm.proxy.common_utils.auth_cache_invalidation_pubsub.coordination_redis_cache", + return_value=_PubSubRedisCache(client), + ): + eviction: Final = asyncio.create_task( + delete_cache_key_objects( + hashed_tokens=hashed_tokens, user_api_key_cache=UserApiKeyCache(), proxy_logging_obj=proxy_logging_obj + ) + ) + await asyncio.sleep(0.1) + broadcast_while_wedged: Final = {json.loads(message)["cache_key"] for message in client.published} + usage_redis.release.set() + await eviction + await _drain_publishes() + + assert broadcast_while_wedged == set(hashed_tokens) From aa1dfd17324b727ff5b0eb747752637b28ecb75e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:11:16 -0700 Subject: [PATCH 5/7] test(proxy): wait on the broadcast instead of a sleep and type the peer eviction tests --- tests/unit/proxy/auth/test_auth_checks.py | 36 +++++++++++-------- .../test_key_management_endpoints.py | 4 +-- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/tests/unit/proxy/auth/test_auth_checks.py b/tests/unit/proxy/auth/test_auth_checks.py index 29a036a4142..361c031a6be 100644 --- a/tests/unit/proxy/auth/test_auth_checks.py +++ b/tests/unit/proxy/auth/test_auth_checks.py @@ -2382,13 +2382,25 @@ async def test_bulk_eviction_clears_the_redis_copies_before_waiting_on_a_wedged_ assert redis.entries == {} -class _WedgedDeleteRedisCache(_AuthRedisCache): +class _SignallingPublishRedisClient(_SlowPublishRedisClient): def __init__(self) -> None: + super().__init__() + self.sent = asyncio.Event() + + async def _send(self, messages: Sequence[str]) -> None: + await super()._send(messages) + self.sent.set() + + +class _DeleteAfterBroadcastRedisCache(_AuthRedisCache): + def __init__(self, broadcast_sent: asyncio.Event) -> None: super().__init__({}) - self.release = asyncio.Event() + self._broadcast_sent = broadcast_sent + self.finished = False async def delete_cache_keys(self, keys: Sequence[str]) -> None: - await self.release.wait() + await self._broadcast_sent.wait() + self.finished = True @pytest.mark.asyncio @@ -2397,9 +2409,8 @@ async def test_bulk_eviction_broadcasts_while_the_usage_cache_redis_is_wedged( ) -> None: monkeypatch.setattr(pubsub_module, "_in_flight_publishes", asyncio.Semaphore(16)) monkeypatch.setattr(pubsub_module, "_pending_publishes", set()) - monkeypatch.setattr(pubsub_module, "_BULK_EVICTION_WAIT_SECONDS", 0.5) - client: Final = _SlowPublishRedisClient() - usage_redis: Final = _WedgedDeleteRedisCache() + client: Final = _SignallingPublishRedisClient() + usage_redis: Final = _DeleteAfterBroadcastRedisCache(broadcast_sent=client.sent) proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=None) proxy_logging_obj.internal_usage_cache = InternalUsageCache( dual_cache=DualCache(redis_cache=usage_redis) # pyright: ignore[reportArgumentType] # a dict-backed fake of the Redis layer @@ -2410,15 +2421,10 @@ async def test_bulk_eviction_broadcasts_while_the_usage_cache_redis_is_wedged( "litellm.proxy.common_utils.auth_cache_invalidation_pubsub.coordination_redis_cache", return_value=_PubSubRedisCache(client), ): - eviction: Final = asyncio.create_task( - delete_cache_key_objects( - hashed_tokens=hashed_tokens, user_api_key_cache=UserApiKeyCache(), proxy_logging_obj=proxy_logging_obj - ) + await delete_cache_key_objects( + hashed_tokens=hashed_tokens, user_api_key_cache=UserApiKeyCache(), proxy_logging_obj=proxy_logging_obj ) - await asyncio.sleep(0.1) - broadcast_while_wedged: Final = {json.loads(message)["cache_key"] for message in client.published} - usage_redis.release.set() - await eviction await _drain_publishes() - assert broadcast_while_wedged == set(hashed_tokens) + assert usage_redis.finished + assert {json.loads(message)["cache_key"] for message in client.published} == set(hashed_tokens) diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index 8032ef4659d..dd6e40c0d62 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -5860,7 +5860,7 @@ async def _until_evicted(cache: UserApiKeyCache, hashed_token: str) -> None: @pytest.mark.asyncio -async def test_delete_verification_tokens_stops_the_key_on_a_peer_worker(monkeypatch): +async def test_delete_verification_tokens_stops_the_key_on_a_peer_worker(monkeypatch: pytest.MonkeyPatch) -> None: """Auth serves key objects cache-first, so a peer worker that is not told about the delete keeps authenticating the deleted key from its own copy until user_api_key_cache_ttl expires""" raw_key: Final = "sk-" + uuid4().hex @@ -5899,7 +5899,7 @@ async def test_delete_verification_tokens_stops_the_key_on_a_peer_worker(monkeyp @pytest.mark.asyncio -async def test_delete_verification_tokens_leaves_unowned_cache_entries_alone_on_every_worker(monkeypatch): +async def test_delete_verification_tokens_leaves_unowned_cache_entries_alone_on_every_worker(monkeypatch: pytest.MonkeyPatch) -> None: """/key/delete accepts any string, and a name that is not a key row passes through unhashed. A key owner padding the request with another team's spend counter name must not wipe that counter from every worker's memory: only the rows the caller was allowed to delete are evicted and broadcast""" From 9ad81ffa8e2ebcc4f02a82a20f7457a19993eb15 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:30:44 -0700 Subject: [PATCH 6/7] test(proxy): wait for the peer to apply the broadcast instead of polling its cache --- .../test_key_management_endpoints.py | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index dd6e40c0d62..462e06be6ed 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -5762,17 +5762,26 @@ async def test_delete_verification_tokens_evicts_jwt_key_mapping_cache(monkeypat class _ChannelSubscription: + """Marks a message done when the subscriber comes back for the next one, so ``messages.join()`` + returns once every delivered message has been applied""" + def __init__(self) -> None: self.messages: asyncio.Queue[dict[str, object]] = asyncio.Queue() + self._holding_message = False async def subscribe(self, *channels: str) -> None: return None async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> dict[str, object] | None: + if self._holding_message: + self._holding_message = False + self.messages.task_done() try: - return await asyncio.wait_for(self.messages.get(), timeout) + message: Final = await asyncio.wait_for(self.messages.get(), timeout) except asyncio.TimeoutError: return None + self._holding_message = True + return message async def aclose(self) -> None: return None @@ -5821,6 +5830,9 @@ class _SharedCoordinationRedis: for subscription in self._subscriptions: subscription.messages.put_nowait({"type": "message", "data": message.encode()}) + async def until_every_subscriber_applied_its_messages(self) -> None: + _ = await asyncio.gather(*(subscription.messages.join() for subscription in self._subscriptions)) + def _owned_key_row(hashed_token: str) -> LiteLLM_VerificationToken: return LiteLLM_VerificationToken( @@ -5852,13 +5864,6 @@ def _prisma_with_key_row(key_row: LiteLLM_VerificationToken) -> AsyncMock: return prisma_client -async def _until_evicted(cache: UserApiKeyCache, hashed_token: str) -> None: - for _ in range(200): - if cache.get_cache(key=hashed_token) is None: - return - await asyncio.sleep(0.01) - - @pytest.mark.asyncio async def test_delete_verification_tokens_stops_the_key_on_a_peer_worker(monkeypatch: pytest.MonkeyPatch) -> None: """Auth serves key objects cache-first, so a peer worker that is not told about the delete keeps @@ -5886,7 +5891,7 @@ async def test_delete_verification_tokens_stops_the_key_on_a_peer_worker(monkeyp user_id="admin-user", api_key="sk-admin", user_role=LitellmUserRoles.PROXY_ADMIN.value ), ) - await _until_evicted(peer_worker_cache, hashed_token) + await coordination_redis.until_every_subscriber_applied_its_messages() finally: await peer_subscriber.stop() @@ -5929,7 +5934,7 @@ async def test_delete_verification_tokens_leaves_unowned_cache_entries_alone_on_ user_id="user-123", api_key="sk-owner", user_role=LitellmUserRoles.INTERNAL_USER.value ), ) - await _until_evicted(peer_worker_cache, hashed_token) + await coordination_redis.until_every_subscriber_applied_its_messages() finally: await peer_subscriber.stop() From 6bdf36c194530cdbcab0d7b218b953e2d1dd9534 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:04:18 -0700 Subject: [PATCH 7/7] test(proxy): cover hashed keys and aliases when deleting a key warmed on a peer --- .../management/test_key_delete_revocation.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/integration/management/test_key_delete_revocation.py b/tests/integration/management/test_key_delete_revocation.py index e3a093b51d7..bdc91dfe5f6 100644 --- a/tests/integration/management/test_key_delete_revocation.py +++ b/tests/integration/management/test_key_delete_revocation.py @@ -171,7 +171,8 @@ def test_deleting_a_key_revokes_it(gateway: Gateway, spelling: str) -> None: assert deleted_rows == (_DeletedKeyRow(token=key_hash),) -def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gateway) -> None: +@pytest.mark.parametrize("spelling", ("keys", "hashed_keys", "key_aliases")) +def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gateway, spelling: str) -> None: backend: Final = "integration-keys-auth-delete" prompts: Final = ( "delete warm primary " + uuid.uuid4().hex, @@ -204,9 +205,13 @@ def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gate ) assert len(warm_responses) == 2 assert tuple((request.method, request.target) for request in wire.drain()) == (_EXPECTED_CHAT,) * 2 - deleted: Final = gateway.request("POST", "/key/delete", {"keys": [key]}) + identifier: Final = {"keys": key, "hashed_keys": sha256(key.encode()).hexdigest(), "key_aliases": alias}[ + spelling + ] + delete_body: Final = {"key_aliases" if spelling == "key_aliases" else "keys": [identifier]} + deleted: Final = gateway.request("POST", "/key/delete", delete_body) assert deleted.status_code == 200, deleted.text - assert _DeleteResponse.model_validate_json(deleted.content).deleted_keys == (key,), deleted.text + assert _DeleteResponse.model_validate_json(deleted.content).deleted_keys == (identifier,), deleted.text peer_refusal: Final = eventually( lambda: _chat(peer, model, key, prompts[2]), lambda response: response.status_code == 401,