diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 448643ef59b..3d0be18683a 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -3435,28 +3435,35 @@ async def delete_cache_key_objects( proxy_logging_obj: ProxyLogging | None, ) -> None: """ - Evict a batch of key objects, for callers that delete keys in bulk rather than through - `/key/delete`. Auth resolves a cached key object without re-reading its team, so a key left - cached after its row is gone keeps buying access until its TTL expires. - - Evicting locally only reaches this worker; `_delete_cache_key_object` itself broadcasts each - token, so a deleted key left in a peer worker's in-memory cache still authenticates there until - its TTL expires. + Evict a batch of deleted key objects here and on every other worker. Auth resolves a cached key + object without re-reading its team, so a key left cached after its row is gone keeps buying + access until its TTL expires. Best-effort per key: the rows are already deleted by the time this runs, so an unreachable cache backend must not abort the caller partway through its own cascade. - Every token is dropped from this worker's memory first, then from the auth cache and the - internal usage cache in one batch each, so a deleted key stops authenticating here the instant - its rows are gone instead of refilling from Redis on the next miss. Every other worker then - gets the eviction in one pipelined broadcast: a publish per key would pass the publisher's - pending cap on a bulk delete, and the dropped keys would keep authenticating there until their - TTL. + Every token is dropped from this worker's memory first, then from the auth cache's Redis layer, + so a deleted key stops authenticating here the instant its rows are gone instead of refilling + from Redis on the next miss. Every other worker then gets the eviction in one pipelined + broadcast: a publish per key would pass the publisher's pending cap on a bulk delete, and the + dropped keys would keep authenticating there until their TTL. The internal usage cache is not + an auth refill source, so its batch delete runs beside that chain and a hung Redis costs one + bounded wait there instead of another in line. """ evict_local(cache_keys=hashed_tokens, user_api_key_cache=user_api_key_cache) + usage_eviction: Final = ( + () + if proxy_logging_obj is None + else (evict_shared(cache_keys=hashed_tokens, cache=proxy_logging_obj.internal_usage_cache.dual_cache),) + ) + _ = await asyncio.gather( + _evict_auth_cache_then_broadcast(hashed_tokens=hashed_tokens, user_api_key_cache=user_api_key_cache), + *usage_eviction, + ) + + +async def _evict_auth_cache_then_broadcast(hashed_tokens: Sequence[str], user_api_key_cache: UserApiKeyCache) -> None: await evict_shared(cache_keys=hashed_tokens, cache=user_api_key_cache) - if proxy_logging_obj is not None: - await evict_shared(cache_keys=hashed_tokens, cache=proxy_logging_obj.internal_usage_cache.dual_cache) await broadcast_evictions(cache_keys=hashed_tokens) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 0ad4258fb04..894cc8a6070 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -58,6 +58,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo _delete_cache_key_object, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export can_team_access_model, delete_cache_key_object, + delete_cache_key_objects, get_jwt_key_mapping_cache_keys_for_token, get_key_end_user_budget_id, get_org_object, @@ -5490,7 +5491,7 @@ async def delete_verification_tokens( - List of keys being deleted, this contains information about the key_alias, token, and user_id being deleted, this is passed down to the KeyManagementEventHooks to delete the keys from the secret manager and handle audit logs """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj failed_tokens: list = [] try: @@ -5574,11 +5575,11 @@ async def delete_verification_tokens( verbose_proxy_logger.debug(traceback.format_exc()) raise e - for key in tokens: - user_api_key_cache.delete_cache(key) - # remove hash token from cache - hashed_token = hash_token(cast(str, key)) - user_api_key_cache.delete_cache(hashed_token) + await delete_cache_key_objects( + hashed_tokens=tuple(key.token for key in authorized_keys if key.token is not None), + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) # After credential invalidation, so a failure here can never keep a deleted key alive. for deleted_key in authorized_keys: diff --git a/tests/integration/management/test_key_delete_revocation.py b/tests/integration/management/test_key_delete_revocation.py index 5b3527a52ae..bdc91dfe5f6 100644 --- a/tests/integration/management/test_key_delete_revocation.py +++ b/tests/integration/management/test_key_delete_revocation.py @@ -171,11 +171,8 @@ def test_deleting_a_key_revokes_it(gateway: Gateway, spelling: str) -> None: assert deleted_rows == (_DeletedKeyRow(token=key_hash),) -def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gateway) -> None: - pytest.skip( - "BUG: POST /key/delete does not broadcast the key eviction, so a warmed peer keeps answering 200 for about " - "60 seconds until its in-memory cache TTL expires" - ) +@pytest.mark.parametrize("spelling", ("keys", "hashed_keys", "key_aliases")) +def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gateway, spelling: str) -> None: backend: Final = "integration-keys-auth-delete" prompts: Final = ( "delete warm primary " + uuid.uuid4().hex, @@ -208,9 +205,13 @@ def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gate ) assert len(warm_responses) == 2 assert tuple((request.method, request.target) for request in wire.drain()) == (_EXPECTED_CHAT,) * 2 - deleted: Final = gateway.request("POST", "/key/delete", {"keys": [key]}) + identifier: Final = {"keys": key, "hashed_keys": sha256(key.encode()).hexdigest(), "key_aliases": alias}[ + spelling + ] + delete_body: Final = {"key_aliases" if spelling == "key_aliases" else "keys": [identifier]} + deleted: Final = gateway.request("POST", "/key/delete", delete_body) assert deleted.status_code == 200, deleted.text - assert _DeleteResponse.model_validate_json(deleted.content).deleted_keys == (key,), deleted.text + assert _DeleteResponse.model_validate_json(deleted.content).deleted_keys == (identifier,), deleted.text peer_refusal: Final = eventually( lambda: _chat(peer, model, key, prompts[2]), lambda response: response.status_code == 401, diff --git a/tests/unit/proxy/auth/test_auth_checks.py b/tests/unit/proxy/auth/test_auth_checks.py index e1afa81d372..361c031a6be 100644 --- a/tests/unit/proxy/auth/test_auth_checks.py +++ b/tests/unit/proxy/auth/test_auth_checks.py @@ -46,7 +46,7 @@ from litellm.proxy.auth.auth_checks import ( _team_soft_budget_check, common_checks, ) -from litellm.proxy.utils import ProxyLogging +from litellm.proxy.utils import InternalUsageCache, ProxyLogging from litellm.proxy.utils import CallInfo from litellm.types.passthrough_endpoints.pass_through_endpoints import LITELLM_PASS_THROUGH_ENDPOINT_MARKER @@ -2380,3 +2380,51 @@ async def test_bulk_eviction_clears_the_redis_copies_before_waiting_on_a_wedged_ assert refilled == [] assert redis.entries == {} + + +class _SignallingPublishRedisClient(_SlowPublishRedisClient): + def __init__(self) -> None: + super().__init__() + self.sent = asyncio.Event() + + async def _send(self, messages: Sequence[str]) -> None: + await super()._send(messages) + self.sent.set() + + +class _DeleteAfterBroadcastRedisCache(_AuthRedisCache): + def __init__(self, broadcast_sent: asyncio.Event) -> None: + super().__init__({}) + self._broadcast_sent = broadcast_sent + self.finished = False + + async def delete_cache_keys(self, keys: Sequence[str]) -> None: + await self._broadcast_sent.wait() + self.finished = True + + +@pytest.mark.asyncio +async def test_bulk_eviction_broadcasts_while_the_usage_cache_redis_is_wedged( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(pubsub_module, "_in_flight_publishes", asyncio.Semaphore(16)) + monkeypatch.setattr(pubsub_module, "_pending_publishes", set()) + client: Final = _SignallingPublishRedisClient() + usage_redis: Final = _DeleteAfterBroadcastRedisCache(broadcast_sent=client.sent) + proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=None) + proxy_logging_obj.internal_usage_cache = InternalUsageCache( + dual_cache=DualCache(redis_cache=usage_redis) # pyright: ignore[reportArgumentType] # a dict-backed fake of the Redis layer + ) + hashed_tokens: Final = ("token-a", "token-b") + + with patch( + "litellm.proxy.common_utils.auth_cache_invalidation_pubsub.coordination_redis_cache", + return_value=_PubSubRedisCache(client), + ): + await delete_cache_key_objects( + hashed_tokens=hashed_tokens, user_api_key_cache=UserApiKeyCache(), proxy_logging_obj=proxy_logging_obj + ) + await _drain_publishes() + + assert usage_redis.finished + assert {json.loads(message)["cache_key"] for message in client.published} == set(hashed_tokens) diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index d6ee9b1d29e..462e06be6ed 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -5,7 +5,7 @@ from typing import Final from types import SimpleNamespace import json from datetime import datetime, timedelta, timezone -from uuid import UUID +from uuid import UUID, uuid4 import litellm import pytest @@ -40,13 +40,17 @@ from litellm.proxy._types import ( RegenerateKeyRequest, ResetSpendRequest, UpdateKeyRequest, + hash_token, ) from litellm.models.object_permission import LiteLLM_ObjectPermissionTable +from litellm.caching.in_memory_cache import InMemoryCache from litellm.proxy.auth.auth_checks import ( delete_cache_key_object, + get_key_object, jwt_key_mapping_cache_key, ) from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import AuthCacheInvalidationSubscriber from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, project_cache_key from litellm.constants import PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker @@ -5757,6 +5761,188 @@ async def test_delete_verification_tokens_evicts_jwt_key_mapping_cache(monkeypat ) +class _ChannelSubscription: + """Marks a message done when the subscriber comes back for the next one, so ``messages.join()`` + returns once every delivered message has been applied""" + + def __init__(self) -> None: + self.messages: asyncio.Queue[dict[str, object]] = asyncio.Queue() + self._holding_message = False + + async def subscribe(self, *channels: str) -> None: + return None + + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> dict[str, object] | None: + if self._holding_message: + self._holding_message = False + self.messages.task_done() + try: + message: Final = await asyncio.wait_for(self.messages.get(), timeout) + except asyncio.TimeoutError: + return None + self._holding_message = True + return message + + async def aclose(self) -> None: + return None + + +class _ChannelPipeline: + def __init__(self, redis: "_SharedCoordinationRedis") -> None: + self._redis = redis + self._messages: tuple[str, ...] = () + + def publish(self, channel: str, message: str) -> "_ChannelPipeline": + self._messages = (*self._messages, message) + return self + + async def execute(self) -> None: + for message in self._messages: + self._redis.deliver(message) + + +class _SharedCoordinationRedis: + """The coordination Redis two workers share: whatever one worker publishes reaches the other's subscriber""" + + namespace = None + + def __init__(self) -> None: + self.published: tuple[str, ...] = () + self._subscriptions: tuple[_ChannelSubscription, ...] = () + + def init_pubsub_client(self) -> "_SharedCoordinationRedis": + return self + + def pubsub(self) -> _ChannelSubscription: + subscription: Final = _ChannelSubscription() + self._subscriptions = (*self._subscriptions, subscription) + return subscription + + def pipeline(self, transaction: bool = True) -> _ChannelPipeline: + return _ChannelPipeline(self) + + async def publish(self, channel: str, message: str) -> int: + self.deliver(message) + return 1 + + def deliver(self, message: str) -> None: + self.published = (*self.published, message) + for subscription in self._subscriptions: + subscription.messages.put_nowait({"type": "message", "data": message.encode()}) + + async def until_every_subscriber_applied_its_messages(self) -> None: + _ = await asyncio.gather(*(subscription.messages.join() for subscription in self._subscriptions)) + + +def _owned_key_row(hashed_token: str) -> LiteLLM_VerificationToken: + return LiteLLM_VerificationToken( + token=hashed_token, + user_id="user-123", + team_id=None, + key_alias="doomed-key", + spend=0.0, + max_budget=None, + models=[], + aliases={}, + config={}, + permissions={}, + metadata={}, + model_max_budget={}, + model_spend={}, + soft_budget_cooldown=False, + allowed_routes=[], + ) + + +def _prisma_with_key_row(key_row: LiteLLM_VerificationToken) -> AsyncMock: + prisma_client: Final = AsyncMock() + prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[key_row]) + prisma_client.db.litellm_jwtkeymapping = _CascadingJWTMappingTable([]) + prisma_client.db.litellm_deletedverificationtoken.create_many = AsyncMock() + prisma_client.delete_data = AsyncMock(return_value=[key_row.token]) + prisma_client.get_data = AsyncMock(return_value=None) + return prisma_client + + +@pytest.mark.asyncio +async def test_delete_verification_tokens_stops_the_key_on_a_peer_worker(monkeypatch: pytest.MonkeyPatch) -> None: + """Auth serves key objects cache-first, so a peer worker that is not told about the delete keeps + authenticating the deleted key from its own copy until user_api_key_cache_ttl expires""" + raw_key: Final = "sk-" + uuid4().hex + hashed_token: Final = hash_token(raw_key) + deleting_worker_cache: Final = UserApiKeyCache() + peer_worker_cache: Final = UserApiKeyCache() + for cache in (deleting_worker_cache, peer_worker_cache): + cache.set_cache(key=hashed_token, value=UserAPIKeyAuth(token=hashed_token, user_id="user-123")) + coordination_redis: Final = _SharedCoordinationRedis() + peer_subscriber: Final = AuthCacheInvalidationSubscriber( + redis_cache=coordination_redis, user_api_key_cache=peer_worker_cache + ) + peer_subscriber.start() + prisma_client: Final = _prisma_with_key_row(_owned_key_row(hashed_token)) + monkeypatch.setattr("litellm.proxy.proxy_server.redis_usage_cache", coordination_redis) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + + try: + await delete_verification_tokens( + tokens=[raw_key], + user_api_key_cache=deleting_worker_cache, + user_api_key_dict=UserAPIKeyAuth( + user_id="admin-user", api_key="sk-admin", user_role=LitellmUserRoles.PROXY_ADMIN.value + ), + ) + await coordination_redis.until_every_subscriber_applied_its_messages() + finally: + await peer_subscriber.stop() + + assert deleting_worker_cache.get_cache(key=hashed_token) is None + with pytest.raises(ProxyException) as denied: + await get_key_object( + hashed_token=hashed_token, prisma_client=prisma_client, user_api_key_cache=peer_worker_cache + ) + assert denied.value.code == "401" + + +@pytest.mark.asyncio +async def test_delete_verification_tokens_leaves_unowned_cache_entries_alone_on_every_worker(monkeypatch: pytest.MonkeyPatch) -> None: + """/key/delete accepts any string, and a name that is not a key row passes through unhashed. A key + owner padding the request with another team's spend counter name must not wipe that counter from + every worker's memory: only the rows the caller was allowed to delete are evicted and broadcast""" + raw_key: Final = "sk-" + uuid4().hex + hashed_token: Final = hash_token(raw_key) + other_team_counter: Final = "spend:team:other-team" + deleting_worker_cache: Final = UserApiKeyCache() + peer_worker_cache: Final = UserApiKeyCache() + peer_spend_counters: Final = InMemoryCache() + peer_spend_counters.set_cache(other_team_counter, 42.0) + peer_worker_cache.set_cache(key=hashed_token, value=UserAPIKeyAuth(token=hashed_token, user_id="user-123")) + coordination_redis: Final = _SharedCoordinationRedis() + peer_subscriber: Final = AuthCacheInvalidationSubscriber( + redis_cache=coordination_redis, + user_api_key_cache=peer_worker_cache, + additional_in_memory_caches=(peer_spend_counters,), + ) + peer_subscriber.start() + monkeypatch.setattr("litellm.proxy.proxy_server.redis_usage_cache", coordination_redis) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _prisma_with_key_row(_owned_key_row(hashed_token))) + + try: + await delete_verification_tokens( + tokens=[other_team_counter, raw_key], + user_api_key_cache=deleting_worker_cache, + user_api_key_dict=UserAPIKeyAuth( + user_id="user-123", api_key="sk-owner", user_role=LitellmUserRoles.INTERNAL_USER.value + ), + ) + await coordination_redis.until_every_subscriber_applied_its_messages() + finally: + await peer_subscriber.stop() + + assert peer_worker_cache.get_cache(key=hashed_token) is None + assert peer_spend_counters.get_cache(other_team_counter) == 42.0 + assert coordination_redis.published == (json.dumps({"cache_key": hashed_token}),) + + @pytest.mark.asyncio async def test_delete_key_fn_persists_deleted_keys(monkeypatch): from litellm.proxy._types import KeyRequest