This commit is contained in:
Etienne Chabert 2026-10-27 02:24:52 +00:00 • committed by GitHub
commit bb7489d7df
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 273 additions and 30 deletions

View file

@ -3435,28 +3435,35 @@ async def delete_cache_key_objects(
proxy_logging_obj: ProxyLogging | None,
) -> None:
"""
Evict a batch of key objects, for callers that delete keys in bulk rather than through
`/key/delete`. Auth resolves a cached key object without re-reading its team, so a key left
cached after its row is gone keeps buying access until its TTL expires.
Evicting locally only reaches this worker; `_delete_cache_key_object` itself broadcasts each
token, so a deleted key left in a peer worker's in-memory cache still authenticates there until
its TTL expires.
Evict a batch of deleted key objects here and on every other worker. Auth resolves a cached key
object without re-reading its team, so a key left cached after its row is gone keeps buying
access until its TTL expires.
Best-effort per key: the rows are already deleted by the time this runs, so an unreachable
cache backend must not abort the caller partway through its own cascade.
Every token is dropped from this worker's memory first, then from the auth cache and the
internal usage cache in one batch each, so a deleted key stops authenticating here the instant
its rows are gone instead of refilling from Redis on the next miss. Every other worker then
gets the eviction in one pipelined broadcast: a publish per key would pass the publisher's
pending cap on a bulk delete, and the dropped keys would keep authenticating there until their
TTL.
Every token is dropped from this worker's memory first, then from the auth cache's Redis layer,
so a deleted key stops authenticating here the instant its rows are gone instead of refilling
from Redis on the next miss. Every other worker then gets the eviction in one pipelined
broadcast: a publish per key would pass the publisher's pending cap on a bulk delete, and the
dropped keys would keep authenticating there until their TTL. The internal usage cache is not
an auth refill source, so its batch delete runs beside that chain and a hung Redis costs one
bounded wait there instead of another in line.
"""
evict_local(cache_keys=hashed_tokens, user_api_key_cache=user_api_key_cache)
usage_eviction: Final = (
()
if proxy_logging_obj is None
else (evict_shared(cache_keys=hashed_tokens, cache=proxy_logging_obj.internal_usage_cache.dual_cache),)
)
_ = await asyncio.gather(
_evict_auth_cache_then_broadcast(hashed_tokens=hashed_tokens, user_api_key_cache=user_api_key_cache),
*usage_eviction,
)
async def _evict_auth_cache_then_broadcast(hashed_tokens: Sequence[str], user_api_key_cache: UserApiKeyCache) -> None:
await evict_shared(cache_keys=hashed_tokens, cache=user_api_key_cache)
if proxy_logging_obj is not None:
await evict_shared(cache_keys=hashed_tokens, cache=proxy_logging_obj.internal_usage_cache.dual_cache)
await broadcast_evictions(cache_keys=hashed_tokens)

View file

@ -58,6 +58,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
_delete_cache_key_object, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
can_team_access_model,
delete_cache_key_object,
delete_cache_key_objects,
get_jwt_key_mapping_cache_keys_for_token,
get_key_end_user_budget_id,
get_org_object,
@ -5490,7 +5491,7 @@ async def delete_verification_tokens(
- List of keys being deleted, this contains information about the key_alias, token, and user_id being deleted,
this is passed down to the KeyManagementEventHooks to delete the keys from the secret manager and handle audit logs
"""
from litellm.proxy.proxy_server import prisma_client
from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj
failed_tokens: list = []
try:
@ -5574,11 +5575,11 @@ async def delete_verification_tokens(
verbose_proxy_logger.debug(traceback.format_exc())
raise e
for key in tokens:
user_api_key_cache.delete_cache(key)
# remove hash token from cache
hashed_token = hash_token(cast(str, key))
user_api_key_cache.delete_cache(hashed_token)
await delete_cache_key_objects(
hashed_tokens=tuple(key.token for key in authorized_keys if key.token is not None),
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
# After credential invalidation, so a failure here can never keep a deleted key alive.
for deleted_key in authorized_keys:

View file

@ -171,11 +171,8 @@ def test_deleting_a_key_revokes_it(gateway: Gateway, spelling: str) -> None:
assert deleted_rows == (_DeletedKeyRow(token=key_hash),)
def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gateway) -> None:
pytest.skip(
"BUG: POST /key/delete does not broadcast the key eviction, so a warmed peer keeps answering 200 for about "
"60 seconds until its in-memory cache TTL expires"
)
@pytest.mark.parametrize("spelling", ("keys", "hashed_keys", "key_aliases"))
def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gateway, spelling: str) -> None:
backend: Final = "integration-keys-auth-delete"
prompts: Final = (
"delete warm primary " + uuid.uuid4().hex,
@ -208,9 +205,13 @@ def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gate
)
assert len(warm_responses) == 2
assert tuple((request.method, request.target) for request in wire.drain()) == (_EXPECTED_CHAT,) * 2
deleted: Final = gateway.request("POST", "/key/delete", {"keys": [key]})
identifier: Final = {"keys": key, "hashed_keys": sha256(key.encode()).hexdigest(), "key_aliases": alias}[
spelling
]
delete_body: Final = {"key_aliases" if spelling == "key_aliases" else "keys": [identifier]}
deleted: Final = gateway.request("POST", "/key/delete", delete_body)
assert deleted.status_code == 200, deleted.text
assert _DeleteResponse.model_validate_json(deleted.content).deleted_keys == (key,), deleted.text
assert _DeleteResponse.model_validate_json(deleted.content).deleted_keys == (identifier,), deleted.text
peer_refusal: Final = eventually(
lambda: _chat(peer, model, key, prompts[2]),
lambda response: response.status_code == 401,

View file

@ -46,7 +46,7 @@ from litellm.proxy.auth.auth_checks import (
_team_soft_budget_check,
common_checks,
)
from litellm.proxy.utils import ProxyLogging
from litellm.proxy.utils import InternalUsageCache, ProxyLogging
from litellm.proxy.utils import CallInfo
from litellm.types.passthrough_endpoints.pass_through_endpoints import LITELLM_PASS_THROUGH_ENDPOINT_MARKER
@ -2380,3 +2380,51 @@ async def test_bulk_eviction_clears_the_redis_copies_before_waiting_on_a_wedged_
assert refilled == []
assert redis.entries == {}
class _SignallingPublishRedisClient(_SlowPublishRedisClient):
def __init__(self) -> None:
super().__init__()
self.sent = asyncio.Event()
async def _send(self, messages: Sequence[str]) -> None:
await super()._send(messages)
self.sent.set()
class _DeleteAfterBroadcastRedisCache(_AuthRedisCache):
def __init__(self, broadcast_sent: asyncio.Event) -> None:
super().__init__({})
self._broadcast_sent = broadcast_sent
self.finished = False
async def delete_cache_keys(self, keys: Sequence[str]) -> None:
await self._broadcast_sent.wait()
self.finished = True
@pytest.mark.asyncio
async def test_bulk_eviction_broadcasts_while_the_usage_cache_redis_is_wedged(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(pubsub_module, "_in_flight_publishes", asyncio.Semaphore(16))
monkeypatch.setattr(pubsub_module, "_pending_publishes", set())
client: Final = _SignallingPublishRedisClient()
usage_redis: Final = _DeleteAfterBroadcastRedisCache(broadcast_sent=client.sent)
proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=None)
proxy_logging_obj.internal_usage_cache = InternalUsageCache(
dual_cache=DualCache(redis_cache=usage_redis) # pyright: ignore[reportArgumentType] # a dict-backed fake of the Redis layer
)
hashed_tokens: Final = ("token-a", "token-b")
with patch(
"litellm.proxy.common_utils.auth_cache_invalidation_pubsub.coordination_redis_cache",
return_value=_PubSubRedisCache(client),
):
await delete_cache_key_objects(
hashed_tokens=hashed_tokens, user_api_key_cache=UserApiKeyCache(), proxy_logging_obj=proxy_logging_obj
)
await _drain_publishes()
assert usage_redis.finished
assert {json.loads(message)["cache_key"] for message in client.published} == set(hashed_tokens)

View file

@ -5,7 +5,7 @@ from typing import Final
from types import SimpleNamespace
import json
from datetime import datetime, timedelta, timezone
from uuid import UUID
from uuid import UUID, uuid4
import litellm
import pytest
@ -40,13 +40,17 @@ from litellm.proxy._types import (
RegenerateKeyRequest,
ResetSpendRequest,
UpdateKeyRequest,
hash_token,
)
from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.proxy.auth.auth_checks import (
delete_cache_key_object,
get_key_object,
jwt_key_mapping_cache_key,
)
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import AuthCacheInvalidationSubscriber
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, project_cache_key
from litellm.constants import PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS
from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker
@ -5757,6 +5761,188 @@ async def test_delete_verification_tokens_evicts_jwt_key_mapping_cache(monkeypat
)
class _ChannelSubscription:
"""Marks a message done when the subscriber comes back for the next one, so ``messages.join()``
returns once every delivered message has been applied"""
def __init__(self) -> None:
self.messages: asyncio.Queue[dict[str, object]] = asyncio.Queue()
self._holding_message = False
async def subscribe(self, *channels: str) -> None:
return None
async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> dict[str, object] | None:
if self._holding_message:
self._holding_message = False
self.messages.task_done()
try:
message: Final = await asyncio.wait_for(self.messages.get(), timeout)
except asyncio.TimeoutError:
return None
self._holding_message = True
return message
async def aclose(self) -> None:
return None
class _ChannelPipeline:
def __init__(self, redis: "_SharedCoordinationRedis") -> None:
self._redis = redis
self._messages: tuple[str, ...] = ()
def publish(self, channel: str, message: str) -> "_ChannelPipeline":
self._messages = (*self._messages, message)
return self
async def execute(self) -> None:
for message in self._messages:
self._redis.deliver(message)
class _SharedCoordinationRedis:
"""The coordination Redis two workers share: whatever one worker publishes reaches the other's subscriber"""
namespace = None
def __init__(self) -> None:
self.published: tuple[str, ...] = ()
self._subscriptions: tuple[_ChannelSubscription, ...] = ()
def init_pubsub_client(self) -> "_SharedCoordinationRedis":
return self
def pubsub(self) -> _ChannelSubscription:
subscription: Final = _ChannelSubscription()
self._subscriptions = (*self._subscriptions, subscription)
return subscription
def pipeline(self, transaction: bool = True) -> _ChannelPipeline:
return _ChannelPipeline(self)
async def publish(self, channel: str, message: str) -> int:
self.deliver(message)
return 1
def deliver(self, message: str) -> None:
self.published = (*self.published, message)
for subscription in self._subscriptions:
subscription.messages.put_nowait({"type": "message", "data": message.encode()})
async def until_every_subscriber_applied_its_messages(self) -> None:
_ = await asyncio.gather(*(subscription.messages.join() for subscription in self._subscriptions))
def _owned_key_row(hashed_token: str) -> LiteLLM_VerificationToken:
return LiteLLM_VerificationToken(
token=hashed_token,
user_id="user-123",
team_id=None,
key_alias="doomed-key",
spend=0.0,
max_budget=None,
models=[],
aliases={},
config={},
permissions={},
metadata={},
model_max_budget={},
model_spend={},
soft_budget_cooldown=False,
allowed_routes=[],
)
def _prisma_with_key_row(key_row: LiteLLM_VerificationToken) -> AsyncMock:
prisma_client: Final = AsyncMock()
prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[key_row])
prisma_client.db.litellm_jwtkeymapping = _CascadingJWTMappingTable([])
prisma_client.db.litellm_deletedverificationtoken.create_many = AsyncMock()
prisma_client.delete_data = AsyncMock(return_value=[key_row.token])
prisma_client.get_data = AsyncMock(return_value=None)
return prisma_client
@pytest.mark.asyncio
async def test_delete_verification_tokens_stops_the_key_on_a_peer_worker(monkeypatch: pytest.MonkeyPatch) -> None:
"""Auth serves key objects cache-first, so a peer worker that is not told about the delete keeps
authenticating the deleted key from its own copy until user_api_key_cache_ttl expires"""
raw_key: Final = "sk-" + uuid4().hex
hashed_token: Final = hash_token(raw_key)
deleting_worker_cache: Final = UserApiKeyCache()
peer_worker_cache: Final = UserApiKeyCache()
for cache in (deleting_worker_cache, peer_worker_cache):
cache.set_cache(key=hashed_token, value=UserAPIKeyAuth(token=hashed_token, user_id="user-123"))
coordination_redis: Final = _SharedCoordinationRedis()
peer_subscriber: Final = AuthCacheInvalidationSubscriber(
redis_cache=coordination_redis, user_api_key_cache=peer_worker_cache
)
peer_subscriber.start()
prisma_client: Final = _prisma_with_key_row(_owned_key_row(hashed_token))
monkeypatch.setattr("litellm.proxy.proxy_server.redis_usage_cache", coordination_redis)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client)
try:
await delete_verification_tokens(
tokens=[raw_key],
user_api_key_cache=deleting_worker_cache,
user_api_key_dict=UserAPIKeyAuth(
user_id="admin-user", api_key="sk-admin", user_role=LitellmUserRoles.PROXY_ADMIN.value
),
)
await coordination_redis.until_every_subscriber_applied_its_messages()
finally:
await peer_subscriber.stop()
assert deleting_worker_cache.get_cache(key=hashed_token) is None
with pytest.raises(ProxyException) as denied:
await get_key_object(
hashed_token=hashed_token, prisma_client=prisma_client, user_api_key_cache=peer_worker_cache
)
assert denied.value.code == "401"
@pytest.mark.asyncio
async def test_delete_verification_tokens_leaves_unowned_cache_entries_alone_on_every_worker(monkeypatch: pytest.MonkeyPatch) -> None:
"""/key/delete accepts any string, and a name that is not a key row passes through unhashed. A key
owner padding the request with another team's spend counter name must not wipe that counter from
every worker's memory: only the rows the caller was allowed to delete are evicted and broadcast"""
raw_key: Final = "sk-" + uuid4().hex
hashed_token: Final = hash_token(raw_key)
other_team_counter: Final = "spend:team:other-team"
deleting_worker_cache: Final = UserApiKeyCache()
peer_worker_cache: Final = UserApiKeyCache()
peer_spend_counters: Final = InMemoryCache()
peer_spend_counters.set_cache(other_team_counter, 42.0)
peer_worker_cache.set_cache(key=hashed_token, value=UserAPIKeyAuth(token=hashed_token, user_id="user-123"))
coordination_redis: Final = _SharedCoordinationRedis()
peer_subscriber: Final = AuthCacheInvalidationSubscriber(
redis_cache=coordination_redis,
user_api_key_cache=peer_worker_cache,
additional_in_memory_caches=(peer_spend_counters,),
)
peer_subscriber.start()
monkeypatch.setattr("litellm.proxy.proxy_server.redis_usage_cache", coordination_redis)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _prisma_with_key_row(_owned_key_row(hashed_token)))
try:
await delete_verification_tokens(
tokens=[other_team_counter, raw_key],
user_api_key_cache=deleting_worker_cache,
user_api_key_dict=UserAPIKeyAuth(
user_id="user-123", api_key="sk-owner", user_role=LitellmUserRoles.INTERNAL_USER.value
),
)
await coordination_redis.until_every_subscriber_applied_its_messages()
finally:
await peer_subscriber.stop()
assert peer_worker_cache.get_cache(key=hashed_token) is None
assert peer_spend_counters.get_cache(other_team_counter) == 42.0
assert coordination_redis.published == (json.dumps({"cache_key": hashed_token}),)
@pytest.mark.asyncio
async def test_delete_key_fn_persists_deleted_keys(monkeypatch):
from litellm.proxy._types import KeyRequest