mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 6bdf36c194 into e16d79d635
This commit is contained in:
commit
bb7489d7df
5 changed files with 273 additions and 30 deletions
|
|
@ -3435,28 +3435,35 @@ async def delete_cache_key_objects(
|
|||
proxy_logging_obj: ProxyLogging | None,
|
||||
) -> None:
|
||||
"""
|
||||
Evict a batch of key objects, for callers that delete keys in bulk rather than through
|
||||
`/key/delete`. Auth resolves a cached key object without re-reading its team, so a key left
|
||||
cached after its row is gone keeps buying access until its TTL expires.
|
||||
|
||||
Evicting locally only reaches this worker; `_delete_cache_key_object` itself broadcasts each
|
||||
token, so a deleted key left in a peer worker's in-memory cache still authenticates there until
|
||||
its TTL expires.
|
||||
Evict a batch of deleted key objects here and on every other worker. Auth resolves a cached key
|
||||
object without re-reading its team, so a key left cached after its row is gone keeps buying
|
||||
access until its TTL expires.
|
||||
|
||||
Best-effort per key: the rows are already deleted by the time this runs, so an unreachable
|
||||
cache backend must not abort the caller partway through its own cascade.
|
||||
|
||||
Every token is dropped from this worker's memory first, then from the auth cache and the
|
||||
internal usage cache in one batch each, so a deleted key stops authenticating here the instant
|
||||
its rows are gone instead of refilling from Redis on the next miss. Every other worker then
|
||||
gets the eviction in one pipelined broadcast: a publish per key would pass the publisher's
|
||||
pending cap on a bulk delete, and the dropped keys would keep authenticating there until their
|
||||
TTL.
|
||||
Every token is dropped from this worker's memory first, then from the auth cache's Redis layer,
|
||||
so a deleted key stops authenticating here the instant its rows are gone instead of refilling
|
||||
from Redis on the next miss. Every other worker then gets the eviction in one pipelined
|
||||
broadcast: a publish per key would pass the publisher's pending cap on a bulk delete, and the
|
||||
dropped keys would keep authenticating there until their TTL. The internal usage cache is not
|
||||
an auth refill source, so its batch delete runs beside that chain and a hung Redis costs one
|
||||
bounded wait there instead of another in line.
|
||||
"""
|
||||
evict_local(cache_keys=hashed_tokens, user_api_key_cache=user_api_key_cache)
|
||||
usage_eviction: Final = (
|
||||
()
|
||||
if proxy_logging_obj is None
|
||||
else (evict_shared(cache_keys=hashed_tokens, cache=proxy_logging_obj.internal_usage_cache.dual_cache),)
|
||||
)
|
||||
_ = await asyncio.gather(
|
||||
_evict_auth_cache_then_broadcast(hashed_tokens=hashed_tokens, user_api_key_cache=user_api_key_cache),
|
||||
*usage_eviction,
|
||||
)
|
||||
|
||||
|
||||
async def _evict_auth_cache_then_broadcast(hashed_tokens: Sequence[str], user_api_key_cache: UserApiKeyCache) -> None:
|
||||
await evict_shared(cache_keys=hashed_tokens, cache=user_api_key_cache)
|
||||
if proxy_logging_obj is not None:
|
||||
await evict_shared(cache_keys=hashed_tokens, cache=proxy_logging_obj.internal_usage_cache.dual_cache)
|
||||
await broadcast_evictions(cache_keys=hashed_tokens)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
|
|||
_delete_cache_key_object, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
can_team_access_model,
|
||||
delete_cache_key_object,
|
||||
delete_cache_key_objects,
|
||||
get_jwt_key_mapping_cache_keys_for_token,
|
||||
get_key_end_user_budget_id,
|
||||
get_org_object,
|
||||
|
|
@ -5490,7 +5491,7 @@ async def delete_verification_tokens(
|
|||
- List of keys being deleted, this contains information about the key_alias, token, and user_id being deleted,
|
||||
this is passed down to the KeyManagementEventHooks to delete the keys from the secret manager and handle audit logs
|
||||
"""
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj
|
||||
|
||||
failed_tokens: list = []
|
||||
try:
|
||||
|
|
@ -5574,11 +5575,11 @@ async def delete_verification_tokens(
|
|||
verbose_proxy_logger.debug(traceback.format_exc())
|
||||
raise e
|
||||
|
||||
for key in tokens:
|
||||
user_api_key_cache.delete_cache(key)
|
||||
# remove hash token from cache
|
||||
hashed_token = hash_token(cast(str, key))
|
||||
user_api_key_cache.delete_cache(hashed_token)
|
||||
await delete_cache_key_objects(
|
||||
hashed_tokens=tuple(key.token for key in authorized_keys if key.token is not None),
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
# After credential invalidation, so a failure here can never keep a deleted key alive.
|
||||
for deleted_key in authorized_keys:
|
||||
|
|
|
|||
|
|
@ -171,11 +171,8 @@ def test_deleting_a_key_revokes_it(gateway: Gateway, spelling: str) -> None:
|
|||
assert deleted_rows == (_DeletedKeyRow(token=key_hash),)
|
||||
|
||||
|
||||
def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gateway) -> None:
|
||||
pytest.skip(
|
||||
"BUG: POST /key/delete does not broadcast the key eviction, so a warmed peer keeps answering 200 for about "
|
||||
"60 seconds until its in-memory cache TTL expires"
|
||||
)
|
||||
@pytest.mark.parametrize("spelling", ("keys", "hashed_keys", "key_aliases"))
|
||||
def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gateway, spelling: str) -> None:
|
||||
backend: Final = "integration-keys-auth-delete"
|
||||
prompts: Final = (
|
||||
"delete warm primary " + uuid.uuid4().hex,
|
||||
|
|
@ -208,9 +205,13 @@ def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gate
|
|||
)
|
||||
assert len(warm_responses) == 2
|
||||
assert tuple((request.method, request.target) for request in wire.drain()) == (_EXPECTED_CHAT,) * 2
|
||||
deleted: Final = gateway.request("POST", "/key/delete", {"keys": [key]})
|
||||
identifier: Final = {"keys": key, "hashed_keys": sha256(key.encode()).hexdigest(), "key_aliases": alias}[
|
||||
spelling
|
||||
]
|
||||
delete_body: Final = {"key_aliases" if spelling == "key_aliases" else "keys": [identifier]}
|
||||
deleted: Final = gateway.request("POST", "/key/delete", delete_body)
|
||||
assert deleted.status_code == 200, deleted.text
|
||||
assert _DeleteResponse.model_validate_json(deleted.content).deleted_keys == (key,), deleted.text
|
||||
assert _DeleteResponse.model_validate_json(deleted.content).deleted_keys == (identifier,), deleted.text
|
||||
peer_refusal: Final = eventually(
|
||||
lambda: _chat(peer, model, key, prompts[2]),
|
||||
lambda response: response.status_code == 401,
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ from litellm.proxy.auth.auth_checks import (
|
|||
_team_soft_budget_check,
|
||||
common_checks,
|
||||
)
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
from litellm.proxy.utils import InternalUsageCache, ProxyLogging
|
||||
from litellm.proxy.utils import CallInfo
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import LITELLM_PASS_THROUGH_ENDPOINT_MARKER
|
||||
|
||||
|
|
@ -2380,3 +2380,51 @@ async def test_bulk_eviction_clears_the_redis_copies_before_waiting_on_a_wedged_
|
|||
|
||||
assert refilled == []
|
||||
assert redis.entries == {}
|
||||
|
||||
|
||||
class _SignallingPublishRedisClient(_SlowPublishRedisClient):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.sent = asyncio.Event()
|
||||
|
||||
async def _send(self, messages: Sequence[str]) -> None:
|
||||
await super()._send(messages)
|
||||
self.sent.set()
|
||||
|
||||
|
||||
class _DeleteAfterBroadcastRedisCache(_AuthRedisCache):
|
||||
def __init__(self, broadcast_sent: asyncio.Event) -> None:
|
||||
super().__init__({})
|
||||
self._broadcast_sent = broadcast_sent
|
||||
self.finished = False
|
||||
|
||||
async def delete_cache_keys(self, keys: Sequence[str]) -> None:
|
||||
await self._broadcast_sent.wait()
|
||||
self.finished = True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bulk_eviction_broadcasts_while_the_usage_cache_redis_is_wedged(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(pubsub_module, "_in_flight_publishes", asyncio.Semaphore(16))
|
||||
monkeypatch.setattr(pubsub_module, "_pending_publishes", set())
|
||||
client: Final = _SignallingPublishRedisClient()
|
||||
usage_redis: Final = _DeleteAfterBroadcastRedisCache(broadcast_sent=client.sent)
|
||||
proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=None)
|
||||
proxy_logging_obj.internal_usage_cache = InternalUsageCache(
|
||||
dual_cache=DualCache(redis_cache=usage_redis) # pyright: ignore[reportArgumentType] # a dict-backed fake of the Redis layer
|
||||
)
|
||||
hashed_tokens: Final = ("token-a", "token-b")
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.common_utils.auth_cache_invalidation_pubsub.coordination_redis_cache",
|
||||
return_value=_PubSubRedisCache(client),
|
||||
):
|
||||
await delete_cache_key_objects(
|
||||
hashed_tokens=hashed_tokens, user_api_key_cache=UserApiKeyCache(), proxy_logging_obj=proxy_logging_obj
|
||||
)
|
||||
await _drain_publishes()
|
||||
|
||||
assert usage_redis.finished
|
||||
assert {json.loads(message)["cache_key"] for message in client.published} == set(hashed_tokens)
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from typing import Final
|
|||
from types import SimpleNamespace
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import UUID
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import litellm
|
||||
import pytest
|
||||
|
|
@ -40,13 +40,17 @@ from litellm.proxy._types import (
|
|||
RegenerateKeyRequest,
|
||||
ResetSpendRequest,
|
||||
UpdateKeyRequest,
|
||||
hash_token,
|
||||
)
|
||||
from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
delete_cache_key_object,
|
||||
get_key_object,
|
||||
jwt_key_mapping_cache_key,
|
||||
)
|
||||
from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import AuthCacheInvalidationSubscriber
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, project_cache_key
|
||||
from litellm.constants import PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS
|
||||
from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker
|
||||
|
|
@ -5757,6 +5761,188 @@ async def test_delete_verification_tokens_evicts_jwt_key_mapping_cache(monkeypat
|
|||
)
|
||||
|
||||
|
||||
class _ChannelSubscription:
|
||||
"""Marks a message done when the subscriber comes back for the next one, so ``messages.join()``
|
||||
returns once every delivered message has been applied"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.messages: asyncio.Queue[dict[str, object]] = asyncio.Queue()
|
||||
self._holding_message = False
|
||||
|
||||
async def subscribe(self, *channels: str) -> None:
|
||||
return None
|
||||
|
||||
async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> dict[str, object] | None:
|
||||
if self._holding_message:
|
||||
self._holding_message = False
|
||||
self.messages.task_done()
|
||||
try:
|
||||
message: Final = await asyncio.wait_for(self.messages.get(), timeout)
|
||||
except asyncio.TimeoutError:
|
||||
return None
|
||||
self._holding_message = True
|
||||
return message
|
||||
|
||||
async def aclose(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _ChannelPipeline:
|
||||
def __init__(self, redis: "_SharedCoordinationRedis") -> None:
|
||||
self._redis = redis
|
||||
self._messages: tuple[str, ...] = ()
|
||||
|
||||
def publish(self, channel: str, message: str) -> "_ChannelPipeline":
|
||||
self._messages = (*self._messages, message)
|
||||
return self
|
||||
|
||||
async def execute(self) -> None:
|
||||
for message in self._messages:
|
||||
self._redis.deliver(message)
|
||||
|
||||
|
||||
class _SharedCoordinationRedis:
|
||||
"""The coordination Redis two workers share: whatever one worker publishes reaches the other's subscriber"""
|
||||
|
||||
namespace = None
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.published: tuple[str, ...] = ()
|
||||
self._subscriptions: tuple[_ChannelSubscription, ...] = ()
|
||||
|
||||
def init_pubsub_client(self) -> "_SharedCoordinationRedis":
|
||||
return self
|
||||
|
||||
def pubsub(self) -> _ChannelSubscription:
|
||||
subscription: Final = _ChannelSubscription()
|
||||
self._subscriptions = (*self._subscriptions, subscription)
|
||||
return subscription
|
||||
|
||||
def pipeline(self, transaction: bool = True) -> _ChannelPipeline:
|
||||
return _ChannelPipeline(self)
|
||||
|
||||
async def publish(self, channel: str, message: str) -> int:
|
||||
self.deliver(message)
|
||||
return 1
|
||||
|
||||
def deliver(self, message: str) -> None:
|
||||
self.published = (*self.published, message)
|
||||
for subscription in self._subscriptions:
|
||||
subscription.messages.put_nowait({"type": "message", "data": message.encode()})
|
||||
|
||||
async def until_every_subscriber_applied_its_messages(self) -> None:
|
||||
_ = await asyncio.gather(*(subscription.messages.join() for subscription in self._subscriptions))
|
||||
|
||||
|
||||
def _owned_key_row(hashed_token: str) -> LiteLLM_VerificationToken:
|
||||
return LiteLLM_VerificationToken(
|
||||
token=hashed_token,
|
||||
user_id="user-123",
|
||||
team_id=None,
|
||||
key_alias="doomed-key",
|
||||
spend=0.0,
|
||||
max_budget=None,
|
||||
models=[],
|
||||
aliases={},
|
||||
config={},
|
||||
permissions={},
|
||||
metadata={},
|
||||
model_max_budget={},
|
||||
model_spend={},
|
||||
soft_budget_cooldown=False,
|
||||
allowed_routes=[],
|
||||
)
|
||||
|
||||
|
||||
def _prisma_with_key_row(key_row: LiteLLM_VerificationToken) -> AsyncMock:
|
||||
prisma_client: Final = AsyncMock()
|
||||
prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[key_row])
|
||||
prisma_client.db.litellm_jwtkeymapping = _CascadingJWTMappingTable([])
|
||||
prisma_client.db.litellm_deletedverificationtoken.create_many = AsyncMock()
|
||||
prisma_client.delete_data = AsyncMock(return_value=[key_row.token])
|
||||
prisma_client.get_data = AsyncMock(return_value=None)
|
||||
return prisma_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_verification_tokens_stops_the_key_on_a_peer_worker(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Auth serves key objects cache-first, so a peer worker that is not told about the delete keeps
|
||||
authenticating the deleted key from its own copy until user_api_key_cache_ttl expires"""
|
||||
raw_key: Final = "sk-" + uuid4().hex
|
||||
hashed_token: Final = hash_token(raw_key)
|
||||
deleting_worker_cache: Final = UserApiKeyCache()
|
||||
peer_worker_cache: Final = UserApiKeyCache()
|
||||
for cache in (deleting_worker_cache, peer_worker_cache):
|
||||
cache.set_cache(key=hashed_token, value=UserAPIKeyAuth(token=hashed_token, user_id="user-123"))
|
||||
coordination_redis: Final = _SharedCoordinationRedis()
|
||||
peer_subscriber: Final = AuthCacheInvalidationSubscriber(
|
||||
redis_cache=coordination_redis, user_api_key_cache=peer_worker_cache
|
||||
)
|
||||
peer_subscriber.start()
|
||||
prisma_client: Final = _prisma_with_key_row(_owned_key_row(hashed_token))
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.redis_usage_cache", coordination_redis)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client)
|
||||
|
||||
try:
|
||||
await delete_verification_tokens(
|
||||
tokens=[raw_key],
|
||||
user_api_key_cache=deleting_worker_cache,
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
user_id="admin-user", api_key="sk-admin", user_role=LitellmUserRoles.PROXY_ADMIN.value
|
||||
),
|
||||
)
|
||||
await coordination_redis.until_every_subscriber_applied_its_messages()
|
||||
finally:
|
||||
await peer_subscriber.stop()
|
||||
|
||||
assert deleting_worker_cache.get_cache(key=hashed_token) is None
|
||||
with pytest.raises(ProxyException) as denied:
|
||||
await get_key_object(
|
||||
hashed_token=hashed_token, prisma_client=prisma_client, user_api_key_cache=peer_worker_cache
|
||||
)
|
||||
assert denied.value.code == "401"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_verification_tokens_leaves_unowned_cache_entries_alone_on_every_worker(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""/key/delete accepts any string, and a name that is not a key row passes through unhashed. A key
|
||||
owner padding the request with another team's spend counter name must not wipe that counter from
|
||||
every worker's memory: only the rows the caller was allowed to delete are evicted and broadcast"""
|
||||
raw_key: Final = "sk-" + uuid4().hex
|
||||
hashed_token: Final = hash_token(raw_key)
|
||||
other_team_counter: Final = "spend:team:other-team"
|
||||
deleting_worker_cache: Final = UserApiKeyCache()
|
||||
peer_worker_cache: Final = UserApiKeyCache()
|
||||
peer_spend_counters: Final = InMemoryCache()
|
||||
peer_spend_counters.set_cache(other_team_counter, 42.0)
|
||||
peer_worker_cache.set_cache(key=hashed_token, value=UserAPIKeyAuth(token=hashed_token, user_id="user-123"))
|
||||
coordination_redis: Final = _SharedCoordinationRedis()
|
||||
peer_subscriber: Final = AuthCacheInvalidationSubscriber(
|
||||
redis_cache=coordination_redis,
|
||||
user_api_key_cache=peer_worker_cache,
|
||||
additional_in_memory_caches=(peer_spend_counters,),
|
||||
)
|
||||
peer_subscriber.start()
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.redis_usage_cache", coordination_redis)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _prisma_with_key_row(_owned_key_row(hashed_token)))
|
||||
|
||||
try:
|
||||
await delete_verification_tokens(
|
||||
tokens=[other_team_counter, raw_key],
|
||||
user_api_key_cache=deleting_worker_cache,
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
user_id="user-123", api_key="sk-owner", user_role=LitellmUserRoles.INTERNAL_USER.value
|
||||
),
|
||||
)
|
||||
await coordination_redis.until_every_subscriber_applied_its_messages()
|
||||
finally:
|
||||
await peer_subscriber.stop()
|
||||
|
||||
assert peer_worker_cache.get_cache(key=hashed_token) is None
|
||||
assert peer_spend_counters.get_cache(other_team_counter) == 42.0
|
||||
assert coordination_redis.published == (json.dumps({"cache_key": hashed_token}),)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_key_fn_persists_deleted_keys(monkeypatch):
|
||||
from litellm.proxy._types import KeyRequest
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue