This commit is contained in:
Kyon 2026-10-27 02:25:33 +00:00 • committed by GitHub
commit 7aad84abe1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 210 additions and 14 deletions

View file

@ -3,7 +3,7 @@ from __future__ import annotations
import asyncio
import re
from collections.abc import Sequence
from typing import TYPE_CHECKING, Any, Final, TypeVar, cast, overload
from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast, overload
from pydantic import BaseModel
@ -34,6 +34,49 @@ def is_user_key_cache_key(key: str) -> bool:
_PIPELINED_SET_OPTIONS: Final = frozenset(("ttl",))
class _AuthRegistryCacheCalls(Protocol):
def get_cache(
self,
key: str,
parent_otel_span: Span | None = None,
local_only: bool = False,
**kwargs: object, # kwargs-ok: preserve arbitrary DualCache cache options
) -> object: ...
async def async_get_cache(
self,
key: str,
parent_otel_span: Span | None = None,
local_only: bool = False,
**kwargs: object, # kwargs-ok: preserve arbitrary DualCache cache options
) -> object: ...
def set_cache(
self,
key: str,
value: object,
local_only: bool = False,
**kwargs: object, # kwargs-ok: preserve arbitrary DualCache cache options
) -> None: ...
async def async_set_cache(
self,
key: str,
value: object,
local_only: bool = False,
**kwargs: object, # kwargs-ok: preserve arbitrary DualCache cache options
) -> None: ...
def delete_cache(self, key: str) -> None: ...
async def async_set_cache_pipeline(
self,
cache_list: Sequence[tuple[str, object]],
local_only: bool = False,
**kwargs: object, # kwargs-ok: preserve arbitrary DualCache cache options
) -> None: ...
class UserApiKeyCache(DualCache):
"""
DualCache wrapper for UserAPIKeyAuth-like payloads.
@ -55,8 +98,8 @@ class UserApiKeyCache(DualCache):
``async_set_cache_pipeline`` applies the same untyped Codec pass as omitting
``model_type`` on ``async_set_cache`` (so ``BaseModel`` rows are dumped before Redis).
User-key objects (see ``is_user_key_cache_key``) live in their own in-memory partition,
``key_object_cache``, so churn in the other management objects cannot evict them. Both
User-key objects and authorization registries use separate in-memory partitions. The registry
partition has capacity four for a fixed three-key whitelist, leaving headroom on rewrites; both
partitions share the same Redis backend and TTL settings.
``get_cache`` / ``async_get_cache`` overloads and implementations must be contiguous
@ -83,8 +126,17 @@ class UserApiKeyCache(DualCache):
default_in_memory_ttl=default_in_memory_ttl,
default_redis_ttl=default_redis_ttl,
)
self.auth_registry_cache: Final = DualCache(
in_memory_cache=InMemoryCache(max_size_in_memory=4),
redis_cache=redis_cache,
default_in_memory_ttl=default_in_memory_ttl,
default_redis_ttl=default_redis_ttl,
)
self._auth_registry_cache_calls: Final[_AuthRegistryCacheCalls] = self.auth_registry_cache
def in_memory_cache_for(self, key: str) -> InMemoryCache:
if is_auth_registry_cache_key(key):
return self.auth_registry_cache.in_memory_cache
return self.key_object_cache.in_memory_cache if is_user_key_cache_key(key) else self.in_memory_cache
def update_cache_ttl(self, default_in_memory_ttl: float | None, default_redis_ttl: float | None) -> None:
@ -92,16 +144,21 @@ class UserApiKeyCache(DualCache):
self.key_object_cache.update_cache_ttl(
default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl
)
self.auth_registry_cache.update_cache_ttl(
default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl
)
def update_in_memory_max_size(self, max_size: int | None) -> None:
super().update_in_memory_max_size(max_size)
self.key_object_cache.update_in_memory_max_size(max_size)
self.auth_registry_cache.update_in_memory_max_size(4)
def attach_redis_cache(
self, redis_cache: RedisCache | None = None, *, default_redis_ttl: float | None = None
) -> None:
super().attach_redis_cache(redis_cache, default_redis_ttl=default_redis_ttl)
self.key_object_cache.attach_redis_cache(redis_cache, default_redis_ttl=default_redis_ttl)
self.auth_registry_cache.attach_redis_cache(redis_cache, default_redis_ttl=default_redis_ttl)
@overload
def get_cache(
@ -135,7 +192,11 @@ class UserApiKeyCache(DualCache):
if model_type is None and "model_type" in kwargs:
model_type = cast(type[BaseModel] | None, kwargs.pop("model_type", None))
cached: Final = (
self.key_object_cache.get_cache(key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs)
self._auth_registry_cache_calls.get_cache(key, parent_otel_span, local_only, **kwargs)
if is_auth_registry_cache_key(key)
else self.key_object_cache.get_cache(
key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs
)
if is_user_key_cache_key(key)
else super().get_cache(key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs)
)
@ -185,7 +246,9 @@ class UserApiKeyCache(DualCache):
if model_type is None and "model_type" in kwargs:
model_type = cast(type[BaseModel] | None, kwargs.pop("model_type", None))
cached: Final = (
await self.key_object_cache.async_get_cache(
await self._auth_registry_cache_calls.async_get_cache(key, parent_otel_span, local_only, **kwargs)
if is_auth_registry_cache_key(key)
else await self.key_object_cache.async_get_cache(
key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs
)
if is_user_key_cache_key(key)
@ -210,6 +273,8 @@ class UserApiKeyCache(DualCache):
def set_cache(self, key: str | None, value: object, local_only: bool = False, **kwargs: object):
model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None))
payload: Final[object] = CacheCodec.serialize(value, model_type=model_type)
if key is not None and is_auth_registry_cache_key(key):
return self._auth_registry_cache_calls.set_cache(key, payload, local_only, **kwargs)
if key is not None and is_user_key_cache_key(key):
return self.key_object_cache.set_cache(key=key, value=payload, local_only=local_only, **kwargs)
return super().set_cache(key=key, value=payload, local_only=local_only, **kwargs)
@ -220,33 +285,50 @@ class UserApiKeyCache(DualCache):
model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None))
payload: Final[object] = CacheCodec.serialize(value, model_type=model_type)
ttl: Final = kwargs.get("ttl")
pipeline_ttl: Final[float | None] = ttl if isinstance(ttl, (int, float)) else None
pipelined: Final = (
key is not None
and not local_only
and kwargs.keys() <= _PIPELINED_SET_OPTIONS
and (ttl is None or isinstance(ttl, (int, float)))
)
if key is not None and is_user_key_cache_key(key):
if pipelined and await self.key_object_cache.async_set_cache_pre_call(key, payload, ttl) is not None:
if pipelined and key is not None:
pre_call_cache: Final[DualCache] = (
self.auth_registry_cache
if is_auth_registry_cache_key(key)
else self.key_object_cache
if is_user_key_cache_key(key)
else self
)
if await pre_call_cache.async_set_cache_pre_call(key, payload, pipeline_ttl) is not None:
return None
if key is not None and is_auth_registry_cache_key(key):
return await self._auth_registry_cache_calls.async_set_cache(key, payload, local_only, **kwargs)
if key is not None and is_user_key_cache_key(key):
return await self.key_object_cache.async_set_cache(key=key, value=payload, local_only=local_only, **kwargs)
if pipelined and await super().async_set_cache_pre_call(key, payload, ttl) is not None:
return None
return await super().async_set_cache(key=key, value=payload, local_only=local_only, **kwargs)
def delete_cache(self, key: str) -> None:
if is_auth_registry_cache_key(key):
self._auth_registry_cache_calls.delete_cache(key)
return
if is_user_key_cache_key(key):
self.key_object_cache.delete_cache(key)
return
super().delete_cache(key)
async def async_delete_cache(self, key: str) -> None:
if is_auth_registry_cache_key(key):
await self.auth_registry_cache.async_delete_cache(key)
return
if is_user_key_cache_key(key):
await self.key_object_cache.async_delete_cache(key)
return
await super().async_delete_cache(key)
async def async_delete_cache_pre_call(self, key: str) -> BatchResult[None] | None:
if is_auth_registry_cache_key(key):
return await self.auth_registry_cache.async_delete_cache_pre_call(key)
if is_user_key_cache_key(key):
return await self.key_object_cache.async_delete_cache_pre_call(key)
return await super().async_delete_cache_pre_call(key)
@ -255,12 +337,18 @@ class UserApiKeyCache(DualCache):
"""Batch twin of ``async_delete_cache``, partitioned like
``async_set_cache_pipeline``.
Both partitions are cleared even when one raises, because a caller
All three partitions are cleared even when one raises, because a caller
batching these has already committed the rows they cache.
"""
key_object_keys: Final = tuple(key for key in keys if is_user_key_cache_key(key))
other_keys: Final = tuple(key for key in keys if not is_user_key_cache_key(key))
registry_keys: Final = tuple(key for key in keys if is_auth_registry_cache_key(key))
key_object_keys: Final = tuple(
key for key in keys if not is_auth_registry_cache_key(key) and is_user_key_cache_key(key)
)
other_keys: Final = tuple(
key for key in keys if not is_auth_registry_cache_key(key) and not is_user_key_cache_key(key)
)
outcomes: Final = await asyncio.gather(
self.auth_registry_cache.async_delete_cache_keys(registry_keys),
self.key_object_cache.async_delete_cache_keys(key_object_keys),
super().async_delete_cache_keys(other_keys),
return_exceptions=True,
@ -272,6 +360,7 @@ class UserApiKeyCache(DualCache):
def flush_cache(self) -> None:
super().flush_cache()
self.key_object_cache.in_memory_cache.flush_cache()
self.auth_registry_cache.in_memory_cache.flush_cache()
async def async_set_cache_pipeline(
self, cache_list: Sequence[tuple[str, object]], local_only: bool = False, **kwargs: object
@ -281,8 +370,19 @@ class UserApiKeyCache(DualCache):
``model_type``: ``BaseModel`` values become JSON-safe dicts; dicts/scalars unchanged.
"""
normalized: Final = tuple((key, CacheCodec.serialize(value, model_type=None)) for key, value in cache_list)
key_object_entries: Final = tuple(entry for entry in normalized if is_user_key_cache_key(entry[0]))
other_entries: Final = tuple(entry for entry in normalized if not is_user_key_cache_key(entry[0]))
registry_entries: Final = tuple(entry for entry in normalized if is_auth_registry_cache_key(entry[0]))
key_object_entries: Final = tuple(
entry
for entry in normalized
if not is_auth_registry_cache_key(entry[0]) and is_user_key_cache_key(entry[0])
)
other_entries: Final = tuple(
entry
for entry in normalized
if not is_auth_registry_cache_key(entry[0]) and not is_user_key_cache_key(entry[0])
)
if registry_entries:
await self._auth_registry_cache_calls.async_set_cache_pipeline(registry_entries, local_only, **kwargs)
if key_object_entries:
await self.key_object_cache.async_set_cache_pipeline(
cache_list=key_object_entries, local_only=local_only, **kwargs
@ -377,6 +477,15 @@ def end_user_restricted_registry_cache_key() -> str:
return "end_user_restricted_registry"
_AUTH_REGISTRY_CACHE_KEYS: Final = frozenset(
(tag_registry_cache_key(), model_access_group_registry_cache_key(), end_user_restricted_registry_cache_key())
)
def is_auth_registry_cache_key(key: str) -> bool:
return key in _AUTH_REGISTRY_CACHE_KEYS
def team_membership_auth_cache_key(team_id: str, user_id: str) -> str:
"""Cache key one team member's ``LiteLLM_TeamMembership`` row is stored under for the admission check."""
return f"{team_id}_{user_id}"

View file

@ -92,10 +92,12 @@ from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from prisma.errors import DataError
from litellm.proxy.common_utils.user_api_key_cache import (
END_USER_RESTRICTED_REGISTRY_OVERFLOW_SENTINEL,
MODEL_ACCESS_GROUP_REGISTRY_OVERFLOW_SENTINEL,
TAG_REGISTRY_OVERFLOW_SENTINEL,
UserApiKeyCache,
end_user_cache_key,
end_user_restricted_registry_cache_key,
model_access_group_registry_cache_key,
object_permission_cache_key,
tag_cache_key,
tag_registry_cache_key,
@ -8213,6 +8215,91 @@ async def test_get_end_user_object_caches_empty_restricted_registry(end_user_reg
mock_prisma.db.litellm_endusertable.find_unique.assert_not_awaited()
@pytest.mark.asyncio
async def test_auth_registries_survive_management_cache_churn():
from litellm.proxy.auth.auth_checks import _cache_registry_answer, _cached_registry, _RegistryNotCached
registry_cases = (
(end_user_restricted_registry_cache_key(), (), END_USER_RESTRICTED_REGISTRY_OVERFLOW_SENTINEL),
(tag_registry_cache_key(), ("tag-a",), TAG_REGISTRY_OVERFLOW_SENTINEL),
(
model_access_group_registry_cache_key(),
MODEL_ACCESS_GROUP_REGISTRY_OVERFLOW_SENTINEL,
MODEL_ACCESS_GROUP_REGISTRY_OVERFLOW_SENTINEL,
),
)
cache = UserApiKeyCache(
in_memory_cache=InMemoryCache(max_size_in_memory=3, clock=lambda: 0.0),
key_object_in_memory_cache=InMemoryCache(max_size_in_memory=3, clock=lambda: 0.0),
default_in_memory_ttl=60,
)
for key, value, _ in registry_cases:
await _cache_registry_answer(key, value, 60, cache)
registry_memory = cache.auth_registry_cache.in_memory_cache
original_ttls = {key: registry_memory.ttl_dict[key] for key, _, _ in registry_cases}
await _cache_registry_answer(registry_cases[-1][0], registry_cases[-1][1], 60, cache)
assert len(registry_memory.cache_dict) <= 3
for key, value, sentinel in registry_cases[:-1]:
expected = None if value == sentinel else frozenset(value)
assert await _cached_registry(key, sentinel, cache) == expected
assert registry_memory.ttl_dict[key] == original_ttls[key]
for index in range(4):
await cache.async_set_cache(key=f"{index + 1:064x}", value={"key_alias": str(index)}, ttl=60)
for key, _, sentinel in registry_cases:
assert not isinstance(await _cached_registry(key, sentinel, cache), _RegistryNotCached)
for index in range(4):
await cache.async_set_cache(key=f"team_id:team-{index}", value={"team_id": f"team-{index}"}, ttl=60)
for key, _, sentinel in registry_cases:
assert not isinstance(await _cached_registry(key, sentinel, cache), _RegistryNotCached)
for key, _, _ in registry_cases:
await cache.async_delete_cache(key)
assert isinstance(await _cached_registry(key, "unused-overflow-sentinel", cache), _RegistryNotCached)
for key, value, _ in registry_cases:
await _cache_registry_answer(key, value, 60, cache)
cache.flush_cache()
for key, _, sentinel in registry_cases:
assert isinstance(await _cached_registry(key, sentinel, cache), _RegistryNotCached)
key = tag_registry_cache_key()
cache.set_cache(key, ("sync-tag",), local_only=True, ttl=60)
assert registry_memory.get_cache(key) == ("sync-tag",)
cache.delete_cache(key)
assert registry_memory.get_cache(key) is None
await cache.async_set_cache_pipeline(((key, ("pipeline-tag",)),), local_only=True, ttl=60)
assert registry_memory.get_cache(key) == ("pipeline-tag",)
assert await cache.async_delete_cache_pre_call(key) is None
assert registry_memory.get_cache(key) == ("pipeline-tag",)
from litellm.caching.redis_batch import request_redis_batch_scope
from tests.unit.caching.test_redis_batch import FakeClient, FakeRedisCache, replies
client = FakeClient(replies)
redis_cache = FakeRedisCache(client)
cache.attach_redis_cache(redis_cache)
with request_redis_batch_scope() as request:
assert await cache.async_set_cache(key, ("queued-tag",), ttl=60) is None
assert request.batch(redis_cache).pending == 1
assert client.pipelines == []
assert redis_cache.alone == []
assert registry_memory.get_cache(key) == ("queued-tag",)
assert cache.in_memory_cache.get_cache(key) is None
assert cache.key_object_cache.in_memory_cache.get_cache(key) is None
assert await cache.async_get_cache(key) == ("queued-tag",)
assert client.pipelines == []
assert redis_cache.alone == []
await request.flush_all()
assert request.batch(redis_cache).pending == 0
assert [pipe.commands for pipe in client.pipelines] == [[("SET", key, json.dumps(["queued-tag"]), 60)]]
assert client.pipelines[0].executed
assert redis_cache.alone == []
@pytest.mark.asyncio
async def test_get_end_user_object_registry_db_error_negative_caches_and_keeps_per_id_fetch(
end_user_registry_skip_enabled,