From 60a99e7ed51eb6d86b4c7158963e22b328bc120f Mon Sep 17 00:00:00 2001 From: skyon0522-ai <251157463+skyon0522-ai@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:49:34 +0900 Subject: [PATCH 1/4] fix(proxy): isolate authorization registries from management cache churn --- .../proxy/common_utils/user_api_key_cache.py | 84 +++++++++++++++++-- ...st_auth_checks_object_access_and_lookup.py | 52 ++++++++++++ 2 files changed, 128 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/common_utils/user_api_key_cache.py b/litellm/proxy/common_utils/user_api_key_cache.py index 8c830816207..e4b2c32703d 100644 --- a/litellm/proxy/common_utils/user_api_key_cache.py +++ b/litellm/proxy/common_utils/user_api_key_cache.py @@ -55,8 +55,8 @@ class UserApiKeyCache(DualCache): ``async_set_cache_pipeline`` applies the same untyped Codec pass as omitting ``model_type`` on ``async_set_cache`` (so ``BaseModel`` rows are dumped before Redis). - User-key objects (see ``is_user_key_cache_key``) live in their own in-memory partition, - ``key_object_cache``, so churn in the other management objects cannot evict them. Both + User-key objects and authorization registries use separate in-memory partitions. The registry + partition has capacity four for a fixed three-key whitelist, leaving headroom on rewrites; both partitions share the same Redis backend and TTL settings. ``get_cache`` / ``async_get_cache`` overloads and implementations must be contiguous @@ -83,8 +83,16 @@ class UserApiKeyCache(DualCache): default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl, ) + self.auth_registry_cache: Final = DualCache( + in_memory_cache=InMemoryCache(max_size_in_memory=4), + redis_cache=redis_cache, + default_in_memory_ttl=default_in_memory_ttl, + default_redis_ttl=default_redis_ttl, + ) def in_memory_cache_for(self, key: str) -> InMemoryCache: + if is_auth_registry_cache_key(key): + return self.auth_registry_cache.in_memory_cache return self.key_object_cache.in_memory_cache if is_user_key_cache_key(key) else self.in_memory_cache def update_cache_ttl(self, default_in_memory_ttl: float | None, default_redis_ttl: float | None) -> None: @@ -92,16 +100,21 @@ class UserApiKeyCache(DualCache): self.key_object_cache.update_cache_ttl( default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl ) + self.auth_registry_cache.update_cache_ttl( + default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl + ) def update_in_memory_max_size(self, max_size: int | None) -> None: super().update_in_memory_max_size(max_size) self.key_object_cache.update_in_memory_max_size(max_size) + self.auth_registry_cache.update_in_memory_max_size(4) def attach_redis_cache( self, redis_cache: RedisCache | None = None, *, default_redis_ttl: float | None = None ) -> None: super().attach_redis_cache(redis_cache, default_redis_ttl=default_redis_ttl) self.key_object_cache.attach_redis_cache(redis_cache, default_redis_ttl=default_redis_ttl) + self.auth_registry_cache.attach_redis_cache(redis_cache, default_redis_ttl=default_redis_ttl) @overload def get_cache( @@ -135,7 +148,13 @@ class UserApiKeyCache(DualCache): if model_type is None and "model_type" in kwargs: model_type = cast(type[BaseModel] | None, kwargs.pop("model_type", None)) cached: Final = ( - self.key_object_cache.get_cache(key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs) + self.auth_registry_cache.get_cache( + key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs + ) + if is_auth_registry_cache_key(key) + else self.key_object_cache.get_cache( + key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs + ) if is_user_key_cache_key(key) else super().get_cache(key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs) ) @@ -185,7 +204,11 @@ class UserApiKeyCache(DualCache): if model_type is None and "model_type" in kwargs: model_type = cast(type[BaseModel] | None, kwargs.pop("model_type", None)) cached: Final = ( - await self.key_object_cache.async_get_cache( + await self.auth_registry_cache.async_get_cache( + key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs + ) + if is_auth_registry_cache_key(key) + else await self.key_object_cache.async_get_cache( key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs ) if is_user_key_cache_key(key) @@ -210,6 +233,8 @@ class UserApiKeyCache(DualCache): def set_cache(self, key: str | None, value: object, local_only: bool = False, **kwargs: object): model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None)) payload: Final[object] = CacheCodec.serialize(value, model_type=model_type) + if key is not None and is_auth_registry_cache_key(key): + return self.auth_registry_cache.set_cache(key=key, value=payload, local_only=local_only, **kwargs) if key is not None and is_user_key_cache_key(key): return self.key_object_cache.set_cache(key=key, value=payload, local_only=local_only, **kwargs) return super().set_cache(key=key, value=payload, local_only=local_only, **kwargs) @@ -226,6 +251,12 @@ class UserApiKeyCache(DualCache): and kwargs.keys() <= _PIPELINED_SET_OPTIONS and (ttl is None or isinstance(ttl, (int, float))) ) + if key is not None and is_auth_registry_cache_key(key): + if pipelined and await self.auth_registry_cache.async_set_cache_pre_call(key, payload, ttl) is not None: + return None + return await self.auth_registry_cache.async_set_cache( + key=key, value=payload, local_only=local_only, **kwargs + ) if key is not None and is_user_key_cache_key(key): if pipelined and await self.key_object_cache.async_set_cache_pre_call(key, payload, ttl) is not None: return None @@ -235,18 +266,26 @@ class UserApiKeyCache(DualCache): return await super().async_set_cache(key=key, value=payload, local_only=local_only, **kwargs) def delete_cache(self, key: str) -> None: + if is_auth_registry_cache_key(key): + self.auth_registry_cache.delete_cache(key) + return if is_user_key_cache_key(key): self.key_object_cache.delete_cache(key) return super().delete_cache(key) async def async_delete_cache(self, key: str) -> None: + if is_auth_registry_cache_key(key): + await self.auth_registry_cache.async_delete_cache(key) + return if is_user_key_cache_key(key): await self.key_object_cache.async_delete_cache(key) return await super().async_delete_cache(key) async def async_delete_cache_pre_call(self, key: str) -> BatchResult[None] | None: + if is_auth_registry_cache_key(key): + return await self.auth_registry_cache.async_delete_cache_pre_call(key) if is_user_key_cache_key(key): return await self.key_object_cache.async_delete_cache_pre_call(key) return await super().async_delete_cache_pre_call(key) @@ -258,9 +297,15 @@ class UserApiKeyCache(DualCache): Both partitions are cleared even when one raises, because a caller batching these has already committed the rows they cache. """ - key_object_keys: Final = tuple(key for key in keys if is_user_key_cache_key(key)) - other_keys: Final = tuple(key for key in keys if not is_user_key_cache_key(key)) + registry_keys: Final = tuple(key for key in keys if is_auth_registry_cache_key(key)) + key_object_keys: Final = tuple( + key for key in keys if not is_auth_registry_cache_key(key) and is_user_key_cache_key(key) + ) + other_keys: Final = tuple( + key for key in keys if not is_auth_registry_cache_key(key) and not is_user_key_cache_key(key) + ) outcomes: Final = await asyncio.gather( + self.auth_registry_cache.async_delete_cache_keys(registry_keys), self.key_object_cache.async_delete_cache_keys(key_object_keys), super().async_delete_cache_keys(other_keys), return_exceptions=True, @@ -272,6 +317,7 @@ class UserApiKeyCache(DualCache): def flush_cache(self) -> None: super().flush_cache() self.key_object_cache.in_memory_cache.flush_cache() + self.auth_registry_cache.in_memory_cache.flush_cache() async def async_set_cache_pipeline( self, cache_list: Sequence[tuple[str, object]], local_only: bool = False, **kwargs: object @@ -281,8 +327,21 @@ class UserApiKeyCache(DualCache): ``model_type``: ``BaseModel`` values become JSON-safe dicts; dicts/scalars unchanged. """ normalized: Final = tuple((key, CacheCodec.serialize(value, model_type=None)) for key, value in cache_list) - key_object_entries: Final = tuple(entry for entry in normalized if is_user_key_cache_key(entry[0])) - other_entries: Final = tuple(entry for entry in normalized if not is_user_key_cache_key(entry[0])) + registry_entries: Final = tuple(entry for entry in normalized if is_auth_registry_cache_key(entry[0])) + key_object_entries: Final = tuple( + entry + for entry in normalized + if not is_auth_registry_cache_key(entry[0]) and is_user_key_cache_key(entry[0]) + ) + other_entries: Final = tuple( + entry + for entry in normalized + if not is_auth_registry_cache_key(entry[0]) and not is_user_key_cache_key(entry[0]) + ) + if registry_entries: + await self.auth_registry_cache.async_set_cache_pipeline( + cache_list=registry_entries, local_only=local_only, **kwargs + ) if key_object_entries: await self.key_object_cache.async_set_cache_pipeline( cache_list=key_object_entries, local_only=local_only, **kwargs @@ -377,6 +436,15 @@ def end_user_restricted_registry_cache_key() -> str: return "end_user_restricted_registry" +_AUTH_REGISTRY_CACHE_KEYS: Final = frozenset( + (tag_registry_cache_key(), model_access_group_registry_cache_key(), end_user_restricted_registry_cache_key()) +) + + +def is_auth_registry_cache_key(key: str) -> bool: + return key in _AUTH_REGISTRY_CACHE_KEYS + + def team_membership_auth_cache_key(team_id: str, user_id: str) -> str: """Cache key one team member's ``LiteLLM_TeamMembership`` row is stored under for the admission check.""" return f"{team_id}_{user_id}" diff --git a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py index e26845e5b2a..032d9f5cc3d 100644 --- a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py +++ b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py @@ -88,10 +88,12 @@ from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from prisma.errors import DataError from litellm.proxy.common_utils.user_api_key_cache import ( END_USER_RESTRICTED_REGISTRY_OVERFLOW_SENTINEL, + MODEL_ACCESS_GROUP_REGISTRY_OVERFLOW_SENTINEL, TAG_REGISTRY_OVERFLOW_SENTINEL, UserApiKeyCache, end_user_cache_key, end_user_restricted_registry_cache_key, + model_access_group_registry_cache_key, object_permission_cache_key, tag_cache_key, tag_registry_cache_key, @@ -7878,6 +7880,56 @@ async def test_get_end_user_object_caches_empty_restricted_registry(end_user_reg mock_prisma.db.litellm_endusertable.find_unique.assert_not_awaited() +@pytest.mark.asyncio +async def test_auth_registries_survive_management_cache_churn(): + from litellm.proxy.auth.auth_checks import _cache_registry_answer, _cached_registry, _RegistryNotCached + + registry_cases = ( + (end_user_restricted_registry_cache_key(), (), END_USER_RESTRICTED_REGISTRY_OVERFLOW_SENTINEL), + (tag_registry_cache_key(), ("tag-a",), TAG_REGISTRY_OVERFLOW_SENTINEL), + ( + model_access_group_registry_cache_key(), + MODEL_ACCESS_GROUP_REGISTRY_OVERFLOW_SENTINEL, + MODEL_ACCESS_GROUP_REGISTRY_OVERFLOW_SENTINEL, + ), + ) + cache = UserApiKeyCache( + in_memory_cache=InMemoryCache(max_size_in_memory=3, clock=lambda: 0.0), + key_object_in_memory_cache=InMemoryCache(max_size_in_memory=3, clock=lambda: 0.0), + default_in_memory_ttl=60, + ) + for key, value, _ in registry_cases: + await _cache_registry_answer(key, value, 60, cache) + + registry_memory = cache.auth_registry_cache.in_memory_cache + original_ttls = {key: registry_memory.ttl_dict[key] for key, _, _ in registry_cases} + await _cache_registry_answer(registry_cases[-1][0], registry_cases[-1][1], 60, cache) + assert len(registry_memory.cache_dict) <= 3 + for key, value, sentinel in registry_cases[:-1]: + expected = None if value == sentinel else frozenset(value) + assert await _cached_registry(key, sentinel, cache) == expected + assert registry_memory.ttl_dict[key] == original_ttls[key] + + for index in range(4): + await cache.async_set_cache(key=f"{index + 1:064x}", value={"key_alias": str(index)}, ttl=60) + for key, _, sentinel in registry_cases: + assert not isinstance(await _cached_registry(key, sentinel, cache), _RegistryNotCached) + + for index in range(4): + await cache.async_set_cache(key=f"team_id:team-{index}", value={"team_id": f"team-{index}"}, ttl=60) + for key, _, sentinel in registry_cases: + assert not isinstance(await _cached_registry(key, sentinel, cache), _RegistryNotCached) + + for key, _, _ in registry_cases: + await cache.async_delete_cache(key) + assert isinstance(await _cached_registry(key, "unused-overflow-sentinel", cache), _RegistryNotCached) + for key, value, _ in registry_cases: + await _cache_registry_answer(key, value, 60, cache) + cache.flush_cache() + for key, _, sentinel in registry_cases: + assert isinstance(await _cached_registry(key, sentinel, cache), _RegistryNotCached) + + @pytest.mark.asyncio async def test_get_end_user_object_registry_db_error_negative_caches_and_keeps_per_id_fetch( end_user_registry_skip_enabled, From 0bb0b2d0549839fd3feae0e2523cd262791ec339 Mon Sep 17 00:00:00 2001 From: skyon0522-ai <251157463+skyon0522-ai@users.noreply.github.com> Date: Thu, 8 Oct 2026 17:05:45 +0900 Subject: [PATCH 2/4] fix(proxy): type registry cache calls and cover write paths --- .../proxy/common_utils/user_api_key_cache.py | 58 ++++++++++++------- ...st_auth_checks_object_access_and_lookup.py | 23 ++++++++ 2 files changed, 59 insertions(+), 22 deletions(-) diff --git a/litellm/proxy/common_utils/user_api_key_cache.py b/litellm/proxy/common_utils/user_api_key_cache.py index e4b2c32703d..7f2af054c11 100644 --- a/litellm/proxy/common_utils/user_api_key_cache.py +++ b/litellm/proxy/common_utils/user_api_key_cache.py @@ -3,7 +3,7 @@ from __future__ import annotations import asyncio import re from collections.abc import Sequence -from typing import TYPE_CHECKING, Any, Final, TypeVar, cast, overload +from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast, overload from pydantic import BaseModel @@ -34,6 +34,21 @@ def is_user_key_cache_key(key: str) -> bool: _PIPELINED_SET_OPTIONS: Final = frozenset(("ttl",)) +class _AuthRegistryCacheCalls(Protocol): + def get_cache( + self, key: str, parent_otel_span: Span | None = None, local_only: bool = False, **kwargs: object + ) -> object: ... + async def async_get_cache( + self, key: str, parent_otel_span: Span | None = None, local_only: bool = False, **kwargs: object + ) -> object: ... + def set_cache(self, key: str, value: object, local_only: bool = False, **kwargs: object) -> None: ... + async def async_set_cache(self, key: str, value: object, local_only: bool = False, **kwargs: object) -> None: ... + def delete_cache(self, key: str) -> None: ... + async def async_set_cache_pipeline( + self, cache_list: Sequence[tuple[str, object]], local_only: bool = False, **kwargs: object + ) -> None: ... + + class UserApiKeyCache(DualCache): """ DualCache wrapper for UserAPIKeyAuth-like payloads. @@ -89,6 +104,10 @@ class UserApiKeyCache(DualCache): default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl, ) + # DualCache leaves these methods untyped; record the exact call surface without a runtime wrapper. + self._auth_registry_cache_calls: Final[_AuthRegistryCacheCalls] = cast( + _AuthRegistryCacheCalls, cast(object, self.auth_registry_cache) + ) def in_memory_cache_for(self, key: str) -> InMemoryCache: if is_auth_registry_cache_key(key): @@ -148,9 +167,7 @@ class UserApiKeyCache(DualCache): if model_type is None and "model_type" in kwargs: model_type = cast(type[BaseModel] | None, kwargs.pop("model_type", None)) cached: Final = ( - self.auth_registry_cache.get_cache( - key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs - ) + self._auth_registry_cache_calls.get_cache(key, parent_otel_span, local_only, **kwargs) if is_auth_registry_cache_key(key) else self.key_object_cache.get_cache( key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs @@ -204,9 +221,7 @@ class UserApiKeyCache(DualCache): if model_type is None and "model_type" in kwargs: model_type = cast(type[BaseModel] | None, kwargs.pop("model_type", None)) cached: Final = ( - await self.auth_registry_cache.async_get_cache( - key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs - ) + await self._auth_registry_cache_calls.async_get_cache(key, parent_otel_span, local_only, **kwargs) if is_auth_registry_cache_key(key) else await self.key_object_cache.async_get_cache( key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs @@ -234,7 +249,7 @@ class UserApiKeyCache(DualCache): model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None)) payload: Final[object] = CacheCodec.serialize(value, model_type=model_type) if key is not None and is_auth_registry_cache_key(key): - return self.auth_registry_cache.set_cache(key=key, value=payload, local_only=local_only, **kwargs) + return self._auth_registry_cache_calls.set_cache(key, payload, local_only, **kwargs) if key is not None and is_user_key_cache_key(key): return self.key_object_cache.set_cache(key=key, value=payload, local_only=local_only, **kwargs) return super().set_cache(key=key, value=payload, local_only=local_only, **kwargs) @@ -245,29 +260,30 @@ class UserApiKeyCache(DualCache): model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None)) payload: Final[object] = CacheCodec.serialize(value, model_type=model_type) ttl: Final = kwargs.get("ttl") + pipeline_ttl: float | None = ttl if isinstance(ttl, (int, float)) else None pipelined: Final = ( key is not None and not local_only and kwargs.keys() <= _PIPELINED_SET_OPTIONS and (ttl is None or isinstance(ttl, (int, float))) ) + if pipelined and key is not None: + pre_call_cache: DualCache = self + if is_auth_registry_cache_key(key): + pre_call_cache = self.auth_registry_cache + elif is_user_key_cache_key(key): + pre_call_cache = self.key_object_cache + if await pre_call_cache.async_set_cache_pre_call(key, payload, pipeline_ttl) is not None: + return None if key is not None and is_auth_registry_cache_key(key): - if pipelined and await self.auth_registry_cache.async_set_cache_pre_call(key, payload, ttl) is not None: - return None - return await self.auth_registry_cache.async_set_cache( - key=key, value=payload, local_only=local_only, **kwargs - ) + return await self._auth_registry_cache_calls.async_set_cache(key, payload, local_only, **kwargs) if key is not None and is_user_key_cache_key(key): - if pipelined and await self.key_object_cache.async_set_cache_pre_call(key, payload, ttl) is not None: - return None return await self.key_object_cache.async_set_cache(key=key, value=payload, local_only=local_only, **kwargs) - if pipelined and await super().async_set_cache_pre_call(key, payload, ttl) is not None: - return None return await super().async_set_cache(key=key, value=payload, local_only=local_only, **kwargs) def delete_cache(self, key: str) -> None: if is_auth_registry_cache_key(key): - self.auth_registry_cache.delete_cache(key) + self._auth_registry_cache_calls.delete_cache(key) return if is_user_key_cache_key(key): self.key_object_cache.delete_cache(key) @@ -294,7 +310,7 @@ class UserApiKeyCache(DualCache): """Batch twin of ``async_delete_cache``, partitioned like ``async_set_cache_pipeline``. - Both partitions are cleared even when one raises, because a caller + All three partitions are cleared even when one raises, because a caller batching these has already committed the rows they cache. """ registry_keys: Final = tuple(key for key in keys if is_auth_registry_cache_key(key)) @@ -339,9 +355,7 @@ class UserApiKeyCache(DualCache): if not is_auth_registry_cache_key(entry[0]) and not is_user_key_cache_key(entry[0]) ) if registry_entries: - await self.auth_registry_cache.async_set_cache_pipeline( - cache_list=registry_entries, local_only=local_only, **kwargs - ) + await self._auth_registry_cache_calls.async_set_cache_pipeline(registry_entries, local_only, **kwargs) if key_object_entries: await self.key_object_cache.async_set_cache_pipeline( cache_list=key_object_entries, local_only=local_only, **kwargs diff --git a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py index 032d9f5cc3d..34c5cb02563 100644 --- a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py +++ b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py @@ -7929,6 +7929,29 @@ async def test_auth_registries_survive_management_cache_churn(): for key, _, sentinel in registry_cases: assert isinstance(await _cached_registry(key, sentinel, cache), _RegistryNotCached) + key = tag_registry_cache_key() + cache.set_cache(key, ("sync-tag",), local_only=True, ttl=60) + assert registry_memory.get_cache(key) == ("sync-tag",) + cache.delete_cache(key) + assert registry_memory.get_cache(key) is None + + await cache.async_set_cache_pipeline(((key, ("pipeline-tag",)),), local_only=True, ttl=60) + assert registry_memory.get_cache(key) == ("pipeline-tag",) + assert await cache.async_delete_cache_pre_call(key) is None + assert registry_memory.get_cache(key) == ("pipeline-tag",) + + with ( + patch.object( + cache.auth_registry_cache, + "async_set_cache_pre_call", + new=AsyncMock(return_value=object()), + ) as set_pre_call, + patch.object(cache.auth_registry_cache, "async_set_cache", new=AsyncMock()) as direct_set, + ): + assert await cache.async_set_cache(key, ("queued-tag",), ttl=60) is None + set_pre_call.assert_awaited_once_with(key, ("queued-tag",), 60) + direct_set.assert_not_awaited() + @pytest.mark.asyncio async def test_get_end_user_object_registry_db_error_negative_caches_and_keeps_per_id_fetch( From 2b12f609f8e59deb85c0149a6e30a1069775b187 Mon Sep 17 00:00:00 2001 From: skyon0522-ai <251157463+skyon0522-ai@users.noreply.github.com> Date: Thu, 8 Oct 2026 17:28:53 +0900 Subject: [PATCH 3/4] fix(proxy): align registry cache typing with project discipline --- .../proxy/common_utils/user_api_key_cache.py | 57 ++++++++++++++----- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/common_utils/user_api_key_cache.py b/litellm/proxy/common_utils/user_api_key_cache.py index 7f2af054c11..a50ab0182cd 100644 --- a/litellm/proxy/common_utils/user_api_key_cache.py +++ b/litellm/proxy/common_utils/user_api_key_cache.py @@ -36,16 +36,44 @@ _PIPELINED_SET_OPTIONS: Final = frozenset(("ttl",)) class _AuthRegistryCacheCalls(Protocol): def get_cache( - self, key: str, parent_otel_span: Span | None = None, local_only: bool = False, **kwargs: object + self, + key: str, + parent_otel_span: Span | None = None, + local_only: bool = False, + **kwargs: object, # kwargs-ok: preserve arbitrary DualCache cache options ) -> object: ... + async def async_get_cache( - self, key: str, parent_otel_span: Span | None = None, local_only: bool = False, **kwargs: object + self, + key: str, + parent_otel_span: Span | None = None, + local_only: bool = False, + **kwargs: object, # kwargs-ok: preserve arbitrary DualCache cache options ) -> object: ... - def set_cache(self, key: str, value: object, local_only: bool = False, **kwargs: object) -> None: ... - async def async_set_cache(self, key: str, value: object, local_only: bool = False, **kwargs: object) -> None: ... + + def set_cache( + self, + key: str, + value: object, + local_only: bool = False, + **kwargs: object, # kwargs-ok: preserve arbitrary DualCache cache options + ) -> None: ... + + async def async_set_cache( + self, + key: str, + value: object, + local_only: bool = False, + **kwargs: object, # kwargs-ok: preserve arbitrary DualCache cache options + ) -> None: ... + def delete_cache(self, key: str) -> None: ... + async def async_set_cache_pipeline( - self, cache_list: Sequence[tuple[str, object]], local_only: bool = False, **kwargs: object + self, + cache_list: Sequence[tuple[str, object]], + local_only: bool = False, + **kwargs: object, # kwargs-ok: preserve arbitrary DualCache cache options ) -> None: ... @@ -104,10 +132,7 @@ class UserApiKeyCache(DualCache): default_in_memory_ttl=default_in_memory_ttl, default_redis_ttl=default_redis_ttl, ) - # DualCache leaves these methods untyped; record the exact call surface without a runtime wrapper. - self._auth_registry_cache_calls: Final[_AuthRegistryCacheCalls] = cast( - _AuthRegistryCacheCalls, cast(object, self.auth_registry_cache) - ) + self._auth_registry_cache_calls: Final[_AuthRegistryCacheCalls] = self.auth_registry_cache def in_memory_cache_for(self, key: str) -> InMemoryCache: if is_auth_registry_cache_key(key): @@ -260,7 +285,7 @@ class UserApiKeyCache(DualCache): model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None)) payload: Final[object] = CacheCodec.serialize(value, model_type=model_type) ttl: Final = kwargs.get("ttl") - pipeline_ttl: float | None = ttl if isinstance(ttl, (int, float)) else None + pipeline_ttl: Final[float | None] = ttl if isinstance(ttl, (int, float)) else None pipelined: Final = ( key is not None and not local_only @@ -268,11 +293,13 @@ class UserApiKeyCache(DualCache): and (ttl is None or isinstance(ttl, (int, float))) ) if pipelined and key is not None: - pre_call_cache: DualCache = self - if is_auth_registry_cache_key(key): - pre_call_cache = self.auth_registry_cache - elif is_user_key_cache_key(key): - pre_call_cache = self.key_object_cache + pre_call_cache: Final[DualCache] = ( + self.auth_registry_cache + if is_auth_registry_cache_key(key) + else self.key_object_cache + if is_user_key_cache_key(key) + else self + ) if await pre_call_cache.async_set_cache_pre_call(key, payload, pipeline_ttl) is not None: return None if key is not None and is_auth_registry_cache_key(key): From beebd0e9384391b2079a907f8179d89d50d199f0 Mon Sep 17 00:00:00 2001 From: Kyon <251157463+skyon0522-ai@users.noreply.github.com> Date: Sun, 11 Oct 2026 23:58:58 +0900 Subject: [PATCH 4/4] Test queued auth registry writes through the real request batch --- ...st_auth_checks_object_access_and_lookup.py | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py index 34c5cb02563..4fd12a3eddd 100644 --- a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py +++ b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py @@ -7940,17 +7940,29 @@ async def test_auth_registries_survive_management_cache_churn(): assert await cache.async_delete_cache_pre_call(key) is None assert registry_memory.get_cache(key) == ("pipeline-tag",) - with ( - patch.object( - cache.auth_registry_cache, - "async_set_cache_pre_call", - new=AsyncMock(return_value=object()), - ) as set_pre_call, - patch.object(cache.auth_registry_cache, "async_set_cache", new=AsyncMock()) as direct_set, - ): + from litellm.caching.redis_batch import request_redis_batch_scope + from tests.unit.caching.test_redis_batch import FakeClient, FakeRedisCache, replies + + client = FakeClient(replies) + redis_cache = FakeRedisCache(client) + cache.attach_redis_cache(redis_cache) + with request_redis_batch_scope() as request: assert await cache.async_set_cache(key, ("queued-tag",), ttl=60) is None - set_pre_call.assert_awaited_once_with(key, ("queued-tag",), 60) - direct_set.assert_not_awaited() + assert request.batch(redis_cache).pending == 1 + assert client.pipelines == [] + assert redis_cache.alone == [] + assert registry_memory.get_cache(key) == ("queued-tag",) + assert cache.in_memory_cache.get_cache(key) is None + assert cache.key_object_cache.in_memory_cache.get_cache(key) is None + assert await cache.async_get_cache(key) == ("queued-tag",) + assert client.pipelines == [] + assert redis_cache.alone == [] + await request.flush_all() + + assert request.batch(redis_cache).pending == 0 + assert [pipe.commands for pipe in client.pipelines] == [[("SET", key, json.dumps(["queued-tag"]), 60)]] + assert client.pipelines[0].executed + assert redis_cache.alone == [] @pytest.mark.asyncio