perf(auth): reuse validated models on UserApiKeyCache memory hits (#51912)

A typed read of the same in-memory payload returns a shallow copy of the model the previous read validated instead of running model_validate again. Writes, deletes and flushes drop the memo, and the Redis and shared store payloads are unchanged.
This commit is contained in:
Nate Armstrong 2026-11-15 15:39:44 -08:00 • committed by GitHub
parent 8c97c243fe
commit eb8a2d224e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 204 additions and 7 deletions

View file

@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import re
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast, overload
from weakref import WeakValueDictionary
@ -48,6 +49,13 @@ def _ttl(kwargs: Mapping[str, object]) -> float | None:
return _TTL.validate_python(kwargs.get("ttl"))
@dataclass(frozen=True, slots=True)
class _DecodedModel:
payload: object
model_type: type[BaseModel]
model: BaseModel
class CacheLease:
__slots__ = ("__weakref__", "key", "shared")
@ -114,6 +122,7 @@ class UserApiKeyCache(DualCache):
self._redis_connection: RedisCache | None = None
self._shared: AuthCacheStore | None = None
self._leases: Final[WeakValueDictionary[str, CacheLease]] = WeakValueDictionary()
self._decoded_models: Final[dict[str, _DecodedModel]] = {} # mutable-ok: per-key memo of validated reads
self.attach_redis_cache(redis_cache, default_redis_ttl=default_redis_ttl)
@property
@ -175,6 +184,7 @@ class UserApiKeyCache(DualCache):
def set_local(self, key: str, value: object, ttl: float | None) -> None:
"""A memory write alone, with ``default_in_memory_ttl`` when ``ttl`` is None."""
self._drop_decoded((key,))
if self._shared is not None:
self._shared.set_local(key, value, ttl)
return
@ -199,6 +209,7 @@ class UserApiKeyCache(DualCache):
"""Caches ``value`` unless the lease's key was evicted since it was taken; False when refused."""
if self._leases.get(lease.key) is not lease:
return False
self._drop_decoded((lease.key,))
shared: Final = self._shared
if shared is None:
await self.async_set_cache(key=lease.key, value=value, ttl=ttl)
@ -214,9 +225,35 @@ class UserApiKeyCache(DualCache):
return True
def _void_leases(self, keys: Sequence[str]) -> None:
self._drop_decoded(keys)
for key in keys:
_ = self._leases.pop(key, None)
def _drop_decoded(self, keys: Sequence[str]) -> None:
for key in keys:
_ = self._decoded_models.pop(key, None)
def _decoded(self, cached: object, key: str, model_type: type[BaseModel] | None, method: str) -> object:
"""In Python memory, a read of the same payload object a previous read validated returns a shallow copy of
that model instead of validating again, so a caller that sets a field cannot change what the next reader
gets. The shared store hands back a new payload on every read, so it is validated each time."""
if model_type is None or cached is None:
return cached
memo: Final = self._decoded_models.get(key)
if memo is not None and memo.payload is cached and memo.model_type is model_type:
return memo.model.model_copy()
decoded: Final = _validated(cached, key, model_type, method)
if decoded is None or decoded is cached or self._shared is not None:
return decoded
_ = self._decoded_models.pop(key, None)
self._decoded_models[key] = _DecodedModel(payload=cached, model_type=model_type, model=decoded)
if len(self._decoded_models) > self._decoded_capacity():
_ = self._decoded_models.pop(next(iter(self._decoded_models)))
return decoded.model_copy()
def _decoded_capacity(self) -> int:
return self.in_memory_cache.max_size_in_memory + self.key_object_cache.in_memory_cache.max_size_in_memory
def delete_local(self, key: str) -> None:
"""Drops this worker's memory copy alone, as a cross-worker eviction broadcast does."""
self._void_leases((key,))
@ -271,7 +308,7 @@ class UserApiKeyCache(DualCache):
if is_user_key_cache_key(key)
else super().get_cache(key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs)
)
return _decoded(cached, key, model_type, "get_cache")
return self._decoded(cached, key, model_type, "get_cache")
@overload
async def async_get_cache(
@ -316,7 +353,7 @@ class UserApiKeyCache(DualCache):
key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs
)
)
return _decoded(cached, key, model_type, "async_get_cache")
return self._decoded(cached, key, model_type, "async_get_cache")
@staticmethod
def _get_shared_sync(shared: AuthCacheStore, key: str, local_only: bool) -> object:
@ -337,6 +374,8 @@ class UserApiKeyCache(DualCache):
def set_cache(self, key: str | None, value: CacheValue, local_only: bool = False, **kwargs: object):
model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None))
payload: Final[object] = CacheCodec.serialize(value, model_type=model_type)
if key is not None:
self._drop_decoded((key,))
shared: Final = self._shared
if shared is not None:
if key is None:
@ -358,6 +397,8 @@ class UserApiKeyCache(DualCache):
else, or with options the pipeline does not carry, it goes to Redis directly as before."""
model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None))
payload: Final[object] = CacheCodec.serialize(value, model_type=model_type)
if key is not None:
self._drop_decoded((key,))
ttl: Final = kwargs.get("ttl")
pipelined: Final = (
key is not None
@ -443,6 +484,7 @@ class UserApiKeyCache(DualCache):
def flush_cache(self) -> None:
self._leases.clear()
self._decoded_models.clear()
self.budget_windows_absent_cache.flush_cache()
if self._shared is not None:
self._shared.flush_local()
@ -458,6 +500,7 @@ class UserApiKeyCache(DualCache):
``model_type``: ``BaseModel`` values become JSON-safe dicts; dicts/scalars unchanged.
"""
normalized: Final = tuple((key, CacheCodec.serialize(value, model_type=None)) for key, value in cache_list)
self._drop_decoded(tuple(key for key, _ in normalized))
shared: Final = self._shared
if shared is not None:
await self._set_many_shared(shared, normalized, local_only=local_only, ttl=_ttl(kwargs))
@ -519,9 +562,7 @@ class UserApiKeyCache(DualCache):
)
def _decoded(cached: object, key: str, model_type: type[BaseModel] | None, method: str) -> object:
if model_type is None or cached is None:
return cached
def _validated(cached: object, key: str, model_type: type[BaseModel], method: str) -> BaseModel | None:
decoded: Final = CacheCodec.deserialize(cached, model_type=model_type)
if decoded is None:
verbose_proxy_logger.error(

View file

@ -1,16 +1,17 @@
import datetime
import hashlib
from collections.abc import Awaitable, Callable
from typing import Any, Final, TypeAlias
from typing import Any, ClassVar, Final, TypeAlias
from unittest.mock import MagicMock
import pytest
from pydantic import BaseModel, model_validator
from litellm.caching.dual_cache import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.caching.redis_cache import RedisCache
from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy._types import LiteLLM_TeamTableCachedObj, LiteLLM_UserTable, UserAPIKeyAuth
from litellm.proxy.common_utils.user_api_key_cache import (
UserApiKeyCache,
budget_windows_absent_cache_key,
@ -587,3 +588,158 @@ async def test_an_eviction_that_lands_while_redis_takes_the_write_keeps_it_out_o
assert await cache.async_set_leased(lease, {"spend": 5.0}, ttl=60) is False
assert await cache.async_get_cache("customer_member_id:m") is None
class _CountedValidation(BaseModel):
validations: ClassVar[int] = 0
spend: float
@model_validator(mode="before")
@classmethod
def _count(cls, data: object) -> object:
_CountedValidation.validations += 1
return data
_TEAM_KEY: Final = "team_id:t1"
def _auth_objects() -> tuple[tuple[str, BaseModel], ...]:
return (
(HASHED_TOKEN, UserAPIKeyAuth(token=HASHED_TOKEN, spend=1.5, models=["gpt-4o"], metadata={"a": 1})),
(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=2.0, models=["gpt-4o"], metadata={"a": 1})),
("u1", LiteLLM_UserTable(user_id="u1", spend=3.0, models=["gpt-4o"], teams=["t1"])),
)
@pytest.mark.asyncio
@pytest.mark.parametrize("new_cache", [_memory_only, _with_redis_auth_cache])
@pytest.mark.parametrize(("key", "stored"), _auth_objects())
async def test_every_hit_returns_an_object_equal_to_the_stored_one(
new_cache: _NewCache, key: str, stored: BaseModel
) -> None:
cache: Final = new_cache()
await cache.async_set_cache(key, stored, model_type=type(stored))
reads: Final = (
await cache.async_get_cache(key, model_type=type(stored)),
await cache.async_get_cache(key, model_type=type(stored)),
cache.get_cache(key, model_type=type(stored)),
)
assert [type(read) for read in reads] == [type(stored)] * 3
assert list(reads) == [stored] * 3
@pytest.mark.asyncio
@pytest.mark.parametrize("new_cache", [_memory_only, _with_redis_auth_cache])
@pytest.mark.parametrize(("key", "stored"), _auth_objects())
async def test_a_field_set_on_a_read_object_does_not_reach_the_next_read(
new_cache: _NewCache, key: str, stored: BaseModel
) -> None:
cache: Final = new_cache()
await cache.async_set_cache(key, stored, model_type=type(stored))
reads: Final = [await cache.async_get_cache(key, model_type=type(stored)) for _ in range(2)]
for read in reads:
assert read is not None
read.spend = 99.0
read.models = ["mutated"]
after_mutation: Final = await cache.async_get_cache(key, model_type=type(stored))
assert after_mutation == stored
@pytest.mark.asyncio
async def test_reads_of_an_unchanged_entry_validate_it_once() -> None:
cache: Final = UserApiKeyCache()
await cache.async_set_cache("k", {"spend": 1.0})
before: Final = _CountedValidation.validations
reads: Final = [await cache.async_get_cache("k", model_type=_CountedValidation) for _ in range(5)]
assert _CountedValidation.validations - before == 1
assert reads == [_CountedValidation(spend=1.0)] * 5
@pytest.mark.asyncio
@pytest.mark.parametrize("new_cache", [_memory_only, _with_redis_auth_cache])
async def test_each_write_changes_what_the_next_read_returns(new_cache: _NewCache) -> None:
cache: Final = new_cache()
await cache.async_set_cache(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=1.0))
assert await cache.async_get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj) is not None
await cache.async_set_cache(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=2.0))
after_async_set: Final = await cache.async_get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj)
cache.set_cache(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=3.0), local_only=True)
after_local_set: Final = cache.get_cache(_TEAM_KEY, local_only=True, model_type=LiteLLM_TeamTableCachedObj)
await cache.async_set_cache_pipeline([(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=4.0))])
after_pipeline: Final = await cache.async_get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj)
assert [after_async_set, after_local_set, after_pipeline] == [
LiteLLM_TeamTableCachedObj(team_id="t1", spend=2.0),
LiteLLM_TeamTableCachedObj(team_id="t1", spend=3.0),
LiteLLM_TeamTableCachedObj(team_id="t1", spend=4.0),
]
@pytest.mark.asyncio
async def test_a_payload_rewritten_in_place_and_set_again_is_read_back() -> None:
cache: Final = UserApiKeyCache()
await cache.async_set_cache("k", {"spend": 1.0})
assert await cache.async_get_cache("k", model_type=_CountedValidation) == _CountedValidation(spend=1.0)
payload: Final = await cache.async_get_cache("k")
payload["spend"] = 2.0
await cache.async_set_cache("k", payload)
assert await cache.async_get_cache("k", model_type=_CountedValidation) == _CountedValidation(spend=2.0)
@pytest.mark.asyncio
async def test_a_payload_written_straight_into_the_memory_layer_is_read_back() -> None:
memory: Final = InMemoryCache()
cache: Final = UserApiKeyCache(in_memory_cache=memory)
await cache.async_set_cache("k", {"spend": 1.0})
assert await cache.async_get_cache("k", model_type=_CountedValidation) == _CountedValidation(spend=1.0)
memory.set_cache("k", {"spend": 2.0})
assert await cache.async_get_cache("k", model_type=_CountedValidation) == _CountedValidation(spend=2.0)
@pytest.mark.asyncio
@pytest.mark.parametrize("new_cache", [_memory_only, _with_redis_auth_cache])
@pytest.mark.parametrize("evict", _EVICTIONS)
async def test_a_typed_read_after_an_eviction_is_a_miss(new_cache: _NewCache, evict: _Evict) -> None:
cache: Final = new_cache()
await cache.async_set_cache(HASHED_TOKEN, UserAPIKeyAuth(token=HASHED_TOKEN), model_type=UserAPIKeyAuth)
assert await cache.async_get_cache(HASHED_TOKEN, model_type=UserAPIKeyAuth) is not None
await evict(cache, HASHED_TOKEN)
assert await cache.async_get_cache(HASHED_TOKEN, local_only=True, model_type=UserAPIKeyAuth) is None
def test_a_typed_read_after_the_ttl_is_a_miss() -> None:
clock: Final = _ManualClock()
cache: Final = UserApiKeyCache(in_memory_cache=InMemoryCache(clock=clock), default_in_memory_ttl=60)
cache.set_cache(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1"))
assert cache.get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj) is not None
clock.advance(61)
assert cache.get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj) is None
@pytest.mark.asyncio
async def test_one_entry_read_as_two_models_returns_each_model() -> None:
cache: Final = UserApiKeyCache()
await cache.async_set_cache(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=1.0))
as_team: Final = await cache.async_get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj)
as_counted: Final = await cache.async_get_cache(_TEAM_KEY, model_type=_CountedValidation)
assert type(as_team) is LiteLLM_TeamTableCachedObj
assert as_counted == _CountedValidation(spend=1.0)