From eb8a2d224e75593899f6af418c0aa5d3b3ce3060 Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Sun, 15 Nov 2026 15:39:44 -0800 Subject: [PATCH] perf(auth): reuse validated models on UserApiKeyCache memory hits (#51912) A typed read of the same in-memory payload returns a shallow copy of the model the previous read validated instead of running model_validate again. Writes, deletes and flushes drop the memo, and the Redis and shared store payloads are unchanged. --- .../proxy/common_utils/user_api_key_cache.py | 51 +++++- .../common_utils/test_user_api_key_cache.py | 160 +++++++++++++++++- 2 files changed, 204 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/common_utils/user_api_key_cache.py b/litellm/proxy/common_utils/user_api_key_cache.py index cabeb866864..1de923db72f 100644 --- a/litellm/proxy/common_utils/user_api_key_cache.py +++ b/litellm/proxy/common_utils/user_api_key_cache.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import re from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast, overload from weakref import WeakValueDictionary @@ -48,6 +49,13 @@ def _ttl(kwargs: Mapping[str, object]) -> float | None: return _TTL.validate_python(kwargs.get("ttl")) +@dataclass(frozen=True, slots=True) +class _DecodedModel: + payload: object + model_type: type[BaseModel] + model: BaseModel + + class CacheLease: __slots__ = ("__weakref__", "key", "shared") @@ -114,6 +122,7 @@ class UserApiKeyCache(DualCache): self._redis_connection: RedisCache | None = None self._shared: AuthCacheStore | None = None self._leases: Final[WeakValueDictionary[str, CacheLease]] = WeakValueDictionary() + self._decoded_models: Final[dict[str, _DecodedModel]] = {} # mutable-ok: per-key memo of validated reads self.attach_redis_cache(redis_cache, default_redis_ttl=default_redis_ttl) @property @@ -175,6 +184,7 @@ class UserApiKeyCache(DualCache): def set_local(self, key: str, value: object, ttl: float | None) -> None: """A memory write alone, with ``default_in_memory_ttl`` when ``ttl`` is None.""" + self._drop_decoded((key,)) if self._shared is not None: self._shared.set_local(key, value, ttl) return @@ -199,6 +209,7 @@ class UserApiKeyCache(DualCache): """Caches ``value`` unless the lease's key was evicted since it was taken; False when refused.""" if self._leases.get(lease.key) is not lease: return False + self._drop_decoded((lease.key,)) shared: Final = self._shared if shared is None: await self.async_set_cache(key=lease.key, value=value, ttl=ttl) @@ -214,9 +225,35 @@ class UserApiKeyCache(DualCache): return True def _void_leases(self, keys: Sequence[str]) -> None: + self._drop_decoded(keys) for key in keys: _ = self._leases.pop(key, None) + def _drop_decoded(self, keys: Sequence[str]) -> None: + for key in keys: + _ = self._decoded_models.pop(key, None) + + def _decoded(self, cached: object, key: str, model_type: type[BaseModel] | None, method: str) -> object: + """In Python memory, a read of the same payload object a previous read validated returns a shallow copy of + that model instead of validating again, so a caller that sets a field cannot change what the next reader + gets. The shared store hands back a new payload on every read, so it is validated each time.""" + if model_type is None or cached is None: + return cached + memo: Final = self._decoded_models.get(key) + if memo is not None and memo.payload is cached and memo.model_type is model_type: + return memo.model.model_copy() + decoded: Final = _validated(cached, key, model_type, method) + if decoded is None or decoded is cached or self._shared is not None: + return decoded + _ = self._decoded_models.pop(key, None) + self._decoded_models[key] = _DecodedModel(payload=cached, model_type=model_type, model=decoded) + if len(self._decoded_models) > self._decoded_capacity(): + _ = self._decoded_models.pop(next(iter(self._decoded_models))) + return decoded.model_copy() + + def _decoded_capacity(self) -> int: + return self.in_memory_cache.max_size_in_memory + self.key_object_cache.in_memory_cache.max_size_in_memory + def delete_local(self, key: str) -> None: """Drops this worker's memory copy alone, as a cross-worker eviction broadcast does.""" self._void_leases((key,)) @@ -271,7 +308,7 @@ class UserApiKeyCache(DualCache): if is_user_key_cache_key(key) else super().get_cache(key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs) ) - return _decoded(cached, key, model_type, "get_cache") + return self._decoded(cached, key, model_type, "get_cache") @overload async def async_get_cache( @@ -316,7 +353,7 @@ class UserApiKeyCache(DualCache): key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs ) ) - return _decoded(cached, key, model_type, "async_get_cache") + return self._decoded(cached, key, model_type, "async_get_cache") @staticmethod def _get_shared_sync(shared: AuthCacheStore, key: str, local_only: bool) -> object: @@ -337,6 +374,8 @@ class UserApiKeyCache(DualCache): def set_cache(self, key: str | None, value: CacheValue, local_only: bool = False, **kwargs: object): model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None)) payload: Final[object] = CacheCodec.serialize(value, model_type=model_type) + if key is not None: + self._drop_decoded((key,)) shared: Final = self._shared if shared is not None: if key is None: @@ -358,6 +397,8 @@ class UserApiKeyCache(DualCache): else, or with options the pipeline does not carry, it goes to Redis directly as before.""" model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None)) payload: Final[object] = CacheCodec.serialize(value, model_type=model_type) + if key is not None: + self._drop_decoded((key,)) ttl: Final = kwargs.get("ttl") pipelined: Final = ( key is not None @@ -443,6 +484,7 @@ class UserApiKeyCache(DualCache): def flush_cache(self) -> None: self._leases.clear() + self._decoded_models.clear() self.budget_windows_absent_cache.flush_cache() if self._shared is not None: self._shared.flush_local() @@ -458,6 +500,7 @@ class UserApiKeyCache(DualCache): ``model_type``: ``BaseModel`` values become JSON-safe dicts; dicts/scalars unchanged. """ normalized: Final = tuple((key, CacheCodec.serialize(value, model_type=None)) for key, value in cache_list) + self._drop_decoded(tuple(key for key, _ in normalized)) shared: Final = self._shared if shared is not None: await self._set_many_shared(shared, normalized, local_only=local_only, ttl=_ttl(kwargs)) @@ -519,9 +562,7 @@ class UserApiKeyCache(DualCache): ) -def _decoded(cached: object, key: str, model_type: type[BaseModel] | None, method: str) -> object: - if model_type is None or cached is None: - return cached +def _validated(cached: object, key: str, model_type: type[BaseModel], method: str) -> BaseModel | None: decoded: Final = CacheCodec.deserialize(cached, model_type=model_type) if decoded is None: verbose_proxy_logger.error( diff --git a/tests/unit/proxy/common_utils/test_user_api_key_cache.py b/tests/unit/proxy/common_utils/test_user_api_key_cache.py index b081a8b88ec..eb98bbc5bf0 100644 --- a/tests/unit/proxy/common_utils/test_user_api_key_cache.py +++ b/tests/unit/proxy/common_utils/test_user_api_key_cache.py @@ -1,16 +1,17 @@ import datetime import hashlib from collections.abc import Awaitable, Callable -from typing import Any, Final, TypeAlias +from typing import Any, ClassVar, Final, TypeAlias from unittest.mock import MagicMock import pytest +from pydantic import BaseModel, model_validator from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import RedisCache from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LiteLLM_TeamTableCachedObj, LiteLLM_UserTable, UserAPIKeyAuth from litellm.proxy.common_utils.user_api_key_cache import ( UserApiKeyCache, budget_windows_absent_cache_key, @@ -587,3 +588,158 @@ async def test_an_eviction_that_lands_while_redis_takes_the_write_keeps_it_out_o assert await cache.async_set_leased(lease, {"spend": 5.0}, ttl=60) is False assert await cache.async_get_cache("customer_member_id:m") is None + + +class _CountedValidation(BaseModel): + validations: ClassVar[int] = 0 + + spend: float + + @model_validator(mode="before") + @classmethod + def _count(cls, data: object) -> object: + _CountedValidation.validations += 1 + return data + + +_TEAM_KEY: Final = "team_id:t1" + + +def _auth_objects() -> tuple[tuple[str, BaseModel], ...]: + return ( + (HASHED_TOKEN, UserAPIKeyAuth(token=HASHED_TOKEN, spend=1.5, models=["gpt-4o"], metadata={"a": 1})), + (_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=2.0, models=["gpt-4o"], metadata={"a": 1})), + ("u1", LiteLLM_UserTable(user_id="u1", spend=3.0, models=["gpt-4o"], teams=["t1"])), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("new_cache", [_memory_only, _with_redis_auth_cache]) +@pytest.mark.parametrize(("key", "stored"), _auth_objects()) +async def test_every_hit_returns_an_object_equal_to_the_stored_one( + new_cache: _NewCache, key: str, stored: BaseModel +) -> None: + cache: Final = new_cache() + await cache.async_set_cache(key, stored, model_type=type(stored)) + + reads: Final = ( + await cache.async_get_cache(key, model_type=type(stored)), + await cache.async_get_cache(key, model_type=type(stored)), + cache.get_cache(key, model_type=type(stored)), + ) + + assert [type(read) for read in reads] == [type(stored)] * 3 + assert list(reads) == [stored] * 3 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("new_cache", [_memory_only, _with_redis_auth_cache]) +@pytest.mark.parametrize(("key", "stored"), _auth_objects()) +async def test_a_field_set_on_a_read_object_does_not_reach_the_next_read( + new_cache: _NewCache, key: str, stored: BaseModel +) -> None: + cache: Final = new_cache() + await cache.async_set_cache(key, stored, model_type=type(stored)) + reads: Final = [await cache.async_get_cache(key, model_type=type(stored)) for _ in range(2)] + for read in reads: + assert read is not None + read.spend = 99.0 + read.models = ["mutated"] + + after_mutation: Final = await cache.async_get_cache(key, model_type=type(stored)) + + assert after_mutation == stored + + +@pytest.mark.asyncio +async def test_reads_of_an_unchanged_entry_validate_it_once() -> None: + cache: Final = UserApiKeyCache() + await cache.async_set_cache("k", {"spend": 1.0}) + before: Final = _CountedValidation.validations + + reads: Final = [await cache.async_get_cache("k", model_type=_CountedValidation) for _ in range(5)] + + assert _CountedValidation.validations - before == 1 + assert reads == [_CountedValidation(spend=1.0)] * 5 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("new_cache", [_memory_only, _with_redis_auth_cache]) +async def test_each_write_changes_what_the_next_read_returns(new_cache: _NewCache) -> None: + cache: Final = new_cache() + await cache.async_set_cache(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=1.0)) + assert await cache.async_get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj) is not None + + await cache.async_set_cache(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=2.0)) + after_async_set: Final = await cache.async_get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj) + cache.set_cache(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=3.0), local_only=True) + after_local_set: Final = cache.get_cache(_TEAM_KEY, local_only=True, model_type=LiteLLM_TeamTableCachedObj) + await cache.async_set_cache_pipeline([(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=4.0))]) + after_pipeline: Final = await cache.async_get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj) + + assert [after_async_set, after_local_set, after_pipeline] == [ + LiteLLM_TeamTableCachedObj(team_id="t1", spend=2.0), + LiteLLM_TeamTableCachedObj(team_id="t1", spend=3.0), + LiteLLM_TeamTableCachedObj(team_id="t1", spend=4.0), + ] + + +@pytest.mark.asyncio +async def test_a_payload_rewritten_in_place_and_set_again_is_read_back() -> None: + cache: Final = UserApiKeyCache() + await cache.async_set_cache("k", {"spend": 1.0}) + assert await cache.async_get_cache("k", model_type=_CountedValidation) == _CountedValidation(spend=1.0) + payload: Final = await cache.async_get_cache("k") + payload["spend"] = 2.0 + + await cache.async_set_cache("k", payload) + + assert await cache.async_get_cache("k", model_type=_CountedValidation) == _CountedValidation(spend=2.0) + + +@pytest.mark.asyncio +async def test_a_payload_written_straight_into_the_memory_layer_is_read_back() -> None: + memory: Final = InMemoryCache() + cache: Final = UserApiKeyCache(in_memory_cache=memory) + await cache.async_set_cache("k", {"spend": 1.0}) + assert await cache.async_get_cache("k", model_type=_CountedValidation) == _CountedValidation(spend=1.0) + + memory.set_cache("k", {"spend": 2.0}) + + assert await cache.async_get_cache("k", model_type=_CountedValidation) == _CountedValidation(spend=2.0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("new_cache", [_memory_only, _with_redis_auth_cache]) +@pytest.mark.parametrize("evict", _EVICTIONS) +async def test_a_typed_read_after_an_eviction_is_a_miss(new_cache: _NewCache, evict: _Evict) -> None: + cache: Final = new_cache() + await cache.async_set_cache(HASHED_TOKEN, UserAPIKeyAuth(token=HASHED_TOKEN), model_type=UserAPIKeyAuth) + assert await cache.async_get_cache(HASHED_TOKEN, model_type=UserAPIKeyAuth) is not None + + await evict(cache, HASHED_TOKEN) + + assert await cache.async_get_cache(HASHED_TOKEN, local_only=True, model_type=UserAPIKeyAuth) is None + + +def test_a_typed_read_after_the_ttl_is_a_miss() -> None: + clock: Final = _ManualClock() + cache: Final = UserApiKeyCache(in_memory_cache=InMemoryCache(clock=clock), default_in_memory_ttl=60) + cache.set_cache(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1")) + assert cache.get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj) is not None + + clock.advance(61) + + assert cache.get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj) is None + + +@pytest.mark.asyncio +async def test_one_entry_read_as_two_models_returns_each_model() -> None: + cache: Final = UserApiKeyCache() + await cache.async_set_cache(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=1.0)) + + as_team: Final = await cache.async_get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj) + as_counted: Final = await cache.async_get_cache(_TEAM_KEY, model_type=_CountedValidation) + + assert type(as_team) is LiteLLM_TeamTableCachedObj + assert as_counted == _CountedValidation(spend=1.0)