mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
perf(auth): reuse validated models on UserApiKeyCache memory hits (#51912)
A typed read of the same in-memory payload returns a shallow copy of the model the previous read validated instead of running model_validate again. Writes, deletes and flushes drop the memo, and the Redis and shared store payloads are unchanged.
This commit is contained in:
parent
8c97c243fe
commit
eb8a2d224e
2 changed files with 204 additions and 7 deletions
|
|
@ -3,6 +3,7 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import re
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast, overload
|
||||
from weakref import WeakValueDictionary
|
||||
|
||||
|
|
@ -48,6 +49,13 @@ def _ttl(kwargs: Mapping[str, object]) -> float | None:
|
|||
return _TTL.validate_python(kwargs.get("ttl"))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _DecodedModel:
|
||||
payload: object
|
||||
model_type: type[BaseModel]
|
||||
model: BaseModel
|
||||
|
||||
|
||||
class CacheLease:
|
||||
__slots__ = ("__weakref__", "key", "shared")
|
||||
|
||||
|
|
@ -114,6 +122,7 @@ class UserApiKeyCache(DualCache):
|
|||
self._redis_connection: RedisCache | None = None
|
||||
self._shared: AuthCacheStore | None = None
|
||||
self._leases: Final[WeakValueDictionary[str, CacheLease]] = WeakValueDictionary()
|
||||
self._decoded_models: Final[dict[str, _DecodedModel]] = {} # mutable-ok: per-key memo of validated reads
|
||||
self.attach_redis_cache(redis_cache, default_redis_ttl=default_redis_ttl)
|
||||
|
||||
@property
|
||||
|
|
@ -175,6 +184,7 @@ class UserApiKeyCache(DualCache):
|
|||
|
||||
def set_local(self, key: str, value: object, ttl: float | None) -> None:
|
||||
"""A memory write alone, with ``default_in_memory_ttl`` when ``ttl`` is None."""
|
||||
self._drop_decoded((key,))
|
||||
if self._shared is not None:
|
||||
self._shared.set_local(key, value, ttl)
|
||||
return
|
||||
|
|
@ -199,6 +209,7 @@ class UserApiKeyCache(DualCache):
|
|||
"""Caches ``value`` unless the lease's key was evicted since it was taken; False when refused."""
|
||||
if self._leases.get(lease.key) is not lease:
|
||||
return False
|
||||
self._drop_decoded((lease.key,))
|
||||
shared: Final = self._shared
|
||||
if shared is None:
|
||||
await self.async_set_cache(key=lease.key, value=value, ttl=ttl)
|
||||
|
|
@ -214,9 +225,35 @@ class UserApiKeyCache(DualCache):
|
|||
return True
|
||||
|
||||
def _void_leases(self, keys: Sequence[str]) -> None:
|
||||
self._drop_decoded(keys)
|
||||
for key in keys:
|
||||
_ = self._leases.pop(key, None)
|
||||
|
||||
def _drop_decoded(self, keys: Sequence[str]) -> None:
|
||||
for key in keys:
|
||||
_ = self._decoded_models.pop(key, None)
|
||||
|
||||
def _decoded(self, cached: object, key: str, model_type: type[BaseModel] | None, method: str) -> object:
|
||||
"""In Python memory, a read of the same payload object a previous read validated returns a shallow copy of
|
||||
that model instead of validating again, so a caller that sets a field cannot change what the next reader
|
||||
gets. The shared store hands back a new payload on every read, so it is validated each time."""
|
||||
if model_type is None or cached is None:
|
||||
return cached
|
||||
memo: Final = self._decoded_models.get(key)
|
||||
if memo is not None and memo.payload is cached and memo.model_type is model_type:
|
||||
return memo.model.model_copy()
|
||||
decoded: Final = _validated(cached, key, model_type, method)
|
||||
if decoded is None or decoded is cached or self._shared is not None:
|
||||
return decoded
|
||||
_ = self._decoded_models.pop(key, None)
|
||||
self._decoded_models[key] = _DecodedModel(payload=cached, model_type=model_type, model=decoded)
|
||||
if len(self._decoded_models) > self._decoded_capacity():
|
||||
_ = self._decoded_models.pop(next(iter(self._decoded_models)))
|
||||
return decoded.model_copy()
|
||||
|
||||
def _decoded_capacity(self) -> int:
|
||||
return self.in_memory_cache.max_size_in_memory + self.key_object_cache.in_memory_cache.max_size_in_memory
|
||||
|
||||
def delete_local(self, key: str) -> None:
|
||||
"""Drops this worker's memory copy alone, as a cross-worker eviction broadcast does."""
|
||||
self._void_leases((key,))
|
||||
|
|
@ -271,7 +308,7 @@ class UserApiKeyCache(DualCache):
|
|||
if is_user_key_cache_key(key)
|
||||
else super().get_cache(key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs)
|
||||
)
|
||||
return _decoded(cached, key, model_type, "get_cache")
|
||||
return self._decoded(cached, key, model_type, "get_cache")
|
||||
|
||||
@overload
|
||||
async def async_get_cache(
|
||||
|
|
@ -316,7 +353,7 @@ class UserApiKeyCache(DualCache):
|
|||
key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs
|
||||
)
|
||||
)
|
||||
return _decoded(cached, key, model_type, "async_get_cache")
|
||||
return self._decoded(cached, key, model_type, "async_get_cache")
|
||||
|
||||
@staticmethod
|
||||
def _get_shared_sync(shared: AuthCacheStore, key: str, local_only: bool) -> object:
|
||||
|
|
@ -337,6 +374,8 @@ class UserApiKeyCache(DualCache):
|
|||
def set_cache(self, key: str | None, value: CacheValue, local_only: bool = False, **kwargs: object):
|
||||
model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None))
|
||||
payload: Final[object] = CacheCodec.serialize(value, model_type=model_type)
|
||||
if key is not None:
|
||||
self._drop_decoded((key,))
|
||||
shared: Final = self._shared
|
||||
if shared is not None:
|
||||
if key is None:
|
||||
|
|
@ -358,6 +397,8 @@ class UserApiKeyCache(DualCache):
|
|||
else, or with options the pipeline does not carry, it goes to Redis directly as before."""
|
||||
model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None))
|
||||
payload: Final[object] = CacheCodec.serialize(value, model_type=model_type)
|
||||
if key is not None:
|
||||
self._drop_decoded((key,))
|
||||
ttl: Final = kwargs.get("ttl")
|
||||
pipelined: Final = (
|
||||
key is not None
|
||||
|
|
@ -443,6 +484,7 @@ class UserApiKeyCache(DualCache):
|
|||
|
||||
def flush_cache(self) -> None:
|
||||
self._leases.clear()
|
||||
self._decoded_models.clear()
|
||||
self.budget_windows_absent_cache.flush_cache()
|
||||
if self._shared is not None:
|
||||
self._shared.flush_local()
|
||||
|
|
@ -458,6 +500,7 @@ class UserApiKeyCache(DualCache):
|
|||
``model_type``: ``BaseModel`` values become JSON-safe dicts; dicts/scalars unchanged.
|
||||
"""
|
||||
normalized: Final = tuple((key, CacheCodec.serialize(value, model_type=None)) for key, value in cache_list)
|
||||
self._drop_decoded(tuple(key for key, _ in normalized))
|
||||
shared: Final = self._shared
|
||||
if shared is not None:
|
||||
await self._set_many_shared(shared, normalized, local_only=local_only, ttl=_ttl(kwargs))
|
||||
|
|
@ -519,9 +562,7 @@ class UserApiKeyCache(DualCache):
|
|||
)
|
||||
|
||||
|
||||
def _decoded(cached: object, key: str, model_type: type[BaseModel] | None, method: str) -> object:
|
||||
if model_type is None or cached is None:
|
||||
return cached
|
||||
def _validated(cached: object, key: str, model_type: type[BaseModel], method: str) -> BaseModel | None:
|
||||
decoded: Final = CacheCodec.deserialize(cached, model_type=model_type)
|
||||
if decoded is None:
|
||||
verbose_proxy_logger.error(
|
||||
|
|
|
|||
|
|
@ -1,16 +1,17 @@
|
|||
import datetime
|
||||
import hashlib
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any, Final, TypeAlias
|
||||
from typing import Any, ClassVar, Final, TypeAlias
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, model_validator
|
||||
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.caching.redis_cache import RedisCache
|
||||
from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy._types import LiteLLM_TeamTableCachedObj, LiteLLM_UserTable, UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.user_api_key_cache import (
|
||||
UserApiKeyCache,
|
||||
budget_windows_absent_cache_key,
|
||||
|
|
@ -587,3 +588,158 @@ async def test_an_eviction_that_lands_while_redis_takes_the_write_keeps_it_out_o
|
|||
|
||||
assert await cache.async_set_leased(lease, {"spend": 5.0}, ttl=60) is False
|
||||
assert await cache.async_get_cache("customer_member_id:m") is None
|
||||
|
||||
|
||||
class _CountedValidation(BaseModel):
|
||||
validations: ClassVar[int] = 0
|
||||
|
||||
spend: float
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def _count(cls, data: object) -> object:
|
||||
_CountedValidation.validations += 1
|
||||
return data
|
||||
|
||||
|
||||
_TEAM_KEY: Final = "team_id:t1"
|
||||
|
||||
|
||||
def _auth_objects() -> tuple[tuple[str, BaseModel], ...]:
|
||||
return (
|
||||
(HASHED_TOKEN, UserAPIKeyAuth(token=HASHED_TOKEN, spend=1.5, models=["gpt-4o"], metadata={"a": 1})),
|
||||
(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=2.0, models=["gpt-4o"], metadata={"a": 1})),
|
||||
("u1", LiteLLM_UserTable(user_id="u1", spend=3.0, models=["gpt-4o"], teams=["t1"])),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("new_cache", [_memory_only, _with_redis_auth_cache])
|
||||
@pytest.mark.parametrize(("key", "stored"), _auth_objects())
|
||||
async def test_every_hit_returns_an_object_equal_to_the_stored_one(
|
||||
new_cache: _NewCache, key: str, stored: BaseModel
|
||||
) -> None:
|
||||
cache: Final = new_cache()
|
||||
await cache.async_set_cache(key, stored, model_type=type(stored))
|
||||
|
||||
reads: Final = (
|
||||
await cache.async_get_cache(key, model_type=type(stored)),
|
||||
await cache.async_get_cache(key, model_type=type(stored)),
|
||||
cache.get_cache(key, model_type=type(stored)),
|
||||
)
|
||||
|
||||
assert [type(read) for read in reads] == [type(stored)] * 3
|
||||
assert list(reads) == [stored] * 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("new_cache", [_memory_only, _with_redis_auth_cache])
|
||||
@pytest.mark.parametrize(("key", "stored"), _auth_objects())
|
||||
async def test_a_field_set_on_a_read_object_does_not_reach_the_next_read(
|
||||
new_cache: _NewCache, key: str, stored: BaseModel
|
||||
) -> None:
|
||||
cache: Final = new_cache()
|
||||
await cache.async_set_cache(key, stored, model_type=type(stored))
|
||||
reads: Final = [await cache.async_get_cache(key, model_type=type(stored)) for _ in range(2)]
|
||||
for read in reads:
|
||||
assert read is not None
|
||||
read.spend = 99.0
|
||||
read.models = ["mutated"]
|
||||
|
||||
after_mutation: Final = await cache.async_get_cache(key, model_type=type(stored))
|
||||
|
||||
assert after_mutation == stored
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reads_of_an_unchanged_entry_validate_it_once() -> None:
|
||||
cache: Final = UserApiKeyCache()
|
||||
await cache.async_set_cache("k", {"spend": 1.0})
|
||||
before: Final = _CountedValidation.validations
|
||||
|
||||
reads: Final = [await cache.async_get_cache("k", model_type=_CountedValidation) for _ in range(5)]
|
||||
|
||||
assert _CountedValidation.validations - before == 1
|
||||
assert reads == [_CountedValidation(spend=1.0)] * 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("new_cache", [_memory_only, _with_redis_auth_cache])
|
||||
async def test_each_write_changes_what_the_next_read_returns(new_cache: _NewCache) -> None:
|
||||
cache: Final = new_cache()
|
||||
await cache.async_set_cache(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=1.0))
|
||||
assert await cache.async_get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj) is not None
|
||||
|
||||
await cache.async_set_cache(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=2.0))
|
||||
after_async_set: Final = await cache.async_get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj)
|
||||
cache.set_cache(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=3.0), local_only=True)
|
||||
after_local_set: Final = cache.get_cache(_TEAM_KEY, local_only=True, model_type=LiteLLM_TeamTableCachedObj)
|
||||
await cache.async_set_cache_pipeline([(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=4.0))])
|
||||
after_pipeline: Final = await cache.async_get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj)
|
||||
|
||||
assert [after_async_set, after_local_set, after_pipeline] == [
|
||||
LiteLLM_TeamTableCachedObj(team_id="t1", spend=2.0),
|
||||
LiteLLM_TeamTableCachedObj(team_id="t1", spend=3.0),
|
||||
LiteLLM_TeamTableCachedObj(team_id="t1", spend=4.0),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_payload_rewritten_in_place_and_set_again_is_read_back() -> None:
|
||||
cache: Final = UserApiKeyCache()
|
||||
await cache.async_set_cache("k", {"spend": 1.0})
|
||||
assert await cache.async_get_cache("k", model_type=_CountedValidation) == _CountedValidation(spend=1.0)
|
||||
payload: Final = await cache.async_get_cache("k")
|
||||
payload["spend"] = 2.0
|
||||
|
||||
await cache.async_set_cache("k", payload)
|
||||
|
||||
assert await cache.async_get_cache("k", model_type=_CountedValidation) == _CountedValidation(spend=2.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_payload_written_straight_into_the_memory_layer_is_read_back() -> None:
|
||||
memory: Final = InMemoryCache()
|
||||
cache: Final = UserApiKeyCache(in_memory_cache=memory)
|
||||
await cache.async_set_cache("k", {"spend": 1.0})
|
||||
assert await cache.async_get_cache("k", model_type=_CountedValidation) == _CountedValidation(spend=1.0)
|
||||
|
||||
memory.set_cache("k", {"spend": 2.0})
|
||||
|
||||
assert await cache.async_get_cache("k", model_type=_CountedValidation) == _CountedValidation(spend=2.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("new_cache", [_memory_only, _with_redis_auth_cache])
|
||||
@pytest.mark.parametrize("evict", _EVICTIONS)
|
||||
async def test_a_typed_read_after_an_eviction_is_a_miss(new_cache: _NewCache, evict: _Evict) -> None:
|
||||
cache: Final = new_cache()
|
||||
await cache.async_set_cache(HASHED_TOKEN, UserAPIKeyAuth(token=HASHED_TOKEN), model_type=UserAPIKeyAuth)
|
||||
assert await cache.async_get_cache(HASHED_TOKEN, model_type=UserAPIKeyAuth) is not None
|
||||
|
||||
await evict(cache, HASHED_TOKEN)
|
||||
|
||||
assert await cache.async_get_cache(HASHED_TOKEN, local_only=True, model_type=UserAPIKeyAuth) is None
|
||||
|
||||
|
||||
def test_a_typed_read_after_the_ttl_is_a_miss() -> None:
|
||||
clock: Final = _ManualClock()
|
||||
cache: Final = UserApiKeyCache(in_memory_cache=InMemoryCache(clock=clock), default_in_memory_ttl=60)
|
||||
cache.set_cache(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1"))
|
||||
assert cache.get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj) is not None
|
||||
|
||||
clock.advance(61)
|
||||
|
||||
assert cache.get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_one_entry_read_as_two_models_returns_each_model() -> None:
|
||||
cache: Final = UserApiKeyCache()
|
||||
await cache.async_set_cache(_TEAM_KEY, LiteLLM_TeamTableCachedObj(team_id="t1", spend=1.0))
|
||||
|
||||
as_team: Final = await cache.async_get_cache(_TEAM_KEY, model_type=LiteLLM_TeamTableCachedObj)
|
||||
as_counted: Final = await cache.async_get_cache(_TEAM_KEY, model_type=_CountedValidation)
|
||||
|
||||
assert type(as_team) is LiteLLM_TeamTableCachedObj
|
||||
assert as_counted == _CountedValidation(spend=1.0)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue