mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(caching): scope exact-match cache keys per virtual key, team and org
The exact-match response cache was shared across tenants on the proxy, so a key could be served another team's cached completion. A client-supplied cache_key, cache.namespace or preset_cache_key could also pick another tenant's entries Every cache key now carries a tag derived from the caller's key, team and org, and SDK calls without a proxy identity keep their current keys Fixes #49313
This commit is contained in:
parent
60fc121ce8
commit
71c07184e2
5 changed files with 118 additions and 34 deletions
|
|
@ -46,6 +46,8 @@ from .s3_cache import S3Cache
|
|||
|
||||
RESPONSE_CACHE_TARGET: Final = "llm_response"
|
||||
_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"})
|
||||
_TENANT_TAG_PREFIX: Final = "tenant-"
|
||||
_TENANT_TAG_DIGEST_LENGTH: Final = 32
|
||||
_DECODED_LITERAL: Final = TypeAdapter(object)
|
||||
_REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
|
||||
list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True)
|
||||
|
|
@ -371,17 +373,52 @@ class Cache:
|
|||
return (*self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS, self._SEMANTIC_CACHE_END_USER_SCOPE_FIELD)
|
||||
return self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS
|
||||
|
||||
def _get_semantic_cache_tenant_scope(self, kwargs: dict) -> str:
|
||||
litellm_params: Final[dict] = kwargs.get("litellm_params") or {}
|
||||
metadata_sources: Final[tuple[dict, ...]] = tuple(
|
||||
source.get(key) or {} for source in (kwargs, litellm_params) for key in ("metadata", "litellm_metadata")
|
||||
def _get_semantic_cache_tenant_scope(self, kwargs: Mapping[str, object]) -> str:
|
||||
return self._get_tenant_scope(kwargs, self._semantic_cache_scope_fields())
|
||||
|
||||
@staticmethod
|
||||
def _get_tenant_scope(kwargs: Mapping[str, object], fields: tuple[str, ...]) -> str:
|
||||
litellm_params: Final = kwargs.get("litellm_params")
|
||||
param_sources: Final = (kwargs, litellm_params if isinstance(litellm_params, Mapping) else {})
|
||||
metadata_sources: Final[tuple[Mapping[str, object], ...]] = tuple(
|
||||
metadata
|
||||
for source in param_sources
|
||||
for key in ("metadata", "litellm_metadata")
|
||||
if isinstance(metadata := source.get(key), Mapping)
|
||||
)
|
||||
scope_values: Final = (
|
||||
(field, next((source[field] for source in metadata_sources if source.get(field) is not None), None))
|
||||
for field in self._semantic_cache_scope_fields()
|
||||
for field in fields
|
||||
)
|
||||
return "".join(f"{field}: {value}" for field, value in scope_values if value is not None)
|
||||
|
||||
def _cache_tenant_scope_fields(self) -> tuple[str, ...]:
|
||||
if self._is_semantic_cache():
|
||||
return self._semantic_cache_scope_fields()
|
||||
return self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS
|
||||
|
||||
def scope_cache_key(self, cache_key: str, kwargs: Mapping[str, object]) -> str:
|
||||
"""
|
||||
Prefix ``cache_key`` with the tag of the tenant (key / team / org from proxy auth) that kwargs carry.
|
||||
|
||||
Idempotent, so every key a tenant uses starts with its own tag, whatever ``cache_key``,
|
||||
``cache.namespace`` or ``preset_cache_key`` the caller sends. No-op without a tenant (SDK use)
|
||||
"""
|
||||
tenant_scope: Final = self._get_tenant_scope(kwargs, self._cache_tenant_scope_fields())
|
||||
if not tenant_scope:
|
||||
return cache_key
|
||||
tenant_digest: Final = hashlib.sha256(tenant_scope.encode()).hexdigest()[:_TENANT_TAG_DIGEST_LENGTH]
|
||||
tenant_prefix: Final = f"{_TENANT_TAG_PREFIX}{tenant_digest}:"
|
||||
if cache_key.startswith(tenant_prefix):
|
||||
return cache_key
|
||||
return f"{tenant_prefix}{cache_key}"
|
||||
|
||||
def resolve_cache_key(self, kwargs: Mapping[str, object]) -> str | None:
|
||||
if "cache_key" not in kwargs:
|
||||
return self.get_cache_key(**kwargs)
|
||||
cache_key: Final = kwargs["cache_key"]
|
||||
return self.scope_cache_key(cache_key, kwargs) if isinstance(cache_key, str) else None
|
||||
|
||||
def get_cache_key(self, **kwargs: object) -> str:
|
||||
"""
|
||||
Get the cache key for the given arguments.
|
||||
|
|
@ -398,7 +435,7 @@ class Cache:
|
|||
preset_cache_key: Final = self.get_preset_cache_key_from_kwargs(**kwargs)
|
||||
if preset_cache_key is not None:
|
||||
verbose_logger.debug("\nReturning preset cache key: %s", preset_cache_key)
|
||||
return preset_cache_key
|
||||
return self.scope_cache_key(preset_cache_key, kwargs)
|
||||
|
||||
combined_kwargs: Final = ModelParamHelper.get_all_llm_api_params()
|
||||
is_semantic_cache: Final = self._is_semantic_cache()
|
||||
|
|
@ -432,6 +469,7 @@ class Cache:
|
|||
cache_key + reasoning_summary_material + tool_call_id_shape_material
|
||||
)
|
||||
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
|
||||
hashed_cache_key = self.scope_cache_key(hashed_cache_key, kwargs)
|
||||
verbose_logger.debug(
|
||||
"\nCreated cache key: %s (source material length: %d)",
|
||||
hashed_cache_key,
|
||||
|
|
@ -683,10 +721,7 @@ class Cache:
|
|||
if self.should_use_cache(**kwargs) is not True:
|
||||
return
|
||||
with response_cache_phase("get"):
|
||||
if "cache_key" in kwargs:
|
||||
cache_key = kwargs["cache_key"]
|
||||
else:
|
||||
cache_key = self.get_cache_key(**kwargs)
|
||||
cache_key = self.resolve_cache_key(kwargs)
|
||||
if cache_key is not None:
|
||||
cache_control_args: Final[DynamicCacheControl] = kwargs.get("cache", {})
|
||||
max_age = cache_control_args.get("s-maxage") or cache_control_args.get("s-max-age") or float("inf")
|
||||
|
|
@ -716,10 +751,7 @@ class Cache:
|
|||
return
|
||||
|
||||
with response_cache_phase("get"):
|
||||
if "cache_key" in kwargs:
|
||||
cache_key = kwargs["cache_key"]
|
||||
else:
|
||||
cache_key = self.get_cache_key(**kwargs)
|
||||
cache_key = self.resolve_cache_key(kwargs)
|
||||
if cache_key is not None:
|
||||
cache_control_args: Final = kwargs.get("cache", {})
|
||||
max_age: Final = cache_control_args.get(
|
||||
|
|
@ -739,10 +771,7 @@ class Cache:
|
|||
Common implementation across sync + async add_cache functions
|
||||
"""
|
||||
try:
|
||||
if "cache_key" in kwargs:
|
||||
cache_key = kwargs["cache_key"]
|
||||
else:
|
||||
cache_key = self.get_cache_key(**kwargs)
|
||||
cache_key = self.resolve_cache_key(kwargs)
|
||||
if cache_key is not None:
|
||||
if isinstance(result, BaseModel):
|
||||
result = result.model_dump_json(exclude_unset=True)
|
||||
|
|
|
|||
|
|
@ -395,11 +395,7 @@ class LLMCachingHandler:
|
|||
end_time=end_time,
|
||||
cache_hit=cache_hit,
|
||||
)
|
||||
cache_key: Final = (
|
||||
self.preset_cache_key
|
||||
or self.request_kwargs.get("cache_key")
|
||||
or litellm.cache.get_cache_key(**self.request_kwargs)
|
||||
)
|
||||
cache_key: Final = self.preset_cache_key or litellm.cache.resolve_cache_key(self.request_kwargs)
|
||||
_set_cached_hidden_param(cached_result, "cache_key", cache_key)
|
||||
return CachingHandlerResponse(cached_result=cached_result)
|
||||
elif (
|
||||
|
|
@ -522,11 +518,7 @@ class LLMCachingHandler:
|
|||
end_time=end_time,
|
||||
cache_hit=cache_hit,
|
||||
)
|
||||
cache_key: Final = (
|
||||
self.preset_cache_key
|
||||
or self.request_kwargs.get("cache_key")
|
||||
or litellm.cache.get_cache_key(**self.request_kwargs)
|
||||
)
|
||||
cache_key: Final = self.preset_cache_key or litellm.cache.resolve_cache_key(self.request_kwargs)
|
||||
_set_cached_hidden_param(cached_result, "cache_key", cache_key)
|
||||
return CachingHandlerResponse(cached_result=cached_result)
|
||||
return CachingHandlerResponse(cached_result=cached_result)
|
||||
|
|
@ -920,7 +912,11 @@ class LLMCachingHandler:
|
|||
request_kwargs.pop("cache_key", None)
|
||||
if litellm.cache.supports_async() is True:
|
||||
## check if dual cache is supported ##
|
||||
self.preset_cache_key = request_cache_key or litellm.cache.get_cache_key(**request_kwargs)
|
||||
self.preset_cache_key = (
|
||||
litellm.cache.scope_cache_key(request_cache_key, request_kwargs)
|
||||
if request_cache_key
|
||||
else litellm.cache.get_cache_key(**request_kwargs)
|
||||
)
|
||||
with response_cache_phase("get"):
|
||||
cached_result = await litellm.cache.async_get_cache(
|
||||
dynamic_cache_object=self.dual_cache,
|
||||
|
|
@ -928,7 +924,11 @@ class LLMCachingHandler:
|
|||
**request_kwargs,
|
||||
)
|
||||
else: # fallback for caches that don't support async
|
||||
self.preset_cache_key = request_cache_key or litellm.cache.get_cache_key(**request_kwargs)
|
||||
self.preset_cache_key = (
|
||||
litellm.cache.scope_cache_key(request_cache_key, request_kwargs)
|
||||
if request_cache_key
|
||||
else litellm.cache.get_cache_key(**request_kwargs)
|
||||
)
|
||||
with response_cache_phase("get"):
|
||||
cached_result = litellm.cache.get_cache(
|
||||
dynamic_cache_object=self.dual_cache,
|
||||
|
|
|
|||
|
|
@ -49,12 +49,15 @@ class _PROXY_BatchRedisRequests(CustomLogger):
|
|||
api_key: Final = user_api_key_dict.api_key
|
||||
|
||||
cache_key_name: Final = f"litellm:{api_key}:{call_type}"
|
||||
scoped_key_prefix: Final = (
|
||||
litellm.cache.scope_cache_key(cache_key_name, data) if litellm.cache is not None else cache_key_name
|
||||
)
|
||||
self.in_memory_cache = cache.in_memory_cache
|
||||
|
||||
key_value_dict = {}
|
||||
in_memory_cache_exists = False
|
||||
for key in cache.in_memory_cache.cache_dict:
|
||||
if isinstance(key, str) and key.startswith(cache_key_name):
|
||||
if isinstance(key, str) and key.startswith(scoped_key_prefix):
|
||||
in_memory_cache_exists = True
|
||||
|
||||
if in_memory_cache_exists is False and litellm.cache is not None:
|
||||
|
|
@ -63,9 +66,9 @@ class _PROXY_BatchRedisRequests(CustomLogger):
|
|||
- Get the relevant values
|
||||
"""
|
||||
if litellm.cache.type is not None and isinstance(litellm.cache.cache, RedisCache):
|
||||
self.print_verbose(f"cache_key_name: {cache_key_name}")
|
||||
self.print_verbose(f"cache_key_name: {scoped_key_prefix}")
|
||||
with response_cache_phase("get"):
|
||||
keys = await litellm.cache.cache.async_scan_iter(pattern=cache_key_name, count=100)
|
||||
keys = await litellm.cache.cache.async_scan_iter(pattern=scoped_key_prefix, count=100)
|
||||
self.print_verbose(f"redis keys: {keys}")
|
||||
if len(keys) > 0:
|
||||
key_value_dict = await litellm.cache.cache.async_batch_get_cache(key_list=keys)
|
||||
|
|
@ -94,7 +97,9 @@ class _PROXY_BatchRedisRequests(CustomLogger):
|
|||
"""
|
||||
try: # never block execution
|
||||
cache_key: str | None = None
|
||||
if "cache_key" in kwargs:
|
||||
if "cache_key" in kwargs and litellm.cache is not None:
|
||||
cache_key = litellm.cache.resolve_cache_key(kwargs)
|
||||
elif "cache_key" in kwargs:
|
||||
cache_key = kwargs["cache_key"]
|
||||
elif litellm.cache is not None:
|
||||
cache_key = litellm.cache.get_cache_key(
|
||||
|
|
|
|||
|
|
@ -1608,3 +1608,51 @@ def test_cache_key_separates_model_ids_unless_a_model_group_owns_the_key() -> No
|
|||
assert key_a != bare
|
||||
assert key_a == key_a_again
|
||||
assert grouped_a == grouped_b
|
||||
|
||||
|
||||
_TENANT_A_IDENTITY: Final = {"user_api_key": "hashed-key-a", "user_api_key_team_id": "team-a"}
|
||||
_TENANT_B_IDENTITY: Final = {"user_api_key": "hashed-key-b", "user_api_key_team_id": "team-b"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"client_cache_options",
|
||||
[
|
||||
pytest.param({}, id="no-client-options"),
|
||||
pytest.param({"cache_key": "client-chosen-key"}, id="same-client-cache-key"),
|
||||
pytest.param({"cache": {"namespace": "shared"}}, id="same-client-namespace"),
|
||||
],
|
||||
)
|
||||
def test_exact_cache_is_not_shared_across_tenants(monkeypatch, client_cache_options):
|
||||
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL))
|
||||
|
||||
def answer(tenant: dict[str, str], content: str) -> str:
|
||||
response = completion(
|
||||
model="gpt-4o-mini",
|
||||
messages=_CACHING_TEST_MESSAGES,
|
||||
caching=True,
|
||||
mock_response=content,
|
||||
metadata=dict(tenant),
|
||||
**client_cache_options,
|
||||
)
|
||||
return response.choices[0].message.content
|
||||
|
||||
assert answer(_TENANT_A_IDENTITY, "tenant a") == "tenant a"
|
||||
assert answer(_TENANT_B_IDENTITY, "tenant b") == "tenant b"
|
||||
assert answer(_TENANT_A_IDENTITY, "tenant a, uncached") == "tenant a"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"])
|
||||
@pytest.mark.parametrize("forged_key_param", ["cache_key", "preset_cache_key"])
|
||||
def test_exact_cache_rejects_another_tenants_cache_key(metadata_field, forged_key_param):
|
||||
cache = Cache(type=LiteLLMCacheType.LOCAL)
|
||||
tenant_a_request = {"model": "gpt-4o-mini", "messages": messages, metadata_field: dict(_TENANT_A_IDENTITY)}
|
||||
cache.add_cache('{"answer": "tenant a"}', **tenant_a_request)
|
||||
tenant_a_key = cache.get_cache_key(**tenant_a_request)
|
||||
forged_key = (
|
||||
{"cache_key": tenant_a_key}
|
||||
if forged_key_param == "cache_key"
|
||||
else {"litellm_params": {"preset_cache_key": tenant_a_key}}
|
||||
)
|
||||
|
||||
assert cache.get_cache(**tenant_a_request) is not None
|
||||
assert cache.get_cache(**forged_key, **{metadata_field: dict(_TENANT_B_IDENTITY)}) is None
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ async def test_async_get_cache_reuses_preset_cache_key_for_responses():
|
|||
mock_cache.supported_call_types = [CallTypes.aresponses.value]
|
||||
mock_cache.supports_async.return_value = True
|
||||
mock_cache.get_cache_key.return_value = "responses-stream-cache-key"
|
||||
mock_cache.scope_cache_key.side_effect = lambda cache_key, kwargs: cache_key
|
||||
mock_cache.async_get_cache = AsyncMock(return_value=None)
|
||||
litellm.cache = mock_cache
|
||||
|
||||
|
|
@ -84,6 +85,7 @@ async def test_async_get_cache_falls_back_to_sync_cache_for_responses():
|
|||
mock_cache.supported_call_types = [CallTypes.aresponses.value]
|
||||
mock_cache.supports_async.return_value = False
|
||||
mock_cache.get_cache_key.return_value = "responses-stream-cache-key"
|
||||
mock_cache.scope_cache_key.side_effect = lambda cache_key, kwargs: cache_key
|
||||
mock_cache.get_cache.return_value = None
|
||||
litellm.cache = mock_cache
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue