scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 109

fix(caching): opt-in team-scoped cache key to prevent cross-tenant cache reuse

Browse source 
Adds an opt-in cache_params flag, add_team_id_to_cache_key, that folds the
requesting team (metadata.user_api_key_team_id, falling back to the hashed api
key) into the response-cache key. Without it the cache key is request-params-
only, so two teams on a multi-tenant proxy share cache entries and one team can
be served another's cached response (visible via x-litellm-cache-key). Default
False preserves existing behavior. Surfaced with Sectum AI; AI-assisted.

Signed-off-by: Dmitry Maranik <dmitrymaranik@gmail.com>
This commit is contained in:
Dmitry Maranik 2026-06-08 11:38:31 -07:00
parent 35f6961526
commit 9232aad656
1 changed files with 34 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
litellm/caching/caching.py
View file

@ -63,6 +63,7 @@ class Cache:
port: Optional[str] = None,
password: Optional[str] = None,
namespace: Optional[str] = None,
add_team_id_to_cache_key: bool = False,
ttl: Optional[float] = None,
default_in_memory_ttl: Optional[float] = None,
default_in_redis_ttl: Optional[float] = None,
@ -257,6 +258,7 @@ class Cache:
self.supported_call_types = supported_call_types # default to ["completion", "acompletion", "embedding", "aembedding"]
self.type = type
self.namespace = namespace
self.add_team_id_to_cache_key = add_team_id_to_cache_key
self.redis_flush_size = redis_flush_size
self.ttl = ttl
self.mode: CacheMode = mode or CacheMode.default_on
@ -309,6 +311,8 @@ class Cache:
param_value = kwargs[param]
cache_key += f"{str(param)}: {str(param_value)}"
cache_key += self._get_team_scope_for_cache_key(**kwargs)
verbose_logger.debug("\nCreated cache key: %s", cache_key)
hashed_cache_key = Cache._get_hashed_cache_key(cache_key)
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
@ -420,6 +424,36 @@ class Cache:
verbose_logger.debug("Hashed cache key (SHA-256): %s", hash_hex)
return hash_hex
def _get_team_scope_for_cache_key(self, **kwargs) -> str:
"""Optionally scope the cache key by the requesting team.
On a multi-tenant proxy the cache key is otherwise derived only from the
request parameters, so two different teams (tenants) sending the same
request share cache entries - one team can be served another's cached
response. When ``add_team_id_to_cache_key`` is enabled (via
``cache_params``), the requesting team id is folded into the cache key so
cache entries are not reused across teams; same-team requests still share
the cache. A request with no team falls back to the (hashed) api key, so
it is still isolated rather than silently sharing the global entry.
Opt-in - the default preserves the existing behavior.
"""
if not self.add_team_id_to_cache_key:
return ""
metadata = kwargs.get("metadata") or {}
litellm_params = kwargs.get("litellm_params") or {}
metadata_in_litellm_params = litellm_params.get("metadata") or {}
team_id = metadata.get(
"user_api_key_team_id"
) or metadata_in_litellm_params.get("user_api_key_team_id")
if team_id:
return f"user_api_key_team_id: {team_id}"
api_key = metadata.get("user_api_key") or metadata_in_litellm_params.get(
"user_api_key"
)
if api_key:
return f"user_api_key: {api_key}"
return ""
def _add_namespace_to_cache_key(self, hash_hex: str, **kwargs) -> str:
"""
If a redis namespace is provided, add it to the cache key