From 71c07184e2f2b6b565bcfd728bac8be70a6de4d9 Mon Sep 17 00:00:00 2001 From: GellaNagaSaiKrishna Date: Sun, 1 Nov 2026 19:22:52 +0530 Subject: [PATCH] fix(caching): scope exact-match cache keys per virtual key, team and org The exact-match response cache was shared across tenants on the proxy, so a key could be served another team's cached completion. A client-supplied cache_key, cache.namespace or preset_cache_key could also pick another tenant's entries Every cache key now carries a tag derived from the caller's key, team and org, and SDK calls without a proxy identity keep their current keys Fixes #49313 --- litellm/caching/caching.py | 65 ++++++++++++++----- litellm/caching/caching_handler.py | 24 +++---- litellm/proxy/hooks/batch_redis_get.py | 13 ++-- tests/unit/caching/test_caching.py | 48 ++++++++++++++ .../test_responses_stream_cache_keys.py | 2 + 5 files changed, 118 insertions(+), 34 deletions(-) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 91cf9bb2cfe..5f6d015fcdf 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -46,6 +46,8 @@ from .s3_cache import S3Cache RESPONSE_CACHE_TARGET: Final = "llm_response" _ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"}) +_TENANT_TAG_PREFIX: Final = "tenant-" +_TENANT_TAG_DIGEST_LENGTH: Final = 32 _DECODED_LITERAL: Final = TypeAdapter(object) _REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter( list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True) @@ -371,17 +373,52 @@ class Cache: return (*self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS, self._SEMANTIC_CACHE_END_USER_SCOPE_FIELD) return self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS - def _get_semantic_cache_tenant_scope(self, kwargs: dict) -> str: - litellm_params: Final[dict] = kwargs.get("litellm_params") or {} - metadata_sources: Final[tuple[dict, ...]] = tuple( - source.get(key) or {} for source in (kwargs, litellm_params) for key in ("metadata", "litellm_metadata") + def _get_semantic_cache_tenant_scope(self, kwargs: Mapping[str, object]) -> str: + return self._get_tenant_scope(kwargs, self._semantic_cache_scope_fields()) + + @staticmethod + def _get_tenant_scope(kwargs: Mapping[str, object], fields: tuple[str, ...]) -> str: + litellm_params: Final = kwargs.get("litellm_params") + param_sources: Final = (kwargs, litellm_params if isinstance(litellm_params, Mapping) else {}) + metadata_sources: Final[tuple[Mapping[str, object], ...]] = tuple( + metadata + for source in param_sources + for key in ("metadata", "litellm_metadata") + if isinstance(metadata := source.get(key), Mapping) ) scope_values: Final = ( (field, next((source[field] for source in metadata_sources if source.get(field) is not None), None)) - for field in self._semantic_cache_scope_fields() + for field in fields ) return "".join(f"{field}: {value}" for field, value in scope_values if value is not None) + def _cache_tenant_scope_fields(self) -> tuple[str, ...]: + if self._is_semantic_cache(): + return self._semantic_cache_scope_fields() + return self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS + + def scope_cache_key(self, cache_key: str, kwargs: Mapping[str, object]) -> str: + """ + Prefix ``cache_key`` with the tag of the tenant (key / team / org from proxy auth) that kwargs carry. + + Idempotent, so every key a tenant uses starts with its own tag, whatever ``cache_key``, + ``cache.namespace`` or ``preset_cache_key`` the caller sends. No-op without a tenant (SDK use) + """ + tenant_scope: Final = self._get_tenant_scope(kwargs, self._cache_tenant_scope_fields()) + if not tenant_scope: + return cache_key + tenant_digest: Final = hashlib.sha256(tenant_scope.encode()).hexdigest()[:_TENANT_TAG_DIGEST_LENGTH] + tenant_prefix: Final = f"{_TENANT_TAG_PREFIX}{tenant_digest}:" + if cache_key.startswith(tenant_prefix): + return cache_key + return f"{tenant_prefix}{cache_key}" + + def resolve_cache_key(self, kwargs: Mapping[str, object]) -> str | None: + if "cache_key" not in kwargs: + return self.get_cache_key(**kwargs) + cache_key: Final = kwargs["cache_key"] + return self.scope_cache_key(cache_key, kwargs) if isinstance(cache_key, str) else None + def get_cache_key(self, **kwargs: object) -> str: """ Get the cache key for the given arguments. @@ -398,7 +435,7 @@ class Cache: preset_cache_key: Final = self.get_preset_cache_key_from_kwargs(**kwargs) if preset_cache_key is not None: verbose_logger.debug("\nReturning preset cache key: %s", preset_cache_key) - return preset_cache_key + return self.scope_cache_key(preset_cache_key, kwargs) combined_kwargs: Final = ModelParamHelper.get_all_llm_api_params() is_semantic_cache: Final = self._is_semantic_cache() @@ -432,6 +469,7 @@ class Cache: cache_key + reasoning_summary_material + tool_call_id_shape_material ) hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs) + hashed_cache_key = self.scope_cache_key(hashed_cache_key, kwargs) verbose_logger.debug( "\nCreated cache key: %s (source material length: %d)", hashed_cache_key, @@ -683,10 +721,7 @@ class Cache: if self.should_use_cache(**kwargs) is not True: return with response_cache_phase("get"): - if "cache_key" in kwargs: - cache_key = kwargs["cache_key"] - else: - cache_key = self.get_cache_key(**kwargs) + cache_key = self.resolve_cache_key(kwargs) if cache_key is not None: cache_control_args: Final[DynamicCacheControl] = kwargs.get("cache", {}) max_age = cache_control_args.get("s-maxage") or cache_control_args.get("s-max-age") or float("inf") @@ -716,10 +751,7 @@ class Cache: return with response_cache_phase("get"): - if "cache_key" in kwargs: - cache_key = kwargs["cache_key"] - else: - cache_key = self.get_cache_key(**kwargs) + cache_key = self.resolve_cache_key(kwargs) if cache_key is not None: cache_control_args: Final = kwargs.get("cache", {}) max_age: Final = cache_control_args.get( @@ -739,10 +771,7 @@ class Cache: Common implementation across sync + async add_cache functions """ try: - if "cache_key" in kwargs: - cache_key = kwargs["cache_key"] - else: - cache_key = self.get_cache_key(**kwargs) + cache_key = self.resolve_cache_key(kwargs) if cache_key is not None: if isinstance(result, BaseModel): result = result.model_dump_json(exclude_unset=True) diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 1d80d6952a4..7bc4fffdad3 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -395,11 +395,7 @@ class LLMCachingHandler: end_time=end_time, cache_hit=cache_hit, ) - cache_key: Final = ( - self.preset_cache_key - or self.request_kwargs.get("cache_key") - or litellm.cache.get_cache_key(**self.request_kwargs) - ) + cache_key: Final = self.preset_cache_key or litellm.cache.resolve_cache_key(self.request_kwargs) _set_cached_hidden_param(cached_result, "cache_key", cache_key) return CachingHandlerResponse(cached_result=cached_result) elif ( @@ -522,11 +518,7 @@ class LLMCachingHandler: end_time=end_time, cache_hit=cache_hit, ) - cache_key: Final = ( - self.preset_cache_key - or self.request_kwargs.get("cache_key") - or litellm.cache.get_cache_key(**self.request_kwargs) - ) + cache_key: Final = self.preset_cache_key or litellm.cache.resolve_cache_key(self.request_kwargs) _set_cached_hidden_param(cached_result, "cache_key", cache_key) return CachingHandlerResponse(cached_result=cached_result) return CachingHandlerResponse(cached_result=cached_result) @@ -920,7 +912,11 @@ class LLMCachingHandler: request_kwargs.pop("cache_key", None) if litellm.cache.supports_async() is True: ## check if dual cache is supported ## - self.preset_cache_key = request_cache_key or litellm.cache.get_cache_key(**request_kwargs) + self.preset_cache_key = ( + litellm.cache.scope_cache_key(request_cache_key, request_kwargs) + if request_cache_key + else litellm.cache.get_cache_key(**request_kwargs) + ) with response_cache_phase("get"): cached_result = await litellm.cache.async_get_cache( dynamic_cache_object=self.dual_cache, @@ -928,7 +924,11 @@ class LLMCachingHandler: **request_kwargs, ) else: # fallback for caches that don't support async - self.preset_cache_key = request_cache_key or litellm.cache.get_cache_key(**request_kwargs) + self.preset_cache_key = ( + litellm.cache.scope_cache_key(request_cache_key, request_kwargs) + if request_cache_key + else litellm.cache.get_cache_key(**request_kwargs) + ) with response_cache_phase("get"): cached_result = litellm.cache.get_cache( dynamic_cache_object=self.dual_cache, diff --git a/litellm/proxy/hooks/batch_redis_get.py b/litellm/proxy/hooks/batch_redis_get.py index 94eb0bb52d1..0480e8712f3 100644 --- a/litellm/proxy/hooks/batch_redis_get.py +++ b/litellm/proxy/hooks/batch_redis_get.py @@ -49,12 +49,15 @@ class _PROXY_BatchRedisRequests(CustomLogger): api_key: Final = user_api_key_dict.api_key cache_key_name: Final = f"litellm:{api_key}:{call_type}" + scoped_key_prefix: Final = ( + litellm.cache.scope_cache_key(cache_key_name, data) if litellm.cache is not None else cache_key_name + ) self.in_memory_cache = cache.in_memory_cache key_value_dict = {} in_memory_cache_exists = False for key in cache.in_memory_cache.cache_dict: - if isinstance(key, str) and key.startswith(cache_key_name): + if isinstance(key, str) and key.startswith(scoped_key_prefix): in_memory_cache_exists = True if in_memory_cache_exists is False and litellm.cache is not None: @@ -63,9 +66,9 @@ class _PROXY_BatchRedisRequests(CustomLogger): - Get the relevant values """ if litellm.cache.type is not None and isinstance(litellm.cache.cache, RedisCache): - self.print_verbose(f"cache_key_name: {cache_key_name}") + self.print_verbose(f"cache_key_name: {scoped_key_prefix}") with response_cache_phase("get"): - keys = await litellm.cache.cache.async_scan_iter(pattern=cache_key_name, count=100) + keys = await litellm.cache.cache.async_scan_iter(pattern=scoped_key_prefix, count=100) self.print_verbose(f"redis keys: {keys}") if len(keys) > 0: key_value_dict = await litellm.cache.cache.async_batch_get_cache(key_list=keys) @@ -94,7 +97,9 @@ class _PROXY_BatchRedisRequests(CustomLogger): """ try: # never block execution cache_key: str | None = None - if "cache_key" in kwargs: + if "cache_key" in kwargs and litellm.cache is not None: + cache_key = litellm.cache.resolve_cache_key(kwargs) + elif "cache_key" in kwargs: cache_key = kwargs["cache_key"] elif litellm.cache is not None: cache_key = litellm.cache.get_cache_key( diff --git a/tests/unit/caching/test_caching.py b/tests/unit/caching/test_caching.py index 0f6ef2304fb..b631ec87c12 100644 --- a/tests/unit/caching/test_caching.py +++ b/tests/unit/caching/test_caching.py @@ -1608,3 +1608,51 @@ def test_cache_key_separates_model_ids_unless_a_model_group_owns_the_key() -> No assert key_a != bare assert key_a == key_a_again assert grouped_a == grouped_b + + +_TENANT_A_IDENTITY: Final = {"user_api_key": "hashed-key-a", "user_api_key_team_id": "team-a"} +_TENANT_B_IDENTITY: Final = {"user_api_key": "hashed-key-b", "user_api_key_team_id": "team-b"} + + +@pytest.mark.parametrize( + "client_cache_options", + [ + pytest.param({}, id="no-client-options"), + pytest.param({"cache_key": "client-chosen-key"}, id="same-client-cache-key"), + pytest.param({"cache": {"namespace": "shared"}}, id="same-client-namespace"), + ], +) +def test_exact_cache_is_not_shared_across_tenants(monkeypatch, client_cache_options): + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL)) + + def answer(tenant: dict[str, str], content: str) -> str: + response = completion( + model="gpt-4o-mini", + messages=_CACHING_TEST_MESSAGES, + caching=True, + mock_response=content, + metadata=dict(tenant), + **client_cache_options, + ) + return response.choices[0].message.content + + assert answer(_TENANT_A_IDENTITY, "tenant a") == "tenant a" + assert answer(_TENANT_B_IDENTITY, "tenant b") == "tenant b" + assert answer(_TENANT_A_IDENTITY, "tenant a, uncached") == "tenant a" + + +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +@pytest.mark.parametrize("forged_key_param", ["cache_key", "preset_cache_key"]) +def test_exact_cache_rejects_another_tenants_cache_key(metadata_field, forged_key_param): + cache = Cache(type=LiteLLMCacheType.LOCAL) + tenant_a_request = {"model": "gpt-4o-mini", "messages": messages, metadata_field: dict(_TENANT_A_IDENTITY)} + cache.add_cache('{"answer": "tenant a"}', **tenant_a_request) + tenant_a_key = cache.get_cache_key(**tenant_a_request) + forged_key = ( + {"cache_key": tenant_a_key} + if forged_key_param == "cache_key" + else {"litellm_params": {"preset_cache_key": tenant_a_key}} + ) + + assert cache.get_cache(**tenant_a_request) is not None + assert cache.get_cache(**forged_key, **{metadata_field: dict(_TENANT_B_IDENTITY)}) is None diff --git a/tests/unit/caching/test_responses_stream_cache_keys.py b/tests/unit/caching/test_responses_stream_cache_keys.py index e836948f05e..76f29ce4c20 100644 --- a/tests/unit/caching/test_responses_stream_cache_keys.py +++ b/tests/unit/caching/test_responses_stream_cache_keys.py @@ -34,6 +34,7 @@ async def test_async_get_cache_reuses_preset_cache_key_for_responses(): mock_cache.supported_call_types = [CallTypes.aresponses.value] mock_cache.supports_async.return_value = True mock_cache.get_cache_key.return_value = "responses-stream-cache-key" + mock_cache.scope_cache_key.side_effect = lambda cache_key, kwargs: cache_key mock_cache.async_get_cache = AsyncMock(return_value=None) litellm.cache = mock_cache @@ -84,6 +85,7 @@ async def test_async_get_cache_falls_back_to_sync_cache_for_responses(): mock_cache.supported_call_types = [CallTypes.aresponses.value] mock_cache.supports_async.return_value = False mock_cache.get_cache_key.return_value = "responses-stream-cache-key" + mock_cache.scope_cache_key.side_effect = lambda cache_key, kwargs: cache_key mock_cache.get_cache.return_value = None litellm.cache = mock_cache