This commit is contained in:
Gella Naga Sai Krishna 2026-11-02 15:37:35 +05:30 • committed by GitHub
commit 202e1b3a48
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 118 additions and 34 deletions

View file

@ -46,6 +46,8 @@ from .s3_cache import S3Cache
RESPONSE_CACHE_TARGET: Final = "llm_response"
_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"})
_TENANT_TAG_PREFIX: Final = "tenant-"
_TENANT_TAG_DIGEST_LENGTH: Final = 32
_DECODED_LITERAL: Final = TypeAdapter(object)
_REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True)
@ -371,17 +373,52 @@ class Cache:
return (*self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS, self._SEMANTIC_CACHE_END_USER_SCOPE_FIELD)
return self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS
def _get_semantic_cache_tenant_scope(self, kwargs: dict) -> str:
litellm_params: Final[dict] = kwargs.get("litellm_params") or {}
metadata_sources: Final[tuple[dict, ...]] = tuple(
source.get(key) or {} for source in (kwargs, litellm_params) for key in ("metadata", "litellm_metadata")
def _get_semantic_cache_tenant_scope(self, kwargs: Mapping[str, object]) -> str:
return self._get_tenant_scope(kwargs, self._semantic_cache_scope_fields())
@staticmethod
def _get_tenant_scope(kwargs: Mapping[str, object], fields: tuple[str, ...]) -> str:
litellm_params: Final = kwargs.get("litellm_params")
param_sources: Final = (kwargs, litellm_params if isinstance(litellm_params, Mapping) else {})
metadata_sources: Final[tuple[Mapping[str, object], ...]] = tuple(
metadata
for source in param_sources
for key in ("metadata", "litellm_metadata")
if isinstance(metadata := source.get(key), Mapping)
)
scope_values: Final = (
(field, next((source[field] for source in metadata_sources if source.get(field) is not None), None))
for field in self._semantic_cache_scope_fields()
for field in fields
)
return "".join(f"{field}: {value}" for field, value in scope_values if value is not None)
def _cache_tenant_scope_fields(self) -> tuple[str, ...]:
if self._is_semantic_cache():
return self._semantic_cache_scope_fields()
return self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS
def scope_cache_key(self, cache_key: str, kwargs: Mapping[str, object]) -> str:
"""
Prefix ``cache_key`` with the tag of the tenant (key / team / org from proxy auth) that kwargs carry.
Idempotent, so every key a tenant uses starts with its own tag, whatever ``cache_key``,
``cache.namespace`` or ``preset_cache_key`` the caller sends. No-op without a tenant (SDK use)
"""
tenant_scope: Final = self._get_tenant_scope(kwargs, self._cache_tenant_scope_fields())
if not tenant_scope:
return cache_key
tenant_digest: Final = hashlib.sha256(tenant_scope.encode()).hexdigest()[:_TENANT_TAG_DIGEST_LENGTH]
tenant_prefix: Final = f"{_TENANT_TAG_PREFIX}{tenant_digest}:"
if cache_key.startswith(tenant_prefix):
return cache_key
return f"{tenant_prefix}{cache_key}"
def resolve_cache_key(self, kwargs: Mapping[str, object]) -> str | None:
if "cache_key" not in kwargs:
return self.get_cache_key(**kwargs)
cache_key: Final = kwargs["cache_key"]
return self.scope_cache_key(cache_key, kwargs) if isinstance(cache_key, str) else None
def get_cache_key(self, **kwargs: object) -> str:
"""
Get the cache key for the given arguments.
@ -398,7 +435,7 @@ class Cache:
preset_cache_key: Final = self.get_preset_cache_key_from_kwargs(**kwargs)
if preset_cache_key is not None:
verbose_logger.debug("\nReturning preset cache key: %s", preset_cache_key)
return preset_cache_key
return self.scope_cache_key(preset_cache_key, kwargs)
combined_kwargs: Final = ModelParamHelper.get_all_llm_api_params()
is_semantic_cache: Final = self._is_semantic_cache()
@ -432,6 +469,7 @@ class Cache:
cache_key + reasoning_summary_material + tool_call_id_shape_material
)
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
hashed_cache_key = self.scope_cache_key(hashed_cache_key, kwargs)
verbose_logger.debug(
"\nCreated cache key: %s (source material length: %d)",
hashed_cache_key,
@ -683,10 +721,7 @@ class Cache:
if self.should_use_cache(**kwargs) is not True:
return
with response_cache_phase("get"):
if "cache_key" in kwargs:
cache_key = kwargs["cache_key"]
else:
cache_key = self.get_cache_key(**kwargs)
cache_key = self.resolve_cache_key(kwargs)
if cache_key is not None:
cache_control_args: Final[DynamicCacheControl] = kwargs.get("cache", {})
max_age = cache_control_args.get("s-maxage") or cache_control_args.get("s-max-age") or float("inf")
@ -716,10 +751,7 @@ class Cache:
return
with response_cache_phase("get"):
if "cache_key" in kwargs:
cache_key = kwargs["cache_key"]
else:
cache_key = self.get_cache_key(**kwargs)
cache_key = self.resolve_cache_key(kwargs)
if cache_key is not None:
cache_control_args: Final = kwargs.get("cache", {})
max_age: Final = cache_control_args.get(
@ -739,10 +771,7 @@ class Cache:
Common implementation across sync + async add_cache functions
"""
try:
if "cache_key" in kwargs:
cache_key = kwargs["cache_key"]
else:
cache_key = self.get_cache_key(**kwargs)
cache_key = self.resolve_cache_key(kwargs)
if cache_key is not None:
if isinstance(result, BaseModel):
result = result.model_dump_json(exclude_unset=True)

View file

@ -395,11 +395,7 @@ class LLMCachingHandler:
end_time=end_time,
cache_hit=cache_hit,
)
cache_key: Final = (
self.preset_cache_key
or self.request_kwargs.get("cache_key")
or litellm.cache.get_cache_key(**self.request_kwargs)
)
cache_key: Final = self.preset_cache_key or litellm.cache.resolve_cache_key(self.request_kwargs)
_set_cached_hidden_param(cached_result, "cache_key", cache_key)
return CachingHandlerResponse(cached_result=cached_result)
elif (
@ -522,11 +518,7 @@ class LLMCachingHandler:
end_time=end_time,
cache_hit=cache_hit,
)
cache_key: Final = (
self.preset_cache_key
or self.request_kwargs.get("cache_key")
or litellm.cache.get_cache_key(**self.request_kwargs)
)
cache_key: Final = self.preset_cache_key or litellm.cache.resolve_cache_key(self.request_kwargs)
_set_cached_hidden_param(cached_result, "cache_key", cache_key)
return CachingHandlerResponse(cached_result=cached_result)
return CachingHandlerResponse(cached_result=cached_result)
@ -920,7 +912,11 @@ class LLMCachingHandler:
request_kwargs.pop("cache_key", None)
if litellm.cache.supports_async() is True:
## check if dual cache is supported ##
self.preset_cache_key = request_cache_key or litellm.cache.get_cache_key(**request_kwargs)
self.preset_cache_key = (
litellm.cache.scope_cache_key(request_cache_key, request_kwargs)
if request_cache_key
else litellm.cache.get_cache_key(**request_kwargs)
)
with response_cache_phase("get"):
cached_result = await litellm.cache.async_get_cache(
dynamic_cache_object=self.dual_cache,
@ -928,7 +924,11 @@ class LLMCachingHandler:
**request_kwargs,
)
else: # fallback for caches that don't support async
self.preset_cache_key = request_cache_key or litellm.cache.get_cache_key(**request_kwargs)
self.preset_cache_key = (
litellm.cache.scope_cache_key(request_cache_key, request_kwargs)
if request_cache_key
else litellm.cache.get_cache_key(**request_kwargs)
)
with response_cache_phase("get"):
cached_result = litellm.cache.get_cache(
dynamic_cache_object=self.dual_cache,

View file

@ -49,12 +49,15 @@ class _PROXY_BatchRedisRequests(CustomLogger):
api_key: Final = user_api_key_dict.api_key
cache_key_name: Final = f"litellm:{api_key}:{call_type}"
scoped_key_prefix: Final = (
litellm.cache.scope_cache_key(cache_key_name, data) if litellm.cache is not None else cache_key_name
)
self.in_memory_cache = cache.in_memory_cache
key_value_dict = {}
in_memory_cache_exists = False
for key in cache.in_memory_cache.cache_dict:
if isinstance(key, str) and key.startswith(cache_key_name):
if isinstance(key, str) and key.startswith(scoped_key_prefix):
in_memory_cache_exists = True
if in_memory_cache_exists is False and litellm.cache is not None:
@ -63,9 +66,9 @@ class _PROXY_BatchRedisRequests(CustomLogger):
- Get the relevant values
"""
if litellm.cache.type is not None and isinstance(litellm.cache.cache, RedisCache):
self.print_verbose(f"cache_key_name: {cache_key_name}")
self.print_verbose(f"cache_key_name: {scoped_key_prefix}")
with response_cache_phase("get"):
keys = await litellm.cache.cache.async_scan_iter(pattern=cache_key_name, count=100)
keys = await litellm.cache.cache.async_scan_iter(pattern=scoped_key_prefix, count=100)
self.print_verbose(f"redis keys: {keys}")
if len(keys) > 0:
key_value_dict = await litellm.cache.cache.async_batch_get_cache(key_list=keys)
@ -94,7 +97,9 @@ class _PROXY_BatchRedisRequests(CustomLogger):
"""
try: # never block execution
cache_key: str | None = None
if "cache_key" in kwargs:
if "cache_key" in kwargs and litellm.cache is not None:
cache_key = litellm.cache.resolve_cache_key(kwargs)
elif "cache_key" in kwargs:
cache_key = kwargs["cache_key"]
elif litellm.cache is not None:
cache_key = litellm.cache.get_cache_key(

View file

@ -1608,3 +1608,51 @@ def test_cache_key_separates_model_ids_unless_a_model_group_owns_the_key() -> No
assert key_a != bare
assert key_a == key_a_again
assert grouped_a == grouped_b
_TENANT_A_IDENTITY: Final = {"user_api_key": "hashed-key-a", "user_api_key_team_id": "team-a"}
_TENANT_B_IDENTITY: Final = {"user_api_key": "hashed-key-b", "user_api_key_team_id": "team-b"}
@pytest.mark.parametrize(
"client_cache_options",
[
pytest.param({}, id="no-client-options"),
pytest.param({"cache_key": "client-chosen-key"}, id="same-client-cache-key"),
pytest.param({"cache": {"namespace": "shared"}}, id="same-client-namespace"),
],
)
def test_exact_cache_is_not_shared_across_tenants(monkeypatch, client_cache_options):
monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL))
def answer(tenant: dict[str, str], content: str) -> str:
response = completion(
model="gpt-4o-mini",
messages=_CACHING_TEST_MESSAGES,
caching=True,
mock_response=content,
metadata=dict(tenant),
**client_cache_options,
)
return response.choices[0].message.content
assert answer(_TENANT_A_IDENTITY, "tenant a") == "tenant a"
assert answer(_TENANT_B_IDENTITY, "tenant b") == "tenant b"
assert answer(_TENANT_A_IDENTITY, "tenant a, uncached") == "tenant a"
@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"])
@pytest.mark.parametrize("forged_key_param", ["cache_key", "preset_cache_key"])
def test_exact_cache_rejects_another_tenants_cache_key(metadata_field, forged_key_param):
cache = Cache(type=LiteLLMCacheType.LOCAL)
tenant_a_request = {"model": "gpt-4o-mini", "messages": messages, metadata_field: dict(_TENANT_A_IDENTITY)}
cache.add_cache('{"answer": "tenant a"}', **tenant_a_request)
tenant_a_key = cache.get_cache_key(**tenant_a_request)
forged_key = (
{"cache_key": tenant_a_key}
if forged_key_param == "cache_key"
else {"litellm_params": {"preset_cache_key": tenant_a_key}}
)
assert cache.get_cache(**tenant_a_request) is not None
assert cache.get_cache(**forged_key, **{metadata_field: dict(_TENANT_B_IDENTITY)}) is None

View file

@ -34,6 +34,7 @@ async def test_async_get_cache_reuses_preset_cache_key_for_responses():
mock_cache.supported_call_types = [CallTypes.aresponses.value]
mock_cache.supports_async.return_value = True
mock_cache.get_cache_key.return_value = "responses-stream-cache-key"
mock_cache.scope_cache_key.side_effect = lambda cache_key, kwargs: cache_key
mock_cache.async_get_cache = AsyncMock(return_value=None)
litellm.cache = mock_cache
@ -84,6 +85,7 @@ async def test_async_get_cache_falls_back_to_sync_cache_for_responses():
mock_cache.supported_call_types = [CallTypes.aresponses.value]
mock_cache.supports_async.return_value = False
mock_cache.get_cache_key.return_value = "responses-stream-cache-key"
mock_cache.scope_cache_key.side_effect = lambda cache_key, kwargs: cache_key
mock_cache.get_cache.return_value = None
litellm.cache = mock_cache