From 5dbc99e7e5935b6e394dfa7fcc3880b5a8cfc330 Mon Sep 17 00:00:00 2001 From: Filipe Andujar Date: Wed, 21 Oct 2026 11:36:23 -0300 Subject: [PATCH 1/4] feat(caching): add cache_scope to isolate response cache hits per team or key The exact-match response cache keys entries on model and params only, so every caller of a proxy shares hits. A caller can also read another tenant's entry by sending its cache_key or preset_cache_key, and the proxy hands that key to everyone in x-litellm-cache-key cache_params.cache_scope: global | team | key (default global, unchanged behaviour) folds the authenticated team id or key hash into the key hash. A non-global scope also drops caller-supplied cache_key, preset_cache_key, and identity fields under litellm_params.metadata. Exposed in /cache/settings and the Admin UI --- litellm/caching/caching.py | 39 +++++++-- litellm/proxy/litellm_pre_call_utils.py | 30 +++++++ litellm/types/caching.py | 6 ++ .../cache_settings_endpoints.py | 13 +++ tests/unit/caching/test_caching.py | 81 +++++++++++++++++++ .../unit/proxy/test_litellm_pre_call_utils.py | 48 +++++++++++ .../cache_settings/cacheSettingsFields.ts | 14 ++++ .../cache_settings/cacheSettingsUtils.test.ts | 10 +++ 8 files changed, 234 insertions(+), 7 deletions(-) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 5f5b8210096..1ea31476e6a 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -19,6 +19,7 @@ from enum import Enum from typing import Any, Final, Literal, cast from pydantic import BaseModel, TypeAdapter +from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13 import litellm from litellm._internal_context import current_service_target, service_target @@ -73,6 +74,12 @@ class CacheMode(str, Enum): #### LiteLLM.Completion / Embedding Cache #### +def _is_str_keyed_mapping( + value: object, +) -> TypeIs[Mapping[str, object]]: # guard-ok: request kwargs and their metadata dicts are str-keyed + return isinstance(value, Mapping) + + def _is_conversation_item(item: object) -> bool: if isinstance(item, BaseModel): return True @@ -142,6 +149,7 @@ class Cache: semantic_cache_embedding_max_input_tokens: int | None = None, semantic_cache_embedding_timeout: float | None = None, semantic_cache_scope: str = SemanticCacheScope.KEY.value, + cache_scope: str = CacheScope.GLOBAL.value, max_messages: int | None = 4, # GCP IAM authentication parameters gcp_service_account: str | None = None, @@ -172,6 +180,7 @@ class Cache: semantic_cache_embedding_max_input_tokens (int, optional): Truncate prompts to this many tokens before embedding them for semantic caching. Defaults to the embedding deployment's configured max_input_tokens. semantic_cache_embedding_timeout (float, optional): Seconds a semantic-cache lookup may spend embedding the prompt before it gives up and lets the request continue to the LLM. Defaults to SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS. semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key". + cache_scope (str, optional): Who can share a cache hit. "global" shares hits between every caller. "team" isolates hits per team, and keys without a team per key. "key" isolates hits per key. On the proxy, a non-global scope also ignores caller-supplied `cache_key` / `preset_cache_key`. Defaults to "global". max_messages (int, optional): Requests with more `messages` (or Responses API `input` items) than this are neither looked up nor stored, so long agent conversations never serve or create a cache entry. None disables the limit. Defaults to 4. # Disk Cache Args @@ -323,6 +332,7 @@ class Cache: self.ttl = ttl self.mode: CacheMode = mode or CacheMode.default_on self.semantic_cache_scope: str = SemanticCacheScope(semantic_cache_scope).value + self.cache_scope: str = CacheScope(cache_scope).value self.max_messages: int | None = max_messages if self.type == LiteLLMCacheType.LOCAL and default_in_memory_ttl is not None: @@ -365,17 +375,31 @@ class Cache: return (*self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS, self._SEMANTIC_CACHE_END_USER_SCOPE_FIELD) return self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS - def _get_semantic_cache_tenant_scope(self, kwargs: dict) -> str: - litellm_params: Final[dict] = kwargs.get("litellm_params") or {} - metadata_sources: Final[tuple[dict, ...]] = tuple( - source.get(key) or {} for source in (kwargs, litellm_params) for key in ("metadata", "litellm_metadata") + @staticmethod + def _auth_metadata_value(kwargs: Mapping[str, object], field: str) -> object | None: + litellm_params: Final = kwargs.get("litellm_params") + containers: Final[tuple[Mapping[str, object], ...]] = ( + (kwargs, litellm_params) if _is_str_keyed_mapping(litellm_params) else (kwargs,) ) + candidates: Final = (container.get(key) for container in containers for key in ("metadata", "litellm_metadata")) + metadata_sources: Final = tuple(source for source in candidates if _is_str_keyed_mapping(source)) + return next((source[field] for source in metadata_sources if source.get(field) is not None), None) + + def _get_semantic_cache_tenant_scope(self, kwargs: Mapping[str, object]) -> str: scope_values: Final = ( - (field, next((source[field] for source in metadata_sources if source.get(field) is not None), None)) - for field in self._semantic_cache_scope_fields() + (field, self._auth_metadata_value(kwargs, field)) for field in self._semantic_cache_scope_fields() ) return "".join(f"{field}: {value}" for field, value in scope_values if value is not None) + def _get_cache_scope_material(self, kwargs: Mapping[str, object]) -> str: + if self.cache_scope == CacheScope.GLOBAL: + return "" + team_id: Final = self._auth_metadata_value(kwargs, "user_api_key_team_id") + if self.cache_scope == CacheScope.TEAM and team_id is not None: + return f"cache_scope_team: {team_id}" + key_hash: Final = self._auth_metadata_value(kwargs, "user_api_key_hash") + return "" if key_hash is None else f"cache_scope_key: {key_hash}" + def get_cache_key(self, **kwargs: object) -> str: """ Get the cache key for the given arguments. @@ -419,7 +443,8 @@ class Cache: reasoning_summary_material: Final = ( "" if reasoning_summary is None else f"reasoning_summary: {reasoning_summary}" ) - hashed_cache_key = Cache._get_hashed_cache_key(cache_key + reasoning_summary_material) + cache_scope_material: Final = self._get_cache_scope_material(kwargs) + hashed_cache_key = Cache._get_hashed_cache_key(cache_key + cache_scope_material + reasoning_summary_material) hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs) verbose_logger.debug( "\nCreated cache key: %s (source material length: %d)", diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 8a45f658f36..bb0fb621e48 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -22,6 +22,7 @@ from fastapi import HTTPException, Request, status from pydantic import TypeAdapter from pydantic import ValidationError as PydanticValidationError from starlette.datastructures import Headers +from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13 import litellm from litellm._logging import verbose_logger, verbose_proxy_logger @@ -87,6 +88,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg safe_get_request_headers, ) from litellm.proxy.spend_tracking.carried_budget_state import carried_budget_metadata +from litellm.types.caching import CacheScope from litellm.types.integrations.anthropic_cache_control_hook import GATEWAY_INJECTED_CACHE_METADATA_KEY # Cache special headers as a frozenset for O(1) lookup performance @@ -807,6 +809,33 @@ def _strip_router_reserved_metadata( ) +_CLIENT_CACHE_KEY_OVERRIDE_FIELDS: Final = frozenset({"cache_key", "preset_cache_key"}) + + +def _is_request_json_object( + value: object, +) -> TypeIs[dict[str, object]]: # guard-ok: parsed JSON request bodies have str keys # mutable-ok: stripped in place + return isinstance(value, dict) + + +def _strip_client_cache_key_overrides( + data: dict[str, object], # mutable-ok: strips in place on the request body the pre-call pipeline threads through +) -> None: + """Under a scoped response cache, the cache key must come from proxy auth, never from the caller.""" + if litellm.cache is None or litellm.cache.cache_scope == CacheScope.GLOBAL: + return + for field in _CLIENT_CACHE_KEY_OVERRIDE_FIELDS & data.keys(): + data.pop(field) + if not _is_request_json_object(litellm_params := data.get("litellm_params")): + return + litellm_params.pop("preset_cache_key", None) + for metadata_key in ("metadata", "litellm_metadata"): + if not _is_request_json_object(metadata := litellm_params.get(metadata_key)): + continue + for field in tuple(key for key in metadata if key.startswith("user_api_key_")): + metadata.pop(field) + + def get_metadata_variable_name(request: Request) -> str: """ Helper to return what the "metadata" field should be called in the request data @@ -2649,6 +2678,7 @@ async def add_litellm_data_to_request( if not _key_or_team_allows_client_pricing_override(user_api_key_dict): _strip_client_pricing_overrides(data) _strip_router_reserved_metadata(data) + _strip_client_cache_key_overrides(data) # Same reason as the strips above: runs after the metadata string-to-dict parse # so JSON-string metadata cannot smuggle callback credentials past the dict guard. diff --git a/litellm/types/caching.py b/litellm/types/caching.py index ced2065a674..ab3358ecba6 100644 --- a/litellm/types/caching.py +++ b/litellm/types/caching.py @@ -24,6 +24,12 @@ class SemanticCacheScope(str, Enum): END_USER = "end_user" +class CacheScope(str, Enum): + GLOBAL = "global" + TEAM = "team" + KEY = "key" + + CachingSupportedCallTypes = Literal[ "completion", "acompletion", diff --git a/litellm/types/management_endpoints/cache_settings_endpoints.py b/litellm/types/management_endpoints/cache_settings_endpoints.py index 3584f8ce1da..edad485df75 100644 --- a/litellm/types/management_endpoints/cache_settings_endpoints.py +++ b/litellm/types/management_endpoints/cache_settings_endpoints.py @@ -200,6 +200,19 @@ CACHE_SETTINGS_FIELDS: Final[list[CacheSettingsField]] = [ ui_field_name="Semantic Cache Scope", redis_type="semantic", ), + CacheSettingsField( + field_name="cache_scope", + field_type="String", + field_value=None, + field_description=( + "Who can share a cache hit. 'global' shares hits between every caller. 'team' isolates hits per team," + " and keys without a team per key. 'key' isolates hits per key." + ), + field_default="global", + options=["global", "team", "key"], + ui_field_name="Cache Scope", + redis_type=None, + ), # GCP IAM authentication fields CacheSettingsField( field_name="gcp_service_account", diff --git a/tests/unit/caching/test_caching.py b/tests/unit/caching/test_caching.py index 91083921130..9dc89c28ffe 100644 --- a/tests/unit/caching/test_caching.py +++ b/tests/unit/caching/test_caching.py @@ -274,6 +274,87 @@ def test_exact_cache_key_still_includes_prompt(): assert key_a != key_b +_TEAM_A_KEY_1: Final = {"user_api_key_hash": "hash-1", "user_api_key_team_id": "team-a"} +_TEAM_A_KEY_2: Final = {"user_api_key_hash": "hash-2", "user_api_key_team_id": "team-a"} +_TEAM_B_KEY_3: Final = {"user_api_key_hash": "hash-3", "user_api_key_team_id": "team-b"} +_NO_TEAM_KEY_4: Final = {"user_api_key_hash": "hash-4"} +_NO_TEAM_KEY_5: Final = {"user_api_key_hash": "hash-5"} + + +def _scoped_keys(cache_scope: str | None, *identities: dict[str, str]) -> list[str]: + cache: Final = Cache(type=LiteLLMCacheType.LOCAL, **({} if cache_scope is None else {"cache_scope": cache_scope})) + messages: Final = [{"role": "user", "content": "What color is the sky?"}] + return [cache.get_cache_key(model="gpt-4o-mini", messages=messages, metadata=dict(i)) for i in identities] + + +@pytest.mark.parametrize( + ("cache_scope", "same_team_shares", "other_team_shares", "teamless_keys_share"), + [ + (None, True, True, True), + ("global", True, True, True), + ("team", True, False, False), + ("key", False, False, False), + ], +) +def test_cache_scope_controls_who_shares_an_exact_cache_hit( + cache_scope, same_team_shares, other_team_shares, teamless_keys_share +): + key_1, key_2, key_3, key_4, key_5 = _scoped_keys( + cache_scope, _TEAM_A_KEY_1, _TEAM_A_KEY_2, _TEAM_B_KEY_3, _NO_TEAM_KEY_4, _NO_TEAM_KEY_5 + ) + assert (key_1 == key_2) is same_team_shares + assert (key_1 == key_3) is other_team_shares + assert (key_4 == key_5) is teamless_keys_share + + +@pytest.mark.parametrize("cache_scope", ["team", "key"]) +def test_cache_scope_is_stable_for_the_same_caller(cache_scope): + first, again = _scoped_keys(cache_scope, _TEAM_A_KEY_1, _TEAM_A_KEY_1) + assert first == again + + +def test_cache_scope_reads_identity_from_litellm_metadata_and_litellm_params(): + cache: Final = Cache(type=LiteLLMCacheType.LOCAL, cache_scope="team") + messages: Final = [{"role": "user", "content": "hi"}] + from_metadata: Final = cache.get_cache_key(model="gpt-4o-mini", messages=messages, metadata=dict(_TEAM_A_KEY_1)) + from_litellm_metadata: Final = cache.get_cache_key( + model="gpt-4o-mini", messages=messages, litellm_metadata=dict(_TEAM_A_KEY_1) + ) + from_litellm_params: Final = cache.get_cache_key( + model="gpt-4o-mini", messages=messages, litellm_params={"metadata": dict(_TEAM_A_KEY_1)} + ) + other_team: Final = cache.get_cache_key( + model="gpt-4o-mini", messages=messages, litellm_metadata=dict(_TEAM_B_KEY_3) + ) + assert from_metadata == from_litellm_metadata == from_litellm_params + assert from_metadata != other_team + + +def test_cache_scope_rejects_unknown_value(): + with pytest.raises(ValueError, match="'org' is not a valid CacheScope"): + Cache(type=LiteLLMCacheType.LOCAL, cache_scope="org") + + +@pytest.mark.asyncio +async def test_team_cache_scope_isolates_acompletion_hits_between_teams(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, cache_scope="team")) + messages: Final = [{"role": "user", "content": f"team scope {uuid.uuid4().hex}"}] + + async def call(identity: dict[str, str], mock_response: str): + return await acompletion( + model="gpt-4o-mini", messages=messages, metadata=dict(identity), mock_response=mock_response + ) + + team_a_first: Final = await call(_TEAM_A_KEY_1, "a1") + await asyncio.gather(*_PENDING_CACHE_WRITES) + team_a_other_key: Final = await call(_TEAM_A_KEY_2, "a2") + team_b: Final = await call(_TEAM_B_KEY_3, "b") + + assert team_a_other_key.id == team_a_first.id + assert team_b.id != team_a_first.id + assert team_b.choices[0].message.content == "b" + + @pytest.mark.parametrize( "anthropic_param", [ diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index d9def23bde8..c0f0d6287f6 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -36,6 +36,8 @@ from litellm.litellm_core_utils.redact_messages import _get_turn_off_message_log from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id from litellm.proxy._types import AddTeamCallback, ProxyException, TeamCallbackMetadata, UserAPIKeyAuth +from litellm.caching.caching import Cache +from litellm.types.caching import LiteLLMCacheType from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES from litellm.proxy.litellm_pre_call_utils import ( KeyAndTeamLoggingSettings, @@ -1181,6 +1183,52 @@ async def test_add_litellm_data_to_request_keeps_the_request_max_fallbacks_cap() assert updated["max_fallbacks"] == 0 +_FORGED_CACHE_KEY_FIELDS: Final = { + "cache_key": "ns:victim-hash", + "preset_cache_key": "ns:victim-hash", + "litellm_params": { + "preset_cache_key": "ns:victim-hash", + "metadata": {"user_api_key_team_id": "victim-team", "user_api_key_hash": "victim-key"}, + }, +} + + +async def _add_data_with_forged_cache_key(monkeypatch: pytest.MonkeyPatch, cache_scope: str) -> dict: + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, cache_scope=cache_scope)) + return await add_litellm_data_to_request( + data={ + "model": "gpt-4o", + "messages": [{"role": "user", "content": "hi"}], + **copy.deepcopy(_FORGED_CACHE_KEY_FIELDS), + }, + request=_make_request_mock("/v1/chat/completions", {"Content-Type": "application/json"}), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", team_id="caller-team"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cache_scope", ["team", "key"]) +async def test_scoped_cache_ignores_caller_supplied_cache_keys(monkeypatch: pytest.MonkeyPatch, cache_scope: str): + updated: Final = await _add_data_with_forged_cache_key(monkeypatch, cache_scope) + + assert "cache_key" not in updated + assert "preset_cache_key" not in updated + assert "preset_cache_key" not in updated["litellm_params"] + assert "user_api_key_team_id" not in updated["litellm_params"]["metadata"] + assert "user_api_key_hash" not in updated["litellm_params"]["metadata"] + assert updated["metadata"]["user_api_key_team_id"] == "caller-team" + + +@pytest.mark.asyncio +async def test_global_cache_scope_keeps_caller_supplied_cache_key(monkeypatch: pytest.MonkeyPatch): + updated: Final = await _add_data_with_forged_cache_key(monkeypatch, "global") + + assert updated["cache_key"] == "ns:victim-hash" + + @pytest.mark.asyncio async def test_add_litellm_data_to_request_strips_user_control_fields(): """Strip untrusted proxy-control fields before guardrails, logging, and headers read metadata.""" diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts index 9ebb9c191ed..dae894437cb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts @@ -241,6 +241,20 @@ export const CACHE_FIELDS: readonly CacheField[] = [ helpText: "Namespace prefix for cache keys", redisType: null, }, + { + name: "cache_scope", + label: "Cache Scope", + type: "select", + section: "cacheManagement", + helpText: + "Who can share a cache hit. Global shares hits between every caller. Team isolates hits per team, and keys without a team per key. Key isolates hits per key.", + redisType: null, + options: [ + { value: "global", label: "Global (shared by every caller)" }, + { value: "team", label: "Team (isolated per team)" }, + { value: "key", label: "Key (isolated per key)" }, + ], + }, { name: "ttl", label: "TTL (seconds)", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts index c851f4aad96..4aceef97ec2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts @@ -83,6 +83,16 @@ describe("buildCachePayload", () => { expect(node).not.toHaveProperty("semantic_cache_scope"); }); + it("should send the cache scope for every redis type and leave it to the server default when unset", () => { + const redisTypes = ["node", "cluster", "sentinel", "semantic"] as const; + for (const redisType of redisTypes) { + const scoped = buildCachePayload(redisType, { cache_scope: "team" }, { forTesting: false }); + expect(scoped.cache_scope).toBe("team"); + } + const unset = buildCachePayload("node", buildInitialValues({}), { forTesting: false }); + expect(unset).not.toHaveProperty("cache_scope"); + }); + it("should keep type redis when testing a semantic cache so the test endpoint accepts it", () => { const payload = buildCachePayload("semantic", { similarity_threshold: 0.9 }, { forTesting: true }); expect(payload.type).toBe("redis"); From 0a807101fbdedd0096c9b1fe3360ce95973223bc Mon Sep 17 00:00:00 2001 From: Filipe Andujar Date: Wed, 21 Oct 2026 18:47:12 -0300 Subject: [PATCH 2/4] fix(proxy): type the request body once before the pre-call strips Passing the untyped request dict to each strip helper added a reportUnknownArgumentType per call, and the new cache key strip pushed the type gate over its ceiling. Cast once to dict[str, object] and pass that to every strip --- litellm/proxy/litellm_pre_call_utils.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index bb0fb621e48..d8bf79cf35a 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2675,17 +2675,18 @@ async def add_litellm_data_to_request( # metadata (sent via multipart/form-data or extra_body) wouldn't be a # dict yet at the earlier strip point and the isinstance(dict) guard # would silently skip the field. + request_body: Final = cast("dict[str, object]", data) # cast-ok: data is the untyped request body dict if not _key_or_team_allows_client_pricing_override(user_api_key_dict): - _strip_client_pricing_overrides(data) - _strip_router_reserved_metadata(data) - _strip_client_cache_key_overrides(data) + _strip_client_pricing_overrides(request_body) + _strip_router_reserved_metadata(request_body) + _strip_client_cache_key_overrides(request_body) # Same reason as the strips above: runs after the metadata string-to-dict parse # so JSON-string metadata cannot smuggle callback credentials past the dict guard. - _strip_client_callback_credentials(data) + _strip_client_callback_credentials(request_body) if not _allow_client_message_redaction_opt_out and litellm.turn_off_message_logging is True: - _strip_client_message_redaction_opt_out(data) + _strip_client_message_redaction_opt_out(request_body) # Fill in the proxy_server_request body snapshot now that metadata has # been parsed. Consumers (standard_logging_payload, lago, From 0bd03f6a5d67adf4bb5372f5b66b33e5f430f454 Mon Sep 17 00:00:00 2001 From: Filipe Andujar Date: Wed, 21 Oct 2026 19:15:01 -0300 Subject: [PATCH 3/4] fix(caching): skip a scoped cache when the request carries no identity to scope by JWT auth resolves api_key to None, so a JWT caller has no key hash. Under cache_scope key, or team for a teamless JWT caller, the scope material was empty and those callers fell back to one shared bucket. A scoped cache now neither reads nor writes when it cannot tell callers apart, the same way per-user Copilot sessions already bypass the response cache --- litellm/caching/caching.py | 3 ++ litellm/caching/caching_handler.py | 3 +- tests/unit/caching/test_caching.py | 61 ++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 1ea31476e6a..299340a818a 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -400,6 +400,9 @@ class Cache: key_hash: Final = self._auth_metadata_value(kwargs, "user_api_key_hash") return "" if key_hash is None else f"cache_scope_key: {key_hash}" + def lacks_cache_scope_identity(self, kwargs: Mapping[str, object]) -> bool: + return self.cache_scope != CacheScope.GLOBAL and self._get_cache_scope_material(kwargs) == "" + def get_cache_key(self, **kwargs: object) -> str: """ Get the cache key for the given arguments. diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 7089ff78835..bd5d51cb41f 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -27,7 +27,7 @@ import litellm from litellm._internal_context import is_internal_call, post_response_phase from litellm._logging import print_verbose, verbose_logger from litellm.caching import InMemoryCache -from litellm.caching.caching import S3Cache, response_cache_phase +from litellm.caching.caching import Cache, S3Cache, response_cache_phase from litellm.constants import CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS, RESPONSE_CACHE_EXCLUDED_PROVIDERS from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( @@ -115,6 +115,7 @@ def _is_response_cache_excluded(model: str | None, kwargs: Mapping[str, object]) (isinstance(custom_llm_provider, str) and custom_llm_provider in RESPONSE_CACHE_EXCLUDED_PROVIDERS) or model_provider in RESPONSE_CACHE_EXCLUDED_PROVIDERS or is_github_copilot_per_user_request(kwargs) + or (isinstance(litellm.cache, Cache) and litellm.cache.lacks_cache_scope_identity(kwargs)) ) diff --git a/tests/unit/caching/test_caching.py b/tests/unit/caching/test_caching.py index 9dc89c28ffe..72f9c0eb3a2 100644 --- a/tests/unit/caching/test_caching.py +++ b/tests/unit/caching/test_caching.py @@ -355,6 +355,67 @@ async def test_team_cache_scope_isolates_acompletion_hits_between_teams(monkeypa assert team_b.choices[0].message.content == "b" +_NO_IDENTITY: Final = {} +_TEAMLESS_JWT_CALLER: Final = {"user_api_key_user_id": "jwt-user"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("cache_scope", "identity"), + [("team", _NO_IDENTITY), ("key", _NO_IDENTITY), ("team", _TEAMLESS_JWT_CALLER), ("key", _TEAMLESS_JWT_CALLER)], +) +async def test_scoped_cache_never_shares_hits_between_callers_without_identity( + monkeypatch: pytest.MonkeyPatch, cache_scope: str, identity: dict[str, str] +) -> None: + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, cache_scope=cache_scope)) + messages: Final = [{"role": "user", "content": f"no identity {uuid.uuid4().hex}"}] + + first: Final = await acompletion(model="gpt-4o-mini", messages=messages, metadata=dict(identity), mock_response="1") + await asyncio.gather(*_PENDING_CACHE_WRITES) + second: Final = await acompletion( + model="gpt-4o-mini", messages=messages, metadata=dict(identity), mock_response="2" + ) + + assert second.id != first.id + assert second.choices[0].message.content == "2" + + +@pytest.mark.asyncio +async def test_scoped_cache_without_identity_does_not_read_a_scoped_callers_entry( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, cache_scope="team")) + messages: Final = [{"role": "user", "content": f"cross read {uuid.uuid4().hex}"}] + + team_a: Final = await acompletion( + model="gpt-4o-mini", messages=messages, metadata=dict(_TEAM_A_KEY_1), mock_response="a" + ) + await asyncio.gather(*_PENDING_CACHE_WRITES) + anonymous: Final = await acompletion(model="gpt-4o-mini", messages=messages, mock_response="anon") + + assert anonymous.id != team_a.id + + +@pytest.mark.asyncio +async def test_team_cache_scope_still_serves_cached_embeddings_to_the_same_team( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, cache_scope="team")) + text: Final = f"embed {uuid.uuid4().hex}" + + await aembedding(model="text-embedding-3-small", input=[text], metadata=dict(_TEAM_A_KEY_1), mock_response=[0.1]) + await asyncio.gather(*_PENDING_CACHE_WRITES) + same_team: Final = await aembedding( + model="text-embedding-3-small", input=[text], metadata=dict(_TEAM_A_KEY_2), mock_response=[0.9] + ) + other_team: Final = await aembedding( + model="text-embedding-3-small", input=[text], metadata=dict(_TEAM_B_KEY_3), mock_response=[0.5] + ) + + assert same_team.data[0]["embedding"] == [0.1] + assert other_team.data[0]["embedding"] == [0.5] + + @pytest.mark.parametrize( "anthropic_param", [ From 566f2ed6fcdc6e99df9cb68385c5b28056b2cfd4 Mon Sep 17 00:00:00 2001 From: Filipe Andujar Date: Thu, 22 Oct 2026 15:18:57 -0300 Subject: [PATCH 4/4] test(proxy): cover the cache key strip on a request without litellm_params Codecov flagged the early return when the body carries no litellm_params as the one uncovered line. The test also pins that the strip never adds a litellm_params slot the caller did not send --- .../unit/proxy/test_litellm_pre_call_utils.py | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index c0f0d6287f6..d4d3ba09dde 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -3,6 +3,7 @@ import copy import json import os import time +from collections.abc import Mapping from datetime import datetime, timezone from types import SimpleNamespace from typing import Final @@ -1193,13 +1194,18 @@ _FORGED_CACHE_KEY_FIELDS: Final = { } -async def _add_data_with_forged_cache_key(monkeypatch: pytest.MonkeyPatch, cache_scope: str) -> dict: +_FORGED_TOP_LEVEL_CACHE_KEY_ONLY: Final = {"cache_key": "ns:victim-hash", "preset_cache_key": "ns:victim-hash"} + + +async def _add_data_with_forged_cache_key( + monkeypatch: pytest.MonkeyPatch, cache_scope: str, forged: Mapping[str, object] = _FORGED_CACHE_KEY_FIELDS +) -> dict: monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL, cache_scope=cache_scope)) return await add_litellm_data_to_request( data={ "model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}], - **copy.deepcopy(_FORGED_CACHE_KEY_FIELDS), + **copy.deepcopy(dict(forged)), }, request=_make_request_mock("/v1/chat/completions", {"Content-Type": "application/json"}), user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", team_id="caller-team"), @@ -1222,6 +1228,18 @@ async def test_scoped_cache_ignores_caller_supplied_cache_keys(monkeypatch: pyte assert updated["metadata"]["user_api_key_team_id"] == "caller-team" +@pytest.mark.asyncio +@pytest.mark.parametrize("cache_scope", ["team", "key"]) +async def test_scoped_cache_strips_a_forged_cache_key_on_a_request_without_litellm_params( + monkeypatch: pytest.MonkeyPatch, cache_scope: str +): + updated: Final = await _add_data_with_forged_cache_key(monkeypatch, cache_scope, _FORGED_TOP_LEVEL_CACHE_KEY_ONLY) + + assert "cache_key" not in updated + assert "preset_cache_key" not in updated + assert "litellm_params" not in updated + + @pytest.mark.asyncio async def test_global_cache_scope_keeps_caller_supplied_cache_key(monkeypatch: pytest.MonkeyPatch): updated: Final = await _add_data_with_forged_cache_key(monkeypatch, "global")