feat(caching): add semantic_cache_scope to isolate semantic cache hits per end user (#39590)

Semantic cache keys omit the prompt, so every end user behind one virtual key
shares a bucket and can be served another user's semantically similar response.
Add an opt-in cache_params.semantic_cache_scope (key | end_user) that appends the
authenticated end-user id to the tenant scope, read from metadata and
litellm_metadata so /v1/chat/completions, /v1/responses and /v1/messages are all
covered, falling back to the key scope when no end-user id is present. Expose the
setting in the cache settings API and the Admin UI cache settings form

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/caching/caching.py

@@ -100,6 +100,7 @@ class Cache:
         qdrant_semantic_cache_vector_size: int | None = None,
         semantic_cache_embedding_max_input_tokens: int | None = None,
         semantic_cache_embedding_timeout: float | None = None,
+        semantic_cache_scope: str = SemanticCacheScope.KEY.value,
         # GCP IAM authentication parameters
         gcp_service_account: str | None = None,
         gcp_ssl_ca_certs: str | None = None,
@@ -127,6 +128,7 @@ class Cache:
             similarity_threshold (float, optional): The similarity threshold for semantic-caching, Required if type is "redis-semantic" or "qdrant-semantic".
             semantic_cache_embedding_max_input_tokens (int, optional): Truncate prompts to this many tokens before embedding them for semantic caching. Defaults to the embedding deployment's configured max_input_tokens.
             semantic_cache_embedding_timeout (float, optional): Seconds a semantic-cache lookup may spend embedding the prompt before it gives up and lets the request continue to the LLM. Defaults to SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS.
+            semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key".
 
             # Disk Cache Args
             disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
@@ -274,6 +276,7 @@ class Cache:
         self.redis_flush_size = redis_flush_size
         self.ttl = ttl
         self.mode: CacheMode = mode or CacheMode.default_on
+        self.semantic_cache_scope: str = SemanticCacheScope(semantic_cache_scope).value
 
         if self.type == LiteLLMCacheType.LOCAL and default_in_memory_ttl is not None:
             self.ttl = default_in_memory_ttl
@@ -301,6 +304,7 @@ class Cache:
         "user_api_key_team_id",
         "user_api_key_org_id",
     )
+    _SEMANTIC_CACHE_END_USER_SCOPE_FIELD: Final = "user_api_key_end_user_id"
 
     def _is_semantic_cache(self) -> bool:
         return self.type in (
@@ -309,19 +313,21 @@ class Cache:
             LiteLLMCacheType.VALKEY_SEMANTIC,
         )
 
-    def _get_semantic_cache_tenant_scope(self, kwargs: dict) -> str:
-        metadata: Final[dict] = kwargs.get("metadata") or {}
-        litellm_params: Final[dict] = kwargs.get("litellm_params") or {}
-        metadata_in_litellm_params: Final[dict] = litellm_params.get("metadata") or {}
+    def _semantic_cache_scope_fields(self) -> tuple[str, ...]:
+        if self.semantic_cache_scope == SemanticCacheScope.END_USER:
+            return (*self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS, self._SEMANTIC_CACHE_END_USER_SCOPE_FIELD)
+        return self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS
 
-        scope = ""
-        for field in self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS:
-            value = metadata.get(field)
-            if value is None:
-                value = metadata_in_litellm_params.get(field)
-            if value is not None:
-                scope += f"{field}: {value}"
-        return scope
+    def _get_semantic_cache_tenant_scope(self, kwargs: dict) -> str:
+        litellm_params: Final[dict] = kwargs.get("litellm_params") or {}
+        metadata_sources: Final[tuple[dict, ...]] = tuple(
+            source.get(key) or {} for source in (kwargs, litellm_params) for key in ("metadata", "litellm_metadata")
+        )
+        scope_values: Final = (
+            (field, next((source[field] for source in metadata_sources if source.get(field) is not None), None))
+            for field in self._semantic_cache_scope_fields()
+        )
+        return "".join(f"{field}: {value}" for field, value in scope_values if value is not None)
 
     def get_cache_key(self, **kwargs) -> str:
         """

litellm/types/caching.py

@@ -18,6 +18,11 @@ class LiteLLMCacheType(str, Enum):
     GCS = "gcs"
 
 
+class SemanticCacheScope(str, Enum):
+    KEY = "key"
+    END_USER = "end_user"
+
+
 CachingSupportedCallTypes = Literal[
     "completion",
     "acompletion",

litellm/types/management_endpoints/cache_settings_endpoints.py

@@ -187,6 +187,19 @@ CACHE_SETTINGS_FIELDS: Final[list[CacheSettingsField]] = [
         ui_field_name="Embedding Model",
         redis_type="semantic",
     ),
+    CacheSettingsField(
+        field_name="semantic_cache_scope",
+        field_type="String",
+        field_value=None,
+        field_description=(
+            "Isolation granularity for semantic cache hits. 'key' shares hits between all end users of a key/team/org."
+            " 'end_user' also isolates per end user; requests without an end user fall back to the key scope."
+        ),
+        field_default="key",
+        options=["key", "end_user"],
+        ui_field_name="Semantic Cache Scope",
+        redis_type="semantic",
+    ),
     # GCP IAM authentication fields
     CacheSettingsField(
         field_name="gcp_service_account",

tests/test_litellm/caching/test_caching.py

@@ -4,7 +4,7 @@ import re
 import pytest
 
 from litellm.caching.caching import Cache
-from litellm.types.caching import LiteLLMCacheType
+from litellm.types.caching import LiteLLMCacheType, SemanticCacheScope
 from litellm.types.utils import Embedding, EmbeddingResponse, Usage
 
 
@@ -80,12 +80,13 @@ def test_get_per_item_prompt_tokens_distributes_with_remainder():
     assert per_item == [4, 3, 3]
 
 
-def _semantic_cache():
+def _semantic_cache(**cache_kwargs):
     return Cache(
         type=LiteLLMCacheType.VALKEY_SEMANTIC,
         host="localhost",
         port="6379",
         similarity_threshold=0.8,
+        **cache_kwargs,
     )
 
 
@@ -139,6 +140,76 @@ def test_semantic_cache_key_isolates_tenants():
     assert key_a != key_team
 
 
+_SEMANTICALLY_IDENTICAL_PROMPTS = (
+    [{"role": "user", "content": "What color is the sky?"}],
+    [{"role": "user", "content": "Tell me the colour of the daytime sky."}],
+)
+
+
+def _end_user_keys(cache, metadata_field, *end_user_ids):
+    return [
+        cache.get_cache_key(
+            model="gpt-4o-mini",
+            messages=messages,
+            **{metadata_field: {"user_api_key": "hash-A", "user_api_key_end_user_id": end_user_id}},
+        )
+        for messages, end_user_id in zip(_SEMANTICALLY_IDENTICAL_PROMPTS, end_user_ids)
+    ]
+
+
+@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"])
+def test_semantic_cache_key_shares_bucket_across_end_users_by_default(metadata_field):
+    key_alice, key_bob = _end_user_keys(_semantic_cache(), metadata_field, "alice", "bob")
+    assert key_alice == key_bob
+
+
+@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"])
+def test_semantic_cache_key_isolates_end_users_under_end_user_scope(metadata_field):
+    cache = _semantic_cache(semantic_cache_scope="end_user")
+    key_alice, key_bob = _end_user_keys(cache, metadata_field, "alice", "bob")
+    key_alice_again, _ = _end_user_keys(cache, metadata_field, "alice", "alice")
+    assert key_alice != key_bob
+    assert key_alice == key_alice_again
+
+
+def test_semantic_cache_key_end_user_scope_without_end_user_falls_back_to_key_scope():
+    cache = _semantic_cache(semantic_cache_scope=SemanticCacheScope.END_USER)
+    messages = [{"role": "user", "content": "What color is the sky?"}]
+    key_scope_only = cache.get_cache_key(model="gpt-4o-mini", messages=messages, metadata={"user_api_key": "hash-A"})
+    end_user_absent = cache.get_cache_key(
+        model="gpt-4o-mini",
+        messages=messages,
+        metadata={"user_api_key": "hash-A", "user_api_key_end_user_id": None},
+    )
+    other_key = cache.get_cache_key(model="gpt-4o-mini", messages=messages, metadata={"user_api_key": "hash-B"})
+    key_alice, _ = _end_user_keys(cache, "metadata", "alice", "alice")
+    default_scope_key = _semantic_cache().get_cache_key(
+        model="gpt-4o-mini", messages=messages, metadata={"user_api_key": "hash-A"}
+    )
+    assert key_scope_only == end_user_absent == default_scope_key
+    assert key_scope_only != other_key
+    assert key_scope_only != key_alice
+
+
+def test_semantic_cache_key_reads_tenant_identity_from_litellm_metadata():
+    cache = _semantic_cache()
+    messages = [{"role": "user", "content": "What color is the sky?"}]
+    key_a = cache.get_cache_key(model="gpt-4o-mini", messages=messages, litellm_metadata={"user_api_key": "hash-A"})
+    key_b = cache.get_cache_key(model="gpt-4o-mini", messages=messages, litellm_metadata={"user_api_key": "hash-B"})
+    key_a_in_litellm_params = cache.get_cache_key(
+        model="gpt-4o-mini",
+        messages=messages,
+        litellm_params={"litellm_metadata": {"user_api_key": "hash-A"}},
+    )
+    assert key_a != key_b
+    assert key_a == key_a_in_litellm_params
+
+
+def test_semantic_cache_scope_rejects_unknown_value():
+    with pytest.raises(ValueError, match="'team' is not a valid SemanticCacheScope"):
+        _semantic_cache(semantic_cache_scope="team")
+
+
 def test_semantic_cache_key_still_separates_models_and_params():
     cache = _semantic_cache()
     messages = [{"role": "user", "content": "hi"}]

ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFormField.tsx

@@ -12,6 +12,7 @@ import {
   ComboboxList,
 } from "@/components/ui/combobox";
 import { Input } from "@/components/ui/input";
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
 import { Switch } from "@/components/ui/switch";
 import { Textarea } from "@/components/ui/textarea";
 import { CacheField } from "./cacheSettingsFields";
@@ -64,6 +65,36 @@ const CacheFormField: React.FC<CacheFormFieldProps> = ({ field, embeddingModels,
             />
           );
         }
+        if (field.type === "select") {
+          const options = field.options ?? [];
+          const { id, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy, name, onBlur, disabled } = rest;
+          return (
+            <Select
+              items={options.map((option) => ({ label: option.label, value: option.value }))}
+              name={name}
+              disabled={disabled}
+              value={typeof value === "string" && value !== "" ? value : null}
+              onValueChange={(selected: string | null) => onChange(selected ?? "")}
+            >
+              <SelectTrigger
+                id={id}
+                aria-invalid={ariaInvalid}
+                aria-describedby={ariaDescribedBy}
+                onBlur={onBlur}
+                className="w-full"
+              >
+                <SelectValue placeholder="Select an option" />
+              </SelectTrigger>
+              <SelectContent>
+                {options.map((option) => (
+                  <SelectItem key={option.value} value={option.value}>
+                    {option.label}
+                  </SelectItem>
+                ))}
+              </SelectContent>
+            </Select>
+          );
+        }
         if (field.type === "model-select") {
           const selected = embeddingModels.find((model) => model.value === value) ?? null;
           return (

ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts

@@ -1,4 +1,17 @@
-export type CacheFieldType = "string" | "password" | "integer" | "float" | "boolean" | "list" | "model-select";
+export type CacheFieldType =
+  | "string"
+  | "password"
+  | "integer"
+  | "float"
+  | "boolean"
+  | "list"
+  | "model-select"
+  | "select";
+
+export interface CacheFieldOption {
+  readonly value: string;
+  readonly label: string;
+}
 
 export type RedisType = "node" | "cluster" | "sentinel" | "semantic";
 
@@ -18,6 +31,7 @@ export interface CacheField {
   readonly helpText: string;
   readonly redisType: RedisType | null;
   readonly defaultValue?: string | number | boolean;
+  readonly options?: readonly CacheFieldOption[];
   readonly rules?: CacheFieldRule[];
   // Credential field: never prefilled into the form, and dropped from the save
   // payload when left untouched so the redacted marker is never persisted.
@@ -179,6 +193,20 @@ export const CACHE_FIELDS: readonly CacheField[] = [
     helpText: "Embedding model for semantic cache",
     redisType: "semantic",
   },
+  {
+    name: "semantic_cache_scope",
+    label: "Semantic Cache Scope",
+    type: "select",
+    section: "semantic",
+    helpText:
+      "Who can share a semantic cache hit. Key shares hits between all end users of a key/team/org. End user also isolates per end user; requests without an end user fall back to the key scope.",
+    redisType: "semantic",
+    defaultValue: "key",
+    options: [
+      { value: "key", label: "Key (shared by all end users of the key/team/org)" },
+      { value: "end_user", label: "End user (isolated per end user)" },
+    ],
+  },
   {
     name: "ssl",
     label: "SSL",

ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts

@@ -25,6 +25,7 @@ describe("buildInitialValues", () => {
     const values = buildInitialValues({});
     expect(values.port).toBe("6379");
     expect(values.similarity_threshold).toBe("0.8");
+    expect(values.semantic_cache_scope).toBe("key");
     expect(values.ssl).toBe(false);
     expect(values.db).toBe("");
   });
@@ -75,6 +76,13 @@ describe("buildCachePayload", () => {
     expect(payload.similarity_threshold).toBe(0.9);
   });
 
+  it("should send the semantic cache scope only for a semantic cache", () => {
+    const semantic = buildCachePayload("semantic", { semantic_cache_scope: "end_user" }, { forTesting: false });
+    expect(semantic.semantic_cache_scope).toBe("end_user");
+    const node = buildCachePayload("node", { semantic_cache_scope: "end_user" }, { forTesting: false });
+    expect(node).not.toHaveProperty("semantic_cache_scope");
+  });
+
   it("should keep type redis when testing a semantic cache so the test endpoint accepts it", () => {
     const payload = buildCachePayload("semantic", { similarity_threshold: 0.9 }, { forTesting: true });
     expect(payload.type).toBe("redis");

ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.integration.test.tsx

@@ -138,6 +138,26 @@ describe("CacheSettings advanced settings round-trip", () => {
     expect(updateCacheSettingsCall.mock.calls[0][1]).not.toHaveProperty("redis_startup_nodes");
   });
 
+  it("saves the semantic cache scope picked from the select and shows the loaded value", async () => {
+    getCacheSettingsCall.mockResolvedValue({
+      current_values: { redis_type: "semantic", host: "redis.internal", semantic_cache_scope: "key" },
+    });
+    const user = userEvent.setup();
+    renderSettings();
+    const trigger = await screen.findByLabelText("Semantic Cache Scope");
+    expect(trigger).toHaveTextContent("Key (shared by all end users of the key/team/org)");
+
+    await user.click(trigger);
+    await user.click(await screen.findByRole("option", { name: "End user (isolated per end user)" }));
+    await save(user);
+
+    await waitFor(() => expect(updateCacheSettingsCall).toHaveBeenCalledTimes(1));
+    expect(updateCacheSettingsCall.mock.calls[0][1]).toMatchObject({
+      type: "redis-semantic",
+      semantic_cache_scope: "end_user",
+    });
+  });
+
   it("does not block the save on a malformed value inside a collapsed advanced section", async () => {
     getCacheSettingsCall.mockResolvedValue({ current_values: { host: "redis.internal" } });
     const user = userEvent.setup();