From 16db51e2cfc28e02bd460481e634a8403ea9265e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 18:44:54 -0700 Subject: [PATCH] feat(caching): add semantic_cache_scope to isolate semantic cache hits per end user (#39590) Semantic cache keys omit the prompt, so every end user behind one virtual key shares a bucket and can be served another user's semantically similar response. Add an opt-in cache_params.semantic_cache_scope (key | end_user) that appends the authenticated end-user id to the tenant scope, read from metadata and litellm_metadata so /v1/chat/completions, /v1/responses and /v1/messages are all covered, falling back to the key scope when no end-user id is present. Expose the setting in the cache settings API and the Admin UI cache settings form Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/caching.py | 30 +++++--- litellm/types/caching.py | 5 ++ .../cache_settings_endpoints.py | 13 ++++ tests/test_litellm/caching/test_caching.py | 75 ++++++++++++++++++- .../cache_settings/CacheFormField.tsx | 31 ++++++++ .../cache_settings/cacheSettingsFields.ts | 30 +++++++- .../cache_settings/cacheSettingsUtils.test.ts | 8 ++ .../cache_settings/index.integration.test.tsx | 20 +++++ 8 files changed, 197 insertions(+), 15 deletions(-) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 754815fce47..884d095793c 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -100,6 +100,7 @@ class Cache: qdrant_semantic_cache_vector_size: int | None = None, semantic_cache_embedding_max_input_tokens: int | None = None, semantic_cache_embedding_timeout: float | None = None, + semantic_cache_scope: str = SemanticCacheScope.KEY.value, # GCP IAM authentication parameters gcp_service_account: str | None = None, gcp_ssl_ca_certs: str | None = None, @@ -127,6 +128,7 @@ class Cache: similarity_threshold (float, optional): The similarity threshold for semantic-caching, Required if type is "redis-semantic" or "qdrant-semantic". semantic_cache_embedding_max_input_tokens (int, optional): Truncate prompts to this many tokens before embedding them for semantic caching. Defaults to the embedding deployment's configured max_input_tokens. semantic_cache_embedding_timeout (float, optional): Seconds a semantic-cache lookup may spend embedding the prompt before it gives up and lets the request continue to the LLM. Defaults to SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS. + semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key". # Disk Cache Args disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None. @@ -274,6 +276,7 @@ class Cache: self.redis_flush_size = redis_flush_size self.ttl = ttl self.mode: CacheMode = mode or CacheMode.default_on + self.semantic_cache_scope: str = SemanticCacheScope(semantic_cache_scope).value if self.type == LiteLLMCacheType.LOCAL and default_in_memory_ttl is not None: self.ttl = default_in_memory_ttl @@ -301,6 +304,7 @@ class Cache: "user_api_key_team_id", "user_api_key_org_id", ) + _SEMANTIC_CACHE_END_USER_SCOPE_FIELD: Final = "user_api_key_end_user_id" def _is_semantic_cache(self) -> bool: return self.type in ( @@ -309,19 +313,21 @@ class Cache: LiteLLMCacheType.VALKEY_SEMANTIC, ) - def _get_semantic_cache_tenant_scope(self, kwargs: dict) -> str: - metadata: Final[dict] = kwargs.get("metadata") or {} - litellm_params: Final[dict] = kwargs.get("litellm_params") or {} - metadata_in_litellm_params: Final[dict] = litellm_params.get("metadata") or {} + def _semantic_cache_scope_fields(self) -> tuple[str, ...]: + if self.semantic_cache_scope == SemanticCacheScope.END_USER: + return (*self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS, self._SEMANTIC_CACHE_END_USER_SCOPE_FIELD) + return self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS - scope = "" - for field in self._SEMANTIC_CACHE_TENANT_SCOPE_FIELDS: - value = metadata.get(field) - if value is None: - value = metadata_in_litellm_params.get(field) - if value is not None: - scope += f"{field}: {value}" - return scope + def _get_semantic_cache_tenant_scope(self, kwargs: dict) -> str: + litellm_params: Final[dict] = kwargs.get("litellm_params") or {} + metadata_sources: Final[tuple[dict, ...]] = tuple( + source.get(key) or {} for source in (kwargs, litellm_params) for key in ("metadata", "litellm_metadata") + ) + scope_values: Final = ( + (field, next((source[field] for source in metadata_sources if source.get(field) is not None), None)) + for field in self._semantic_cache_scope_fields() + ) + return "".join(f"{field}: {value}" for field, value in scope_values if value is not None) def get_cache_key(self, **kwargs) -> str: """ diff --git a/litellm/types/caching.py b/litellm/types/caching.py index 10c83376a6c..747f202d35e 100644 --- a/litellm/types/caching.py +++ b/litellm/types/caching.py @@ -18,6 +18,11 @@ class LiteLLMCacheType(str, Enum): GCS = "gcs" +class SemanticCacheScope(str, Enum): + KEY = "key" + END_USER = "end_user" + + CachingSupportedCallTypes = Literal[ "completion", "acompletion", diff --git a/litellm/types/management_endpoints/cache_settings_endpoints.py b/litellm/types/management_endpoints/cache_settings_endpoints.py index 2e6d0be9071..32b32991449 100644 --- a/litellm/types/management_endpoints/cache_settings_endpoints.py +++ b/litellm/types/management_endpoints/cache_settings_endpoints.py @@ -187,6 +187,19 @@ CACHE_SETTINGS_FIELDS: Final[list[CacheSettingsField]] = [ ui_field_name="Embedding Model", redis_type="semantic", ), + CacheSettingsField( + field_name="semantic_cache_scope", + field_type="String", + field_value=None, + field_description=( + "Isolation granularity for semantic cache hits. 'key' shares hits between all end users of a key/team/org." + " 'end_user' also isolates per end user; requests without an end user fall back to the key scope." + ), + field_default="key", + options=["key", "end_user"], + ui_field_name="Semantic Cache Scope", + redis_type="semantic", + ), # GCP IAM authentication fields CacheSettingsField( field_name="gcp_service_account", diff --git a/tests/test_litellm/caching/test_caching.py b/tests/test_litellm/caching/test_caching.py index 955b0e531bc..4d0ec0fb677 100644 --- a/tests/test_litellm/caching/test_caching.py +++ b/tests/test_litellm/caching/test_caching.py @@ -4,7 +4,7 @@ import re import pytest from litellm.caching.caching import Cache -from litellm.types.caching import LiteLLMCacheType +from litellm.types.caching import LiteLLMCacheType, SemanticCacheScope from litellm.types.utils import Embedding, EmbeddingResponse, Usage @@ -80,12 +80,13 @@ def test_get_per_item_prompt_tokens_distributes_with_remainder(): assert per_item == [4, 3, 3] -def _semantic_cache(): +def _semantic_cache(**cache_kwargs): return Cache( type=LiteLLMCacheType.VALKEY_SEMANTIC, host="localhost", port="6379", similarity_threshold=0.8, + **cache_kwargs, ) @@ -139,6 +140,76 @@ def test_semantic_cache_key_isolates_tenants(): assert key_a != key_team +_SEMANTICALLY_IDENTICAL_PROMPTS = ( + [{"role": "user", "content": "What color is the sky?"}], + [{"role": "user", "content": "Tell me the colour of the daytime sky."}], +) + + +def _end_user_keys(cache, metadata_field, *end_user_ids): + return [ + cache.get_cache_key( + model="gpt-4o-mini", + messages=messages, + **{metadata_field: {"user_api_key": "hash-A", "user_api_key_end_user_id": end_user_id}}, + ) + for messages, end_user_id in zip(_SEMANTICALLY_IDENTICAL_PROMPTS, end_user_ids) + ] + + +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +def test_semantic_cache_key_shares_bucket_across_end_users_by_default(metadata_field): + key_alice, key_bob = _end_user_keys(_semantic_cache(), metadata_field, "alice", "bob") + assert key_alice == key_bob + + +@pytest.mark.parametrize("metadata_field", ["metadata", "litellm_metadata"]) +def test_semantic_cache_key_isolates_end_users_under_end_user_scope(metadata_field): + cache = _semantic_cache(semantic_cache_scope="end_user") + key_alice, key_bob = _end_user_keys(cache, metadata_field, "alice", "bob") + key_alice_again, _ = _end_user_keys(cache, metadata_field, "alice", "alice") + assert key_alice != key_bob + assert key_alice == key_alice_again + + +def test_semantic_cache_key_end_user_scope_without_end_user_falls_back_to_key_scope(): + cache = _semantic_cache(semantic_cache_scope=SemanticCacheScope.END_USER) + messages = [{"role": "user", "content": "What color is the sky?"}] + key_scope_only = cache.get_cache_key(model="gpt-4o-mini", messages=messages, metadata={"user_api_key": "hash-A"}) + end_user_absent = cache.get_cache_key( + model="gpt-4o-mini", + messages=messages, + metadata={"user_api_key": "hash-A", "user_api_key_end_user_id": None}, + ) + other_key = cache.get_cache_key(model="gpt-4o-mini", messages=messages, metadata={"user_api_key": "hash-B"}) + key_alice, _ = _end_user_keys(cache, "metadata", "alice", "alice") + default_scope_key = _semantic_cache().get_cache_key( + model="gpt-4o-mini", messages=messages, metadata={"user_api_key": "hash-A"} + ) + assert key_scope_only == end_user_absent == default_scope_key + assert key_scope_only != other_key + assert key_scope_only != key_alice + + +def test_semantic_cache_key_reads_tenant_identity_from_litellm_metadata(): + cache = _semantic_cache() + messages = [{"role": "user", "content": "What color is the sky?"}] + key_a = cache.get_cache_key(model="gpt-4o-mini", messages=messages, litellm_metadata={"user_api_key": "hash-A"}) + key_b = cache.get_cache_key(model="gpt-4o-mini", messages=messages, litellm_metadata={"user_api_key": "hash-B"}) + key_a_in_litellm_params = cache.get_cache_key( + model="gpt-4o-mini", + messages=messages, + litellm_params={"litellm_metadata": {"user_api_key": "hash-A"}}, + ) + assert key_a != key_b + assert key_a == key_a_in_litellm_params + + +def test_semantic_cache_scope_rejects_unknown_value(): + with pytest.raises(ValueError, match="'team' is not a valid SemanticCacheScope"): + _semantic_cache(semantic_cache_scope="team") + + def test_semantic_cache_key_still_separates_models_and_params(): cache = _semantic_cache() messages = [{"role": "user", "content": "hi"}] diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFormField.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFormField.tsx index 24ec9043d21..2de078c8a33 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFormField.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFormField.tsx @@ -12,6 +12,7 @@ import { ComboboxList, } from "@/components/ui/combobox"; import { Input } from "@/components/ui/input"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Switch } from "@/components/ui/switch"; import { Textarea } from "@/components/ui/textarea"; import { CacheField } from "./cacheSettingsFields"; @@ -64,6 +65,36 @@ const CacheFormField: React.FC = ({ field, embeddingModels, /> ); } + if (field.type === "select") { + const options = field.options ?? []; + const { id, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy, name, onBlur, disabled } = rest; + return ( + + ); + } if (field.type === "model-select") { const selected = embeddingModels.find((model) => model.value === value) ?? null; return ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts index 04dd6ff6038..9ebb9c191ed 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts @@ -1,4 +1,17 @@ -export type CacheFieldType = "string" | "password" | "integer" | "float" | "boolean" | "list" | "model-select"; +export type CacheFieldType = + | "string" + | "password" + | "integer" + | "float" + | "boolean" + | "list" + | "model-select" + | "select"; + +export interface CacheFieldOption { + readonly value: string; + readonly label: string; +} export type RedisType = "node" | "cluster" | "sentinel" | "semantic"; @@ -18,6 +31,7 @@ export interface CacheField { readonly helpText: string; readonly redisType: RedisType | null; readonly defaultValue?: string | number | boolean; + readonly options?: readonly CacheFieldOption[]; readonly rules?: CacheFieldRule[]; // Credential field: never prefilled into the form, and dropped from the save // payload when left untouched so the redacted marker is never persisted. @@ -179,6 +193,20 @@ export const CACHE_FIELDS: readonly CacheField[] = [ helpText: "Embedding model for semantic cache", redisType: "semantic", }, + { + name: "semantic_cache_scope", + label: "Semantic Cache Scope", + type: "select", + section: "semantic", + helpText: + "Who can share a semantic cache hit. Key shares hits between all end users of a key/team/org. End user also isolates per end user; requests without an end user fall back to the key scope.", + redisType: "semantic", + defaultValue: "key", + options: [ + { value: "key", label: "Key (shared by all end users of the key/team/org)" }, + { value: "end_user", label: "End user (isolated per end user)" }, + ], + }, { name: "ssl", label: "SSL", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts index c530519ee06..c851f4aad96 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts @@ -25,6 +25,7 @@ describe("buildInitialValues", () => { const values = buildInitialValues({}); expect(values.port).toBe("6379"); expect(values.similarity_threshold).toBe("0.8"); + expect(values.semantic_cache_scope).toBe("key"); expect(values.ssl).toBe(false); expect(values.db).toBe(""); }); @@ -75,6 +76,13 @@ describe("buildCachePayload", () => { expect(payload.similarity_threshold).toBe(0.9); }); + it("should send the semantic cache scope only for a semantic cache", () => { + const semantic = buildCachePayload("semantic", { semantic_cache_scope: "end_user" }, { forTesting: false }); + expect(semantic.semantic_cache_scope).toBe("end_user"); + const node = buildCachePayload("node", { semantic_cache_scope: "end_user" }, { forTesting: false }); + expect(node).not.toHaveProperty("semantic_cache_scope"); + }); + it("should keep type redis when testing a semantic cache so the test endpoint accepts it", () => { const payload = buildCachePayload("semantic", { similarity_threshold: 0.9 }, { forTesting: true }); expect(payload.type).toBe("redis"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.integration.test.tsx index 07cc73cddc4..918d8947151 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.integration.test.tsx @@ -138,6 +138,26 @@ describe("CacheSettings advanced settings round-trip", () => { expect(updateCacheSettingsCall.mock.calls[0][1]).not.toHaveProperty("redis_startup_nodes"); }); + it("saves the semantic cache scope picked from the select and shows the loaded value", async () => { + getCacheSettingsCall.mockResolvedValue({ + current_values: { redis_type: "semantic", host: "redis.internal", semantic_cache_scope: "key" }, + }); + const user = userEvent.setup(); + renderSettings(); + const trigger = await screen.findByLabelText("Semantic Cache Scope"); + expect(trigger).toHaveTextContent("Key (shared by all end users of the key/team/org)"); + + await user.click(trigger); + await user.click(await screen.findByRole("option", { name: "End user (isolated per end user)" })); + await save(user); + + await waitFor(() => expect(updateCacheSettingsCall).toHaveBeenCalledTimes(1)); + expect(updateCacheSettingsCall.mock.calls[0][1]).toMatchObject({ + type: "redis-semantic", + semantic_cache_scope: "end_user", + }); + }); + it("does not block the save on a malformed value inside a collapsed advanced section", async () => { getCacheSettingsCall.mockResolvedValue({ current_values: { host: "redis.internal" } }); const user = userEvent.setup();