From 07af4a512a3cb46571c9771d6e518c7b05ae876d Mon Sep 17 00:00:00 2001 From: Rehan Sanjay Date: Mon, 31 Aug 2026 11:39:56 +0530 Subject: [PATCH] fix(ui): decouple Semantic from the Redis Type dropdown in Cache Settings Closes #32621. "Semantic" sat in the Redis Type dropdown next to Node, Cluster and Sentinel. Those three are network topologies; semantic caching is a feature that runs on top of any of them. Picking it meant you could not also say whether your Redis was a cluster, and the payload derived `type: redis-semantic` from the topology dropdown. Semantic caching is now its own toggle: - `RedisType` is `node | cluster | sentinel`, and the semantic fields are no longer gated on a topology. - `buildCachePayload` takes `semanticEnabled` and decides `redis-semantic` from it rather than from `redisType`. - Turning the toggle off sends explicit nulls for the semantic fields. They are always visible now, so omitting them would leave the previous values sitting on the server. - Configs saved under the old `redis_type: "semantic"` still load with the toggle on, as do configs that only have the semantic values set. Rebased onto current litellm_internal_staging. The whole cache_settings suite passes (37 tests), including four new ones covering the toggle, the explicit-null clear, and both backward-compatible load paths. --- .../cache_settings/RedisTypeSelector.tsx | 1 - .../cache_settings/cacheSettingsFields.ts | 11 ++-- .../cache_settings/cacheSettingsUtils.test.ts | 51 +++++++++++++++---- .../cache_settings/cacheSettingsUtils.ts | 13 +++-- .../cache_settings/index.integration.test.tsx | 4 ++ .../_components/cache_settings/index.test.tsx | 31 ++++++++++- .../_components/cache_settings/index.tsx | 32 +++++++++--- 7 files changed, 115 insertions(+), 28 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.tsx index 40f553e6fde..e20eca7dbe2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.tsx @@ -11,7 +11,6 @@ const REDIS_TYPE_LABELS: Readonly> = { node: "Node (Single Instance)", cluster: "Cluster", sentinel: "Sentinel", - semantic: "Semantic", }; const RedisTypeSelector: React.FC = ({ redisType, redisTypeDescriptions, onTypeChange }) => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts index 9ebb9c191ed..46fffcf94ff 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts @@ -13,7 +13,7 @@ export interface CacheFieldOption { readonly label: string; } -export type RedisType = "node" | "cluster" | "sentinel" | "semantic"; +export type RedisType = "node" | "cluster" | "sentinel"; export type CacheSection = "connection" | "cluster" | "sentinel" | "semantic" | "ssl" | "cacheManagement" | "gcp"; @@ -38,13 +38,12 @@ export interface CacheField { readonly secret?: boolean; } -export const REDIS_TYPES: readonly RedisType[] = ["node", "cluster", "sentinel", "semantic"]; +export const REDIS_TYPES: readonly RedisType[] = ["node", "cluster", "sentinel"]; export const REDIS_TYPE_DESCRIPTIONS: Readonly> = { node: "Standard Redis node/single instance", cluster: "Redis Cluster mode for high availability and horizontal scaling", sentinel: "Redis Sentinel mode for high availability with automatic failover", - semantic: "Semantic caching that reuses responses for similar prompts", }; const isBlank = (value: unknown): boolean => value === undefined || value === null || String(value).trim() === ""; @@ -181,7 +180,7 @@ export const CACHE_FIELDS: readonly CacheField[] = [ type: "float", section: "semantic", helpText: "Similarity threshold for semantic cache", - redisType: "semantic", + redisType: null, defaultValue: 0.8, rules: [numberRule], }, @@ -191,7 +190,7 @@ export const CACHE_FIELDS: readonly CacheField[] = [ type: "model-select", section: "semantic", helpText: "Embedding model for semantic cache", - redisType: "semantic", + redisType: null, }, { name: "semantic_cache_scope", @@ -200,7 +199,7 @@ export const CACHE_FIELDS: readonly CacheField[] = [ section: "semantic", helpText: "Who can share a semantic cache hit. Key shares hits between all end users of a key/team/org. End user also isolates per end user; requests without an end user fall back to the key scope.", - redisType: "semantic", + redisType: null, defaultValue: "key", options: [ { value: "key", label: "Key (shared by all end users of the key/team/org)" }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts index c851f4aad96..86a20f0ba2d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts @@ -51,6 +51,10 @@ describe("buildCachePayload", () => { port: "6379", ssl: false, ssl_check_hostname: false, + // semantic caching is off, so its fields are explicitly cleared + similarity_threshold: null, + redis_semantic_cache_embedding_model: null, + semantic_cache_scope: null, }); expect(payload).not.toHaveProperty("redis_type"); expect(payload).not.toHaveProperty("username"); @@ -70,21 +74,50 @@ describe("buildCachePayload", () => { expect(payload).not.toHaveProperty("redis_startup_nodes"); }); - it("should send type redis-semantic when saving a semantic cache", () => { - const payload = buildCachePayload("semantic", { similarity_threshold: 0.9 }, { forTesting: false }); + it("should send type redis-semantic when saving with semantic caching enabled", () => { + const payload = buildCachePayload( + "node", + { similarity_threshold: 0.9 }, + { forTesting: false, semanticEnabled: true }, + ); expect(payload.type).toBe("redis-semantic"); expect(payload.similarity_threshold).toBe(0.9); }); - it("should send the semantic cache scope only for a semantic cache", () => { - const semantic = buildCachePayload("semantic", { semantic_cache_scope: "end_user" }, { forTesting: false }); - expect(semantic.semantic_cache_scope).toBe("end_user"); - const node = buildCachePayload("node", { semantic_cache_scope: "end_user" }, { forTesting: false }); - expect(node).not.toHaveProperty("semantic_cache_scope"); + it("should send null for semantic fields when semantic caching is disabled, so turning it off clears them", () => { + const payload = buildCachePayload( + "node", + { similarity_threshold: 0.9 }, + { forTesting: false, semanticEnabled: false }, + ); + expect(payload.type).toBe("redis"); + expect(payload.similarity_threshold).toBe(null); + expect(payload.redis_semantic_cache_embedding_model).toBe(null); }); - it("should keep type redis when testing a semantic cache so the test endpoint accepts it", () => { - const payload = buildCachePayload("semantic", { similarity_threshold: 0.9 }, { forTesting: true }); + it("should send the semantic cache scope only when semantic caching is enabled", () => { + const enabled = buildCachePayload( + "node", + { semantic_cache_scope: "end_user" }, + { forTesting: false, semanticEnabled: true }, + ); + expect(enabled.semantic_cache_scope).toBe("end_user"); + // Disabled sends an explicit null rather than omitting the field, so a scope + // configured earlier is cleared on the server instead of silently surviving. + const disabled = buildCachePayload( + "node", + { semantic_cache_scope: "end_user" }, + { forTesting: false, semanticEnabled: false }, + ); + expect(disabled.semantic_cache_scope).toBe(null); + }); + + it("should keep type redis when testing with semantic caching enabled so the test endpoint accepts it", () => { + const payload = buildCachePayload( + "node", + { similarity_threshold: 0.9 }, + { forTesting: true, semanticEnabled: true }, + ); expect(payload.type).toBe("redis"); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts index 8da54e3ee78..1571d31f088 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts @@ -2,7 +2,7 @@ import { CACHE_FIELDS, CacheField, CacheSection, REDACTED_VALUE, RedisType } fro export type CacheFormValue = string | number | boolean | null | undefined; export type CacheFormValues = Record; -export type CacheSavePayloadValue = string | number | boolean | unknown[]; +export type CacheSavePayloadValue = string | number | boolean | unknown[] | null; export type CacheSavePayload = Record; export const isFieldVisible = (field: CacheField, redisType: RedisType): boolean => @@ -89,13 +89,18 @@ const saveValueForField = (field: CacheField, raw: CacheFormValue): CacheSavePay export const buildCachePayload = ( redisType: RedisType, values: CacheFormValues, - { forTesting }: { forTesting: boolean }, + { forTesting, semanticEnabled = false }: { forTesting: boolean; semanticEnabled?: boolean }, ): CacheSavePayload => { - const type = !forTesting && redisType === "semantic" ? "redis-semantic" : "redis"; + const type = !forTesting && semanticEnabled ? "redis-semantic" : "redis"; const entries = CACHE_FIELDS.filter((field) => isFieldVisible(field, redisType)).flatMap((field) => { + // Semantic fields are always visible now, so clearing the toggle has to send an + // explicit null - omitting them would leave the previous values on the server. + if (field.section === "semantic" && !semanticEnabled) { + return [[field.name, null] as [string, CacheSavePayloadValue]]; + } const value = saveValueForField(field, values[field.name]); - return value === undefined ? [] : [[field.name, value] as const]; + return value === undefined ? [] : [[field.name, value] as [string, CacheSavePayloadValue]]; }); return { type, ...Object.fromEntries(entries) }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.integration.test.tsx index 0d1da4dc8ce..26ff5179e17 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.integration.test.tsx @@ -61,6 +61,10 @@ describe("CacheSettings advanced settings round-trip", () => { namespace: "prod-ns", ttl: 300, max_connections: 50, + // semantic caching is off, so its fields are cleared explicitly + similarity_threshold: null, + redis_semantic_cache_embedding_model: null, + semantic_cache_scope: null, }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.test.tsx index b1ed2b8a5ab..80ab2c86ca2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.test.tsx @@ -63,13 +63,36 @@ describe("CacheSettings", () => { }); }); - describe("when the redis type is semantic", () => { - it("should reveal the semantic fields", async () => { + describe("semantic caching", () => { + it("should not offer Semantic as a redis type, since it is not a topology", async () => { + renderSettings(); + await screen.findByText("Redis Type"); + expect(screen.queryByText("Semantic")).not.toBeInTheDocument(); + }); + + it("should stay collapsed until the toggle is switched on", async () => { + const user = userEvent.setup(); + renderSettings(); + expect(await screen.findByText("Enable Semantic Caching")).toBeInTheDocument(); + expect(screen.queryByText("Similarity Threshold")).not.toBeInTheDocument(); + + await user.click(screen.getByRole("switch")); + expect(await screen.findByText("Similarity Threshold")).toBeInTheDocument(); + expect(screen.getByText("Embedding Model")).toBeInTheDocument(); + }); + + it("should reveal the semantic fields for a config saved under the old redis_type", async () => { getCacheSettingsCall.mockResolvedValue({ current_values: { redis_type: "semantic" } }); renderSettings(); expect(await screen.findByText("Similarity Threshold")).toBeInTheDocument(); expect(screen.getByText("Embedding Model")).toBeInTheDocument(); }); + + it("should reveal the semantic fields when only the semantic values are set", async () => { + getCacheSettingsCall.mockResolvedValue({ current_values: { redis_type: "node", similarity_threshold: 0.9 } }); + renderSettings(); + expect(await screen.findByText("Similarity Threshold")).toBeInTheDocument(); + }); }); describe("when a field fails inline validation", () => { @@ -128,6 +151,10 @@ describe("CacheSettings", () => { port: "6379", ssl: false, ssl_check_hostname: false, + // semantic caching is off, so its fields are cleared explicitly + similarity_threshold: null, + redis_semantic_cache_embedding_model: null, + semantic_cache_scope: null, }), ); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx index d26cc1b40f8..318bbf69156 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx @@ -6,6 +6,7 @@ import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/component import { getCacheSettingsCall, testCacheConnectionCall, updateCacheSettingsCall } from "@/components/networking"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; import { toast } from "@/lib/toast"; +import { Switch } from "@/components/ui/switch"; import RedisTypeSelector from "./RedisTypeSelector"; import CacheFieldSection from "./CacheFieldSection"; import { EmbeddingModelOption } from "./CacheFormField"; @@ -37,6 +38,7 @@ const CacheSettings: React.FC = ({ accessToken }) => { const [isTesting, setIsTesting] = useState(false); const [isSaving, setIsSaving] = useState(false); const [configuredSecrets, setConfiguredSecrets] = useState>(new Set()); + const [semanticEnabled, setSemanticEnabled] = useState(false); const loadCacheSettings = useCallback(async () => { if (!accessToken) { @@ -48,6 +50,12 @@ const CacheSettings: React.FC = ({ accessToken }) => { form.reset(buildInitialValues(currentValues)); setConfiguredSecrets(configuredSecretFields(currentValues)); setRedisType(toRedisType(currentValues.redis_type)); + // "semantic" is no longer a redis_type, but existing configs were saved with it. + setSemanticEnabled( + currentValues.redis_type === "semantic" || + currentValues.similarity_threshold != null || + currentValues.redis_semantic_cache_embedding_model != null, + ); } catch (error) { console.error("Failed to load cache settings:", error); toast.fromError("Failed to load cache settings"); @@ -102,7 +110,7 @@ const CacheSettings: React.FC = ({ accessToken }) => { try { const result = await testCacheConnectionCall( accessToken, - buildCachePayload(redisType, values, { forTesting: true }), + buildCachePayload(redisType, values, { forTesting: true, semanticEnabled }), ); if (result.status === "success") { toast.success("Cache connection test successful!"); @@ -128,7 +136,10 @@ const CacheSettings: React.FC = ({ accessToken }) => { setIsSaving(true); try { - await updateCacheSettingsCall(accessToken, buildCachePayload(redisType, values, { forTesting: false })); + await updateCacheSettingsCall( + accessToken, + buildCachePayload(redisType, values, { forTesting: false, semanticEnabled }), + ); toast.success("Cache settings updated successfully"); await loadCacheSettings(); } catch (error) { @@ -192,16 +203,25 @@ const CacheSettings: React.FC = ({ accessToken }) => { )} - {redisType === "semantic" && ( -
+
+
+ +
+ Enable Semantic Caching +

+ Reuse responses for semantically similar prompts using embedding vectors +

+
+
+ {semanticEnabled && ( -
- )} + )} +