diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.tsx index 40f553e6fde..e20eca7dbe2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.tsx @@ -11,7 +11,6 @@ const REDIS_TYPE_LABELS: Readonly> = { node: "Node (Single Instance)", cluster: "Cluster", sentinel: "Sentinel", - semantic: "Semantic", }; const RedisTypeSelector: React.FC = ({ redisType, redisTypeDescriptions, onTypeChange }) => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts index 9ebb9c191ed..46fffcf94ff 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts @@ -13,7 +13,7 @@ export interface CacheFieldOption { readonly label: string; } -export type RedisType = "node" | "cluster" | "sentinel" | "semantic"; +export type RedisType = "node" | "cluster" | "sentinel"; export type CacheSection = "connection" | "cluster" | "sentinel" | "semantic" | "ssl" | "cacheManagement" | "gcp"; @@ -38,13 +38,12 @@ export interface CacheField { readonly secret?: boolean; } -export const REDIS_TYPES: readonly RedisType[] = ["node", "cluster", "sentinel", "semantic"]; +export const REDIS_TYPES: readonly RedisType[] = ["node", "cluster", "sentinel"]; export const REDIS_TYPE_DESCRIPTIONS: Readonly> = { node: "Standard Redis node/single instance", cluster: "Redis Cluster mode for high availability and horizontal scaling", sentinel: "Redis Sentinel mode for high availability with automatic failover", - semantic: "Semantic caching that reuses responses for similar prompts", }; const isBlank = (value: unknown): boolean => value === undefined || value === null || String(value).trim() === ""; @@ -181,7 +180,7 @@ export const CACHE_FIELDS: readonly CacheField[] = [ type: "float", section: "semantic", helpText: "Similarity threshold for semantic cache", - redisType: "semantic", + redisType: null, defaultValue: 0.8, rules: [numberRule], }, @@ -191,7 +190,7 @@ export const CACHE_FIELDS: readonly CacheField[] = [ type: "model-select", section: "semantic", helpText: "Embedding model for semantic cache", - redisType: "semantic", + redisType: null, }, { name: "semantic_cache_scope", @@ -200,7 +199,7 @@ export const CACHE_FIELDS: readonly CacheField[] = [ section: "semantic", helpText: "Who can share a semantic cache hit. Key shares hits between all end users of a key/team/org. End user also isolates per end user; requests without an end user fall back to the key scope.", - redisType: "semantic", + redisType: null, defaultValue: "key", options: [ { value: "key", label: "Key (shared by all end users of the key/team/org)" }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts index c851f4aad96..03acb1690f6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts @@ -70,21 +70,62 @@ describe("buildCachePayload", () => { expect(payload).not.toHaveProperty("redis_startup_nodes"); }); - it("should send type redis-semantic when saving a semantic cache", () => { - const payload = buildCachePayload("semantic", { similarity_threshold: 0.9 }, { forTesting: false }); + it("should send type redis-semantic when saving with semantic caching enabled", () => { + const payload = buildCachePayload( + "node", + { similarity_threshold: 0.9 }, + { forTesting: false, semanticEnabled: true }, + ); expect(payload.type).toBe("redis-semantic"); expect(payload.similarity_threshold).toBe(0.9); }); - it("should send the semantic cache scope only for a semantic cache", () => { - const semantic = buildCachePayload("semantic", { semantic_cache_scope: "end_user" }, { forTesting: false }); - expect(semantic.semantic_cache_scope).toBe("end_user"); - const node = buildCachePayload("node", { semantic_cache_scope: "end_user" }, { forTesting: false }); - expect(node).not.toHaveProperty("semantic_cache_scope"); + it.each(["cluster", "sentinel"] as const)( + "should never send redis-semantic for %s, which the semantic cache cannot connect to", + (redisType) => { + const payload = buildCachePayload( + redisType, + { similarity_threshold: 0.9, semantic_cache_scope: "end_user" }, + { forTesting: false, semanticEnabled: true }, + ); + expect(payload.type).toBe("redis"); + expect(payload).not.toHaveProperty("similarity_threshold"); + expect(payload).not.toHaveProperty("semantic_cache_scope"); + }, + ); + + it("should omit semantic fields when semantic caching is disabled, even if they hold values", () => { + const payload = buildCachePayload( + "node", + { similarity_threshold: 0.9, redis_semantic_cache_embedding_model: "text-embedding-3-small" }, + { forTesting: false, semanticEnabled: false }, + ); + expect(payload.type).toBe("redis"); + expect(payload).not.toHaveProperty("similarity_threshold"); + expect(payload).not.toHaveProperty("redis_semantic_cache_embedding_model"); }); - it("should keep type redis when testing a semantic cache so the test endpoint accepts it", () => { - const payload = buildCachePayload("semantic", { similarity_threshold: 0.9 }, { forTesting: true }); + it("should send the semantic cache scope only when semantic caching is enabled", () => { + const enabled = buildCachePayload( + "node", + { semantic_cache_scope: "end_user" }, + { forTesting: false, semanticEnabled: true }, + ); + expect(enabled.semantic_cache_scope).toBe("end_user"); + const disabled = buildCachePayload( + "node", + { semantic_cache_scope: "end_user" }, + { forTesting: false, semanticEnabled: false }, + ); + expect(disabled).not.toHaveProperty("semantic_cache_scope"); + }); + + it("should keep type redis when testing with semantic caching enabled so the test endpoint accepts it", () => { + const payload = buildCachePayload( + "node", + { similarity_threshold: 0.9 }, + { forTesting: true, semanticEnabled: true }, + ); expect(payload.type).toBe("redis"); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts index 8da54e3ee78..4d7e0cc2d98 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts @@ -2,7 +2,7 @@ import { CACHE_FIELDS, CacheField, CacheSection, REDACTED_VALUE, RedisType } fro export type CacheFormValue = string | number | boolean | null | undefined; export type CacheFormValues = Record; -export type CacheSavePayloadValue = string | number | boolean | unknown[]; +export type CacheSavePayloadValue = string | number | boolean | unknown[] | null; export type CacheSavePayload = Record; export const isFieldVisible = (field: CacheField, redisType: RedisType): boolean => @@ -86,16 +86,22 @@ const saveValueForField = (field: CacheField, raw: CacheFormValue): CacheSavePay return trimmed === "" ? undefined : trimmed; }; +export const supportsSemanticCache = (redisType: RedisType): boolean => redisType === "node"; + export const buildCachePayload = ( redisType: RedisType, values: CacheFormValues, - { forTesting }: { forTesting: boolean }, + { forTesting, semanticEnabled = false }: { forTesting: boolean; semanticEnabled?: boolean }, ): CacheSavePayload => { - const type = !forTesting && redisType === "semantic" ? "redis-semantic" : "redis"; + const semantic = semanticEnabled && supportsSemanticCache(redisType); + const type = !forTesting && semantic ? "redis-semantic" : "redis"; const entries = CACHE_FIELDS.filter((field) => isFieldVisible(field, redisType)).flatMap((field) => { + if (field.section === "semantic" && !semantic) { + return []; + } const value = saveValueForField(field, values[field.name]); - return value === undefined ? [] : [[field.name, value] as const]; + return value === undefined ? [] : [[field.name, value] as [string, CacheSavePayloadValue]]; }); return { type, ...Object.fromEntries(entries) }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.test.tsx index b1ed2b8a5ab..ebdf4a7b8ae 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.test.tsx @@ -63,13 +63,56 @@ describe("CacheSettings", () => { }); }); - describe("when the redis type is semantic", () => { - it("should reveal the semantic fields", async () => { + describe("semantic caching", () => { + it("should not offer Semantic as a redis type, since it is not a topology", async () => { + renderSettings(); + await screen.findByText("Redis Type"); + expect(screen.queryByText("Semantic")).not.toBeInTheDocument(); + }); + + it("should stay collapsed until the toggle is switched on", async () => { + const user = userEvent.setup(); + renderSettings(); + expect(await screen.findByText("Enable Semantic Caching")).toBeInTheDocument(); + expect(screen.queryByText("Similarity Threshold")).not.toBeInTheDocument(); + + await user.click(screen.getByRole("switch")); + expect(await screen.findByText("Similarity Threshold")).toBeInTheDocument(); + expect(screen.getByText("Embedding Model")).toBeInTheDocument(); + }); + + it("should reveal the semantic fields for a config saved under the old redis_type", async () => { getCacheSettingsCall.mockResolvedValue({ current_values: { redis_type: "semantic" } }); renderSettings(); expect(await screen.findByText("Similarity Threshold")).toBeInTheDocument(); expect(screen.getByText("Embedding Model")).toBeInTheDocument(); }); + + it("should reveal the semantic fields when only the semantic values are set", async () => { + getCacheSettingsCall.mockResolvedValue({ current_values: { redis_type: "node", similarity_threshold: 0.9 } }); + renderSettings(); + expect(await screen.findByText("Similarity Threshold")).toBeInTheDocument(); + }); + + it("should load a saved redis-semantic config as a node with the toggle on", async () => { + getCacheSettingsCall.mockResolvedValue({ current_values: { type: "redis-semantic", host: "localhost" } }); + renderSettings(); + expect(await screen.findByText("Similarity Threshold")).toBeInTheDocument(); + expect(screen.getByText("Node (Single Instance)")).toBeInTheDocument(); + expect(screen.getByRole("switch")).not.toHaveAttribute("data-disabled"); + }); + + it.each([ + ["cluster", "Startup Nodes"], + ["sentinel", "Sentinel Nodes"], + ])("should disable the toggle and hide the semantic fields for %s", async (redisType, topologyField) => { + getCacheSettingsCall.mockResolvedValue({ current_values: { redis_type: redisType, similarity_threshold: 0.9 } }); + renderSettings(); + expect(await screen.findByText(topologyField)).toBeInTheDocument(); + expect(screen.getByRole("switch")).toHaveAttribute("data-disabled"); + expect(screen.getByText(/Semantic caching needs a single Redis node/)).toBeInTheDocument(); + expect(screen.queryByText("Similarity Threshold")).not.toBeInTheDocument(); + }); }); describe("when a field fails inline validation", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx index d26cc1b40f8..2e538be55c5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx @@ -6,6 +6,7 @@ import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/component import { getCacheSettingsCall, testCacheConnectionCall, updateCacheSettingsCall } from "@/components/networking"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; import { toast } from "@/lib/toast"; +import { Switch } from "@/components/ui/switch"; import RedisTypeSelector from "./RedisTypeSelector"; import CacheFieldSection from "./CacheFieldSection"; import { EmbeddingModelOption } from "./CacheFormField"; @@ -16,6 +17,7 @@ import { CacheFormValues, configuredSecretFields, isFieldVisible, + supportsSemanticCache, } from "./cacheSettingsUtils"; const ADVANCED_SECTIONS = ["ssl", "cacheManagement", "gcp"] as const; @@ -37,6 +39,9 @@ const CacheSettings: React.FC = ({ accessToken }) => { const [isTesting, setIsTesting] = useState(false); const [isSaving, setIsSaving] = useState(false); const [configuredSecrets, setConfiguredSecrets] = useState>(new Set()); + const [semanticEnabled, setSemanticEnabled] = useState(false); + const semanticAvailable = supportsSemanticCache(redisType); + const semanticActive = semanticEnabled && semanticAvailable; const loadCacheSettings = useCallback(async () => { if (!accessToken) { @@ -48,6 +53,12 @@ const CacheSettings: React.FC = ({ accessToken }) => { form.reset(buildInitialValues(currentValues)); setConfiguredSecrets(configuredSecretFields(currentValues)); setRedisType(toRedisType(currentValues.redis_type)); + setSemanticEnabled( + currentValues.type === "redis-semantic" || + currentValues.redis_type === "semantic" || + currentValues.similarity_threshold != null || + currentValues.redis_semantic_cache_embedding_model != null, + ); } catch (error) { console.error("Failed to load cache settings:", error); toast.fromError("Failed to load cache settings"); @@ -102,7 +113,7 @@ const CacheSettings: React.FC = ({ accessToken }) => { try { const result = await testCacheConnectionCall( accessToken, - buildCachePayload(redisType, values, { forTesting: true }), + buildCachePayload(redisType, values, { forTesting: true, semanticEnabled: semanticActive }), ); if (result.status === "success") { toast.success("Cache connection test successful!"); @@ -128,7 +139,10 @@ const CacheSettings: React.FC = ({ accessToken }) => { setIsSaving(true); try { - await updateCacheSettingsCall(accessToken, buildCachePayload(redisType, values, { forTesting: false })); + await updateCacheSettingsCall( + accessToken, + buildCachePayload(redisType, values, { forTesting: false, semanticEnabled: semanticActive }), + ); toast.success("Cache settings updated successfully"); await loadCacheSettings(); } catch (error) { @@ -192,16 +206,27 @@ const CacheSettings: React.FC = ({ accessToken }) => { )} - {redisType === "semantic" && ( -
+
+
+ +
+ Enable Semantic Caching +

+ {semanticAvailable + ? "Reuse responses for semantically similar prompts using embedding vectors" + : "Semantic caching needs a single Redis node, so it is unavailable for Cluster and Sentinel"} +

+
+
+ {semanticActive && ( -
- )} + )} +