Merge 40aa2bf639 into 8e6d99d74a

ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.tsx

@@ -11,7 +11,6 @@ const REDIS_TYPE_LABELS: Readonly<Record<string, string>> = {
   node: "Node (Single Instance)",
   cluster: "Cluster",
   sentinel: "Sentinel",
-  semantic: "Semantic",
 };
 
 const RedisTypeSelector: React.FC<RedisTypeSelectorProps> = ({ redisType, redisTypeDescriptions, onTypeChange }) => {

ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts

@@ -13,7 +13,7 @@ export interface CacheFieldOption {
   readonly label: string;
 }
 
-export type RedisType = "node" | "cluster" | "sentinel" | "semantic";
+export type RedisType = "node" | "cluster" | "sentinel";
 
 export type CacheSection = "connection" | "cluster" | "sentinel" | "semantic" | "ssl" | "cacheManagement" | "gcp";
 
@@ -38,13 +38,12 @@ export interface CacheField {
   readonly secret?: boolean;
 }
 
-export const REDIS_TYPES: readonly RedisType[] = ["node", "cluster", "sentinel", "semantic"];
+export const REDIS_TYPES: readonly RedisType[] = ["node", "cluster", "sentinel"];
 
 export const REDIS_TYPE_DESCRIPTIONS: Readonly<Record<RedisType, string>> = {
   node: "Standard Redis node/single instance",
   cluster: "Redis Cluster mode for high availability and horizontal scaling",
   sentinel: "Redis Sentinel mode for high availability with automatic failover",
-  semantic: "Semantic caching that reuses responses for similar prompts",
 };
 
 const isBlank = (value: unknown): boolean => value === undefined || value === null || String(value).trim() === "";
@@ -181,7 +180,7 @@ export const CACHE_FIELDS: readonly CacheField[] = [
     type: "float",
     section: "semantic",
     helpText: "Similarity threshold for semantic cache",
-    redisType: "semantic",
+    redisType: null,
     defaultValue: 0.8,
     rules: [numberRule],
   },
@@ -191,7 +190,7 @@ export const CACHE_FIELDS: readonly CacheField[] = [
     type: "model-select",
     section: "semantic",
     helpText: "Embedding model for semantic cache",
-    redisType: "semantic",
+    redisType: null,
   },
   {
     name: "semantic_cache_scope",
@@ -200,7 +199,7 @@ export const CACHE_FIELDS: readonly CacheField[] = [
     section: "semantic",
     helpText:
       "Who can share a semantic cache hit. Key shares hits between all end users of a key/team/org. End user also isolates per end user; requests without an end user fall back to the key scope.",
-    redisType: "semantic",
+    redisType: null,
     defaultValue: "key",
     options: [
       { value: "key", label: "Key (shared by all end users of the key/team/org)" },

ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts

@@ -70,21 +70,62 @@ describe("buildCachePayload", () => {
     expect(payload).not.toHaveProperty("redis_startup_nodes");
   });
 
-  it("should send type redis-semantic when saving a semantic cache", () => {
-    const payload = buildCachePayload("semantic", { similarity_threshold: 0.9 }, { forTesting: false });
+  it("should send type redis-semantic when saving with semantic caching enabled", () => {
+    const payload = buildCachePayload(
+      "node",
+      { similarity_threshold: 0.9 },
+      { forTesting: false, semanticEnabled: true },
+    );
     expect(payload.type).toBe("redis-semantic");
     expect(payload.similarity_threshold).toBe(0.9);
   });
 
-  it("should send the semantic cache scope only for a semantic cache", () => {
-    const semantic = buildCachePayload("semantic", { semantic_cache_scope: "end_user" }, { forTesting: false });
-    expect(semantic.semantic_cache_scope).toBe("end_user");
-    const node = buildCachePayload("node", { semantic_cache_scope: "end_user" }, { forTesting: false });
-    expect(node).not.toHaveProperty("semantic_cache_scope");
+  it.each(["cluster", "sentinel"] as const)(
+    "should never send redis-semantic for %s, which the semantic cache cannot connect to",
+    (redisType) => {
+      const payload = buildCachePayload(
+        redisType,
+        { similarity_threshold: 0.9, semantic_cache_scope: "end_user" },
+        { forTesting: false, semanticEnabled: true },
+      );
+      expect(payload.type).toBe("redis");
+      expect(payload).not.toHaveProperty("similarity_threshold");
+      expect(payload).not.toHaveProperty("semantic_cache_scope");
+    },
+  );
+
+  it("should omit semantic fields when semantic caching is disabled, even if they hold values", () => {
+    const payload = buildCachePayload(
+      "node",
+      { similarity_threshold: 0.9, redis_semantic_cache_embedding_model: "text-embedding-3-small" },
+      { forTesting: false, semanticEnabled: false },
+    );
+    expect(payload.type).toBe("redis");
+    expect(payload).not.toHaveProperty("similarity_threshold");
+    expect(payload).not.toHaveProperty("redis_semantic_cache_embedding_model");
   });
 
-  it("should keep type redis when testing a semantic cache so the test endpoint accepts it", () => {
-    const payload = buildCachePayload("semantic", { similarity_threshold: 0.9 }, { forTesting: true });
+  it("should send the semantic cache scope only when semantic caching is enabled", () => {
+    const enabled = buildCachePayload(
+      "node",
+      { semantic_cache_scope: "end_user" },
+      { forTesting: false, semanticEnabled: true },
+    );
+    expect(enabled.semantic_cache_scope).toBe("end_user");
+    const disabled = buildCachePayload(
+      "node",
+      { semantic_cache_scope: "end_user" },
+      { forTesting: false, semanticEnabled: false },
+    );
+    expect(disabled).not.toHaveProperty("semantic_cache_scope");
+  });
+
+  it("should keep type redis when testing with semantic caching enabled so the test endpoint accepts it", () => {
+    const payload = buildCachePayload(
+      "node",
+      { similarity_threshold: 0.9 },
+      { forTesting: true, semanticEnabled: true },
+    );
     expect(payload.type).toBe("redis");
   });
 

ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts

@@ -2,7 +2,7 @@ import { CACHE_FIELDS, CacheField, CacheSection, REDACTED_VALUE, RedisType } fro
 
 export type CacheFormValue = string | number | boolean | null | undefined;
 export type CacheFormValues = Record<string, CacheFormValue>;
-export type CacheSavePayloadValue = string | number | boolean | unknown[];
+export type CacheSavePayloadValue = string | number | boolean | unknown[] | null;
 export type CacheSavePayload = Record<string, CacheSavePayloadValue>;
 
 export const isFieldVisible = (field: CacheField, redisType: RedisType): boolean =>
@@ -86,16 +86,22 @@ const saveValueForField = (field: CacheField, raw: CacheFormValue): CacheSavePay
   return trimmed === "" ? undefined : trimmed;
 };
 
+export const supportsSemanticCache = (redisType: RedisType): boolean => redisType === "node";
+
 export const buildCachePayload = (
   redisType: RedisType,
   values: CacheFormValues,
-  { forTesting }: { forTesting: boolean },
+  { forTesting, semanticEnabled = false }: { forTesting: boolean; semanticEnabled?: boolean },
 ): CacheSavePayload => {
-  const type = !forTesting && redisType === "semantic" ? "redis-semantic" : "redis";
+  const semantic = semanticEnabled && supportsSemanticCache(redisType);
+  const type = !forTesting && semantic ? "redis-semantic" : "redis";
 
   const entries = CACHE_FIELDS.filter((field) => isFieldVisible(field, redisType)).flatMap((field) => {
+    if (field.section === "semantic" && !semantic) {
+      return [];
+    }
     const value = saveValueForField(field, values[field.name]);
-    return value === undefined ? [] : [[field.name, value] as const];
+    return value === undefined ? [] : [[field.name, value] as [string, CacheSavePayloadValue]];
   });
 
   return { type, ...Object.fromEntries(entries) };

ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.test.tsx

@@ -63,13 +63,56 @@ describe("CacheSettings", () => {
     });
   });
 
-  describe("when the redis type is semantic", () => {
-    it("should reveal the semantic fields", async () => {
+  describe("semantic caching", () => {
+    it("should not offer Semantic as a redis type, since it is not a topology", async () => {
+      renderSettings();
+      await screen.findByText("Redis Type");
+      expect(screen.queryByText("Semantic")).not.toBeInTheDocument();
+    });
+
+    it("should stay collapsed until the toggle is switched on", async () => {
+      const user = userEvent.setup();
+      renderSettings();
+      expect(await screen.findByText("Enable Semantic Caching")).toBeInTheDocument();
+      expect(screen.queryByText("Similarity Threshold")).not.toBeInTheDocument();
+
+      await user.click(screen.getByRole("switch"));
+      expect(await screen.findByText("Similarity Threshold")).toBeInTheDocument();
+      expect(screen.getByText("Embedding Model")).toBeInTheDocument();
+    });
+
+    it("should reveal the semantic fields for a config saved under the old redis_type", async () => {
       getCacheSettingsCall.mockResolvedValue({ current_values: { redis_type: "semantic" } });
       renderSettings();
       expect(await screen.findByText("Similarity Threshold")).toBeInTheDocument();
       expect(screen.getByText("Embedding Model")).toBeInTheDocument();
     });
+
+    it("should reveal the semantic fields when only the semantic values are set", async () => {
+      getCacheSettingsCall.mockResolvedValue({ current_values: { redis_type: "node", similarity_threshold: 0.9 } });
+      renderSettings();
+      expect(await screen.findByText("Similarity Threshold")).toBeInTheDocument();
+    });
+
+    it("should load a saved redis-semantic config as a node with the toggle on", async () => {
+      getCacheSettingsCall.mockResolvedValue({ current_values: { type: "redis-semantic", host: "localhost" } });
+      renderSettings();
+      expect(await screen.findByText("Similarity Threshold")).toBeInTheDocument();
+      expect(screen.getByText("Node (Single Instance)")).toBeInTheDocument();
+      expect(screen.getByRole("switch")).not.toHaveAttribute("data-disabled");
+    });
+
+    it.each([
+      ["cluster", "Startup Nodes"],
+      ["sentinel", "Sentinel Nodes"],
+    ])("should disable the toggle and hide the semantic fields for %s", async (redisType, topologyField) => {
+      getCacheSettingsCall.mockResolvedValue({ current_values: { redis_type: redisType, similarity_threshold: 0.9 } });
+      renderSettings();
+      expect(await screen.findByText(topologyField)).toBeInTheDocument();
+      expect(screen.getByRole("switch")).toHaveAttribute("data-disabled");
+      expect(screen.getByText(/Semantic caching needs a single Redis node/)).toBeInTheDocument();
+      expect(screen.queryByText("Similarity Threshold")).not.toBeInTheDocument();
+    });
   });
 
   describe("when a field fails inline validation", () => {

ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx

@@ -6,6 +6,7 @@ import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/component
 import { getCacheSettingsCall, testCacheConnectionCall, updateCacheSettingsCall } from "@/components/networking";
 import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models";
 import { toast } from "@/lib/toast";
+import { Switch } from "@/components/ui/switch";
 import RedisTypeSelector from "./RedisTypeSelector";
 import CacheFieldSection from "./CacheFieldSection";
 import { EmbeddingModelOption } from "./CacheFormField";
@@ -16,6 +17,7 @@ import {
   CacheFormValues,
   configuredSecretFields,
   isFieldVisible,
+  supportsSemanticCache,
 } from "./cacheSettingsUtils";
 
 const ADVANCED_SECTIONS = ["ssl", "cacheManagement", "gcp"] as const;
@@ -37,6 +39,9 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
   const [isTesting, setIsTesting] = useState<boolean>(false);
   const [isSaving, setIsSaving] = useState<boolean>(false);
   const [configuredSecrets, setConfiguredSecrets] = useState<ReadonlySet<string>>(new Set());
+  const [semanticEnabled, setSemanticEnabled] = useState<boolean>(false);
+  const semanticAvailable = supportsSemanticCache(redisType);
+  const semanticActive = semanticEnabled && semanticAvailable;
 
   const loadCacheSettings = useCallback(async () => {
     if (!accessToken) {
@@ -48,6 +53,12 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
       form.reset(buildInitialValues(currentValues));
       setConfiguredSecrets(configuredSecretFields(currentValues));
       setRedisType(toRedisType(currentValues.redis_type));
+      setSemanticEnabled(
+        currentValues.type === "redis-semantic" ||
+          currentValues.redis_type === "semantic" ||
+          currentValues.similarity_threshold != null ||
+          currentValues.redis_semantic_cache_embedding_model != null,
+      );
     } catch (error) {
       console.error("Failed to load cache settings:", error);
       toast.fromError("Failed to load cache settings");
@@ -102,7 +113,7 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
     try {
       const result = await testCacheConnectionCall(
         accessToken,
-        buildCachePayload(redisType, values, { forTesting: true }),
+        buildCachePayload(redisType, values, { forTesting: true, semanticEnabled: semanticActive }),
       );
       if (result.status === "success") {
         toast.success("Cache connection test successful!");
@@ -128,7 +139,10 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
 
     setIsSaving(true);
     try {
-      await updateCacheSettingsCall(accessToken, buildCachePayload(redisType, values, { forTesting: false }));
+      await updateCacheSettingsCall(
+        accessToken,
+        buildCachePayload(redisType, values, { forTesting: false, semanticEnabled: semanticActive }),
+      );
       toast.success("Cache settings updated successfully");
       await loadCacheSettings();
     } catch (error) {
@@ -192,16 +206,27 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
             </div>
           )}
 
-          {redisType === "semantic" && (
-            <div className="pt-4 border-t border-border">
+          <div className="pt-4 border-t border-border">
+            <div className="mb-4 flex items-center gap-3">
+              <Switch checked={semanticActive} disabled={!semanticAvailable} onCheckedChange={setSemanticEnabled} />
+              <div>
+                <span className="text-sm font-medium text-foreground">Enable Semantic Caching</span>
+                <p className="text-xs text-muted-foreground">
+                  {semanticAvailable
+                    ? "Reuse responses for semantically similar prompts using embedding vectors"
+                    : "Semantic caching needs a single Redis node, so it is unavailable for Cluster and Sentinel"}
+                </p>
+              </div>
+            </div>
+            {semanticActive && (
               <CacheFieldSection
                 title="Semantic Configuration"
                 section="semantic"
                 redisType={redisType}
                 embeddingModels={embeddingModels}
               />
-            </div>
-          )}
+            )}
+          </div>
 
           <Collapsible open={advancedOpen} onOpenChange={setAdvancedOpen} className="mt-4">
             <CollapsibleTrigger className="group flex w-full items-center justify-between py-2 text-left">