diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 2448348bbce..a551c9a61d6 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -53,11 +53,6 @@ "count": 1 } }, - "src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/caching/_components/cache_settings/index.tsx": { "no-restricted-imports": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFieldSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFieldSection.tsx index ced822cd796..c4f88154775 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFieldSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFieldSection.tsx @@ -1,11 +1,12 @@ import React from "react"; import CacheFormField, { EmbeddingModelOption } from "./CacheFormField"; import { fieldsForSection } from "./cacheSettingsUtils"; -import { CacheSection, RedisType } from "./cacheSettingsFields"; +import { CacheSection, CacheType, RedisType } from "./cacheSettingsFields"; interface CacheFieldSectionProps { title: string; section: CacheSection; + cacheType: CacheType; redisType: RedisType; embeddingModels: EmbeddingModelOption[]; gridCols?: string; @@ -15,12 +16,13 @@ interface CacheFieldSectionProps { const CacheFieldSection: React.FC = ({ title, section, + cacheType, redisType, embeddingModels, gridCols = "grid-cols-1 gap-6 sm:grid-cols-2", headingLevel = "h4", }) => { - const fields = fieldsForSection(section, redisType); + const fields = fieldsForSection(section, cacheType, redisType); if (fields.length === 0) { return null; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheOptionSelector.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheOptionSelector.test.tsx new file mode 100644 index 00000000000..089d248a18b --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheOptionSelector.test.tsx @@ -0,0 +1,48 @@ +import { describe, expect, it, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import CacheOptionSelector from "./CacheOptionSelector"; + +const OPTIONS = [ + { value: "node", label: "Node (Single Instance)" }, + { value: "cluster", label: "Cluster" }, +] as const; + +const DESCRIPTIONS = { node: "single instance", cluster: "cluster mode" }; + +describe("CacheOptionSelector", () => { + it("should show the description for the selected value", () => { + render( + {}} + />, + ); + expect(screen.getByText("cluster mode")).toBeInTheDocument(); + expect(screen.getByText("Redis Type")).toBeInTheDocument(); + }); + + it("should emit the chosen option value on selection", async () => { + const user = userEvent.setup(); + const onChange = vi.fn(); + render( + , + ); + + await user.click(document.querySelector(".ant-select-selector") as HTMLElement); + await user.click(await screen.findByText("Cluster")); + + expect(onChange).toHaveBeenCalledWith("cluster"); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheOptionSelector.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheOptionSelector.tsx new file mode 100644 index 00000000000..272a2922b68 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheOptionSelector.tsx @@ -0,0 +1,38 @@ +import React from "react"; +import { Select } from "antd"; + +export interface CacheSelectOption { + readonly value: string; + readonly label: string; +} + +interface CacheOptionSelectorProps { + label: string; + value: string; + options: readonly CacheSelectOption[]; + descriptions: Readonly>; + fallbackDescription: string; + onChange: (value: string) => void; +} + +const CacheOptionSelector: React.FC = ({ + label, + value, + options, + descriptions, + fallbackDescription, + onChange, +}) => ( +
+ + - Node (Single Instance) - Cluster - Sentinel - Semantic - -

- {redisTypeDescriptions[redisType] || "Select the type of Redis deployment you're using"} -

-
- ); -}; - -export default RedisTypeSelector; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts index 1f5b566fc5f..ec110bf447f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts @@ -2,7 +2,9 @@ import type { FormItemProps } from "antd"; export type CacheFieldType = "string" | "password" | "integer" | "float" | "boolean" | "list" | "model-select"; -export type RedisType = "node" | "cluster" | "sentinel" | "semantic"; +export type CacheType = "standard" | "semantic"; + +export type RedisType = "node" | "cluster" | "sentinel"; export type CacheSection = "connection" | "cluster" | "sentinel" | "semantic" | "ssl" | "cacheManagement" | "gcp"; @@ -15,17 +17,35 @@ export interface CacheField { readonly section: CacheSection; readonly helpText: string; readonly redisType: RedisType | null; + readonly cacheType?: CacheType; readonly defaultValue?: string | number | boolean; readonly rules?: CacheFieldRule[]; } -export const REDIS_TYPES: readonly RedisType[] = ["node", "cluster", "sentinel", "semantic"]; +export const CACHE_TYPES: readonly CacheType[] = ["standard", "semantic"]; + +export const CACHE_TYPE_LABELS: Readonly> = { + standard: "Standard", + semantic: "Semantic", +}; + +export const CACHE_TYPE_DESCRIPTIONS: Readonly> = { + standard: "Exact-match caching keyed on the request", + semantic: "Reuses responses for semantically similar prompts using embedding vectors", +}; + +export const REDIS_TYPES: readonly RedisType[] = ["node", "cluster", "sentinel"]; + +export const REDIS_TYPE_LABELS: Readonly> = { + node: "Node (Single Instance)", + cluster: "Cluster", + sentinel: "Sentinel", +}; export const REDIS_TYPE_DESCRIPTIONS: Readonly> = { node: "Standard Redis node/single instance", cluster: "Redis Cluster mode for high availability and horizontal scaling", sentinel: "Redis Sentinel mode for high availability with automatic failover", - semantic: "Semantic caching that reuses responses for similar prompts", }; const portRule: CacheFieldRule = { @@ -177,7 +197,8 @@ export const CACHE_FIELDS: readonly CacheField[] = [ type: "float", section: "semantic", helpText: "Similarity threshold for semantic cache", - redisType: "semantic", + redisType: null, + cacheType: "semantic", defaultValue: 0.8, rules: [numberRule], }, @@ -187,7 +208,8 @@ export const CACHE_FIELDS: readonly CacheField[] = [ type: "model-select", section: "semantic", helpText: "Embedding model for semantic cache", - redisType: "semantic", + redisType: null, + cacheType: "semantic", }, { name: "ssl", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts index 79f28a97842..b3c65a2a8c6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts @@ -2,13 +2,21 @@ import { describe, it, expect } from "vitest"; import { buildCachePayload, buildInitialValues, fieldsForSection } from "./cacheSettingsUtils"; describe("fieldsForSection", () => { - it("should only include a redis-type-specific field when that type is selected", () => { - expect(fieldsForSection("cluster", "cluster").map((f) => f.name)).toEqual(["redis_startup_nodes"]); - expect(fieldsForSection("cluster", "node")).toEqual([]); + it("should only include a redis-type-specific field when that topology is selected", () => { + expect(fieldsForSection("cluster", "standard", "cluster").map((f) => f.name)).toEqual(["redis_startup_nodes"]); + expect(fieldsForSection("cluster", "standard", "node")).toEqual([]); + }); + + it("should only include semantic fields when the cache type is semantic", () => { + expect(fieldsForSection("semantic", "semantic", "node").map((f) => f.name)).toEqual([ + "similarity_threshold", + "redis_semantic_cache_embedding_model", + ]); + expect(fieldsForSection("semantic", "standard", "node")).toEqual([]); }); it("should include connection fields for every redis type in schema order", () => { - expect(fieldsForSection("connection", "node").map((f) => f.name)).toEqual([ + expect(fieldsForSection("connection", "standard", "node").map((f) => f.name)).toEqual([ "url", "host", "port", @@ -42,7 +50,12 @@ describe("buildInitialValues", () => { describe("buildCachePayload", () => { it("should tag the payload as redis and drop empty fields and the UI-only redis_type", () => { - const payload = buildCachePayload("node", { host: "localhost", port: "6379", username: "" }, { forTesting: false }); + const payload = buildCachePayload( + "standard", + "node", + { host: "localhost", port: "6379", username: "" }, + { forTesting: false }, + ); expect(payload).toEqual({ type: "redis", host: "localhost", @@ -56,6 +69,7 @@ describe("buildCachePayload", () => { it("should parse list fields from their textarea string into arrays", () => { const payload = buildCachePayload( + "standard", "cluster", { redis_startup_nodes: '[{"host":"127.0.0.1","port":"7001"}]' }, { forTesting: false }, @@ -64,23 +78,38 @@ describe("buildCachePayload", () => { }); it("should omit a list field whose textarea holds invalid JSON", () => { - const payload = buildCachePayload("cluster", { redis_startup_nodes: "not json" }, { forTesting: false }); + const payload = buildCachePayload( + "standard", + "cluster", + { redis_startup_nodes: "not json" }, + { forTesting: false }, + ); expect(payload).not.toHaveProperty("redis_startup_nodes"); }); it("should send type redis-semantic when saving a semantic cache", () => { - const payload = buildCachePayload("semantic", { similarity_threshold: 0.9 }, { forTesting: false }); + const payload = buildCachePayload("semantic", "node", { similarity_threshold: 0.9 }, { forTesting: false }); expect(payload.type).toBe("redis-semantic"); expect(payload.similarity_threshold).toBe(0.9); }); it("should keep type redis when testing a semantic cache so the test endpoint accepts it", () => { - const payload = buildCachePayload("semantic", { similarity_threshold: 0.9 }, { forTesting: true }); + const payload = buildCachePayload("semantic", "node", { similarity_threshold: 0.9 }, { forTesting: true }); expect(payload.type).toBe("redis"); }); - it("should exclude fields that do not belong to the selected redis type", () => { - const payload = buildCachePayload("node", { sentinel_nodes: '[["localhost",26379]]' }, { forTesting: false }); + it("should exclude topology fields that do not belong to the selected redis type", () => { + const payload = buildCachePayload( + "standard", + "node", + { sentinel_nodes: '[["localhost",26379]]' }, + { forTesting: false }, + ); expect(payload).not.toHaveProperty("sentinel_nodes"); }); + + it("should exclude semantic fields when the cache type is standard", () => { + const payload = buildCachePayload("standard", "node", { similarity_threshold: 0.9 }, { forTesting: false }); + expect(payload).not.toHaveProperty("similarity_threshold"); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts index 088da21961c..b5d51245c41 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts @@ -1,15 +1,18 @@ -import { CACHE_FIELDS, CacheField, CacheSection, RedisType } from "./cacheSettingsFields"; +import { CACHE_FIELDS, CacheField, CacheSection, CacheType, RedisType } from "./cacheSettingsFields"; export type CacheFormValue = string | number | boolean | undefined; export type CacheFormValues = Record; export type CacheSavePayloadValue = string | number | boolean | unknown[]; export type CacheSavePayload = Record; -export const isFieldVisible = (field: CacheField, redisType: RedisType): boolean => - field.redisType === null || field.redisType === redisType; +export const isFieldVisible = (field: CacheField, cacheType: CacheType, redisType: RedisType): boolean => { + const matchesCacheType = field.cacheType === undefined || field.cacheType === cacheType; + const matchesRedisType = field.redisType === null || field.redisType === redisType; + return matchesCacheType && matchesRedisType; +}; -export const fieldsForSection = (section: CacheSection, redisType: RedisType): CacheField[] => - CACHE_FIELDS.filter((field) => field.section === section && isFieldVisible(field, redisType)); +export const fieldsForSection = (section: CacheSection, cacheType: CacheType, redisType: RedisType): CacheField[] => + CACHE_FIELDS.filter((field) => field.section === section && isFieldVisible(field, cacheType, redisType)); const initialValueForField = (field: CacheField, raw: unknown): CacheFormValue => { const source = raw ?? field.defaultValue; @@ -66,13 +69,14 @@ const saveValueForField = (field: CacheField, raw: CacheFormValue): CacheSavePay }; export const buildCachePayload = ( + cacheType: CacheType, redisType: RedisType, values: CacheFormValues, { forTesting }: { forTesting: boolean }, ): CacheSavePayload => { - const type = !forTesting && redisType === "semantic" ? "redis-semantic" : "redis"; + const type = !forTesting && cacheType === "semantic" ? "redis-semantic" : "redis"; - const entries = CACHE_FIELDS.filter((field) => isFieldVisible(field, redisType)).flatMap((field) => { + const entries = CACHE_FIELDS.filter((field) => isFieldVisible(field, cacheType, redisType)).flatMap((field) => { const value = saveValueForField(field, values[field.name]); return value === undefined ? [] : [[field.name, value] as const]; }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.test.tsx index 0f768372ad9..d7ad744c950 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.test.tsx @@ -63,13 +63,38 @@ describe("CacheSettings", () => { }); }); - describe("when the redis type is semantic", () => { - it("should reveal the semantic fields", async () => { - getCacheSettingsCall.mockResolvedValue({ current_values: { redis_type: "semantic" } }); + describe("when the cache type is semantic", () => { + it("should reveal the semantic fields when the stored type is redis-semantic", async () => { + getCacheSettingsCall.mockResolvedValue({ current_values: { type: "redis-semantic" } }); renderSettings(); expect(await screen.findByText("Similarity Threshold")).toBeInTheDocument(); expect(screen.getByText("Embedding Model")).toBeInTheDocument(); }); + + it("should hide the Redis Type topology selector because semantic only supports a node connection", async () => { + getCacheSettingsCall.mockResolvedValue({ current_values: { type: "redis-semantic" } }); + renderSettings(); + expect(await screen.findByText("Cache Type")).toBeInTheDocument(); + expect(screen.queryByText("Redis Type")).not.toBeInTheDocument(); + }); + }); + + describe("cache strategy and redis topology are separate dimensions", () => { + it("should move Semantic out of the Redis Type dropdown into its own Cache Type selector", async () => { + const user = userEvent.setup(); + renderSettings(); + + expect(await screen.findByText("Redis Type")).toBeInTheDocument(); + expect(screen.queryByText("Similarity Threshold")).not.toBeInTheDocument(); + + const cacheTypeSelect = screen.getByText("Cache Type").parentElement?.querySelector(".ant-select-selector"); + expect(cacheTypeSelect).not.toBeNull(); + await user.click(cacheTypeSelect as HTMLElement); + await user.click(await screen.findByText("Semantic")); + + await waitFor(() => expect(screen.queryByText("Redis Type")).not.toBeInTheDocument()); + expect(screen.getByText("Similarity Threshold")).toBeInTheDocument(); + }); }); describe("when a field fails inline validation", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx index 4382769ae9c..f51ed3aa7c5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx @@ -4,10 +4,19 @@ import { Form } from "antd"; import { getCacheSettingsCall, testCacheConnectionCall, updateCacheSettingsCall } from "@/components/networking"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; import NotificationsManager from "@/components/molecules/notifications_manager"; -import RedisTypeSelector from "./RedisTypeSelector"; +import CacheOptionSelector from "./CacheOptionSelector"; import CacheFieldSection from "./CacheFieldSection"; import { EmbeddingModelOption } from "./CacheFormField"; -import { REDIS_TYPES, REDIS_TYPE_DESCRIPTIONS, RedisType } from "./cacheSettingsFields"; +import { + CACHE_TYPES, + CACHE_TYPE_DESCRIPTIONS, + CACHE_TYPE_LABELS, + CacheType, + REDIS_TYPES, + REDIS_TYPE_DESCRIPTIONS, + REDIS_TYPE_LABELS, + RedisType, +} from "./cacheSettingsFields"; import { buildCachePayload, buildInitialValues, CacheFormValues } from "./cacheSettingsUtils"; interface CacheSettingsProps { @@ -19,8 +28,14 @@ interface CacheSettingsProps { const toRedisType = (value: unknown): RedisType => REDIS_TYPES.includes(value as RedisType) ? (value as RedisType) : "node"; +const toCacheType = (value: unknown): CacheType => (value === "redis-semantic" ? "semantic" : "standard"); + +const CACHE_TYPE_OPTIONS = CACHE_TYPES.map((value) => ({ value, label: CACHE_TYPE_LABELS[value] })); +const REDIS_TYPE_OPTIONS = REDIS_TYPES.map((value) => ({ value, label: REDIS_TYPE_LABELS[value] })); + const CacheSettings: React.FC = ({ accessToken }) => { const [form] = Form.useForm(); + const [cacheType, setCacheType] = useState("standard"); const [redisType, setRedisType] = useState("node"); const [embeddingModels, setEmbeddingModels] = useState([]); const [isTesting, setIsTesting] = useState(false); @@ -34,6 +49,7 @@ const CacheSettings: React.FC = ({ accessToken }) => { const data = (await getCacheSettingsCall(accessToken)) as { current_values?: Record }; const currentValues = data.current_values ?? {}; form.setFieldsValue(buildInitialValues(currentValues)); + setCacheType(toCacheType(currentValues.type)); setRedisType(toRedisType(currentValues.redis_type)); } catch (error) { console.error("Failed to load cache settings:", error); @@ -81,7 +97,7 @@ const CacheSettings: React.FC = ({ accessToken }) => { try { const result = await testCacheConnectionCall( accessToken, - buildCachePayload(redisType, values, { forTesting: true }), + buildCachePayload(cacheType, redisType, values, { forTesting: true }), ); if (result.status === "success") { NotificationsManager.success("Cache connection test successful!"); @@ -109,7 +125,10 @@ const CacheSettings: React.FC = ({ accessToken }) => { setIsSaving(true); try { - await updateCacheSettingsCall(accessToken, buildCachePayload(redisType, values, { forTesting: false })); + await updateCacheSettingsCall( + accessToken, + buildCachePayload(cacheType, redisType, values, { forTesting: false }), + ); NotificationsManager.success("Cache settings updated successfully"); await loadCacheSettings(); } catch (error) { @@ -132,26 +151,42 @@ const CacheSettings: React.FC = ({ accessToken }) => {

Configure Redis cache for LiteLLM

- setRedisType(toRedisType(type))} + setCacheType(type === "semantic" ? "semantic" : "standard")} /> + {cacheType === "standard" && ( + setRedisType(toRedisType(type))} + /> + )} +
- {redisType === "cluster" && ( + {cacheType === "standard" && redisType === "cluster" && (
= ({ accessToken }) => {
)} - {redisType === "sentinel" && ( + {cacheType === "standard" && redisType === "sentinel" && (
)} - {redisType === "semantic" && ( + {cacheType === "semantic" && (
@@ -190,6 +227,7 @@ const CacheSettings: React.FC = ({ accessToken }) => { = ({ accessToken }) => { = ({ accessToken }) => {