scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

fix(ui): separate semantic cache strategy from Redis topology dropdown

Browse source 
Semantic caching is a retrieval strategy, not a Redis deployment topology,
so listing it in the Cache Settings 'Redis Type' dropdown alongside Node,
Cluster, and Sentinel confused users (issue #32621). Split the two dimensions
into a 'Cache Type' selector (Standard vs Semantic) and a topology-only
'Redis Type' selector that only shows for standard caches, matching what the
backend actually supports (redis-semantic uses a plain node connection)
This commit is contained in:
Devin AI 2026-07-10 23:22:05 +00:00
parent 4e3c437631
commit 3b313de223
11 changed files with 246 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
ui/litellm-dashboard/eslint-suppressions.json
View file

@ -53,11 +53,6 @@
"count": 1
}
},
"src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.tsx": {
"no-restricted-imports": {
"count": 1
}
},
"src/app/(dashboard)/caching/_components/cache_settings/index.tsx": {
"no-restricted-imports": {
"count": 1

6
ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFieldSection.tsx
View file

@ -1,11 +1,12 @@
import React from "react";
import CacheFormField, { EmbeddingModelOption } from "./CacheFormField";
import { fieldsForSection } from "./cacheSettingsUtils";
import { CacheSection, RedisType } from "./cacheSettingsFields";
import { CacheSection, CacheType, RedisType } from "./cacheSettingsFields";
interface CacheFieldSectionProps {
title: string;
section: CacheSection;
cacheType: CacheType;
redisType: RedisType;
embeddingModels: EmbeddingModelOption[];
gridCols?: string;
@ -15,12 +16,13 @@ interface CacheFieldSectionProps {
const CacheFieldSection: React.FC<CacheFieldSectionProps> = ({
title,
section,
cacheType,
redisType,
embeddingModels,
gridCols = "grid-cols-1 gap-6 sm:grid-cols-2",
headingLevel = "h4",
}) => {
const fields = fieldsForSection(section, redisType);
const fields = fieldsForSection(section, cacheType, redisType);
if (fields.length === 0) {
return null;
}

48
ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheOptionSelector.test.tsx Normal file
View file

@ -0,0 +1,48 @@
import { describe, expect, it, vi } from "vitest";
import { render, screen } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import CacheOptionSelector from "./CacheOptionSelector";
const OPTIONS = [
{ value: "node", label: "Node (Single Instance)" },
{ value: "cluster", label: "Cluster" },
] as const;
const DESCRIPTIONS = { node: "single instance", cluster: "cluster mode" };
describe("CacheOptionSelector", () => {
it("should show the description for the selected value", () => {
render(
<CacheOptionSelector
label="Redis Type"
value="cluster"
options={OPTIONS}
descriptions={DESCRIPTIONS}
fallbackDescription="fallback"
onChange={() => {}}
/>,
);
expect(screen.getByText("cluster mode")).toBeInTheDocument();
expect(screen.getByText("Redis Type")).toBeInTheDocument();
});
it("should emit the chosen option value on selection", async () => {
const user = userEvent.setup();
const onChange = vi.fn();
render(
<CacheOptionSelector
label="Redis Type"
value="node"
options={OPTIONS}
descriptions={DESCRIPTIONS}
fallbackDescription="fallback"
onChange={onChange}
/>,
);
await user.click(document.querySelector(".ant-select-selector") as HTMLElement);
await user.click(await screen.findByText("Cluster"));
expect(onChange).toHaveBeenCalledWith("cluster");
});
});

38
ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheOptionSelector.tsx Normal file
View file

@ -0,0 +1,38 @@
import React from "react";
import { Select } from "antd";
export interface CacheSelectOption {
readonly value: string;
readonly label: string;
}
interface CacheOptionSelectorProps {
label: string;
value: string;
options: readonly CacheSelectOption[];
descriptions: Readonly<Record<string, string>>;
fallbackDescription: string;
onChange: (value: string) => void;
}
const CacheOptionSelector: React.FC<CacheOptionSelectorProps> = ({
label,
value,
options,
descriptions,
fallbackDescription,
onChange,
}) => (
<div className="space-y-2">
<label className="text-sm font-medium text-gray-700">{label}</label>
<Select
value={value}
onChange={(next) => onChange(next)}
style={{ width: "100%" }}
options={options.map((option) => ({ value: option.value, label: option.label }))}
/>
<p className="text-xs text-gray-500">{descriptions[value] || fallbackDescription}</p>
</div>
);
export default CacheOptionSelector;

12
ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.test.tsx
View file

@ -1,12 +0,0 @@
import { describe, expect, it } from "vitest";
import RedisTypeSelector from "./RedisTypeSelector";
import { render } from "@testing-library/react";
describe("RedisTypeSelector", () => {
it("should render the component", () => {
const { getAllByText } = render(
<RedisTypeSelector redisType="redis" redisTypeDescriptions={{}} onTypeChange={() => {}} />,
);
expect(getAllByText(/Redis/i).length).toBeGreaterThan(0);
});
});

27
ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.tsx
View file

@ -1,27 +0,0 @@
import React from "react";
import { Select, SelectItem } from "@tremor/react";
interface RedisTypeSelectorProps {
redisType: string;
redisTypeDescriptions: Readonly<Record<string, string>>;
onTypeChange: (type: string) => void;
}
const RedisTypeSelector: React.FC<RedisTypeSelectorProps> = ({ redisType, redisTypeDescriptions, onTypeChange }) => {
return (
<div className="space-y-2">
<label className="text-sm font-medium text-gray-700">Redis Type</label>
<Select value={redisType} onValueChange={onTypeChange}>
<SelectItem value="node">Node (Single Instance)</SelectItem>
<SelectItem value="cluster">Cluster</SelectItem>
<SelectItem value="sentinel">Sentinel</SelectItem>
<SelectItem value="semantic">Semantic</SelectItem>
</Select>
<p className="text-xs text-gray-500">
{redisTypeDescriptions[redisType] || "Select the type of Redis deployment you're using"}
</p>
</div>
);
};
export default RedisTypeSelector;

32
ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts
View file

@ -2,7 +2,9 @@ import type { FormItemProps } from "antd";
export type CacheFieldType = "string" | "password" | "integer" | "float" | "boolean" | "list" | "model-select";
export type RedisType = "node" | "cluster" | "sentinel" | "semantic";
export type CacheType = "standard" | "semantic";
export type RedisType = "node" | "cluster" | "sentinel";
export type CacheSection = "connection" | "cluster" | "sentinel" | "semantic" | "ssl" | "cacheManagement" | "gcp";
@ -15,17 +17,35 @@ export interface CacheField {
readonly section: CacheSection;
readonly helpText: string;
readonly redisType: RedisType | null;
readonly cacheType?: CacheType;
readonly defaultValue?: string | number | boolean;
readonly rules?: CacheFieldRule[];
}
export const REDIS_TYPES: readonly RedisType[] = ["node", "cluster", "sentinel", "semantic"];
export const CACHE_TYPES: readonly CacheType[] = ["standard", "semantic"];
export const CACHE_TYPE_LABELS: Readonly<Record<CacheType, string>> = {
standard: "Standard",
semantic: "Semantic",
};
export const CACHE_TYPE_DESCRIPTIONS: Readonly<Record<CacheType, string>> = {
standard: "Exact-match caching keyed on the request",
semantic: "Reuses responses for semantically similar prompts using embedding vectors",
};
export const REDIS_TYPES: readonly RedisType[] = ["node", "cluster", "sentinel"];
export const REDIS_TYPE_LABELS: Readonly<Record<RedisType, string>> = {
node: "Node (Single Instance)",
cluster: "Cluster",
sentinel: "Sentinel",
};
export const REDIS_TYPE_DESCRIPTIONS: Readonly<Record<RedisType, string>> = {
node: "Standard Redis node/single instance",
cluster: "Redis Cluster mode for high availability and horizontal scaling",
sentinel: "Redis Sentinel mode for high availability with automatic failover",
semantic: "Semantic caching that reuses responses for similar prompts",
};
const portRule: CacheFieldRule = {
@ -177,7 +197,8 @@ export const CACHE_FIELDS: readonly CacheField[] = [
type: "float",
section: "semantic",
helpText: "Similarity threshold for semantic cache",
redisType: "semantic",
redisType: null,
cacheType: "semantic",
defaultValue: 0.8,
rules: [numberRule],
},
@ -187,7 +208,8 @@ export const CACHE_FIELDS: readonly CacheField[] = [
type: "model-select",
section: "semantic",
helpText: "Embedding model for semantic cache",
redisType: "semantic",
redisType: null,
cacheType: "semantic",
},
{
name: "ssl",

49
ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts
View file

@ -2,13 +2,21 @@ import { describe, it, expect } from "vitest";
import { buildCachePayload, buildInitialValues, fieldsForSection } from "./cacheSettingsUtils";
describe("fieldsForSection", () => {
it("should only include a redis-type-specific field when that type is selected", () => {
expect(fieldsForSection("cluster", "cluster").map((f) => f.name)).toEqual(["redis_startup_nodes"]);
expect(fieldsForSection("cluster", "node")).toEqual([]);
it("should only include a redis-type-specific field when that topology is selected", () => {
expect(fieldsForSection("cluster", "standard", "cluster").map((f) => f.name)).toEqual(["redis_startup_nodes"]);
expect(fieldsForSection("cluster", "standard", "node")).toEqual([]);
});
it("should only include semantic fields when the cache type is semantic", () => {
expect(fieldsForSection("semantic", "semantic", "node").map((f) => f.name)).toEqual([
"similarity_threshold",
"redis_semantic_cache_embedding_model",
]);
expect(fieldsForSection("semantic", "standard", "node")).toEqual([]);
});
it("should include connection fields for every redis type in schema order", () => {
expect(fieldsForSection("connection", "node").map((f) => f.name)).toEqual([
expect(fieldsForSection("connection", "standard", "node").map((f) => f.name)).toEqual([
"url",
"host",
"port",
@ -42,7 +50,12 @@ describe("buildInitialValues", () => {
describe("buildCachePayload", () => {
it("should tag the payload as redis and drop empty fields and the UI-only redis_type", () => {
const payload = buildCachePayload("node", { host: "localhost", port: "6379", username: "" }, { forTesting: false });
const payload = buildCachePayload(
"standard",
"node",
{ host: "localhost", port: "6379", username: "" },
{ forTesting: false },
);
expect(payload).toEqual({
type: "redis",
host: "localhost",
@ -56,6 +69,7 @@ describe("buildCachePayload", () => {
it("should parse list fields from their textarea string into arrays", () => {
const payload = buildCachePayload(
"standard",
"cluster",
{ redis_startup_nodes: '[{"host":"127.0.0.1","port":"7001"}]' },
{ forTesting: false },
@ -64,23 +78,38 @@ describe("buildCachePayload", () => {
});
it("should omit a list field whose textarea holds invalid JSON", () => {
const payload = buildCachePayload("cluster", { redis_startup_nodes: "not json" }, { forTesting: false });
const payload = buildCachePayload(
"standard",
"cluster",
{ redis_startup_nodes: "not json" },
{ forTesting: false },
);
expect(payload).not.toHaveProperty("redis_startup_nodes");
});
it("should send type redis-semantic when saving a semantic cache", () => {
const payload = buildCachePayload("semantic", { similarity_threshold: 0.9 }, { forTesting: false });
const payload = buildCachePayload("semantic", "node", { similarity_threshold: 0.9 }, { forTesting: false });
expect(payload.type).toBe("redis-semantic");
expect(payload.similarity_threshold).toBe(0.9);
});
it("should keep type redis when testing a semantic cache so the test endpoint accepts it", () => {
const payload = buildCachePayload("semantic", { similarity_threshold: 0.9 }, { forTesting: true });
const payload = buildCachePayload("semantic", "node", { similarity_threshold: 0.9 }, { forTesting: true });
expect(payload.type).toBe("redis");
});
it("should exclude fields that do not belong to the selected redis type", () => {
const payload = buildCachePayload("node", { sentinel_nodes: '[["localhost",26379]]' }, { forTesting: false });
it("should exclude topology fields that do not belong to the selected redis type", () => {
const payload = buildCachePayload(
"standard",
"node",
{ sentinel_nodes: '[["localhost",26379]]' },
{ forTesting: false },
);
expect(payload).not.toHaveProperty("sentinel_nodes");
});
it("should exclude semantic fields when the cache type is standard", () => {
const payload = buildCachePayload("standard", "node", { similarity_threshold: 0.9 }, { forTesting: false });
expect(payload).not.toHaveProperty("similarity_threshold");
});
});

18
ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts
View file

@ -1,15 +1,18 @@
import { CACHE_FIELDS, CacheField, CacheSection, RedisType } from "./cacheSettingsFields";
import { CACHE_FIELDS, CacheField, CacheSection, CacheType, RedisType } from "./cacheSettingsFields";
export type CacheFormValue = string | number | boolean | undefined;
export type CacheFormValues = Record<string, CacheFormValue>;
export type CacheSavePayloadValue = string | number | boolean | unknown[];
export type CacheSavePayload = Record<string, CacheSavePayloadValue>;
export const isFieldVisible = (field: CacheField, redisType: RedisType): boolean =>
field.redisType === null || field.redisType === redisType;
export const isFieldVisible = (field: CacheField, cacheType: CacheType, redisType: RedisType): boolean => {
const matchesCacheType = field.cacheType === undefined || field.cacheType === cacheType;
const matchesRedisType = field.redisType === null || field.redisType === redisType;
return matchesCacheType && matchesRedisType;
};
export const fieldsForSection = (section: CacheSection, redisType: RedisType): CacheField[] =>
CACHE_FIELDS.filter((field) => field.section === section && isFieldVisible(field, redisType));
export const fieldsForSection = (section: CacheSection, cacheType: CacheType, redisType: RedisType): CacheField[] =>
CACHE_FIELDS.filter((field) => field.section === section && isFieldVisible(field, cacheType, redisType));
const initialValueForField = (field: CacheField, raw: unknown): CacheFormValue => {
const source = raw ?? field.defaultValue;
@ -66,13 +69,14 @@ const saveValueForField = (field: CacheField, raw: CacheFormValue): CacheSavePay
};
export const buildCachePayload = (
cacheType: CacheType,
redisType: RedisType,
values: CacheFormValues,
{ forTesting }: { forTesting: boolean },
): CacheSavePayload => {
const type = !forTesting && redisType === "semantic" ? "redis-semantic" : "redis";
const type = !forTesting && cacheType === "semantic" ? "redis-semantic" : "redis";
const entries = CACHE_FIELDS.filter((field) => isFieldVisible(field, redisType)).flatMap((field) => {
const entries = CACHE_FIELDS.filter((field) => isFieldVisible(field, cacheType, redisType)).flatMap((field) => {
const value = saveValueForField(field, values[field.name]);
return value === undefined ? [] : [[field.name, value] as const];
});

31
ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.test.tsx
View file

@ -63,13 +63,38 @@ describe("CacheSettings", () => {
});
});
describe("when the redis type is semantic", () => {
it("should reveal the semantic fields", async () => {
getCacheSettingsCall.mockResolvedValue({ current_values: { redis_type: "semantic" } });
describe("when the cache type is semantic", () => {
it("should reveal the semantic fields when the stored type is redis-semantic", async () => {
getCacheSettingsCall.mockResolvedValue({ current_values: { type: "redis-semantic" } });
renderSettings();
expect(await screen.findByText("Similarity Threshold")).toBeInTheDocument();
expect(screen.getByText("Embedding Model")).toBeInTheDocument();
});
it("should hide the Redis Type topology selector because semantic only supports a node connection", async () => {
getCacheSettingsCall.mockResolvedValue({ current_values: { type: "redis-semantic" } });
renderSettings();
expect(await screen.findByText("Cache Type")).toBeInTheDocument();
expect(screen.queryByText("Redis Type")).not.toBeInTheDocument();
});
});
describe("cache strategy and redis topology are separate dimensions", () => {
it("should move Semantic out of the Redis Type dropdown into its own Cache Type selector", async () => {
const user = userEvent.setup();
renderSettings();
expect(await screen.findByText("Redis Type")).toBeInTheDocument();
expect(screen.queryByText("Similarity Threshold")).not.toBeInTheDocument();
const cacheTypeSelect = screen.getByText("Cache Type").parentElement?.querySelector(".ant-select-selector");
expect(cacheTypeSelect).not.toBeNull();
await user.click(cacheTypeSelect as HTMLElement);
await user.click(await screen.findByText("Semantic"));
await waitFor(() => expect(screen.queryByText("Redis Type")).not.toBeInTheDocument());
expect(screen.getByText("Similarity Threshold")).toBeInTheDocument();
});
});
describe("when a field fails inline validation", () => {

62
ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx
View file

@ -4,10 +4,19 @@ import { Form } from "antd";
import { getCacheSettingsCall, testCacheConnectionCall, updateCacheSettingsCall } from "@/components/networking";
import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models";
import NotificationsManager from "@/components/molecules/notifications_manager";
import RedisTypeSelector from "./RedisTypeSelector";
import CacheOptionSelector from "./CacheOptionSelector";
import CacheFieldSection from "./CacheFieldSection";
import { EmbeddingModelOption } from "./CacheFormField";
import { REDIS_TYPES, REDIS_TYPE_DESCRIPTIONS, RedisType } from "./cacheSettingsFields";
import {
CACHE_TYPES,
CACHE_TYPE_DESCRIPTIONS,
CACHE_TYPE_LABELS,
CacheType,
REDIS_TYPES,
REDIS_TYPE_DESCRIPTIONS,
REDIS_TYPE_LABELS,
RedisType,
} from "./cacheSettingsFields";
import { buildCachePayload, buildInitialValues, CacheFormValues } from "./cacheSettingsUtils";
interface CacheSettingsProps {
@ -19,8 +28,14 @@ interface CacheSettingsProps {
const toRedisType = (value: unknown): RedisType =>
REDIS_TYPES.includes(value as RedisType) ? (value as RedisType) : "node";
const toCacheType = (value: unknown): CacheType => (value === "redis-semantic" ? "semantic" : "standard");
const CACHE_TYPE_OPTIONS = CACHE_TYPES.map((value) => ({ value, label: CACHE_TYPE_LABELS[value] }));
const REDIS_TYPE_OPTIONS = REDIS_TYPES.map((value) => ({ value, label: REDIS_TYPE_LABELS[value] }));
const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
const [form] = Form.useForm<CacheFormValues>();
const [cacheType, setCacheType] = useState<CacheType>("standard");
const [redisType, setRedisType] = useState<RedisType>("node");
const [embeddingModels, setEmbeddingModels] = useState<EmbeddingModelOption[]>([]);
const [isTesting, setIsTesting] = useState<boolean>(false);
@ -34,6 +49,7 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
const data = (await getCacheSettingsCall(accessToken)) as { current_values?: Record<string, unknown> };
const currentValues = data.current_values ?? {};
form.setFieldsValue(buildInitialValues(currentValues));
setCacheType(toCacheType(currentValues.type));
setRedisType(toRedisType(currentValues.redis_type));
} catch (error) {
console.error("Failed to load cache settings:", error);
@ -81,7 +97,7 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
try {
const result = await testCacheConnectionCall(
accessToken,
buildCachePayload(redisType, values, { forTesting: true }),
buildCachePayload(cacheType, redisType, values, { forTesting: true }),
);
if (result.status === "success") {
NotificationsManager.success("Cache connection test successful!");
@ -109,7 +125,10 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
setIsSaving(true);
try {
await updateCacheSettingsCall(accessToken, buildCachePayload(redisType, values, { forTesting: false }));
await updateCacheSettingsCall(
accessToken,
buildCachePayload(cacheType, redisType, values, { forTesting: false }),
);
NotificationsManager.success("Cache settings updated successfully");
await loadCacheSettings();
} catch (error) {
@ -132,26 +151,42 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
<p className="text-xs text-gray-500 mt-1">Configure Redis cache for LiteLLM</p>
</div>
<RedisTypeSelector
redisType={redisType}
redisTypeDescriptions={REDIS_TYPE_DESCRIPTIONS}
onTypeChange={(type) => setRedisType(toRedisType(type))}
<CacheOptionSelector
label="Cache Type"
value={cacheType}
options={CACHE_TYPE_OPTIONS}
descriptions={CACHE_TYPE_DESCRIPTIONS}
fallbackDescription="Select how cache lookups are performed"
onChange={(type) => setCacheType(type === "semantic" ? "semantic" : "standard")}
/>
{cacheType === "standard" && (
<CacheOptionSelector
label="Redis Type"
value={redisType}
options={REDIS_TYPE_OPTIONS}
descriptions={REDIS_TYPE_DESCRIPTIONS}
fallbackDescription="Select the type of Redis deployment you're using"
onChange={(type) => setRedisType(toRedisType(type))}
/>
)}
<div className="pt-4 border-t border-gray-200">
<CacheFieldSection
title="Connection Settings"
section="connection"
cacheType={cacheType}
redisType={redisType}
embeddingModels={embeddingModels}
/>
</div>
{redisType === "cluster" && (
{cacheType === "standard" && redisType === "cluster" && (
<div className="pt-4 border-t border-gray-200">
<CacheFieldSection
title="Cluster Configuration"
section="cluster"
cacheType={cacheType}
redisType={redisType}
embeddingModels={embeddingModels}
gridCols="grid-cols-1 gap-6"
@ -159,22 +194,24 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
</div>
)}
{redisType === "sentinel" && (
{cacheType === "standard" && redisType === "sentinel" && (
<div className="pt-4 border-t border-gray-200">
<CacheFieldSection
title="Sentinel Configuration"
section="sentinel"
cacheType={cacheType}
redisType={redisType}
embeddingModels={embeddingModels}
/>
</div>
)}
{redisType === "semantic" && (
{cacheType === "semantic" && (
<div className="pt-4 border-t border-gray-200">
<CacheFieldSection
title="Semantic Configuration"
section="semantic"
cacheType={cacheType}
redisType={redisType}
embeddingModels={embeddingModels}
/>
@ -190,6 +227,7 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
<CacheFieldSection
title="SSL Settings"
section="ssl"
cacheType={cacheType}
redisType={redisType}
embeddingModels={embeddingModels}
headingLevel="h5"
@ -197,6 +235,7 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
<CacheFieldSection
title="Cache Management"
section="cacheManagement"
cacheType={cacheType}
redisType={redisType}
embeddingModels={embeddingModels}
headingLevel="h5"
@ -204,6 +243,7 @@ const CacheSettings: React.FC<CacheSettingsProps> = ({ accessToken }) => {
<CacheFieldSection
title="GCP Authentication"
section="gcp"
cacheType={cacheType}
redisType={redisType}
embeddingModels={embeddingModels}
headingLevel="h5"