scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-17 23:51:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 139

feat(ui): reuse the Add Model cache control widget on Router Settings

Browse source 
default_litellm_params.cache_control_injection_points was only editable as
raw JSON on the Router Settings page, requiring an admin to hand-write
[{"location": "message", "role": "system"}] to enable prompt cache routing -
clunky next to the structured Switch + row editor already used for the same
field on the Add Model page.

Extracted the row editor (location/role/index inputs, add/remove) out of
add_model/cache_control_settings.tsx into a form-agnostic shared component,
CacheControlInjectionPointsEditor, driven by plain value/onChange props
instead of antd Form bindings. cache_control_settings.tsx now delegates to it
(and picks up a real fix along the way: its role/index onChange handlers read
form.getFieldValue("cache_control_points"), a field that was never
registered under that name, so those edits silently never synced into
litellm_extra_params - now reads the correct "cache_control_injection_points"
field via Form.useWatch).

New DefaultLitellmParamsSection renders that same editor for Router Settings
plus a JSON textarea for the remaining default_litellm_params keys
(timeout, max_retries, metadata, ...). Fully controlled through React state
(matching the optional_pre_call_checks pattern) rather than the page's
generic DOM-read save path, and excluded from ReliabilityRetriesSection's
raw-JSON rendering so it isn't shown twice.

Could not visually verify in a live browser this session (a chrome-extension
tooling conflict blocked screenshots/typing); verified via 174 passing
frontend tests covering both components and their Add Model / Router
Settings integrations, plus a clean tsc typecheck.
This commit is contained in:
Krrish Dholakia 2026-07-13 20:47:23 -07:00
parent ac09e37ca8
commit f6d67f025d
9 changed files with 410 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

117
ui/litellm-dashboard/src/components/add_model/cache_control_settings.tsx
View file

@ -1,16 +1,11 @@
import React from "react";
import { Form, Switch, Select, Typography } from "antd";
import { PlusOutlined, MinusCircleOutlined } from "@ant-design/icons";
import NumericalInput from "../shared/numerical_input";
import { Form, Switch, Typography } from "antd";
import CacheControlInjectionPointsEditor, {
CacheControlInjectionPoint,
} from "../shared/cache_control_injection_points_editor";
const { Text } = Typography;
interface CacheControlInjectionPoint {
location: "message";
role?: "user" | "system" | "assistant";
index?: number;
}
interface CacheControlSettingsProps {
form: any; // Form instance from parent
showCacheControl: boolean;
@ -23,24 +18,29 @@ const CacheControlSettings: React.FC<CacheControlSettingsProps> = ({
onCacheControlChange,
}) => {
const updateCacheControlPoints = (injectionPoints: CacheControlInjectionPoint[]) => {
form.setFieldValue("cache_control_injection_points", injectionPoints);
const currentParams = form.getFieldValue("litellm_extra_params");
try {
let paramsObj = currentParams ? JSON.parse(currentParams) : {};
const paramsObj = currentParams ? JSON.parse(currentParams) : {};
if (injectionPoints.length > 0) {
paramsObj.cache_control_injection_points = injectionPoints;
} else {
delete paramsObj.cache_control_injection_points;
}
if (Object.keys(paramsObj).length > 0) {
form.setFieldValue("litellm_extra_params", JSON.stringify(paramsObj, null, 2));
} else {
form.setFieldValue("litellm_extra_params", "");
}
form.setFieldValue(
"litellm_extra_params",
Object.keys(paramsObj).length > 0 ? JSON.stringify(paramsObj, null, 2) : "",
);
} catch (error) {
console.error("Error updating cache control points:", error);
}
};
const cacheControlInjectionPoints = Form.useWatch("cache_control_injection_points", form) || [
{ location: "message" as const },
];
return (
<>
<Form.Item
@ -60,92 +60,7 @@ const CacheControlSettings: React.FC<CacheControlSettingsProps> = ({
litellm can automatically add them for you as a cost saving feature.
</Text>
<Form.List name="cache_control_injection_points" initialValue={[{ location: "message" }]}>
{(fields, { add, remove }) => (
<>
{fields.map((field, index) => (
<div key={field.key} className="flex items-center mb-4 gap-4">
<Form.Item
{...field}
label="Type"
name={[field.name, "location"]}
initialValue="message"
className="mb-0"
style={{ width: "180px" }}
>
<Select disabled options={[{ value: "message", label: "Message" }]} />
</Form.Item>
<Form.Item
{...field}
label="Role"
name={[field.name, "role"]}
className="mb-0"
style={{ width: "180px" }}
tooltip="LiteLLM will mark all messages of this role as cacheable"
>
<Select
placeholder="Select a role"
allowClear
options={[
{ value: "user", label: "User" },
{ value: "system", label: "System" },
{ value: "assistant", label: "Assistant" },
]}
onChange={() => {
const values = form.getFieldValue("cache_control_points");
updateCacheControlPoints(values);
}}
/>
</Form.Item>
<Form.Item
{...field}
label="Index"
name={[field.name, "index"]}
className="mb-0"
style={{ width: "180px" }}
tooltip="(Optional) If set litellm will mark the message at this index as cacheable"
>
<NumericalInput
type="number"
placeholder="Optional"
step={1}
onChange={() => {
const values = form.getFieldValue("cache_control_points");
updateCacheControlPoints(values);
}}
/>
</Form.Item>
{fields.length > 1 && (
<MinusCircleOutlined
className="text-red-500 cursor-pointer text-lg ml-12"
onClick={() => {
remove(field.name);
setTimeout(() => {
const values = form.getFieldValue("cache_control_points");
updateCacheControlPoints(values);
}, 0);
}}
/>
)}
</div>
))}
<Form.Item>
<button
type="button"
className="flex items-center justify-center w-full border border-dashed border-gray-300 py-2 px-4 text-gray-600 hover:text-blue-600 hover:border-blue-300 transition-all rounded-sm"
onClick={() => add()}
>
<PlusOutlined className="mr-2" />
Add Injection Point
</button>
</Form.Item>
</>
)}
</Form.List>
<CacheControlInjectionPointsEditor value={cacheControlInjectionPoints} onChange={updateCacheControlPoints} />
</div>
)}
</>

93
ui/litellm-dashboard/src/components/router_settings/DefaultLitellmParamsSection.test.tsx Normal file
View file

@ -0,0 +1,93 @@
import { describe, it, expect, vi } from "vitest";
import { render, screen } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import DefaultLitellmParamsSection from "./DefaultLitellmParamsSection";
describe("DefaultLitellmParamsSection", () => {
it("should render the non-cache-control keys as JSON in the textarea", () => {
render(
<DefaultLitellmParamsSection
value={{ timeout: 30, max_retries: 0 }}
routerFieldsMetadata={{}}
onChange={vi.fn()}
/>,
);
const textarea = screen.getByRole("textbox") as HTMLTextAreaElement;
expect(textarea.value).toContain('"timeout": 30');
expect(textarea.value).toContain('"max_retries": 0');
});
it("should not show the cache control editor when no injection points are set", () => {
render(<DefaultLitellmParamsSection value={{ timeout: 30 }} routerFieldsMetadata={{}} onChange={vi.fn()} />);
expect(screen.queryByTestId("cache-control-location-select-0")).not.toBeInTheDocument();
});
it("should show the cache control editor pre-populated when injection points are already set", () => {
render(
<DefaultLitellmParamsSection
value={{ cache_control_injection_points: [{ location: "message", role: "system" }] }}
routerFieldsMetadata={{}}
onChange={vi.fn()}
/>,
);
expect(screen.getByTestId("cache-control-location-select-0")).toBeInTheDocument();
});
it("should call onChange with cache_control_injection_points added when the toggle is switched on", async () => {
const onChange = vi.fn();
const user = userEvent.setup();
render(<DefaultLitellmParamsSection value={{ timeout: 30 }} routerFieldsMetadata={{}} onChange={onChange} />);
await user.click(screen.getByRole("switch"));
expect(onChange).toHaveBeenCalledWith({ timeout: 30, cache_control_injection_points: [{ location: "message" }] });
});
it("should call onChange with cache_control_injection_points removed when the toggle is switched off", async () => {
const onChange = vi.fn();
const user = userEvent.setup();
render(
<DefaultLitellmParamsSection
value={{ timeout: 30, cache_control_injection_points: [{ location: "message", role: "system" }] }}
routerFieldsMetadata={{}}
onChange={onChange}
/>,
);
await user.click(screen.getByRole("switch"));
expect(onChange).toHaveBeenCalledWith({ timeout: 30 });
});
it("should merge edited JSON textarea content with cache_control_injection_points on blur", async () => {
const onChange = vi.fn();
const user = userEvent.setup();
render(
<DefaultLitellmParamsSection
value={{ timeout: 30, cache_control_injection_points: [{ location: "message" }] }}
routerFieldsMetadata={{}}
onChange={onChange}
/>,
);
const textarea = screen.getByRole("textbox") as HTMLTextAreaElement;
await user.clear(textarea);
await user.type(textarea, '{{"timeout": 60}');
await user.tab();
expect(onChange).toHaveBeenCalledWith({ timeout: 60, cache_control_injection_points: [{ location: "message" }] });
});
it("should not call onChange with invalid JSON left in the textarea on blur", async () => {
const onChange = vi.fn();
const user = userEvent.setup();
render(<DefaultLitellmParamsSection value={{ timeout: 30 }} routerFieldsMetadata={{}} onChange={onChange} />);
const textarea = screen.getByRole("textbox") as HTMLTextAreaElement;
await user.clear(textarea);
await user.type(textarea, "not json");
await user.tab();
expect(onChange).not.toHaveBeenCalled();
});
});

92
ui/litellm-dashboard/src/components/router_settings/DefaultLitellmParamsSection.tsx Normal file
View file

@ -0,0 +1,92 @@
import React from "react";
import { Input, Switch } from "antd";
import CacheControlInjectionPointsEditor, {
CacheControlInjectionPoint,
} from "../shared/cache_control_injection_points_editor";
interface DefaultLitellmParamsSectionProps {
value: { [key: string]: any };
routerFieldsMetadata: { [key: string]: any };
onChange: (value: { [key: string]: any }) => void;
}
const DefaultLitellmParamsSection: React.FC<DefaultLitellmParamsSectionProps> = ({
value,
routerFieldsMetadata,
onChange,
}) => {
const meta = routerFieldsMetadata["default_litellm_params"];
const { cache_control_injection_points, ...otherParams } = value || {};
const [otherParamsText, setOtherParamsText] = React.useState(() => JSON.stringify(otherParams, null, 2));
const [showCacheControl, setShowCacheControl] = React.useState((cache_control_injection_points?.length ?? 0) > 0);
const parseOtherParams = (): { [key: string]: any } => {
try {
return JSON.parse(otherParamsText || "{}");
} catch {
return otherParams;
}
};
const handleOtherParamsBlur = () => {
try {
const parsed = JSON.parse(otherParamsText || "{}");
onChange({ ...parsed, cache_control_injection_points });
} catch (error) {
console.error("Error parsing default_litellm_params JSON:", error);
}
};
const handleCacheControlPointsChange = (points: CacheControlInjectionPoint[]) => {
const base = parseOtherParams();
onChange(points.length > 0 ? { ...base, cache_control_injection_points: points } : base);
};
const handleCacheControlToggle = (checked: boolean) => {
setShowCacheControl(checked);
handleCacheControlPointsChange(checked ? [{ location: "message" }] : []);
};
return (
<div className="space-y-6">
<div className="max-w-3xl space-y-2">
<span className="text-xs font-medium text-gray-700 uppercase tracking-wide">
{meta?.ui_field_name || "default_litellm_params"}
</span>
<p className="text-xs text-gray-500 mt-0.5 mb-2">{meta?.field_description || ""}</p>
<Input.TextArea
value={otherParamsText}
onChange={(e) => setOtherParamsText(e.target.value)}
onBlur={handleOtherParamsBlur}
autoSize={{ minRows: 2 }}
className="font-mono text-sm w-full"
/>
</div>
<div className="max-w-3xl">
<div className="flex items-center justify-between mb-2">
<span className="text-xs font-medium text-gray-700 uppercase tracking-wide">
Cache Control Injection Points
</span>
<Switch
checked={showCacheControl}
onChange={handleCacheControlToggle}
className="bg-gray-600"
aria-label="Cache Control Injection Points"
/>
</div>
{showCacheControl && (
<div className="ml-6 pl-4 border-l-2 border-gray-200">
<CacheControlInjectionPointsEditor
value={cache_control_injection_points || [{ location: "message" }]}
onChange={handleCacheControlPointsChange}
/>
</div>
)}
</div>
</div>
);
};
export default DefaultLitellmParamsSection;

3
ui/litellm-dashboard/src/components/router_settings/ReliabilityRetriesSection.tsx
View file

@ -29,7 +29,8 @@ const ReliabilityRetriesSection: React.FC<ReliabilityRetriesSectionProps> = ({
param != "retry_policy" &&
param != "model_group_retry_policy" &&
param != "routing_groups" &&
param != "optional_pre_call_checks",
param != "optional_pre_call_checks" &&
param != "default_litellm_params",
)
.map(([param, value]) => (
<div key={param} className="space-y-2">

35
ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.test.tsx
View file

@ -171,4 +171,39 @@ describe("RouterSettingsForm", () => {
}),
);
});
it("should not show the Default LiteLLM Params section before router_settings has loaded", () => {
render(<RouterSettingsForm {...baseProps} />);
expect(screen.queryByText("Cache Control Injection Points")).not.toBeInTheDocument();
});
it("should show the Default LiteLLM Params section once router_settings has loaded", () => {
const props = {
...baseProps,
value: { ...defaultValue, routerSettings: { default_litellm_params: { timeout: 30 } } },
};
render(<RouterSettingsForm {...props} />);
expect(screen.getByText("Cache Control Injection Points")).toBeInTheDocument();
});
it("should call onChange with the updated default_litellm_params when the section changes", async () => {
const onChange = vi.fn();
const user = userEvent.setup();
const props = {
...baseProps,
onChange,
value: { ...defaultValue, routerSettings: { default_litellm_params: { timeout: 30 } } },
};
render(<RouterSettingsForm {...props} />);
await user.click(screen.getByRole("switch", { name: "Cache Control Injection Points" }));
expect(onChange).toHaveBeenCalledWith(
expect.objectContaining({
routerSettings: expect.objectContaining({
default_litellm_params: { timeout: 30, cache_control_injection_points: [{ location: "message" }] },
}),
}),
);
});
});

20
ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.tsx
View file

@ -1,4 +1,5 @@
import React from "react";
import DefaultLitellmParamsSection from "./DefaultLitellmParamsSection";
import LatencyBasedConfiguration from "./LatencyBasedConfiguration";
import OptionalPreCallChecksSelector from "./OptionalPreCallChecksSelector";
import ReliabilityRetriesSection from "./ReliabilityRetriesSection";
@ -47,6 +48,13 @@ const RouterSettingsForm: React.FC<RouterSettingsFormProps> = ({
});
};
const handleDefaultLitellmParamsChange = (params: { [key: string]: any }) => {
onChange({
...value,
routerSettings: { ...value.routerSettings, default_litellm_params: params },
});
};
const optionalPreCallCheckOptions: string[] = routerFieldsMetadata["optional_pre_call_checks"]?.options || [];
return (
@ -95,6 +103,18 @@ const RouterSettingsForm: React.FC<RouterSettingsFormProps> = ({
<LatencyBasedConfiguration routingStrategyArgs={value.routerSettings["routing_strategy_args"]} />
)}
{/* Default LiteLLM Params */}
{"default_litellm_params" in value.routerSettings && (
<>
<DefaultLitellmParamsSection
value={value.routerSettings.default_litellm_params || {}}
routerFieldsMetadata={routerFieldsMetadata}
onChange={handleDefaultLitellmParamsChange}
/>
<div className="border-t border-gray-200" />
</>
)}
{/* Other Settings */}
<ReliabilityRetriesSection routerSettings={value.routerSettings} routerFieldsMetadata={routerFieldsMetadata} />
</div>

2
ui/litellm-dashboard/src/components/router_settings/index.tsx
View file

@ -88,7 +88,7 @@ const RouterSettings: React.FC<RouterSettingsProps> = ({ accessToken, userRole,
const router_settings = formValue.routerSettings;
const numberKeys = new Set(["allowed_fails", "cooldown_time", "num_retries", "timeout", "retry_after"]);
const jsonKeys = new Set(["model_group_alias", "default_litellm_params"]);
const jsonKeys = new Set(["model_group_alias"]);
// retry_policy and model_group_retry_policy are owned by the Model Retry Settings tab;
// routing_groups is owned by the Routing Groups tab. This page must not read or write them.
const tabOwnedKeys = new Set(["retry_policy", "model_group_retry_policy", "routing_groups"]);

61
ui/litellm-dashboard/src/components/shared/cache_control_injection_points_editor.test.tsx Normal file
View file

@ -0,0 +1,61 @@
import { describe, it, expect, vi } from "vitest";
import { render, screen } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import CacheControlInjectionPointsEditor from "./cache_control_injection_points_editor";
describe("CacheControlInjectionPointsEditor", () => {
it("should render one row per point", () => {
render(
<CacheControlInjectionPointsEditor
value={[
{ location: "message", role: "system" },
{ location: "message", index: 0 },
]}
onChange={vi.fn()}
/>,
);
expect(screen.getByTestId("cache-control-location-select-0")).toBeInTheDocument();
expect(screen.getByTestId("cache-control-location-select-1")).toBeInTheDocument();
});
it("should render a single default row when value is empty", () => {
render(<CacheControlInjectionPointsEditor value={[]} onChange={vi.fn()} />);
expect(screen.getByTestId("cache-control-location-select-0")).toBeInTheDocument();
expect(screen.queryByTestId("cache-control-location-select-1")).not.toBeInTheDocument();
});
it("should add a new row with the Add Injection Point button", async () => {
const onChange = vi.fn();
const user = userEvent.setup();
render(<CacheControlInjectionPointsEditor value={[{ location: "message" }]} onChange={onChange} />);
await user.click(screen.getByRole("button", { name: /add injection point/i }));
expect(onChange).toHaveBeenCalledWith([{ location: "message" }, { location: "message" }]);
});
it("should not render a remove button when there is only one row", () => {
render(<CacheControlInjectionPointsEditor value={[{ location: "message" }]} onChange={vi.fn()} />);
expect(document.querySelector(".anticon-minus-circle")).not.toBeInTheDocument();
});
it("should remove a row when its remove icon is clicked", async () => {
const onChange = vi.fn();
const user = userEvent.setup();
render(
<CacheControlInjectionPointsEditor
value={[
{ location: "message", role: "system" },
{ location: "message", role: "user" },
]}
onChange={onChange}
/>,
);
const removeIcons = document.querySelectorAll(".anticon-minus-circle");
expect(removeIcons).toHaveLength(2);
await user.click(removeIcons[0] as HTMLElement);
expect(onChange).toHaveBeenCalledWith([{ location: "message", role: "user" }]);
});
});

90
ui/litellm-dashboard/src/components/shared/cache_control_injection_points_editor.tsx Normal file
View file

@ -0,0 +1,90 @@
import React from "react";
import { Select } from "antd";
import { PlusOutlined, MinusCircleOutlined } from "@ant-design/icons";
import NumericalInput from "./numerical_input";
export interface CacheControlInjectionPoint {
location: "message";
role?: "user" | "system" | "assistant";
index?: number;
}
interface CacheControlInjectionPointsEditorProps {
value: CacheControlInjectionPoint[];
onChange: (points: CacheControlInjectionPoint[]) => void;
}
const CacheControlInjectionPointsEditor: React.FC<CacheControlInjectionPointsEditorProps> = ({ value, onChange }) => {
const points = value.length > 0 ? value : [{ location: "message" as const }];
const updatePoint = (index: number, patch: Partial<CacheControlInjectionPoint>) => {
onChange(points.map((point, i) => (i === index ? { ...point, ...patch } : point)));
};
return (
<>
{points.map((point, index) => (
<div key={index} className="flex items-center mb-4 gap-4">
<div style={{ width: "180px" }}>
<span className="text-xs text-gray-500">Type</span>
<Select
disabled
value="message"
options={[{ value: "message", label: "Message" }]}
className="w-full"
data-testid={`cache-control-location-select-${index}`}
/>
</div>
<div style={{ width: "180px" }}>
<span className="text-xs text-gray-500">Role</span>
<Select
placeholder="Select a role"
allowClear
value={point.role}
onChange={(role) => updatePoint(index, { role })}
options={[
{ value: "user", label: "User" },
{ value: "system", label: "System" },
{ value: "assistant", label: "Assistant" },
]}
className="w-full"
data-testid={`cache-control-role-select-${index}`}
/>
</div>
<div style={{ width: "180px" }}>
<span className="text-xs text-gray-500">Index</span>
<NumericalInput
type="number"
placeholder="Optional"
step={1}
value={point.index}
onChange={(newIndex: string) =>
updatePoint(index, { index: newIndex === "" ? undefined : Number(newIndex) })
}
/>
</div>
{points.length > 1 && (
<MinusCircleOutlined
className="text-red-500 cursor-pointer text-lg ml-12"
onClick={() => onChange(points.filter((_, i) => i !== index))}
/>
)}
</div>
))}
<button
type="button"
className="flex items-center justify-center w-full border border-dashed border-gray-300 py-2 px-4 text-gray-600 hover:text-blue-600 hover:border-blue-300 transition-all rounded"
onClick={() => onChange([...points, { location: "message" as const }])}
>
<PlusOutlined className="mr-2" />
Add Injection Point
</button>
</>
);
};
export default CacheControlInjectionPointsEditor;