diff --git a/ui/litellm-dashboard/src/components/add_model/cache_control_settings.tsx b/ui/litellm-dashboard/src/components/add_model/cache_control_settings.tsx index 3691f888a91..f9a7b4aa2b6 100644 --- a/ui/litellm-dashboard/src/components/add_model/cache_control_settings.tsx +++ b/ui/litellm-dashboard/src/components/add_model/cache_control_settings.tsx @@ -1,16 +1,11 @@ import React from "react"; -import { Form, Switch, Select, Typography } from "antd"; -import { PlusOutlined, MinusCircleOutlined } from "@ant-design/icons"; -import NumericalInput from "../shared/numerical_input"; +import { Form, Switch, Typography } from "antd"; +import CacheControlInjectionPointsEditor, { + CacheControlInjectionPoint, +} from "../shared/cache_control_injection_points_editor"; const { Text } = Typography; -interface CacheControlInjectionPoint { - location: "message"; - role?: "user" | "system" | "assistant"; - index?: number; -} - interface CacheControlSettingsProps { form: any; // Form instance from parent showCacheControl: boolean; @@ -23,24 +18,29 @@ const CacheControlSettings: React.FC = ({ onCacheControlChange, }) => { const updateCacheControlPoints = (injectionPoints: CacheControlInjectionPoint[]) => { + form.setFieldValue("cache_control_injection_points", injectionPoints); + const currentParams = form.getFieldValue("litellm_extra_params"); try { - let paramsObj = currentParams ? JSON.parse(currentParams) : {}; + const paramsObj = currentParams ? JSON.parse(currentParams) : {}; if (injectionPoints.length > 0) { paramsObj.cache_control_injection_points = injectionPoints; } else { delete paramsObj.cache_control_injection_points; } - if (Object.keys(paramsObj).length > 0) { - form.setFieldValue("litellm_extra_params", JSON.stringify(paramsObj, null, 2)); - } else { - form.setFieldValue("litellm_extra_params", ""); - } + form.setFieldValue( + "litellm_extra_params", + Object.keys(paramsObj).length > 0 ? JSON.stringify(paramsObj, null, 2) : "", + ); } catch (error) { console.error("Error updating cache control points:", error); } }; + const cacheControlInjectionPoints = Form.useWatch("cache_control_injection_points", form) || [ + { location: "message" as const }, + ]; + return ( <> = ({ litellm can automatically add them for you as a cost saving feature. - - {(fields, { add, remove }) => ( - <> - {fields.map((field, index) => ( -
- - { - const values = form.getFieldValue("cache_control_points"); - updateCacheControlPoints(values); - }} - /> - - - - { - const values = form.getFieldValue("cache_control_points"); - updateCacheControlPoints(values); - }} - /> - - - {fields.length > 1 && ( - { - remove(field.name); - setTimeout(() => { - const values = form.getFieldValue("cache_control_points"); - updateCacheControlPoints(values); - }, 0); - }} - /> - )} -
- ))} - - - - - - )} - + )} diff --git a/ui/litellm-dashboard/src/components/router_settings/DefaultLitellmParamsSection.test.tsx b/ui/litellm-dashboard/src/components/router_settings/DefaultLitellmParamsSection.test.tsx new file mode 100644 index 00000000000..f95214d96d2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/router_settings/DefaultLitellmParamsSection.test.tsx @@ -0,0 +1,93 @@ +import { describe, it, expect, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import DefaultLitellmParamsSection from "./DefaultLitellmParamsSection"; + +describe("DefaultLitellmParamsSection", () => { + it("should render the non-cache-control keys as JSON in the textarea", () => { + render( + , + ); + const textarea = screen.getByRole("textbox") as HTMLTextAreaElement; + expect(textarea.value).toContain('"timeout": 30'); + expect(textarea.value).toContain('"max_retries": 0'); + }); + + it("should not show the cache control editor when no injection points are set", () => { + render(); + expect(screen.queryByTestId("cache-control-location-select-0")).not.toBeInTheDocument(); + }); + + it("should show the cache control editor pre-populated when injection points are already set", () => { + render( + , + ); + expect(screen.getByTestId("cache-control-location-select-0")).toBeInTheDocument(); + }); + + it("should call onChange with cache_control_injection_points added when the toggle is switched on", async () => { + const onChange = vi.fn(); + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("switch")); + + expect(onChange).toHaveBeenCalledWith({ timeout: 30, cache_control_injection_points: [{ location: "message" }] }); + }); + + it("should call onChange with cache_control_injection_points removed when the toggle is switched off", async () => { + const onChange = vi.fn(); + const user = userEvent.setup(); + render( + , + ); + + await user.click(screen.getByRole("switch")); + + expect(onChange).toHaveBeenCalledWith({ timeout: 30 }); + }); + + it("should merge edited JSON textarea content with cache_control_injection_points on blur", async () => { + const onChange = vi.fn(); + const user = userEvent.setup(); + render( + , + ); + + const textarea = screen.getByRole("textbox") as HTMLTextAreaElement; + await user.clear(textarea); + await user.type(textarea, '{{"timeout": 60}'); + await user.tab(); + + expect(onChange).toHaveBeenCalledWith({ timeout: 60, cache_control_injection_points: [{ location: "message" }] }); + }); + + it("should not call onChange with invalid JSON left in the textarea on blur", async () => { + const onChange = vi.fn(); + const user = userEvent.setup(); + render(); + + const textarea = screen.getByRole("textbox") as HTMLTextAreaElement; + await user.clear(textarea); + await user.type(textarea, "not json"); + await user.tab(); + + expect(onChange).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/router_settings/DefaultLitellmParamsSection.tsx b/ui/litellm-dashboard/src/components/router_settings/DefaultLitellmParamsSection.tsx new file mode 100644 index 00000000000..3795f80a768 --- /dev/null +++ b/ui/litellm-dashboard/src/components/router_settings/DefaultLitellmParamsSection.tsx @@ -0,0 +1,92 @@ +import React from "react"; +import { Input, Switch } from "antd"; +import CacheControlInjectionPointsEditor, { + CacheControlInjectionPoint, +} from "../shared/cache_control_injection_points_editor"; + +interface DefaultLitellmParamsSectionProps { + value: { [key: string]: any }; + routerFieldsMetadata: { [key: string]: any }; + onChange: (value: { [key: string]: any }) => void; +} + +const DefaultLitellmParamsSection: React.FC = ({ + value, + routerFieldsMetadata, + onChange, +}) => { + const meta = routerFieldsMetadata["default_litellm_params"]; + const { cache_control_injection_points, ...otherParams } = value || {}; + + const [otherParamsText, setOtherParamsText] = React.useState(() => JSON.stringify(otherParams, null, 2)); + const [showCacheControl, setShowCacheControl] = React.useState((cache_control_injection_points?.length ?? 0) > 0); + + const parseOtherParams = (): { [key: string]: any } => { + try { + return JSON.parse(otherParamsText || "{}"); + } catch { + return otherParams; + } + }; + + const handleOtherParamsBlur = () => { + try { + const parsed = JSON.parse(otherParamsText || "{}"); + onChange({ ...parsed, cache_control_injection_points }); + } catch (error) { + console.error("Error parsing default_litellm_params JSON:", error); + } + }; + + const handleCacheControlPointsChange = (points: CacheControlInjectionPoint[]) => { + const base = parseOtherParams(); + onChange(points.length > 0 ? { ...base, cache_control_injection_points: points } : base); + }; + + const handleCacheControlToggle = (checked: boolean) => { + setShowCacheControl(checked); + handleCacheControlPointsChange(checked ? [{ location: "message" }] : []); + }; + + return ( +
+
+ + {meta?.ui_field_name || "default_litellm_params"} + +

{meta?.field_description || ""}

+ setOtherParamsText(e.target.value)} + onBlur={handleOtherParamsBlur} + autoSize={{ minRows: 2 }} + className="font-mono text-sm w-full" + /> +
+ +
+
+ + Cache Control Injection Points + + +
+ {showCacheControl && ( +
+ +
+ )} +
+
+ ); +}; + +export default DefaultLitellmParamsSection; diff --git a/ui/litellm-dashboard/src/components/router_settings/ReliabilityRetriesSection.tsx b/ui/litellm-dashboard/src/components/router_settings/ReliabilityRetriesSection.tsx index 477c393f743..b6ae17dac4c 100644 --- a/ui/litellm-dashboard/src/components/router_settings/ReliabilityRetriesSection.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/ReliabilityRetriesSection.tsx @@ -29,7 +29,8 @@ const ReliabilityRetriesSection: React.FC = ({ param != "retry_policy" && param != "model_group_retry_policy" && param != "routing_groups" && - param != "optional_pre_call_checks", + param != "optional_pre_call_checks" && + param != "default_litellm_params", ) .map(([param, value]) => (
diff --git a/ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.test.tsx b/ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.test.tsx index ad3a0dcbf07..727354742ca 100644 --- a/ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.test.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.test.tsx @@ -171,4 +171,39 @@ describe("RouterSettingsForm", () => { }), ); }); + + it("should not show the Default LiteLLM Params section before router_settings has loaded", () => { + render(); + expect(screen.queryByText("Cache Control Injection Points")).not.toBeInTheDocument(); + }); + + it("should show the Default LiteLLM Params section once router_settings has loaded", () => { + const props = { + ...baseProps, + value: { ...defaultValue, routerSettings: { default_litellm_params: { timeout: 30 } } }, + }; + render(); + expect(screen.getByText("Cache Control Injection Points")).toBeInTheDocument(); + }); + + it("should call onChange with the updated default_litellm_params when the section changes", async () => { + const onChange = vi.fn(); + const user = userEvent.setup(); + const props = { + ...baseProps, + onChange, + value: { ...defaultValue, routerSettings: { default_litellm_params: { timeout: 30 } } }, + }; + render(); + + await user.click(screen.getByRole("switch", { name: "Cache Control Injection Points" })); + + expect(onChange).toHaveBeenCalledWith( + expect.objectContaining({ + routerSettings: expect.objectContaining({ + default_litellm_params: { timeout: 30, cache_control_injection_points: [{ location: "message" }] }, + }), + }), + ); + }); }); diff --git a/ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.tsx b/ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.tsx index ffa81e0246d..1c8445d30c2 100644 --- a/ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.tsx @@ -1,4 +1,5 @@ import React from "react"; +import DefaultLitellmParamsSection from "./DefaultLitellmParamsSection"; import LatencyBasedConfiguration from "./LatencyBasedConfiguration"; import OptionalPreCallChecksSelector from "./OptionalPreCallChecksSelector"; import ReliabilityRetriesSection from "./ReliabilityRetriesSection"; @@ -47,6 +48,13 @@ const RouterSettingsForm: React.FC = ({ }); }; + const handleDefaultLitellmParamsChange = (params: { [key: string]: any }) => { + onChange({ + ...value, + routerSettings: { ...value.routerSettings, default_litellm_params: params }, + }); + }; + const optionalPreCallCheckOptions: string[] = routerFieldsMetadata["optional_pre_call_checks"]?.options || []; return ( @@ -95,6 +103,18 @@ const RouterSettingsForm: React.FC = ({ )} + {/* Default LiteLLM Params */} + {"default_litellm_params" in value.routerSettings && ( + <> + +
+ + )} + {/* Other Settings */}
diff --git a/ui/litellm-dashboard/src/components/router_settings/index.tsx b/ui/litellm-dashboard/src/components/router_settings/index.tsx index 2dec5c0c3dc..3d5c6a1855c 100644 --- a/ui/litellm-dashboard/src/components/router_settings/index.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/index.tsx @@ -88,7 +88,7 @@ const RouterSettings: React.FC = ({ accessToken, userRole, const router_settings = formValue.routerSettings; const numberKeys = new Set(["allowed_fails", "cooldown_time", "num_retries", "timeout", "retry_after"]); - const jsonKeys = new Set(["model_group_alias", "default_litellm_params"]); + const jsonKeys = new Set(["model_group_alias"]); // retry_policy and model_group_retry_policy are owned by the Model Retry Settings tab; // routing_groups is owned by the Routing Groups tab. This page must not read or write them. const tabOwnedKeys = new Set(["retry_policy", "model_group_retry_policy", "routing_groups"]); diff --git a/ui/litellm-dashboard/src/components/shared/cache_control_injection_points_editor.test.tsx b/ui/litellm-dashboard/src/components/shared/cache_control_injection_points_editor.test.tsx new file mode 100644 index 00000000000..b0464d7c88b --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/cache_control_injection_points_editor.test.tsx @@ -0,0 +1,61 @@ +import { describe, it, expect, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import CacheControlInjectionPointsEditor from "./cache_control_injection_points_editor"; + +describe("CacheControlInjectionPointsEditor", () => { + it("should render one row per point", () => { + render( + , + ); + expect(screen.getByTestId("cache-control-location-select-0")).toBeInTheDocument(); + expect(screen.getByTestId("cache-control-location-select-1")).toBeInTheDocument(); + }); + + it("should render a single default row when value is empty", () => { + render(); + expect(screen.getByTestId("cache-control-location-select-0")).toBeInTheDocument(); + expect(screen.queryByTestId("cache-control-location-select-1")).not.toBeInTheDocument(); + }); + + it("should add a new row with the Add Injection Point button", async () => { + const onChange = vi.fn(); + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("button", { name: /add injection point/i })); + + expect(onChange).toHaveBeenCalledWith([{ location: "message" }, { location: "message" }]); + }); + + it("should not render a remove button when there is only one row", () => { + render(); + expect(document.querySelector(".anticon-minus-circle")).not.toBeInTheDocument(); + }); + + it("should remove a row when its remove icon is clicked", async () => { + const onChange = vi.fn(); + const user = userEvent.setup(); + render( + , + ); + + const removeIcons = document.querySelectorAll(".anticon-minus-circle"); + expect(removeIcons).toHaveLength(2); + await user.click(removeIcons[0] as HTMLElement); + + expect(onChange).toHaveBeenCalledWith([{ location: "message", role: "user" }]); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/cache_control_injection_points_editor.tsx b/ui/litellm-dashboard/src/components/shared/cache_control_injection_points_editor.tsx new file mode 100644 index 00000000000..5880b8a12ef --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/cache_control_injection_points_editor.tsx @@ -0,0 +1,90 @@ +import React from "react"; +import { Select } from "antd"; +import { PlusOutlined, MinusCircleOutlined } from "@ant-design/icons"; +import NumericalInput from "./numerical_input"; + +export interface CacheControlInjectionPoint { + location: "message"; + role?: "user" | "system" | "assistant"; + index?: number; +} + +interface CacheControlInjectionPointsEditorProps { + value: CacheControlInjectionPoint[]; + onChange: (points: CacheControlInjectionPoint[]) => void; +} + +const CacheControlInjectionPointsEditor: React.FC = ({ value, onChange }) => { + const points = value.length > 0 ? value : [{ location: "message" as const }]; + + const updatePoint = (index: number, patch: Partial) => { + onChange(points.map((point, i) => (i === index ? { ...point, ...patch } : point))); + }; + + return ( + <> + {points.map((point, index) => ( +
+
+ Type + updatePoint(index, { role })} + options={[ + { value: "user", label: "User" }, + { value: "system", label: "System" }, + { value: "assistant", label: "Assistant" }, + ]} + className="w-full" + data-testid={`cache-control-role-select-${index}`} + /> +
+ +
+ Index + + updatePoint(index, { index: newIndex === "" ? undefined : Number(newIndex) }) + } + /> +
+ + {points.length > 1 && ( + onChange(points.filter((_, i) => i !== index))} + /> + )} +
+ ))} + + + + ); +}; + +export default CacheControlInjectionPointsEditor;