diff --git a/litellm/types/management_endpoints/router_settings_endpoints.py b/litellm/types/management_endpoints/router_settings_endpoints.py index fb3d6e10fcf..929f2fde132 100644 --- a/litellm/types/management_endpoints/router_settings_endpoints.py +++ b/litellm/types/management_endpoints/router_settings_endpoints.py @@ -224,36 +224,9 @@ ROUTER_SETTINGS_FIELDS: List[RouterSettingsField] = [ field_name="default_litellm_params", field_type="Dictionary", field_value=None, - field_description=( - "Default parameters for Router.chat.completion.create. E.g. set " - "cache_control_injection_points here to enable Anthropic/Bedrock " - "prompt caching for every model on this proxy." - ), + field_description="Default parameters for Router.chat.completion.create", field_default=None, ui_field_name="Default LiteLLM Params", - link="https://docs.litellm.ai/docs/tutorials/claude_code_prompt_cache_routing", - ), - RouterSettingsField( - field_name="optional_pre_call_checks", - field_type="List", - field_value=None, - field_description=( - "Extra checks the router runs before picking a deployment. Add " - "'prompt_caching' to route repeat requests back to the deployment " - "that cached the prompt." - ), - field_default=[], - options=[ - "prompt_caching", - "router_budget_limiting", - "responses_api_deployment_check", - "deployment_affinity", - "session_affinity", - "enforce_model_rate_limits", - "encrypted_content_affinity", - ], - ui_field_name="Optional Pre-call Checks", - link="https://docs.litellm.ai/docs/tutorials/claude_code_prompt_cache_routing", ), RouterSettingsField( field_name="set_verbose", diff --git a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py index a924e2a49b8..b62f077a62e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_router_settings_endpoints.py @@ -78,46 +78,6 @@ class TestRouterSettingsEndpoints: assert isinstance(routing_strategy_field["options"], list) assert len(routing_strategy_field["options"]) > 0 - @pytest.mark.asyncio - async def test_get_router_fields_includes_optional_pre_call_checks(self): - """ - Regression test: `optional_pre_call_checks` (e.g. "prompt_caching", used for - Claude Code prompt cache routing) must be exposed as a configurable field so - the Admin UI can render and save it, not just `default_litellm_params`. - """ - response = client.get( - "/router/fields", headers={"Authorization": "Bearer sk-1234"} - ) - assert response.status_code == 200 - - fields = response.json()["fields"] - field = next( - (f for f in fields if f["field_name"] == "optional_pre_call_checks"), None - ) - assert field is not None - assert "prompt_caching" in field["options"] - - @pytest.mark.asyncio - async def test_get_router_fields_excludes_unimplemented_forward_client_headers_check(self): - """ - Regression test: "forward_client_headers_by_model_group" is a literal in the - OptionalPreCallChecks type union, but Router.add_optional_pre_call_checks has - no handler for it - selecting it from the Admin UI's multi-select would save - successfully and show as enabled while doing nothing. It must not be offered - as an option until it's actually implemented. - """ - response = client.get( - "/router/fields", headers={"Authorization": "Bearer sk-1234"} - ) - assert response.status_code == 200 - - fields = response.json()["fields"] - field = next( - (f for f in fields if f["field_name"] == "optional_pre_call_checks"), None - ) - assert field is not None - assert "forward_client_headers_by_model_group" not in field["options"] - @pytest.mark.asyncio async def test_get_router_settings_includes_routing_groups_from_live_router( self, monkeypatch diff --git a/ui/litellm-dashboard/src/components/router_settings/DefaultLitellmParamsSection.test.tsx b/ui/litellm-dashboard/src/components/router_settings/DefaultLitellmParamsSection.test.tsx index 94a80d89cc7..9638b81d8b1 100644 --- a/ui/litellm-dashboard/src/components/router_settings/DefaultLitellmParamsSection.test.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/DefaultLitellmParamsSection.test.tsx @@ -1,24 +1,18 @@ import { describe, it, expect, vi } from "vitest"; -import { render, screen } from "@testing-library/react"; +import { fireEvent, render, screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import DefaultLitellmParamsSection from "./DefaultLitellmParamsSection"; describe("DefaultLitellmParamsSection", () => { it("should render the non-cache-control keys as JSON in the textarea", () => { - render( - , - ); + render(); const textarea = screen.getByRole("textbox") as HTMLTextAreaElement; expect(textarea.value).toContain('"timeout": 30'); expect(textarea.value).toContain('"max_retries": 0'); }); it("should not show the cache control editor when no injection points are set", () => { - render(); + render(); expect(screen.queryByTestId("cache-control-location-select-0")).not.toBeInTheDocument(); }); @@ -26,17 +20,29 @@ describe("DefaultLitellmParamsSection", () => { render( , ); expect(screen.getByTestId("cache-control-location-select-0")).toBeInTheDocument(); }); + it("should preserve unsupported cache control points in the JSON editor", () => { + render( + , + ); + + const textarea = screen.getByRole("textbox") as HTMLTextAreaElement; + expect(textarea.value).toContain('"location": "tool_config"'); + expect(screen.queryByTestId("cache-control-location-select-0")).not.toBeInTheDocument(); + }); + it("should call onChange with cache_control_injection_points added when the toggle is switched on", async () => { const onChange = vi.fn(); const user = userEvent.setup(); - render(); + render(); await user.click(screen.getByRole("switch")); @@ -49,7 +55,6 @@ describe("DefaultLitellmParamsSection", () => { render( , ); @@ -65,7 +70,6 @@ describe("DefaultLitellmParamsSection", () => { render( , ); @@ -81,7 +85,7 @@ describe("DefaultLitellmParamsSection", () => { it("should not call onChange and should flag the field as invalid when the textarea has invalid JSON on blur", async () => { const onChange = vi.fn(); const user = userEvent.setup(); - render(); + render(); const textarea = screen.getByRole("textbox") as HTMLTextAreaElement; await user.clear(textarea); @@ -95,7 +99,7 @@ describe("DefaultLitellmParamsSection", () => { it("should clear the invalid state once the textarea is edited again", async () => { const onChange = vi.fn(); const user = userEvent.setup(); - render(); + render(); const textarea = screen.getByRole("textbox") as HTMLTextAreaElement; await user.clear(textarea); @@ -108,4 +112,16 @@ describe("DefaultLitellmParamsSection", () => { expect(textarea).not.toHaveClass("ant-input-status-error"); }); + + it("should reject valid JSON that is not an object", async () => { + const onChange = vi.fn(); + render(); + + const textarea = screen.getByRole("textbox") as HTMLTextAreaElement; + fireEvent.change(textarea, { target: { value: "[]" } }); + fireEvent.blur(textarea); + + expect(onChange).not.toHaveBeenCalled(); + expect(textarea).toHaveClass("ant-input-status-error"); + }); }); diff --git a/ui/litellm-dashboard/src/components/router_settings/DefaultLitellmParamsSection.tsx b/ui/litellm-dashboard/src/components/router_settings/DefaultLitellmParamsSection.tsx index 273d0c65aed..3676f3f28ee 100644 --- a/ui/litellm-dashboard/src/components/router_settings/DefaultLitellmParamsSection.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/DefaultLitellmParamsSection.tsx @@ -6,59 +6,104 @@ import CacheControlInjectionPointsEditor, { import NotificationsManager from "../molecules/notifications_manager"; interface DefaultLitellmParamsSectionProps { - value: { [key: string]: any }; - routerFieldsMetadata: { [key: string]: any }; - onChange: (value: { [key: string]: any }) => void; + value: Record; + onChange: (value: Record) => void; } -const DefaultLitellmParamsSection: React.FC = ({ - value, - routerFieldsMetadata, - onChange, -}) => { - const meta = routerFieldsMetadata["default_litellm_params"]; - const { cache_control_injection_points, ...otherParams } = value || {}; +type ParsedDefaultParams = { status: "valid"; value: Record } | { status: "invalid"; message: string }; + +const CACHE_CONTROL_ROLES = ["user", "system", "assistant"] as const; + +const parseDefaultParams = (text: string): ParsedDefaultParams => { + try { + const parsed: unknown = JSON.parse(text || "{}"); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + return { status: "invalid", message: "Expected a JSON object" }; + } + return { status: "valid", value: parsed as Record }; + } catch (error) { + return { status: "invalid", message: error instanceof Error ? error.message : "Invalid JSON" }; + } +}; + +const isCacheControlInjectionPoint = (value: unknown): value is CacheControlInjectionPoint => { + if (typeof value !== "object" || value === null) { + return false; + } + if (!("location" in value) || value.location !== "message") { + return false; + } + const role = "role" in value ? value.role : undefined; + const index = "index" in value ? value.index : undefined; + const hasValidRole = role === undefined || CACHE_CONTROL_ROLES.some((validRole) => validRole === role); + const hasValidIndex = index === undefined || (typeof index === "number" && Number.isInteger(index)); + return hasValidRole && hasValidIndex; +}; + +const DefaultLitellmParamsSection: React.FC = ({ value, onChange }) => { + const cacheControlInjectionPoints = React.useMemo( + () => + Array.isArray(value.cache_control_injection_points) && + value.cache_control_injection_points.every(isCacheControlInjectionPoint) + ? value.cache_control_injection_points + : undefined, + [value.cache_control_injection_points], + ); + const otherParams = React.useMemo( + () => + Object.fromEntries( + Object.entries(value).filter( + ([key]) => key !== "cache_control_injection_points" || cacheControlInjectionPoints === undefined, + ), + ), + [value, cacheControlInjectionPoints], + ); const [otherParamsText, setOtherParamsText] = React.useState(() => JSON.stringify(otherParams, null, 2)); - const [showCacheControl, setShowCacheControl] = React.useState((cache_control_injection_points?.length ?? 0) > 0); const [hasInvalidJson, setHasInvalidJson] = React.useState(false); - - const parseOtherParams = (): { [key: string]: any } => { - try { - return JSON.parse(otherParamsText || "{}"); - } catch { - return otherParams; - } - }; + const showCacheControl = (cacheControlInjectionPoints?.length ?? 0) > 0; const handleOtherParamsBlur = () => { - try { - const parsed = JSON.parse(otherParamsText || "{}"); - setHasInvalidJson(false); - onChange({ ...parsed, cache_control_injection_points }); - } catch (error) { + const parsed = parseDefaultParams(otherParamsText); + if (parsed.status === "invalid") { setHasInvalidJson(true); - NotificationsManager.warning(`Default LiteLLM Params is not valid JSON, change not saved: ${error}`); + NotificationsManager.warning(`Default LiteLLM Params is not valid JSON: ${parsed.message}`); + return; } + setHasInvalidJson(false); + onChange( + cacheControlInjectionPoints + ? { ...parsed.value, cache_control_injection_points: cacheControlInjectionPoints } + : parsed.value, + ); }; const handleCacheControlPointsChange = (points: CacheControlInjectionPoint[]) => { - const base = parseOtherParams(); + const parsed = parseDefaultParams(otherParamsText); + const base = parsed.status === "valid" ? parsed.value : otherParams; onChange(points.length > 0 ? { ...base, cache_control_injection_points: points } : base); }; const handleCacheControlToggle = (checked: boolean) => { - setShowCacheControl(checked); handleCacheControlPointsChange(checked ? [{ location: "message" }] : []); }; return (
- - {meta?.ui_field_name || "default_litellm_params"} - -

{meta?.field_description || ""}

+ Default LiteLLM Params +

+ Default parameters for Router.chat.completion.create. Set cache control injection points to enable prompt + caching for every model on this proxy.{" "} + + Learn more + +

{ @@ -87,7 +132,7 @@ const DefaultLitellmParamsSection: React.FC = {showCacheControl && (
diff --git a/ui/litellm-dashboard/src/components/router_settings/OptionalPreCallChecksSelector.test.tsx b/ui/litellm-dashboard/src/components/router_settings/OptionalPreCallChecksSelector.test.tsx index ccda8a9d778..87127d437bc 100644 --- a/ui/litellm-dashboard/src/components/router_settings/OptionalPreCallChecksSelector.test.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/OptionalPreCallChecksSelector.test.tsx @@ -1,7 +1,7 @@ import { describe, it, expect, vi } from "vitest"; import { render, screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; -import OptionalPreCallChecksSelector from "./OptionalPreCallChecksSelector"; +import OptionalPreCallChecksSelector, { OPTIONAL_PRE_CALL_CHECK_OPTIONS } from "./OptionalPreCallChecksSelector"; vi.mock("antd", async (importOriginal) => { const actual = await importOriginal(); @@ -24,57 +24,29 @@ vi.mock("antd", async (importOriginal) => { }; }); -const baseMetadata = { - optional_pre_call_checks: { - ui_field_name: "Optional Pre-call Checks", - field_description: "Extra checks the router runs before picking a deployment", - link: null, - }, -}; - -const options = ["prompt_caching", "router_budget_limiting", "session_affinity"]; - describe("OptionalPreCallChecksSelector", () => { - it("should render one option per entry in options", () => { - render(); + it("should render the frontend-defined options", () => { + render(); const select = screen.getByTestId("optional-pre-call-checks-select") as HTMLSelectElement; - expect(Array.from(select.options).map((o) => o.value)).toEqual(options); + expect(Array.from(select.options).map((o) => o.value)).toEqual(OPTIONAL_PRE_CALL_CHECK_OPTIONS); }); - it("should display default label when no metadata is provided", () => { - render(); + it("should display the frontend-defined label and description", () => { + render(); expect(screen.getByText("Optional Pre-call Checks")).toBeInTheDocument(); + expect(screen.getByText(/extra checks the router runs before picking a deployment/i)).toBeInTheDocument(); }); - it("should display the label and description from metadata when provided", () => { - render( - , - ); - expect(screen.getByText("Extra checks the router runs before picking a deployment")).toBeInTheDocument(); - }); - - it("should render a Learn more link when metadata provides one", () => { - const metadata = { - optional_pre_call_checks: { ...baseMetadata.optional_pre_call_checks, link: "https://docs.example.com/checks" }, - }; - render( - , - ); + it("should render the prompt caching documentation link", () => { + render(); const link = screen.getByRole("link", { name: /learn more/i }); - expect(link).toHaveAttribute("href", "https://docs.example.com/checks"); + expect(link).toHaveAttribute("href", "https://docs.litellm.ai/docs/tutorials/claude_code_prompt_cache_routing"); }); it("should call onChange with the selected checks", async () => { const onChange = vi.fn(); const user = userEvent.setup(); - render( - , - ); + render(); await user.selectOptions(screen.getByTestId("optional-pre-call-checks-select"), "prompt_caching"); @@ -82,14 +54,7 @@ describe("OptionalPreCallChecksSelector", () => { }); it("should reflect an already-selected value", () => { - render( - , - ); + render(); const select = screen.getByTestId("optional-pre-call-checks-select") as HTMLSelectElement; const selected = Array.from(select.selectedOptions).map((o) => o.value); expect(selected).toEqual(["router_budget_limiting"]); diff --git a/ui/litellm-dashboard/src/components/router_settings/OptionalPreCallChecksSelector.tsx b/ui/litellm-dashboard/src/components/router_settings/OptionalPreCallChecksSelector.tsx index ab52bad644d..ed644c21710 100644 --- a/ui/litellm-dashboard/src/components/router_settings/OptionalPreCallChecksSelector.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/OptionalPreCallChecksSelector.tsx @@ -1,48 +1,43 @@ import React from "react"; import { Select } from "antd"; +export const OPTIONAL_PRE_CALL_CHECK_OPTIONS = [ + "prompt_caching", + "router_budget_limiting", + "responses_api_deployment_check", + "deployment_affinity", + "session_affinity", + "enforce_model_rate_limits", + "encrypted_content_affinity", +] as const; + interface OptionalPreCallChecksSelectorProps { value: string[]; - options: string[]; - routerFieldsMetadata: { [key: string]: any }; onChange: (value: string[]) => void; } -const OptionalPreCallChecksSelector: React.FC = ({ - value, - options, - routerFieldsMetadata, - onChange, -}) => { - const meta = routerFieldsMetadata["optional_pre_call_checks"]; - +const OptionalPreCallChecksSelector: React.FC = ({ value, onChange }) => { return (
-
- Role +
+ + Role +