From d418752267e55be1ed83d2f8a39cf2ff6e156180 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 14 Jul 2026 17:12:30 +0000 Subject: [PATCH] fix(ui): restore cache control router fields Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...acheControlInjectionPointsSection.test.tsx | 70 ++++++++++ .../CacheControlInjectionPointsSection.tsx | 122 ++++++++++++++++++ .../RouterSettingsForm.test.tsx | 43 +++++- .../router_settings/RouterSettingsForm.tsx | 21 +++ .../components/router_settings/index.test.tsx | 69 +++++++++- .../src/components/router_settings/index.tsx | 4 + 6 files changed, 322 insertions(+), 7 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/router_settings/CacheControlInjectionPointsSection.test.tsx create mode 100644 ui/litellm-dashboard/src/components/router_settings/CacheControlInjectionPointsSection.tsx diff --git a/ui/litellm-dashboard/src/components/router_settings/CacheControlInjectionPointsSection.test.tsx b/ui/litellm-dashboard/src/components/router_settings/CacheControlInjectionPointsSection.test.tsx new file mode 100644 index 00000000000..68272e7026b --- /dev/null +++ b/ui/litellm-dashboard/src/components/router_settings/CacheControlInjectionPointsSection.test.tsx @@ -0,0 +1,70 @@ +import { describe, expect, it, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import CacheControlInjectionPointsSection from "./CacheControlInjectionPointsSection"; + +describe("CacheControlInjectionPointsSection", () => { + it("should render the frontend-owned field without exposing the raw default params input", () => { + const onChange = vi.fn(); + render(); + + expect(screen.getByText("Cache Control Injection Points")).toBeInTheDocument(); + expect(screen.queryByText("Default LiteLLM Params")).not.toBeInTheDocument(); + expect(screen.queryByTestId("cache-control-location-select-0")).not.toBeInTheDocument(); + expect(onChange).not.toHaveBeenCalled(); + }); + + it("should render persisted injection point fields", () => { + render( + , + ); + + expect(screen.getByTestId("cache-control-location-select-0")).toBeInTheDocument(); + expect(screen.getByTestId("cache-control-index-input-0")).toHaveValue("-2"); + }); + + it("should add injection points without replacing other default params", async () => { + const onChange = vi.fn(); + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("switch", { name: "Cache Control Injection Points" })); + + expect(onChange).toHaveBeenCalledWith({ + timeout: 30, + cache_control_injection_points: [{ location: "message" }], + }); + }); + + it("should remove injection points without replacing other default params", async () => { + const onChange = vi.fn(); + const user = userEvent.setup(); + render( + , + ); + + await user.click(screen.getByRole("switch", { name: "Cache Control Injection Points" })); + + expect(onChange).toHaveBeenCalledWith({ timeout: 30 }); + }); + + it("should preserve unsupported injection point values", () => { + const onChange = vi.fn(); + render( + , + ); + + expect(screen.getByRole("switch", { name: "Cache Control Injection Points" })).toBeDisabled(); + expect(screen.getByText(/will be preserved unchanged/i)).toBeInTheDocument(); + expect(onChange).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/router_settings/CacheControlInjectionPointsSection.tsx b/ui/litellm-dashboard/src/components/router_settings/CacheControlInjectionPointsSection.tsx new file mode 100644 index 00000000000..ee113f32ed6 --- /dev/null +++ b/ui/litellm-dashboard/src/components/router_settings/CacheControlInjectionPointsSection.tsx @@ -0,0 +1,122 @@ +import React from "react"; +import { Switch } from "antd"; +import CacheControlInjectionPointsEditor, { + CacheControlInjectionPoint, +} from "../shared/cache_control_injection_points_editor"; + +interface CacheControlInjectionPointsSectionProps { + value: Record; + onChange: (value: Record) => void; +} + +const CACHE_CONTROL_ROLES = ["user", "system", "assistant"] as const; +type CacheControlRole = (typeof CACHE_CONTROL_ROLES)[number]; + +const isCacheControlRole = (value: unknown): value is CacheControlRole => + CACHE_CONTROL_ROLES.some((role) => role === value); + +const parseInjectionPoint = (value: unknown): CacheControlInjectionPoint | undefined => { + if (typeof value !== "object" || value === null) { + return undefined; + } + if (!("location" in value) || value.location !== "message") { + return undefined; + } + + const role = "role" in value ? value.role : undefined; + const index = "index" in value ? value.index : undefined; + const hasRole = role !== undefined && role !== null; + if (hasRole && !isCacheControlRole(role)) { + return undefined; + } + const hasIndex = index !== undefined && index !== null; + if (hasIndex && typeof index !== "number") { + return undefined; + } + if (typeof index === "number" && !Number.isInteger(index)) { + return undefined; + } + + return { + location: "message", + ...(isCacheControlRole(role) ? { role } : {}), + ...(typeof index === "number" ? { index } : {}), + }; +}; + +const parseInjectionPoints = (value: unknown): CacheControlInjectionPoint[] | undefined => { + if (value === undefined) { + return []; + } + if (!Array.isArray(value)) { + return undefined; + } + + const points = value.map(parseInjectionPoint); + return points.every((point) => point !== undefined) + ? points.filter((point): point is CacheControlInjectionPoint => point !== undefined) + : undefined; +}; + +const CacheControlInjectionPointsSection: React.FC = ({ value, onChange }) => { + const injectionPoints = React.useMemo( + () => parseInjectionPoints(value.cache_control_injection_points), + [value.cache_control_injection_points], + ); + const enabled = injectionPoints !== undefined && injectionPoints.length > 0; + + const handlePointsChange = (points: CacheControlInjectionPoint[]) => { + onChange({ ...value, cache_control_injection_points: points }); + }; + + const handleToggle = (checked: boolean) => { + if (checked) { + handlePointsChange([{ location: "message" }]); + return; + } + + const nextValue = Object.fromEntries( + Object.entries(value).filter(([key]) => key !== "cache_control_injection_points"), + ); + onChange(nextValue); + }; + + return ( +
+
+
+ + Cache Control Injection Points + +

+ Choose message locations where LiteLLM should inject cache-control markers.{" "} + + Learn more + +

+
+ +
+ + {injectionPoints === undefined && ( +

+ The configured injection points use values this editor does not support and will be preserved unchanged. +

+ )} + + {enabled && } +
+ ); +}; + +export default CacheControlInjectionPointsSection; diff --git a/ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.test.tsx b/ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.test.tsx index 90f2edb6b48..8a8e468a821 100644 --- a/ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.test.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.test.tsx @@ -156,14 +156,51 @@ describe("RouterSettingsForm", () => { ); }); - it("should not render Default LiteLLM Params when persisted values exist", () => { + it("should render cache-control fields without rendering the raw Default LiteLLM Params input", () => { const props = { ...baseProps, - value: { ...defaultValue, routerSettings: { default_litellm_params: { timeout: 30 } } }, + value: { + ...defaultValue, + routerSettings: { + default_litellm_params: { + timeout: 30, + cache_control_injection_points: [{ location: "message", role: "system", index: -1 }], + }, + }, + }, }; render(); expect(screen.queryByText("Default LiteLLM Params")).not.toBeInTheDocument(); - expect(screen.queryByText("Cache Control Injection Points")).not.toBeInTheDocument(); + expect(screen.getByText("Cache Control Injection Points")).toBeInTheDocument(); + expect(screen.getByTestId("cache-control-index-input-0")).toHaveValue("-1"); + }); + + it("should mark default_litellm_params as modified when cache control changes", async () => { + const onChange = vi.fn(); + const user = userEvent.setup(); + const props = { + ...baseProps, + onChange, + value: { + ...defaultValue, + routerSettings: { default_litellm_params: { timeout: 30 } }, + }, + }; + render(); + + await user.click(screen.getByRole("switch", { name: "Cache Control Injection Points" })); + + expect(onChange).toHaveBeenCalledWith( + expect.objectContaining({ + routerSettings: { + default_litellm_params: { + timeout: 30, + cache_control_injection_points: [{ location: "message" }], + }, + }, + modifiedRouterSettings: ["default_litellm_params"], + }), + ); }); }); diff --git a/ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.tsx b/ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.tsx index b1d95d228fb..d006d813d1f 100644 --- a/ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/RouterSettingsForm.tsx @@ -1,4 +1,5 @@ import React from "react"; +import CacheControlInjectionPointsSection from "./CacheControlInjectionPointsSection"; import LatencyBasedConfiguration from "./LatencyBasedConfiguration"; import OptionalPreCallChecksSelector from "./OptionalPreCallChecksSelector"; import ReliabilityRetriesSection from "./ReliabilityRetriesSection"; @@ -9,6 +10,7 @@ export interface RouterSettingsFormValue { routerSettings: { [key: string]: any }; selectedStrategy: string | null; enableTagFiltering: boolean; + modifiedRouterSettings?: string[]; } interface RouterSettingsFormProps { @@ -19,6 +21,9 @@ interface RouterSettingsFormProps { routingStrategyDescriptions: { [key: string]: string }; } +const markSettingModified = (modifiedSettings: string[] | undefined, setting: string): string[] => + modifiedSettings?.includes(setting) ? modifiedSettings : [...(modifiedSettings || []), setting]; + const RouterSettingsForm: React.FC = ({ value, onChange, @@ -44,6 +49,15 @@ const RouterSettingsForm: React.FC = ({ onChange({ ...value, routerSettings: { ...value.routerSettings, optional_pre_call_checks: checks }, + modifiedRouterSettings: markSettingModified(value.modifiedRouterSettings, "optional_pre_call_checks"), + }); + }; + + const handleCacheControlInjectionPointsChange = (defaultLitellmParams: Record) => { + onChange({ + ...value, + routerSettings: { ...value.routerSettings, default_litellm_params: defaultLitellmParams }, + modifiedRouterSettings: markSettingModified(value.modifiedRouterSettings, "default_litellm_params"), }); }; @@ -88,6 +102,13 @@ const RouterSettingsForm: React.FC = ({
+ + +
+ {/* Other Settings */}
diff --git a/ui/litellm-dashboard/src/components/router_settings/index.test.tsx b/ui/litellm-dashboard/src/components/router_settings/index.test.tsx index 46d0bdaeb8c..55bd76709af 100644 --- a/ui/litellm-dashboard/src/components/router_settings/index.test.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/index.test.tsx @@ -140,7 +140,7 @@ describe("RouterSettings", () => { expect(screen.getByTestId("strategy-select")).toBeInTheDocument(); }); expect(screen.queryByText("Default LiteLLM Params")).not.toBeInTheDocument(); - expect(screen.queryByText("Cache Control Injection Points")).not.toBeInTheDocument(); + expect(screen.getByText("Cache Control Injection Points")).toBeInTheDocument(); await user.click(screen.getByRole("button", { name: /save changes/i })); @@ -208,7 +208,7 @@ describe("RouterSettings", () => { expect(NotificationsManager.success).not.toHaveBeenCalled(); }); - it("should round-trip default_litellm_params and optional_pre_call_checks unmodified on save", async () => { + it("should preserve untouched default_litellm_params by omitting it from an unrelated save", async () => { vi.mocked(getCallbacksCall).mockResolvedValue({ router_settings: { ...mockCallbacksResponse.router_settings, @@ -227,15 +227,76 @@ describe("RouterSettings", () => { await user.click(screen.getByRole("button", { name: /save changes/i })); + await waitFor(() => + expect(setCallbacksCall).toHaveBeenCalledWith( + "test-token", + expect.objectContaining({ + router_settings: expect.not.objectContaining({ + default_litellm_params: expect.anything(), + }), + }), + ), + ); + }); + + it("should save frontend-owned cache-control fields after an explicit edit", async () => { + vi.mocked(getCallbacksCall).mockResolvedValue({ + router_settings: { + ...mockCallbacksResponse.router_settings, + default_litellm_params: { timeout: 30 }, + }, + }); + const user = userEvent.setup(); + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByRole("switch", { name: "Cache Control Injection Points" })).toBeInTheDocument(); + }); + + await user.click(screen.getByRole("switch", { name: "Cache Control Injection Points" })); + await user.click(screen.getByRole("button", { name: /save changes/i })); + await waitFor(() => expect(setCallbacksCall).toHaveBeenCalledWith( "test-token", expect.objectContaining({ router_settings: expect.objectContaining({ default_litellm_params: { - cache_control_injection_points: [{ location: "message", role: "system", index: 2 }], + timeout: 30, + cache_control_injection_points: [{ location: "message" }], }, - optional_pre_call_checks: ["prompt_caching"], + }), + }), + ), + ); + }); + + it("should remove cache-control fields after they are explicitly disabled", async () => { + vi.mocked(getCallbacksCall).mockResolvedValue({ + router_settings: { + ...mockCallbacksResponse.router_settings, + default_litellm_params: { + timeout: 30, + cache_control_injection_points: [{ location: "message", index: -1 }], + }, + }, + }); + const user = userEvent.setup(); + renderWithProviders(); + + await waitFor(() => { + expect(screen.getByTestId("cache-control-index-input-0")).toHaveValue("-1"); + }); + + await user.click(screen.getByRole("switch", { name: "Cache Control Injection Points" })); + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => + expect(setCallbacksCall).toHaveBeenCalledWith( + "test-token", + expect.objectContaining({ + router_settings: expect.objectContaining({ + default_litellm_params: { timeout: 30 }, }), }), ), diff --git a/ui/litellm-dashboard/src/components/router_settings/index.tsx b/ui/litellm-dashboard/src/components/router_settings/index.tsx index aea08425388..a78863558dc 100644 --- a/ui/litellm-dashboard/src/components/router_settings/index.tsx +++ b/ui/litellm-dashboard/src/components/router_settings/index.tsx @@ -85,6 +85,7 @@ const RouterSettings: React.FC = ({ accessToken, userRole, } const router_settings = formValue.routerSettings; + const modifiedRouterSettings = formValue.modifiedRouterSettings || []; const numberKeys = new Set(["allowed_fails", "cooldown_time", "num_retries", "timeout", "retry_after"]); const jsonKeys = new Set(["model_group_alias"]); @@ -131,6 +132,9 @@ const RouterSettings: React.FC = ({ accessToken, userRole, if (tabOwnedKeys.has(key)) { return null; } + if (key === "default_litellm_params" && !modifiedRouterSettings.includes(key)) { + return null; + } if (key !== "routing_strategy_args" && key !== "routing_strategy" && key !== "enable_tag_filtering") { const inputEl = document.querySelector(`input[name="${key}"]`) as HTMLInputElement | null; const parsed = parseInputValue(key, inputEl?.value, value);