From 73cbbdd51defe21a6db5beadf2b3ee73454677be Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Fri, 17 Jul 2026 11:38:24 -0700 Subject: [PATCH] feat(ui): move Anthropic prompt caching to its own Router Settings tab Rather than mixing the flag and its ttl into the generic General settings table (which also surfaced the confusing Not Set / In Config / In DB provenance badges), give prompt caching a dedicated tab with a purpose-built toggle and ttl dropdown. Each registry field gains an optional tab, surfaced as ConfigList.field_tab, so the General tab renders the ungrouped fields and the caching fields render on their own tab. The update, persist and reset endpoints are unchanged. --- litellm/proxy/_types.py | 1 + litellm/proxy/proxy_server.py | 4 + tests/test_litellm/proxy/test_proxy_server.py | 6 ++ .../_components/general_settings.tsx | 78 ++++++++++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 + 5 files changed, 90 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e07c7b9ae78..b47b43411c5 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2123,6 +2123,7 @@ class ConfigList(LiteLLMPydanticObjectBase): premium_field: bool = False nested_fields: Optional[List[FieldDetail]] = None # For nested dictionary or Pydantic fields field_options: Optional[list[str]] = None # Allowed values, for field_type == "Select" + field_tab: Optional[str] = None # Admin UI sub-tab this field renders under; None groups it with the rest class UserHeaderMapping(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ad1617017f9..6725ecdb584 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14809,6 +14809,7 @@ class GeneralSettingsUILiteLLMFieldSpec(TypedDict): type: Literal["Float", "Boolean", "Select"] description: str options: NotRequired[tuple[str, ...]] + tab: NotRequired[str] # Admin UI sub-tab this field renders under; None groups it with the rest _GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, GeneralSettingsUILiteLLMFieldSpec] = { @@ -14822,6 +14823,7 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, GeneralSettingsUILiteLLMFieldSpec }, "enable_anthropic_prompt_caching": { "type": "Boolean", + "tab": "prompt_caching", "description": ( "Automatically add Anthropic cache_control breakpoints to the system prompt and the " "trailing turn, for Anthropic and Bedrock Claude models that support prompt caching. " @@ -14836,6 +14838,7 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, GeneralSettingsUILiteLLMFieldSpec "anthropic_prompt_caching_ttl": { "type": "Select", "options": ("5m", "1h"), + "tab": "prompt_caching", "description": ( "Cache lifetime for the breakpoints added by 'enable_anthropic_prompt_caching'. " "Leave empty for Anthropic's 5m default. 1h suits long agentic sessions but doubles " @@ -15093,6 +15096,7 @@ async def get_config_list( stored_in_db=stored_in_db_litellm, field_default_value=default_value, field_options=list(spec.get("options", ())) or None, + field_tab=spec.get("tab"), nested_fields=None, ) ) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 86e807447c1..56cf213f103 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -9019,6 +9019,12 @@ def test_get_config_list_includes_anthropic_prompt_caching_fields(monkeypatch): assert fields["anthropic_prompt_caching_ttl"]["field_type"] == "Select" assert fields["anthropic_prompt_caching_ttl"]["field_value"] == "1h" assert fields["anthropic_prompt_caching_ttl"]["field_options"] == ["5m", "1h"] + + # Both caching fields carry their sub-tab so the Admin UI can render them on a + # dedicated Prompt Caching tab, while ungrouped fields stay on General. + assert fields["enable_anthropic_prompt_caching"]["field_tab"] == "prompt_caching" + assert fields["anthropic_prompt_caching_ttl"]["field_tab"] == "prompt_caching" + assert fields["budget_exceeded_throttle_percentage"]["field_tab"] is None finally: app.dependency_overrides.clear() diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx index af6bbdde8b9..8cea529d25d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx @@ -7,6 +7,7 @@ import { TableHeaderCell, TableCell, TableBody, + Title, Text, Button, Icon, @@ -21,6 +22,11 @@ import { StatusBadge } from "@/components/shared/table_cells"; import RouterSettings from "@/components/router_settings"; import Fallbacks from "@/components/Settings/RouterSettings/Fallbacks/Fallbacks"; import RoutingGroups from "@/components/routing_groups"; + +const PROMPT_CACHING_TAB = "prompt_caching"; +const ENABLE_ANTHROPIC_PROMPT_CACHING = "enable_anthropic_prompt_caching"; +const ANTHROPIC_PROMPT_CACHING_TTL = "anthropic_prompt_caching_ttl"; + interface GeneralSettingsPageProps { accessToken: string | null; userRole: string | null; @@ -34,6 +40,7 @@ interface generalSettingsItem { field_description: string; stored_in_db: boolean | null; field_options?: string[] | null; + field_tab?: string | null; } const SettingValueEditor: React.FC<{ @@ -83,6 +90,71 @@ const SettingValueEditor: React.FC<{ return null; }; +const PromptCachingPanel: React.FC<{ + accessToken: string; + settings: generalSettingsItem[]; + onChange: (fieldName: string, newValue: any) => void; +}> = ({ accessToken, settings, onChange }) => { + const enableSetting = settings.find((s) => s.field_name === ENABLE_ANTHROPIC_PROMPT_CACHING); + const ttlSetting = settings.find((s) => s.field_name === ANTHROPIC_PROMPT_CACHING_TTL); + + // The two rows come from the same registry the General tab reads; if they + // are not loaded yet there is nothing to render. + if (!enableSetting) { + return null; + } + + const enabled = enableSetting.field_value === true || enableSetting.field_value === "true"; + + // Apply immediately: a toggle and a dropdown are direct controls, so there is + // no separate Update button. Clearing the ttl resets it to the provider default. + const persist = (fieldName: string, value: any) => { + onChange(fieldName, value); + if (value === "" || value === null || value === undefined) { + deleteConfigFieldSetting(accessToken, fieldName); + } else { + updateConfigFieldSetting(accessToken, fieldName, value); + } + }; + + return ( + + Prompt Caching + + Automatically inject Anthropic prompt caching for every Anthropic and Bedrock Claude model, so clients that + never set cache_control themselves still get cached prompts. This is a single + gateway-wide switch; there is no per-model setup. + + +
+
+ Automatic Anthropic prompt caching +

{enableSetting.field_description}

+
+ persist(ENABLE_ANTHROPIC_PROMPT_CACHING, checked)} /> +
+ + {ttlSetting && ( +
+
+ Cache lifetime (TTL) +

{ttlSetting.field_description}

+
+ ({ label: option, value: option }))} + onChange={(newValue) => persist(ANTHROPIC_PROMPT_CACHING_TTL, newValue ?? "")} + /> +
+ )} + + ); +}; + const GeneralSettings: React.FC = ({ accessToken, userRole, userID }) => { const [generalSettings, setGeneralSettings] = useState([]); @@ -156,6 +228,7 @@ const GeneralSettings: React.FC = ({ accessToken, user Loadbalancing Routing Groups Fallbacks + Prompt Caching General @@ -168,6 +241,9 @@ const GeneralSettings: React.FC = ({ accessToken, user + + + @@ -181,7 +257,7 @@ const GeneralSettings: React.FC = ({ accessToken, user {generalSettings - .filter((value) => value.field_type !== "TypedDictionary") + .filter((value) => value.field_type !== "TypedDictionary" && value.field_tab !== PROMPT_CACHING_TAB) .map((value, index) => ( diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 45b06c9e44f..6dc63762e5c 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -22713,6 +22713,8 @@ export interface components { field_name: string; /** Field Options */ field_options?: string[] | null; + /** Field Tab */ + field_tab?: string | null; /** Field Type */ field_type: string; /** Field Value */