feat(ui): move Anthropic prompt caching to its own Router Settings tab

Rather than mixing the flag and its ttl into the generic General settings table
(which also surfaced the confusing Not Set / In Config / In DB provenance badges),
give prompt caching a dedicated tab with a purpose-built toggle and ttl dropdown.

Each registry field gains an optional tab, surfaced as ConfigList.field_tab, so
the General tab renders the ungrouped fields and the caching fields render on
their own tab. The update, persist and reset endpoints are unchanged.
This commit is contained in:
Tin Chi Lo 2026-07-17 11:38:24 -07:00
parent 16e39542a0
commit 73cbbdd51d
5 changed files with 90 additions and 1 deletions

View file

@ -2123,6 +2123,7 @@ class ConfigList(LiteLLMPydanticObjectBase):
premium_field: bool = False
nested_fields: Optional[List[FieldDetail]] = None # For nested dictionary or Pydantic fields
field_options: Optional[list[str]] = None # Allowed values, for field_type == "Select"
field_tab: Optional[str] = None # Admin UI sub-tab this field renders under; None groups it with the rest
class UserHeaderMapping(LiteLLMPydanticObjectBase):

View file

@ -14809,6 +14809,7 @@ class GeneralSettingsUILiteLLMFieldSpec(TypedDict):
type: Literal["Float", "Boolean", "Select"]
description: str
options: NotRequired[tuple[str, ...]]
tab: NotRequired[str] # Admin UI sub-tab this field renders under; None groups it with the rest
_GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, GeneralSettingsUILiteLLMFieldSpec] = {
@ -14822,6 +14823,7 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, GeneralSettingsUILiteLLMFieldSpec
},
"enable_anthropic_prompt_caching": {
"type": "Boolean",
"tab": "prompt_caching",
"description": (
"Automatically add Anthropic cache_control breakpoints to the system prompt and the "
"trailing turn, for Anthropic and Bedrock Claude models that support prompt caching. "
@ -14836,6 +14838,7 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, GeneralSettingsUILiteLLMFieldSpec
"anthropic_prompt_caching_ttl": {
"type": "Select",
"options": ("5m", "1h"),
"tab": "prompt_caching",
"description": (
"Cache lifetime for the breakpoints added by 'enable_anthropic_prompt_caching'. "
"Leave empty for Anthropic's 5m default. 1h suits long agentic sessions but doubles "
@ -15093,6 +15096,7 @@ async def get_config_list(
stored_in_db=stored_in_db_litellm,
field_default_value=default_value,
field_options=list(spec.get("options", ())) or None,
field_tab=spec.get("tab"),
nested_fields=None,
)
)

View file

@ -9019,6 +9019,12 @@ def test_get_config_list_includes_anthropic_prompt_caching_fields(monkeypatch):
assert fields["anthropic_prompt_caching_ttl"]["field_type"] == "Select"
assert fields["anthropic_prompt_caching_ttl"]["field_value"] == "1h"
assert fields["anthropic_prompt_caching_ttl"]["field_options"] == ["5m", "1h"]
# Both caching fields carry their sub-tab so the Admin UI can render them on a
# dedicated Prompt Caching tab, while ungrouped fields stay on General.
assert fields["enable_anthropic_prompt_caching"]["field_tab"] == "prompt_caching"
assert fields["anthropic_prompt_caching_ttl"]["field_tab"] == "prompt_caching"
assert fields["budget_exceeded_throttle_percentage"]["field_tab"] is None
finally:
app.dependency_overrides.clear()

View file

@ -7,6 +7,7 @@ import {
TableHeaderCell,
TableCell,
TableBody,
Title,
Text,
Button,
Icon,
@ -21,6 +22,11 @@ import { StatusBadge } from "@/components/shared/table_cells";
import RouterSettings from "@/components/router_settings";
import Fallbacks from "@/components/Settings/RouterSettings/Fallbacks/Fallbacks";
import RoutingGroups from "@/components/routing_groups";
const PROMPT_CACHING_TAB = "prompt_caching";
const ENABLE_ANTHROPIC_PROMPT_CACHING = "enable_anthropic_prompt_caching";
const ANTHROPIC_PROMPT_CACHING_TTL = "anthropic_prompt_caching_ttl";
interface GeneralSettingsPageProps {
accessToken: string | null;
userRole: string | null;
@ -34,6 +40,7 @@ interface generalSettingsItem {
field_description: string;
stored_in_db: boolean | null;
field_options?: string[] | null;
field_tab?: string | null;
}
const SettingValueEditor: React.FC<{
@ -83,6 +90,71 @@ const SettingValueEditor: React.FC<{
return null;
};
const PromptCachingPanel: React.FC<{
accessToken: string;
settings: generalSettingsItem[];
onChange: (fieldName: string, newValue: any) => void;
}> = ({ accessToken, settings, onChange }) => {
const enableSetting = settings.find((s) => s.field_name === ENABLE_ANTHROPIC_PROMPT_CACHING);
const ttlSetting = settings.find((s) => s.field_name === ANTHROPIC_PROMPT_CACHING_TTL);
// The two rows come from the same registry the General tab reads; if they
// are not loaded yet there is nothing to render.
if (!enableSetting) {
return null;
}
const enabled = enableSetting.field_value === true || enableSetting.field_value === "true";
// Apply immediately: a toggle and a dropdown are direct controls, so there is
// no separate Update button. Clearing the ttl resets it to the provider default.
const persist = (fieldName: string, value: any) => {
onChange(fieldName, value);
if (value === "" || value === null || value === undefined) {
deleteConfigFieldSetting(accessToken, fieldName);
} else {
updateConfigFieldSetting(accessToken, fieldName, value);
}
};
return (
<Card>
<Title>Prompt Caching</Title>
<Text className="mt-2">
Automatically inject Anthropic prompt caching for every Anthropic and Bedrock Claude model, so clients that
never set <span className="font-mono">cache_control</span> themselves still get cached prompts. This is a single
gateway-wide switch; there is no per-model setup.
</Text>
<div className="mt-6 flex items-start justify-between gap-8">
<div className="max-w-2xl">
<Text className="font-medium">Automatic Anthropic prompt caching</Text>
<p className="mt-1 text-xs text-gray-500">{enableSetting.field_description}</p>
</div>
<Switch checked={enabled} onChange={(checked) => persist(ENABLE_ANTHROPIC_PROMPT_CACHING, checked)} />
</div>
{ttlSetting && (
<div className="mt-6 flex items-start justify-between gap-8">
<div className="max-w-2xl">
<Text className={`font-medium ${enabled ? "" : "text-gray-400"}`}>Cache lifetime (TTL)</Text>
<p className="mt-1 text-xs text-gray-500">{ttlSetting.field_description}</p>
</div>
<AntdSelect
allowClear
disabled={!enabled}
style={{ minWidth: "10rem" }}
placeholder="5m (default)"
value={ttlSetting.field_value || undefined}
options={(ttlSetting.field_options ?? []).map((option) => ({ label: option, value: option }))}
onChange={(newValue) => persist(ANTHROPIC_PROMPT_CACHING_TTL, newValue ?? "")}
/>
</div>
)}
</Card>
);
};
const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({ accessToken, userRole, userID }) => {
const [generalSettings, setGeneralSettings] = useState<generalSettingsItem[]>([]);
@ -156,6 +228,7 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({ accessToken, user
<Tab value="1">Loadbalancing</Tab>
<Tab value="2">Routing Groups</Tab>
<Tab value="3">Fallbacks</Tab>
<Tab value="5">Prompt Caching</Tab>
<Tab value="4">General</Tab>
</TabList>
<TabPanels className="px-8 py-6">
@ -168,6 +241,9 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({ accessToken, user
<TabPanel>
<Fallbacks accessToken={accessToken} userRole={userRole} userID={userID} />
</TabPanel>
<TabPanel>
<PromptCachingPanel accessToken={accessToken} settings={generalSettings} onChange={handleInputChange} />
</TabPanel>
<TabPanel>
<Card>
<Table>
@ -181,7 +257,7 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({ accessToken, user
</TableHead>
<TableBody>
{generalSettings
.filter((value) => value.field_type !== "TypedDictionary")
.filter((value) => value.field_type !== "TypedDictionary" && value.field_tab !== PROMPT_CACHING_TAB)
.map((value, index) => (
<TableRow key={index}>
<TableCell>

View file

@ -22713,6 +22713,8 @@ export interface components {
field_name: string;
/** Field Options */
field_options?: string[] | null;
/** Field Tab */
field_tab?: string | null;
/** Field Type */
field_type: string;
/** Field Value */