From 12919628501340c8b7b596d33492bd9f5ef6eff0 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 16 Jul 2026 13:23:41 -0700 Subject: [PATCH 1/6] feat(ui): configure Anthropic automatic prompt caching from the Admin UI Register enable_anthropic_prompt_caching and anthropic_prompt_caching_ttl on the General Settings table so caching can be turned on without hand-writing config. The registry could not express either field: validation was hardcoded to a float in (0, 1], reset set every field to None (not a bool for a boolean flag), and the listing reported any non-None value as 'In Config', which a False default would always trip. Validation now dispatches on the declared type and reset restores each field's own default. ConfigList carries field_options so the table can render a Select for enums instead of no editor at all. --- litellm/proxy/_types.py | 3 +- litellm/proxy/proxy_server.py | 96 +++++++-- tests/test_litellm/proxy/test_proxy_server.py | 204 ++++++++++++++++++ .../_components/general_settings.tsx | 15 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 + 5 files changed, 300 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d102c1d1e37..e07c7b9ae78 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1011,10 +1011,10 @@ class LiteLLM_ObjectPermissionBase(LiteLLMPydanticObjectBase): mcp_tool_search_enabled: Optional[bool] = None +from litellm.models.team import BudgetLimitEntry as BudgetLimitEntry # noqa: E402 from litellm.types.object_permission import ( # noqa: E402 ObjectPermissionDict as ObjectPermissionDict, ) -from litellm.models.team import BudgetLimitEntry as BudgetLimitEntry # noqa: E402 class GenerateRequestBase(LiteLLMPydanticObjectBase): @@ -2122,6 +2122,7 @@ class ConfigList(LiteLLMPydanticObjectBase): field_default_value: Any premium_field: bool = False nested_fields: Optional[List[FieldDetail]] = None # For nested dictionary or Pydantic fields + field_options: Optional[list[str]] = None # Allowed values, for field_type == "Select" class UserHeaderMapping(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index dbdfdd5fdd3..ae91eb70427 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -28,6 +28,7 @@ from typing import ( Optional, Set, Tuple, + TypedDict, Union, cast, get_args, @@ -39,6 +40,7 @@ import anyio import websockets import websockets.exceptions from pydantic import BaseModel, Json, JsonValue +from typing_extensions import NotRequired, assert_never from litellm._uuid import uuid from litellm.constants import ( @@ -363,15 +365,15 @@ from litellm.proxy.management_endpoints.cache_settings_endpoints import ( from litellm.proxy.management_endpoints.callback_management_endpoints import ( router as callback_management_endpoints_router, ) -from litellm.proxy.management_endpoints.coordination_redis_endpoints import ( - get_persisted_coordination_redis_settings, - router as coordination_redis_settings_router, -) from litellm.proxy.management_endpoints.common_utils import ( _user_has_admin_privileges, _user_has_admin_view, admin_can_invite_user, ) +from litellm.proxy.management_endpoints.coordination_redis_endpoints import ( + get_persisted_coordination_redis_settings, + router as coordination_redis_settings_router, +) from litellm.proxy.management_endpoints.cost_tracking_settings import ( router as cost_tracking_settings_router, ) @@ -14800,7 +14802,16 @@ async def get_config_general_settings( ) -_GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, dict[str, str]] = { +GeneralSettingsUILiteLLMValue = Union[float, bool, str, None] + + +class GeneralSettingsUILiteLLMFieldSpec(TypedDict): + type: Literal["Float", "Boolean", "Select"] + description: str + options: NotRequired[tuple[str, ...]] + + +_GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, GeneralSettingsUILiteLLMFieldSpec] = { "budget_exceeded_throttle_percentage": { "type": "Float", "description": ( @@ -14809,18 +14820,64 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, dict[str, str]] = { "over-budget keys." ), }, + "enable_anthropic_prompt_caching": { + "type": "Boolean", + "description": ( + "Automatically add Anthropic cache_control breakpoints to the system prompt and the " + "trailing turn, for Anthropic and Bedrock Claude models that support prompt caching. " + "Lets clients that never set cache_control themselves still get cached prompts. " + "Requests that already carry their own cache_control are left untouched." + ), + }, + "anthropic_prompt_caching_ttl": { + "type": "Select", + "options": ("5m", "1h"), + "description": ( + "Cache lifetime for the breakpoints added by 'enable_anthropic_prompt_caching'. " + "Leave empty for Anthropic's 5m default. 1h suits long agentic sessions but doubles " + "the cache write premium." + ), + }, } -def _validate_general_settings_ui_litellm_value(field_name: str, value: Any) -> Optional[float]: +def _general_settings_ui_litellm_default( + field_type: Literal["Float", "Boolean", "Select"], +) -> GeneralSettingsUILiteLLMValue: + """The value a field falls back to when it is cleared or reset.""" + return False if field_type == "Boolean" else None + + +def _validate_general_settings_ui_litellm_value(field_name: str, value: Any) -> GeneralSettingsUILiteLLMValue: + spec = _GENERAL_SETTINGS_UI_LITELLM_FIELDS[field_name] + field_type = spec["type"] if value is None or value == "": - return None - if isinstance(value, bool) or not isinstance(value, (int, float)) or not (0 < float(value) <= 1): - raise HTTPException( - status_code=400, - detail={"error": f"{field_name} must be a number in (0, 1] or empty"}, - ) - return float(value) + return _general_settings_ui_litellm_default(field_type) + match field_type: + case "Boolean": + if not isinstance(value, bool): + raise HTTPException( + status_code=400, + detail={"error": f"{field_name} must be true or false"}, + ) + return value + case "Select": + options = spec.get("options", ()) + if value not in options: + raise HTTPException( + status_code=400, + detail={"error": f"{field_name} must be one of: {', '.join(options)}, or empty"}, + ) + return cast(str, value) # cast-ok: membership in options proves it is one of the option strings + case "Float": + if isinstance(value, bool) or not isinstance(value, (int, float)) or not (0 < float(value) <= 1): + raise HTTPException( + status_code=400, + detail={"error": f"{field_name} must be a number in (0, 1] or empty"}, + ) + return float(value) + case _: + assert_never(field_type) async def _persist_general_settings_ui_litellm_field( @@ -14841,11 +14898,12 @@ async def _persist_general_settings_ui_litellm_field( async def _reset_general_settings_ui_litellm_field(field_name: str, user_api_key_dict: UserAPIKeyAuth) -> dict: config = await proxy_config.get_config() before_value = config.get("litellm_settings", {}).get(field_name) - setattr(litellm, field_name, None) + default_value = _general_settings_ui_litellm_default(_GENERAL_SETTINGS_UI_LITELLM_FIELDS[field_name]["type"]) + setattr(litellm, field_name, default_value) if "litellm_settings" in config: config["litellm_settings"].pop(field_name, None) await proxy_config.save_config(new_config=config) - asyncio.create_task(create_config_audit_log(field_name, "deleted", before_value, None, user_api_key_dict)) + asyncio.create_task(create_config_audit_log(field_name, "deleted", before_value, default_value, user_api_key_dict)) return {"message": f"Field {field_name} reset", "status": "success"} @@ -15013,11 +15071,12 @@ async def get_config_list( else {} ) for litellm_field_name, spec in _GENERAL_SETTINGS_UI_LITELLM_FIELDS.items(): - current_value: Optional[float] = getattr(litellm, litellm_field_name, None) + current_value: GeneralSettingsUILiteLLMValue = getattr(litellm, litellm_field_name, None) + default_value = _general_settings_ui_litellm_default(spec["type"]) stored_in_db_litellm: Optional[bool] if litellm_field_name in db_litellm_settings: stored_in_db_litellm = True - elif current_value is not None: + elif current_value != default_value: stored_in_db_litellm = False else: stored_in_db_litellm = None @@ -15028,7 +15087,8 @@ async def get_config_list( field_description=spec["description"], field_value=current_value, stored_in_db=stored_in_db_litellm, - field_default_value=None, + field_default_value=default_value, + field_options=list(spec.get("options", ())) or None, nested_fields=None, ) ) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 54db0c0fd4f..86e807447c1 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -8984,6 +8984,210 @@ async def test_update_config_field_throttle_persists_to_litellm_settings(monkeyp assert saved["litellm_settings"]["budget_exceeded_throttle_percentage"] == 0.1 +def test_get_config_list_includes_anthropic_prompt_caching_fields(monkeypatch): + """The auto prompt caching flag and its ttl are litellm_settings globals surfaced on the + General Settings table, so an admin can turn caching on without hand-writing config. The + ttl is a Select and must ship its allowed values, or the table renders no editor for it.""" + import types + from unittest.mock import AsyncMock, MagicMock + + from fastapi.testclient import TestClient + + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.proxy_server import app + + mock_prisma = MagicMock() + mock_config_table = MagicMock() + mock_config_table.find_first = AsyncMock(return_value=None) + mock_prisma.db = types.SimpleNamespace(litellm_config=mock_config_table) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + monkeypatch.setattr(litellm, "anthropic_prompt_caching_ttl", "1h") + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN + ) + try: + client = TestClient(app) + resp = client.get("/config/list", params={"config_type": "general_settings"}) + assert resp.status_code == 200, resp.text + fields = {item["field_name"]: item for item in resp.json()} + + assert fields["enable_anthropic_prompt_caching"]["field_type"] == "Boolean" + assert fields["enable_anthropic_prompt_caching"]["field_value"] is True + + assert fields["anthropic_prompt_caching_ttl"]["field_type"] == "Select" + assert fields["anthropic_prompt_caching_ttl"]["field_value"] == "1h" + assert fields["anthropic_prompt_caching_ttl"]["field_options"] == ["5m", "1h"] + finally: + app.dependency_overrides.clear() + + +def test_get_config_list_marks_untouched_prompt_caching_flag_as_not_set(monkeypatch): + """The flag defaults to False rather than None, so a plain 'is not None' check would + report the default as 'In Config' and imply an admin had set it.""" + import types + from unittest.mock import AsyncMock, MagicMock + + from fastapi.testclient import TestClient + + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.proxy_server import app + + mock_prisma = MagicMock() + mock_config_table = MagicMock() + mock_config_table.find_first = AsyncMock(return_value=None) + mock_prisma.db = types.SimpleNamespace(litellm_config=mock_config_table) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", False) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN + ) + try: + client = TestClient(app) + resp = client.get("/config/list", params={"config_type": "general_settings"}) + fields = {item["field_name"]: item for item in resp.json()} + assert fields["enable_anthropic_prompt_caching"]["stored_in_db"] is None + finally: + app.dependency_overrides.clear() + + +@pytest.mark.parametrize( + "field_name, field_value", + [ + ("enable_anthropic_prompt_caching", True), + ("enable_anthropic_prompt_caching", False), + ("anthropic_prompt_caching_ttl", "5m"), + ("anthropic_prompt_caching_ttl", "1h"), + ], +) +@pytest.mark.asyncio +async def test_update_config_field_prompt_caching_persists_to_litellm_settings(monkeypatch, field_name, field_value): + """Toggling either row must set litellm. live and persist under litellm_settings, + so the running proxy caches immediately and still does after a restart.""" + from unittest.mock import MagicMock + + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import ( + ConfigFieldUpdate, + LitellmUserRoles, + UserAPIKeyAuth, + ) + from litellm.proxy.proxy_server import update_config_general_settings + + saved: dict = {} + + async def fake_get_config(): + return {"litellm_settings": {}} + + async def fake_save_config(new_config=None): + saved.update(new_config or {}) + + monkeypatch.setattr(ps.proxy_config, "get_config", fake_get_config) + monkeypatch.setattr(ps.proxy_config, "save_config", fake_save_config) + monkeypatch.setattr(ps, "prisma_client", MagicMock()) + monkeypatch.setattr(litellm, "store_audit_logs", False) + monkeypatch.setattr(litellm, field_name, None) + + admin = UserAPIKeyAuth(api_key="k", user_id="a", user_role=LitellmUserRoles.PROXY_ADMIN) + await update_config_general_settings( + data=ConfigFieldUpdate(field_name=field_name, field_value=field_value, config_type="general_settings"), + user_api_key_dict=admin, + ) + + assert getattr(litellm, field_name) == field_value + assert saved["litellm_settings"][field_name] == field_value + + +@pytest.mark.parametrize( + "field_name, bad_value", + [ + ("enable_anthropic_prompt_caching", "yes"), + ("enable_anthropic_prompt_caching", 1), + ("anthropic_prompt_caching_ttl", "10m"), + ("anthropic_prompt_caching_ttl", "1H"), + ("anthropic_prompt_caching_ttl", 3600), + ], +) +@pytest.mark.asyncio +async def test_update_config_field_prompt_caching_rejects_invalid(monkeypatch, field_name, bad_value): + """An unsupported ttl must be refused here rather than reaching Anthropic verbatim.""" + from unittest.mock import MagicMock + + from fastapi import HTTPException + + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import ( + ConfigFieldUpdate, + LitellmUserRoles, + UserAPIKeyAuth, + ) + from litellm.proxy.proxy_server import update_config_general_settings + + async def fake_get_config(): + return {"litellm_settings": {}} + + monkeypatch.setattr(ps.proxy_config, "get_config", fake_get_config) + monkeypatch.setattr(ps, "prisma_client", MagicMock()) + monkeypatch.setattr(litellm, field_name, None) + + admin = UserAPIKeyAuth(api_key="k", user_id="a", user_role=LitellmUserRoles.PROXY_ADMIN) + with pytest.raises(HTTPException) as exc: + await update_config_general_settings( + data=ConfigFieldUpdate(field_name=field_name, field_value=bad_value, config_type="general_settings"), + user_api_key_dict=admin, + ) + assert exc.value.status_code == 400 + assert getattr(litellm, field_name) is None + + +@pytest.mark.parametrize( + "field_name, expected_default", + [ + ("enable_anthropic_prompt_caching", False), + ("anthropic_prompt_caching_ttl", None), + ("budget_exceeded_throttle_percentage", None), + ], +) +@pytest.mark.asyncio +async def test_reset_config_field_restores_type_default(monkeypatch, field_name, expected_default): + """Reset must restore each field's own default. Blanket None would leave the boolean flag + set to None, which is not a bool and would read as neither on nor off.""" + from unittest.mock import MagicMock + + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import ( + ConfigFieldDelete, + LitellmUserRoles, + UserAPIKeyAuth, + ) + from litellm.proxy.proxy_server import delete_config_general_settings + + saved: dict = {} + + async def fake_get_config(): + return {"litellm_settings": {field_name: "stale"}} + + async def fake_save_config(new_config=None): + saved.update(new_config or {}) + + monkeypatch.setattr(ps.proxy_config, "get_config", fake_get_config) + monkeypatch.setattr(ps.proxy_config, "save_config", fake_save_config) + monkeypatch.setattr(ps, "prisma_client", MagicMock()) + monkeypatch.setattr(litellm, "store_audit_logs", False) + monkeypatch.setattr(litellm, field_name, "stale") + + admin = UserAPIKeyAuth(api_key="k", user_id="a", user_role=LitellmUserRoles.PROXY_ADMIN) + await delete_config_general_settings( + data=ConfigFieldDelete(field_name=field_name, config_type="general_settings"), + user_api_key_dict=admin, + ) + + assert getattr(litellm, field_name) is expected_default + assert field_name not in saved["litellm_settings"] + + @pytest.mark.parametrize("bad_value", [0, -0.1, 1.5, True]) @pytest.mark.asyncio async def test_update_config_field_throttle_rejects_invalid(monkeypatch, bad_value): diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx index 3955e80f5e9..a1ef8252afa 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx @@ -14,7 +14,7 @@ import { } from "@tremor/react"; import { TabPanel, TabPanels, TabGroup, TabList, Tab } from "@tremor/react"; import { getGeneralSettingsCall, updateConfigFieldSetting, deleteConfigFieldSetting } from "@/components/networking"; -import { InputNumber } from "antd"; +import { InputNumber, Select as AntdSelect } from "antd"; import { TrashIcon } from "@heroicons/react/outline"; import { StatusBadge } from "@/components/shared/table_cells"; @@ -33,6 +33,7 @@ interface generalSettingsItem { field_value: any; field_description: string; stored_in_db: boolean | null; + field_options?: string[] | null; } const GeneralSettings: React.FC = ({ accessToken, userRole, userID }) => { @@ -169,6 +170,18 @@ const GeneralSettings: React.FC = ({ accessToken, user value={value.field_value} onChange={(newValue) => handleInputChange(value.field_name, newValue)} /> + ) : value.field_type == "Select" ? ( + ({ + label: option, + value: option, + }))} + onChange={(newValue) => handleInputChange(value.field_name, newValue ?? "")} + /> ) : null} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 0d8f55164f9..45b06c9e44f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -22711,6 +22711,8 @@ export interface components { field_description: string; /** Field Name */ field_name: string; + /** Field Options */ + field_options?: string[] | null; /** Field Type */ field_type: string; /** Field Value */ From 9f7f53a82a938b0471e41c89be967d49b3275434 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 16 Jul 2026 13:28:34 -0700 Subject: [PATCH 2/6] refactor(ui): extract the General Settings value editor into a component The value cell was a ternary chain over field_type; adding Select made it a fourth level and tripped no-nested-ternary. Early returns read better than a deeper chain and let the suppression baseline ratchet down. --- ui/litellm-dashboard/eslint-suppressions.json | 2 +- .../_components/general_settings.tsx | 80 +++++++++++-------- 2 files changed, 49 insertions(+), 33 deletions(-) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 32e9a03da95..90b0c84244e 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1079,7 +1079,7 @@ }, "src/app/(dashboard)/router-settings/_components/general_settings.tsx": { "no-nested-ternary": { - "count": 3 + "count": 1 }, "no-restricted-imports": { "count": 2 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx index a1ef8252afa..af6bbdde8b9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx @@ -36,6 +36,53 @@ interface generalSettingsItem { field_options?: string[] | null; } +const SettingValueEditor: React.FC<{ + setting: generalSettingsItem; + onChange: (fieldName: string, newValue: any) => void; +}> = ({ setting, onChange }) => { + if (setting.field_type === "Integer") { + return ( + onChange(setting.field_name, newValue)} + /> + ); + } + if (setting.field_type === "Boolean") { + return ( + onChange(setting.field_name, checked)} + /> + ); + } + if (setting.field_type === "Float") { + return ( + onChange(setting.field_name, newValue)} + /> + ); + } + if (setting.field_type === "Select") { + return ( + ({ label: option, value: option }))} + onChange={(newValue) => onChange(setting.field_name, newValue ?? "")} + /> + ); + } + return null; +}; + const GeneralSettings: React.FC = ({ accessToken, userRole, userID }) => { const [generalSettings, setGeneralSettings] = useState([]); @@ -151,38 +198,7 @@ const GeneralSettings: React.FC = ({ accessToken, user

- {value.field_type == "Integer" ? ( - handleInputChange(value.field_name, newValue)} - /> - ) : value.field_type == "Boolean" ? ( - handleInputChange(value.field_name, checked)} - /> - ) : value.field_type == "Float" ? ( - handleInputChange(value.field_name, newValue)} - /> - ) : value.field_type == "Select" ? ( - ({ - label: option, - value: option, - }))} - onChange={(newValue) => handleInputChange(value.field_name, newValue ?? "")} - /> - ) : null} + {value.stored_in_db == true ? ( From 16e39542a095c0f4aaf1a7835d8598b664dd6716 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 16 Jul 2026 15:54:53 -0700 Subject: [PATCH 3/6] docs(ui): state that Anthropic prompt caches are shared per upstream credential The provider caches a prefix against the credentials that sent it, not per end user, so turning the flag on makes every caller's prompts cacheable on that shared account. Surface that where the toggle is, since it is the operator's call to make. --- litellm/proxy/proxy_server.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ae91eb70427..ad1617017f9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14826,7 +14826,11 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, GeneralSettingsUILiteLLMFieldSpec "Automatically add Anthropic cache_control breakpoints to the system prompt and the " "trailing turn, for Anthropic and Bedrock Claude models that support prompt caching. " "Lets clients that never set cache_control themselves still get cached prompts. " - "Requests that already carry their own cache_control are left untouched." + "Requests that already carry their own cache_control are left untouched. " + "The provider caches a prefix against the upstream credentials that sent it, not per " + "end user, so this makes every caller's prompts cacheable on that shared account. " + "Leave this off if callers sharing a set of credentials must not learn whether " + "another caller recently sent a given prompt." ), }, "anthropic_prompt_caching_ttl": { From 73cbbdd51defe21a6db5beadf2b3ee73454677be Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Fri, 17 Jul 2026 11:38:24 -0700 Subject: [PATCH 4/6] feat(ui): move Anthropic prompt caching to its own Router Settings tab Rather than mixing the flag and its ttl into the generic General settings table (which also surfaced the confusing Not Set / In Config / In DB provenance badges), give prompt caching a dedicated tab with a purpose-built toggle and ttl dropdown. Each registry field gains an optional tab, surfaced as ConfigList.field_tab, so the General tab renders the ungrouped fields and the caching fields render on their own tab. The update, persist and reset endpoints are unchanged. --- litellm/proxy/_types.py | 1 + litellm/proxy/proxy_server.py | 4 + tests/test_litellm/proxy/test_proxy_server.py | 6 ++ .../_components/general_settings.tsx | 78 ++++++++++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 + 5 files changed, 90 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e07c7b9ae78..b47b43411c5 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2123,6 +2123,7 @@ class ConfigList(LiteLLMPydanticObjectBase): premium_field: bool = False nested_fields: Optional[List[FieldDetail]] = None # For nested dictionary or Pydantic fields field_options: Optional[list[str]] = None # Allowed values, for field_type == "Select" + field_tab: Optional[str] = None # Admin UI sub-tab this field renders under; None groups it with the rest class UserHeaderMapping(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ad1617017f9..6725ecdb584 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14809,6 +14809,7 @@ class GeneralSettingsUILiteLLMFieldSpec(TypedDict): type: Literal["Float", "Boolean", "Select"] description: str options: NotRequired[tuple[str, ...]] + tab: NotRequired[str] # Admin UI sub-tab this field renders under; None groups it with the rest _GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, GeneralSettingsUILiteLLMFieldSpec] = { @@ -14822,6 +14823,7 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, GeneralSettingsUILiteLLMFieldSpec }, "enable_anthropic_prompt_caching": { "type": "Boolean", + "tab": "prompt_caching", "description": ( "Automatically add Anthropic cache_control breakpoints to the system prompt and the " "trailing turn, for Anthropic and Bedrock Claude models that support prompt caching. " @@ -14836,6 +14838,7 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, GeneralSettingsUILiteLLMFieldSpec "anthropic_prompt_caching_ttl": { "type": "Select", "options": ("5m", "1h"), + "tab": "prompt_caching", "description": ( "Cache lifetime for the breakpoints added by 'enable_anthropic_prompt_caching'. " "Leave empty for Anthropic's 5m default. 1h suits long agentic sessions but doubles " @@ -15093,6 +15096,7 @@ async def get_config_list( stored_in_db=stored_in_db_litellm, field_default_value=default_value, field_options=list(spec.get("options", ())) or None, + field_tab=spec.get("tab"), nested_fields=None, ) ) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 86e807447c1..56cf213f103 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -9019,6 +9019,12 @@ def test_get_config_list_includes_anthropic_prompt_caching_fields(monkeypatch): assert fields["anthropic_prompt_caching_ttl"]["field_type"] == "Select" assert fields["anthropic_prompt_caching_ttl"]["field_value"] == "1h" assert fields["anthropic_prompt_caching_ttl"]["field_options"] == ["5m", "1h"] + + # Both caching fields carry their sub-tab so the Admin UI can render them on a + # dedicated Prompt Caching tab, while ungrouped fields stay on General. + assert fields["enable_anthropic_prompt_caching"]["field_tab"] == "prompt_caching" + assert fields["anthropic_prompt_caching_ttl"]["field_tab"] == "prompt_caching" + assert fields["budget_exceeded_throttle_percentage"]["field_tab"] is None finally: app.dependency_overrides.clear() diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx index af6bbdde8b9..8cea529d25d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx @@ -7,6 +7,7 @@ import { TableHeaderCell, TableCell, TableBody, + Title, Text, Button, Icon, @@ -21,6 +22,11 @@ import { StatusBadge } from "@/components/shared/table_cells"; import RouterSettings from "@/components/router_settings"; import Fallbacks from "@/components/Settings/RouterSettings/Fallbacks/Fallbacks"; import RoutingGroups from "@/components/routing_groups"; + +const PROMPT_CACHING_TAB = "prompt_caching"; +const ENABLE_ANTHROPIC_PROMPT_CACHING = "enable_anthropic_prompt_caching"; +const ANTHROPIC_PROMPT_CACHING_TTL = "anthropic_prompt_caching_ttl"; + interface GeneralSettingsPageProps { accessToken: string | null; userRole: string | null; @@ -34,6 +40,7 @@ interface generalSettingsItem { field_description: string; stored_in_db: boolean | null; field_options?: string[] | null; + field_tab?: string | null; } const SettingValueEditor: React.FC<{ @@ -83,6 +90,71 @@ const SettingValueEditor: React.FC<{ return null; }; +const PromptCachingPanel: React.FC<{ + accessToken: string; + settings: generalSettingsItem[]; + onChange: (fieldName: string, newValue: any) => void; +}> = ({ accessToken, settings, onChange }) => { + const enableSetting = settings.find((s) => s.field_name === ENABLE_ANTHROPIC_PROMPT_CACHING); + const ttlSetting = settings.find((s) => s.field_name === ANTHROPIC_PROMPT_CACHING_TTL); + + // The two rows come from the same registry the General tab reads; if they + // are not loaded yet there is nothing to render. + if (!enableSetting) { + return null; + } + + const enabled = enableSetting.field_value === true || enableSetting.field_value === "true"; + + // Apply immediately: a toggle and a dropdown are direct controls, so there is + // no separate Update button. Clearing the ttl resets it to the provider default. + const persist = (fieldName: string, value: any) => { + onChange(fieldName, value); + if (value === "" || value === null || value === undefined) { + deleteConfigFieldSetting(accessToken, fieldName); + } else { + updateConfigFieldSetting(accessToken, fieldName, value); + } + }; + + return ( + + Prompt Caching + + Automatically inject Anthropic prompt caching for every Anthropic and Bedrock Claude model, so clients that + never set cache_control themselves still get cached prompts. This is a single + gateway-wide switch; there is no per-model setup. + + +
+
+ Automatic Anthropic prompt caching +

{enableSetting.field_description}

+
+ persist(ENABLE_ANTHROPIC_PROMPT_CACHING, checked)} /> +
+ + {ttlSetting && ( +
+
+ Cache lifetime (TTL) +

{ttlSetting.field_description}

+
+ ({ label: option, value: option }))} + onChange={(newValue) => persist(ANTHROPIC_PROMPT_CACHING_TTL, newValue ?? "")} + /> +
+ )} + + ); +}; + const GeneralSettings: React.FC = ({ accessToken, userRole, userID }) => { const [generalSettings, setGeneralSettings] = useState([]); @@ -156,6 +228,7 @@ const GeneralSettings: React.FC = ({ accessToken, user Loadbalancing Routing Groups Fallbacks + Prompt Caching General @@ -168,6 +241,9 @@ const GeneralSettings: React.FC = ({ accessToken, user + + + @@ -181,7 +257,7 @@ const GeneralSettings: React.FC = ({ accessToken, user {generalSettings - .filter((value) => value.field_type !== "TypedDictionary") + .filter((value) => value.field_type !== "TypedDictionary" && value.field_tab !== PROMPT_CACHING_TAB) .map((value, index) => ( diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 45b06c9e44f..6dc63762e5c 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -22713,6 +22713,8 @@ export interface components { field_name: string; /** Field Options */ field_options?: string[] | null; + /** Field Tab */ + field_tab?: string | null; /** Field Type */ field_type: string; /** Field Value */ From 4e5f4884523ea124c6a252563624d104b4dc394c Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Fri, 17 Jul 2026 12:16:09 -0700 Subject: [PATCH 5/6] feat(ui): tighten the Prompt Caching descriptions The toggle and ttl descriptions were a wall of text, with a panel intro that mostly repeated the toggle description. Drop the intro and cut both descriptions to one or two lines, keeping a one-clause note that the cache is shared across callers on the same upstream credentials. --- litellm/proxy/proxy_server.py | 16 +++------------- .../_components/general_settings.tsx | 5 ----- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6725ecdb584..7d87207f7a9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14825,25 +14825,15 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, GeneralSettingsUILiteLLMFieldSpec "type": "Boolean", "tab": "prompt_caching", "description": ( - "Automatically add Anthropic cache_control breakpoints to the system prompt and the " - "trailing turn, for Anthropic and Bedrock Claude models that support prompt caching. " - "Lets clients that never set cache_control themselves still get cached prompts. " - "Requests that already carry their own cache_control are left untouched. " - "The provider caches a prefix against the upstream credentials that sent it, not per " - "end user, so this makes every caller's prompts cacheable on that shared account. " - "Leave this off if callers sharing a set of credentials must not learn whether " - "another caller recently sent a given prompt." + "Auto-adds cache_control to the system prompt and trailing turn for supported Anthropic " + "and Bedrock Claude models. The cache is shared across callers on the same upstream credentials." ), }, "anthropic_prompt_caching_ttl": { "type": "Select", "options": ("5m", "1h"), "tab": "prompt_caching", - "description": ( - "Cache lifetime for the breakpoints added by 'enable_anthropic_prompt_caching'. " - "Leave empty for Anthropic's 5m default. 1h suits long agentic sessions but doubles " - "the cache write premium." - ), + "description": "Empty uses Anthropic's 5m default. 1h suits long sessions but doubles the cache write cost.", }, } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx index 8cea529d25d..1e8658d5104 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx @@ -120,11 +120,6 @@ const PromptCachingPanel: React.FC<{ return ( Prompt Caching - - Automatically inject Anthropic prompt caching for every Anthropic and Bedrock Claude model, so clients that - never set cache_control themselves still get cached prompts. This is a single - gateway-wide switch; there is no per-model setup. -
From 47ba9e76121dd7dbf572e112d7df5ebad5414bac Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Fri, 17 Jul 2026 19:38:42 -0700 Subject: [PATCH 6/6] fix(proxy): propagate the caching flag across workers via the safe-override allowlist enable_anthropic_prompt_caching and anthropic_prompt_caching_ttl are set as live litellm attributes on the worker that handles the UI save, exactly like budget_exceeded_throttle_percentage, but they were missing from LITELLM_SETTINGS_SAFE_DB_OVERRIDES, so a peer worker's config reload merged the DB value without applying it to the live attribute and stayed stale. Add both to the allowlist so they behave like the sibling field, and add test_general_settings_ui_fields_are_db_overridable so the UI registry and the override allowlist cannot drift again (the exact omission that caused this), plus a regression test that the flag flips on a simulated peer-worker reload. --- litellm/constants.py | 6 +++ tests/test_litellm/proxy/test_proxy_server.py | 45 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/litellm/constants.py b/litellm/constants.py index e104c937a9b..6432e2176c7 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1517,6 +1517,12 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [ "cost_discount_config", "cost_margin_config", "budget_exceeded_throttle_percentage", + # Every field editable from the Admin UI (proxy_server._GENERAL_SETTINGS_UI_LITELLM_FIELDS) + # must be listed here so a DB write from one worker overrides the live litellm attribute on + # the others when config reloads; otherwise peer workers stay on their startup value. + # test_general_settings_ui_fields_are_db_overridable enforces that pairing. + "enable_anthropic_prompt_caching", + "anthropic_prompt_caching_ttl", ] SPECIAL_LITELLM_AUTH_TOKEN = ["ui-token"] DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 56cf213f103..a100e7837f4 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -9029,6 +9029,51 @@ def test_get_config_list_includes_anthropic_prompt_caching_fields(monkeypatch): app.dependency_overrides.clear() +def test_general_settings_ui_fields_are_db_overridable(): + """Every field the Admin UI can edit is a `litellm.` set via setattr on the handling + worker (`_persist_general_settings_ui_litellm_field`). Unless it is also in + LITELLM_SETTINGS_SAFE_DB_OVERRIDES, a config reload on a peer worker merges the DB value but + never applies it to the live attribute, so peer workers stay on their startup value. + + This invariant is the guard against the two registries drifting: adding a UI-editable field + without enrolling it in the DB-override allowlist silently breaks cross-worker propagation. + """ + from litellm.constants import LITELLM_SETTINGS_SAFE_DB_OVERRIDES + from litellm.proxy.proxy_server import _GENERAL_SETTINGS_UI_LITELLM_FIELDS + + missing = set(_GENERAL_SETTINGS_UI_LITELLM_FIELDS) - set(LITELLM_SETTINGS_SAFE_DB_OVERRIDES) + assert not missing, ( + f"UI-editable litellm_settings fields missing from LITELLM_SETTINGS_SAFE_DB_OVERRIDES: {sorted(missing)}. " + "Add them, or they will not propagate to other workers when changed from the UI." + ) + + +@pytest.mark.parametrize( + "field_name, db_value", + [ + ("enable_anthropic_prompt_caching", True), + ("anthropic_prompt_caching_ttl", "1h"), + ], +) +def test_prompt_caching_settings_propagate_on_config_reload(monkeypatch, field_name, db_value): + """A UI toggle on one worker persists to the DB; a peer worker picks it up only when the + config reload applies the safe-override allowlist. Regression for the fields being absent + from that allowlist, which left peer workers stale.""" + import litellm.proxy.proxy_server as ps + + # peer worker booted with the opposite/absent value + monkeypatch.setattr(litellm, field_name, False if isinstance(db_value, bool) else None) + + pc = ps.ProxyConfig() + pc._update_config_fields( + current_config={"litellm_settings": {}}, + param_name="litellm_settings", + db_param_value={field_name: db_value}, + ) + + assert getattr(litellm, field_name) == db_value + + def test_get_config_list_marks_untouched_prompt_caching_flag_as_not_set(monkeypatch): """The flag defaults to False rather than None, so a plain 'is not None' check would report the default as 'In Config' and imply an admin had set it."""