From 12919628501340c8b7b596d33492bd9f5ef6eff0 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 16 Jul 2026 13:23:41 -0700 Subject: [PATCH] feat(ui): configure Anthropic automatic prompt caching from the Admin UI Register enable_anthropic_prompt_caching and anthropic_prompt_caching_ttl on the General Settings table so caching can be turned on without hand-writing config. The registry could not express either field: validation was hardcoded to a float in (0, 1], reset set every field to None (not a bool for a boolean flag), and the listing reported any non-None value as 'In Config', which a False default would always trip. Validation now dispatches on the declared type and reset restores each field's own default. ConfigList carries field_options so the table can render a Select for enums instead of no editor at all. --- litellm/proxy/_types.py | 3 +- litellm/proxy/proxy_server.py | 96 +++++++-- tests/test_litellm/proxy/test_proxy_server.py | 204 ++++++++++++++++++ .../_components/general_settings.tsx | 15 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 + 5 files changed, 300 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d102c1d1e37..e07c7b9ae78 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1011,10 +1011,10 @@ class LiteLLM_ObjectPermissionBase(LiteLLMPydanticObjectBase): mcp_tool_search_enabled: Optional[bool] = None +from litellm.models.team import BudgetLimitEntry as BudgetLimitEntry # noqa: E402 from litellm.types.object_permission import ( # noqa: E402 ObjectPermissionDict as ObjectPermissionDict, ) -from litellm.models.team import BudgetLimitEntry as BudgetLimitEntry # noqa: E402 class GenerateRequestBase(LiteLLMPydanticObjectBase): @@ -2122,6 +2122,7 @@ class ConfigList(LiteLLMPydanticObjectBase): field_default_value: Any premium_field: bool = False nested_fields: Optional[List[FieldDetail]] = None # For nested dictionary or Pydantic fields + field_options: Optional[list[str]] = None # Allowed values, for field_type == "Select" class UserHeaderMapping(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index dbdfdd5fdd3..ae91eb70427 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -28,6 +28,7 @@ from typing import ( Optional, Set, Tuple, + TypedDict, Union, cast, get_args, @@ -39,6 +40,7 @@ import anyio import websockets import websockets.exceptions from pydantic import BaseModel, Json, JsonValue +from typing_extensions import NotRequired, assert_never from litellm._uuid import uuid from litellm.constants import ( @@ -363,15 +365,15 @@ from litellm.proxy.management_endpoints.cache_settings_endpoints import ( from litellm.proxy.management_endpoints.callback_management_endpoints import ( router as callback_management_endpoints_router, ) -from litellm.proxy.management_endpoints.coordination_redis_endpoints import ( - get_persisted_coordination_redis_settings, - router as coordination_redis_settings_router, -) from litellm.proxy.management_endpoints.common_utils import ( _user_has_admin_privileges, _user_has_admin_view, admin_can_invite_user, ) +from litellm.proxy.management_endpoints.coordination_redis_endpoints import ( + get_persisted_coordination_redis_settings, + router as coordination_redis_settings_router, +) from litellm.proxy.management_endpoints.cost_tracking_settings import ( router as cost_tracking_settings_router, ) @@ -14800,7 +14802,16 @@ async def get_config_general_settings( ) -_GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, dict[str, str]] = { +GeneralSettingsUILiteLLMValue = Union[float, bool, str, None] + + +class GeneralSettingsUILiteLLMFieldSpec(TypedDict): + type: Literal["Float", "Boolean", "Select"] + description: str + options: NotRequired[tuple[str, ...]] + + +_GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, GeneralSettingsUILiteLLMFieldSpec] = { "budget_exceeded_throttle_percentage": { "type": "Float", "description": ( @@ -14809,18 +14820,64 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, dict[str, str]] = { "over-budget keys." ), }, + "enable_anthropic_prompt_caching": { + "type": "Boolean", + "description": ( + "Automatically add Anthropic cache_control breakpoints to the system prompt and the " + "trailing turn, for Anthropic and Bedrock Claude models that support prompt caching. " + "Lets clients that never set cache_control themselves still get cached prompts. " + "Requests that already carry their own cache_control are left untouched." + ), + }, + "anthropic_prompt_caching_ttl": { + "type": "Select", + "options": ("5m", "1h"), + "description": ( + "Cache lifetime for the breakpoints added by 'enable_anthropic_prompt_caching'. " + "Leave empty for Anthropic's 5m default. 1h suits long agentic sessions but doubles " + "the cache write premium." + ), + }, } -def _validate_general_settings_ui_litellm_value(field_name: str, value: Any) -> Optional[float]: +def _general_settings_ui_litellm_default( + field_type: Literal["Float", "Boolean", "Select"], +) -> GeneralSettingsUILiteLLMValue: + """The value a field falls back to when it is cleared or reset.""" + return False if field_type == "Boolean" else None + + +def _validate_general_settings_ui_litellm_value(field_name: str, value: Any) -> GeneralSettingsUILiteLLMValue: + spec = _GENERAL_SETTINGS_UI_LITELLM_FIELDS[field_name] + field_type = spec["type"] if value is None or value == "": - return None - if isinstance(value, bool) or not isinstance(value, (int, float)) or not (0 < float(value) <= 1): - raise HTTPException( - status_code=400, - detail={"error": f"{field_name} must be a number in (0, 1] or empty"}, - ) - return float(value) + return _general_settings_ui_litellm_default(field_type) + match field_type: + case "Boolean": + if not isinstance(value, bool): + raise HTTPException( + status_code=400, + detail={"error": f"{field_name} must be true or false"}, + ) + return value + case "Select": + options = spec.get("options", ()) + if value not in options: + raise HTTPException( + status_code=400, + detail={"error": f"{field_name} must be one of: {', '.join(options)}, or empty"}, + ) + return cast(str, value) # cast-ok: membership in options proves it is one of the option strings + case "Float": + if isinstance(value, bool) or not isinstance(value, (int, float)) or not (0 < float(value) <= 1): + raise HTTPException( + status_code=400, + detail={"error": f"{field_name} must be a number in (0, 1] or empty"}, + ) + return float(value) + case _: + assert_never(field_type) async def _persist_general_settings_ui_litellm_field( @@ -14841,11 +14898,12 @@ async def _persist_general_settings_ui_litellm_field( async def _reset_general_settings_ui_litellm_field(field_name: str, user_api_key_dict: UserAPIKeyAuth) -> dict: config = await proxy_config.get_config() before_value = config.get("litellm_settings", {}).get(field_name) - setattr(litellm, field_name, None) + default_value = _general_settings_ui_litellm_default(_GENERAL_SETTINGS_UI_LITELLM_FIELDS[field_name]["type"]) + setattr(litellm, field_name, default_value) if "litellm_settings" in config: config["litellm_settings"].pop(field_name, None) await proxy_config.save_config(new_config=config) - asyncio.create_task(create_config_audit_log(field_name, "deleted", before_value, None, user_api_key_dict)) + asyncio.create_task(create_config_audit_log(field_name, "deleted", before_value, default_value, user_api_key_dict)) return {"message": f"Field {field_name} reset", "status": "success"} @@ -15013,11 +15071,12 @@ async def get_config_list( else {} ) for litellm_field_name, spec in _GENERAL_SETTINGS_UI_LITELLM_FIELDS.items(): - current_value: Optional[float] = getattr(litellm, litellm_field_name, None) + current_value: GeneralSettingsUILiteLLMValue = getattr(litellm, litellm_field_name, None) + default_value = _general_settings_ui_litellm_default(spec["type"]) stored_in_db_litellm: Optional[bool] if litellm_field_name in db_litellm_settings: stored_in_db_litellm = True - elif current_value is not None: + elif current_value != default_value: stored_in_db_litellm = False else: stored_in_db_litellm = None @@ -15028,7 +15087,8 @@ async def get_config_list( field_description=spec["description"], field_value=current_value, stored_in_db=stored_in_db_litellm, - field_default_value=None, + field_default_value=default_value, + field_options=list(spec.get("options", ())) or None, nested_fields=None, ) ) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 54db0c0fd4f..86e807447c1 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -8984,6 +8984,210 @@ async def test_update_config_field_throttle_persists_to_litellm_settings(monkeyp assert saved["litellm_settings"]["budget_exceeded_throttle_percentage"] == 0.1 +def test_get_config_list_includes_anthropic_prompt_caching_fields(monkeypatch): + """The auto prompt caching flag and its ttl are litellm_settings globals surfaced on the + General Settings table, so an admin can turn caching on without hand-writing config. The + ttl is a Select and must ship its allowed values, or the table renders no editor for it.""" + import types + from unittest.mock import AsyncMock, MagicMock + + from fastapi.testclient import TestClient + + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.proxy_server import app + + mock_prisma = MagicMock() + mock_config_table = MagicMock() + mock_config_table.find_first = AsyncMock(return_value=None) + mock_prisma.db = types.SimpleNamespace(litellm_config=mock_config_table) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + monkeypatch.setattr(litellm, "anthropic_prompt_caching_ttl", "1h") + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN + ) + try: + client = TestClient(app) + resp = client.get("/config/list", params={"config_type": "general_settings"}) + assert resp.status_code == 200, resp.text + fields = {item["field_name"]: item for item in resp.json()} + + assert fields["enable_anthropic_prompt_caching"]["field_type"] == "Boolean" + assert fields["enable_anthropic_prompt_caching"]["field_value"] is True + + assert fields["anthropic_prompt_caching_ttl"]["field_type"] == "Select" + assert fields["anthropic_prompt_caching_ttl"]["field_value"] == "1h" + assert fields["anthropic_prompt_caching_ttl"]["field_options"] == ["5m", "1h"] + finally: + app.dependency_overrides.clear() + + +def test_get_config_list_marks_untouched_prompt_caching_flag_as_not_set(monkeypatch): + """The flag defaults to False rather than None, so a plain 'is not None' check would + report the default as 'In Config' and imply an admin had set it.""" + import types + from unittest.mock import AsyncMock, MagicMock + + from fastapi.testclient import TestClient + + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.proxy_server import app + + mock_prisma = MagicMock() + mock_config_table = MagicMock() + mock_config_table.find_first = AsyncMock(return_value=None) + mock_prisma.db = types.SimpleNamespace(litellm_config=mock_config_table) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", False) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN + ) + try: + client = TestClient(app) + resp = client.get("/config/list", params={"config_type": "general_settings"}) + fields = {item["field_name"]: item for item in resp.json()} + assert fields["enable_anthropic_prompt_caching"]["stored_in_db"] is None + finally: + app.dependency_overrides.clear() + + +@pytest.mark.parametrize( + "field_name, field_value", + [ + ("enable_anthropic_prompt_caching", True), + ("enable_anthropic_prompt_caching", False), + ("anthropic_prompt_caching_ttl", "5m"), + ("anthropic_prompt_caching_ttl", "1h"), + ], +) +@pytest.mark.asyncio +async def test_update_config_field_prompt_caching_persists_to_litellm_settings(monkeypatch, field_name, field_value): + """Toggling either row must set litellm. live and persist under litellm_settings, + so the running proxy caches immediately and still does after a restart.""" + from unittest.mock import MagicMock + + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import ( + ConfigFieldUpdate, + LitellmUserRoles, + UserAPIKeyAuth, + ) + from litellm.proxy.proxy_server import update_config_general_settings + + saved: dict = {} + + async def fake_get_config(): + return {"litellm_settings": {}} + + async def fake_save_config(new_config=None): + saved.update(new_config or {}) + + monkeypatch.setattr(ps.proxy_config, "get_config", fake_get_config) + monkeypatch.setattr(ps.proxy_config, "save_config", fake_save_config) + monkeypatch.setattr(ps, "prisma_client", MagicMock()) + monkeypatch.setattr(litellm, "store_audit_logs", False) + monkeypatch.setattr(litellm, field_name, None) + + admin = UserAPIKeyAuth(api_key="k", user_id="a", user_role=LitellmUserRoles.PROXY_ADMIN) + await update_config_general_settings( + data=ConfigFieldUpdate(field_name=field_name, field_value=field_value, config_type="general_settings"), + user_api_key_dict=admin, + ) + + assert getattr(litellm, field_name) == field_value + assert saved["litellm_settings"][field_name] == field_value + + +@pytest.mark.parametrize( + "field_name, bad_value", + [ + ("enable_anthropic_prompt_caching", "yes"), + ("enable_anthropic_prompt_caching", 1), + ("anthropic_prompt_caching_ttl", "10m"), + ("anthropic_prompt_caching_ttl", "1H"), + ("anthropic_prompt_caching_ttl", 3600), + ], +) +@pytest.mark.asyncio +async def test_update_config_field_prompt_caching_rejects_invalid(monkeypatch, field_name, bad_value): + """An unsupported ttl must be refused here rather than reaching Anthropic verbatim.""" + from unittest.mock import MagicMock + + from fastapi import HTTPException + + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import ( + ConfigFieldUpdate, + LitellmUserRoles, + UserAPIKeyAuth, + ) + from litellm.proxy.proxy_server import update_config_general_settings + + async def fake_get_config(): + return {"litellm_settings": {}} + + monkeypatch.setattr(ps.proxy_config, "get_config", fake_get_config) + monkeypatch.setattr(ps, "prisma_client", MagicMock()) + monkeypatch.setattr(litellm, field_name, None) + + admin = UserAPIKeyAuth(api_key="k", user_id="a", user_role=LitellmUserRoles.PROXY_ADMIN) + with pytest.raises(HTTPException) as exc: + await update_config_general_settings( + data=ConfigFieldUpdate(field_name=field_name, field_value=bad_value, config_type="general_settings"), + user_api_key_dict=admin, + ) + assert exc.value.status_code == 400 + assert getattr(litellm, field_name) is None + + +@pytest.mark.parametrize( + "field_name, expected_default", + [ + ("enable_anthropic_prompt_caching", False), + ("anthropic_prompt_caching_ttl", None), + ("budget_exceeded_throttle_percentage", None), + ], +) +@pytest.mark.asyncio +async def test_reset_config_field_restores_type_default(monkeypatch, field_name, expected_default): + """Reset must restore each field's own default. Blanket None would leave the boolean flag + set to None, which is not a bool and would read as neither on nor off.""" + from unittest.mock import MagicMock + + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import ( + ConfigFieldDelete, + LitellmUserRoles, + UserAPIKeyAuth, + ) + from litellm.proxy.proxy_server import delete_config_general_settings + + saved: dict = {} + + async def fake_get_config(): + return {"litellm_settings": {field_name: "stale"}} + + async def fake_save_config(new_config=None): + saved.update(new_config or {}) + + monkeypatch.setattr(ps.proxy_config, "get_config", fake_get_config) + monkeypatch.setattr(ps.proxy_config, "save_config", fake_save_config) + monkeypatch.setattr(ps, "prisma_client", MagicMock()) + monkeypatch.setattr(litellm, "store_audit_logs", False) + monkeypatch.setattr(litellm, field_name, "stale") + + admin = UserAPIKeyAuth(api_key="k", user_id="a", user_role=LitellmUserRoles.PROXY_ADMIN) + await delete_config_general_settings( + data=ConfigFieldDelete(field_name=field_name, config_type="general_settings"), + user_api_key_dict=admin, + ) + + assert getattr(litellm, field_name) is expected_default + assert field_name not in saved["litellm_settings"] + + @pytest.mark.parametrize("bad_value", [0, -0.1, 1.5, True]) @pytest.mark.asyncio async def test_update_config_field_throttle_rejects_invalid(monkeypatch, bad_value): diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx index 3955e80f5e9..a1ef8252afa 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx @@ -14,7 +14,7 @@ import { } from "@tremor/react"; import { TabPanel, TabPanels, TabGroup, TabList, Tab } from "@tremor/react"; import { getGeneralSettingsCall, updateConfigFieldSetting, deleteConfigFieldSetting } from "@/components/networking"; -import { InputNumber } from "antd"; +import { InputNumber, Select as AntdSelect } from "antd"; import { TrashIcon } from "@heroicons/react/outline"; import { StatusBadge } from "@/components/shared/table_cells"; @@ -33,6 +33,7 @@ interface generalSettingsItem { field_value: any; field_description: string; stored_in_db: boolean | null; + field_options?: string[] | null; } const GeneralSettings: React.FC = ({ accessToken, userRole, userID }) => { @@ -169,6 +170,18 @@ const GeneralSettings: React.FC = ({ accessToken, user value={value.field_value} onChange={(newValue) => handleInputChange(value.field_name, newValue)} /> + ) : value.field_type == "Select" ? ( + ({ + label: option, + value: option, + }))} + onChange={(newValue) => handleInputChange(value.field_name, newValue ?? "")} + /> ) : null} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 0d8f55164f9..45b06c9e44f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -22711,6 +22711,8 @@ export interface components { field_description: string; /** Field Name */ field_name: string; + /** Field Options */ + field_options?: string[] | null; /** Field Type */ field_type: string; /** Field Value */