scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

feat(ui): configure Anthropic automatic prompt caching from the Admin UI

Browse source 
Register enable_anthropic_prompt_caching and anthropic_prompt_caching_ttl on the
General Settings table so caching can be turned on without hand-writing config.

The registry could not express either field: validation was hardcoded to a float in
(0, 1], reset set every field to None (not a bool for a boolean flag), and the listing
reported any non-None value as 'In Config', which a False default would always trip.
Validation now dispatches on the declared type and reset restores each field's own
default. ConfigList carries field_options so the table can render a Select for enums
instead of no editor at all.
This commit is contained in:
Tin Chi Lo 2026-07-16 13:23:41 -07:00
parent a7d01cb1ac
commit 1291962850
5 changed files with 300 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
litellm/proxy/_types.py
View file

@ -1011,10 +1011,10 @@ class LiteLLM_ObjectPermissionBase(LiteLLMPydanticObjectBase):
mcp_tool_search_enabled: Optional[bool] = None
from litellm.models.team import BudgetLimitEntry as BudgetLimitEntry # noqa: E402
from litellm.types.object_permission import ( # noqa: E402
ObjectPermissionDict as ObjectPermissionDict,
)
from litellm.models.team import BudgetLimitEntry as BudgetLimitEntry # noqa: E402
class GenerateRequestBase(LiteLLMPydanticObjectBase):
@ -2122,6 +2122,7 @@ class ConfigList(LiteLLMPydanticObjectBase):
field_default_value: Any
premium_field: bool = False
nested_fields: Optional[List[FieldDetail]] = None # For nested dictionary or Pydantic fields
field_options: Optional[list[str]] = None # Allowed values, for field_type == "Select"
class UserHeaderMapping(LiteLLMPydanticObjectBase):

96
litellm/proxy/proxy_server.py
View file

@ -28,6 +28,7 @@ from typing import (
Optional,
Set,
Tuple,
TypedDict,
Union,
cast,
get_args,
@ -39,6 +40,7 @@ import anyio
import websockets
import websockets.exceptions
from pydantic import BaseModel, Json, JsonValue
from typing_extensions import NotRequired, assert_never
from litellm._uuid import uuid
from litellm.constants import (
@ -363,15 +365,15 @@ from litellm.proxy.management_endpoints.cache_settings_endpoints import (
from litellm.proxy.management_endpoints.callback_management_endpoints import (
router as callback_management_endpoints_router,
)
from litellm.proxy.management_endpoints.coordination_redis_endpoints import (
get_persisted_coordination_redis_settings,
router as coordination_redis_settings_router,
)
from litellm.proxy.management_endpoints.common_utils import (
_user_has_admin_privileges,
_user_has_admin_view,
admin_can_invite_user,
)
from litellm.proxy.management_endpoints.coordination_redis_endpoints import (
get_persisted_coordination_redis_settings,
router as coordination_redis_settings_router,
)
from litellm.proxy.management_endpoints.cost_tracking_settings import (
router as cost_tracking_settings_router,
)
@ -14800,7 +14802,16 @@ async def get_config_general_settings(
)
_GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, dict[str, str]] = {
GeneralSettingsUILiteLLMValue = Union[float, bool, str, None]
class GeneralSettingsUILiteLLMFieldSpec(TypedDict):
type: Literal["Float", "Boolean", "Select"]
description: str
options: NotRequired[tuple[str, ...]]
_GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, GeneralSettingsUILiteLLMFieldSpec] = {
"budget_exceeded_throttle_percentage": {
"type": "Float",
"description": (
@ -14809,18 +14820,64 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, dict[str, str]] = {
"over-budget keys."
),
},
"enable_anthropic_prompt_caching": {
"type": "Boolean",
"description": (
"Automatically add Anthropic cache_control breakpoints to the system prompt and the "
"trailing turn, for Anthropic and Bedrock Claude models that support prompt caching. "
"Lets clients that never set cache_control themselves still get cached prompts. "
"Requests that already carry their own cache_control are left untouched."
),
},
"anthropic_prompt_caching_ttl": {
"type": "Select",
"options": ("5m", "1h"),
"description": (
"Cache lifetime for the breakpoints added by 'enable_anthropic_prompt_caching'. "
"Leave empty for Anthropic's 5m default. 1h suits long agentic sessions but doubles "
"the cache write premium."
),
},
}
def _validate_general_settings_ui_litellm_value(field_name: str, value: Any) -> Optional[float]:
def _general_settings_ui_litellm_default(
field_type: Literal["Float", "Boolean", "Select"],
) -> GeneralSettingsUILiteLLMValue:
"""The value a field falls back to when it is cleared or reset."""
return False if field_type == "Boolean" else None
def _validate_general_settings_ui_litellm_value(field_name: str, value: Any) -> GeneralSettingsUILiteLLMValue:
spec = _GENERAL_SETTINGS_UI_LITELLM_FIELDS[field_name]
field_type = spec["type"]
if value is None or value == "":
return None
if isinstance(value, bool) or not isinstance(value, (int, float)) or not (0 < float(value) <= 1):
raise HTTPException(
status_code=400,
detail={"error": f"{field_name} must be a number in (0, 1] or empty"},
)
return float(value)
return _general_settings_ui_litellm_default(field_type)
match field_type:
case "Boolean":
if not isinstance(value, bool):
raise HTTPException(
status_code=400,
detail={"error": f"{field_name} must be true or false"},
)
return value
case "Select":
options = spec.get("options", ())
if value not in options:
raise HTTPException(
status_code=400,
detail={"error": f"{field_name} must be one of: {', '.join(options)}, or empty"},
)
return cast(str, value) # cast-ok: membership in options proves it is one of the option strings
case "Float":
if isinstance(value, bool) or not isinstance(value, (int, float)) or not (0 < float(value) <= 1):
raise HTTPException(
status_code=400,
detail={"error": f"{field_name} must be a number in (0, 1] or empty"},
)
return float(value)
case _:
assert_never(field_type)
async def _persist_general_settings_ui_litellm_field(
@ -14841,11 +14898,12 @@ async def _persist_general_settings_ui_litellm_field(
async def _reset_general_settings_ui_litellm_field(field_name: str, user_api_key_dict: UserAPIKeyAuth) -> dict:
config = await proxy_config.get_config()
before_value = config.get("litellm_settings", {}).get(field_name)
setattr(litellm, field_name, None)
default_value = _general_settings_ui_litellm_default(_GENERAL_SETTINGS_UI_LITELLM_FIELDS[field_name]["type"])
setattr(litellm, field_name, default_value)
if "litellm_settings" in config:
config["litellm_settings"].pop(field_name, None)
await proxy_config.save_config(new_config=config)
asyncio.create_task(create_config_audit_log(field_name, "deleted", before_value, None, user_api_key_dict))
asyncio.create_task(create_config_audit_log(field_name, "deleted", before_value, default_value, user_api_key_dict))
return {"message": f"Field {field_name} reset", "status": "success"}
@ -15013,11 +15071,12 @@ async def get_config_list(
else {}
)
for litellm_field_name, spec in _GENERAL_SETTINGS_UI_LITELLM_FIELDS.items():
current_value: Optional[float] = getattr(litellm, litellm_field_name, None)
current_value: GeneralSettingsUILiteLLMValue = getattr(litellm, litellm_field_name, None)
default_value = _general_settings_ui_litellm_default(spec["type"])
stored_in_db_litellm: Optional[bool]
if litellm_field_name in db_litellm_settings:
stored_in_db_litellm = True
elif current_value is not None:
elif current_value != default_value:
stored_in_db_litellm = False
else:
stored_in_db_litellm = None
@ -15028,7 +15087,8 @@ async def get_config_list(
field_description=spec["description"],
field_value=current_value,
stored_in_db=stored_in_db_litellm,
field_default_value=None,
field_default_value=default_value,
field_options=list(spec.get("options", ())) or None,
nested_fields=None,
)
)

204
tests/test_litellm/proxy/test_proxy_server.py
View file

@ -8984,6 +8984,210 @@ async def test_update_config_field_throttle_persists_to_litellm_settings(monkeyp
assert saved["litellm_settings"]["budget_exceeded_throttle_percentage"] == 0.1
def test_get_config_list_includes_anthropic_prompt_caching_fields(monkeypatch):
"""The auto prompt caching flag and its ttl are litellm_settings globals surfaced on the
General Settings table, so an admin can turn caching on without hand-writing config. The
ttl is a Select and must ship its allowed values, or the table renders no editor for it."""
import types
from unittest.mock import AsyncMock, MagicMock
from fastapi.testclient import TestClient
import litellm.proxy.proxy_server as ps
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.proxy_server import app
mock_prisma = MagicMock()
mock_config_table = MagicMock()
mock_config_table.find_first = AsyncMock(return_value=None)
mock_prisma.db = types.SimpleNamespace(litellm_config=mock_config_table)
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
monkeypatch.setattr(litellm, "anthropic_prompt_caching_ttl", "1h")
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN
)
try:
client = TestClient(app)
resp = client.get("/config/list", params={"config_type": "general_settings"})
assert resp.status_code == 200, resp.text
fields = {item["field_name"]: item for item in resp.json()}
assert fields["enable_anthropic_prompt_caching"]["field_type"] == "Boolean"
assert fields["enable_anthropic_prompt_caching"]["field_value"] is True
assert fields["anthropic_prompt_caching_ttl"]["field_type"] == "Select"
assert fields["anthropic_prompt_caching_ttl"]["field_value"] == "1h"
assert fields["anthropic_prompt_caching_ttl"]["field_options"] == ["5m", "1h"]
finally:
app.dependency_overrides.clear()
def test_get_config_list_marks_untouched_prompt_caching_flag_as_not_set(monkeypatch):
"""The flag defaults to False rather than None, so a plain 'is not None' check would
report the default as 'In Config' and imply an admin had set it."""
import types
from unittest.mock import AsyncMock, MagicMock
from fastapi.testclient import TestClient
import litellm.proxy.proxy_server as ps
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.proxy_server import app
mock_prisma = MagicMock()
mock_config_table = MagicMock()
mock_config_table.find_first = AsyncMock(return_value=None)
mock_prisma.db = types.SimpleNamespace(litellm_config=mock_config_table)
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", False)
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN
)
try:
client = TestClient(app)
resp = client.get("/config/list", params={"config_type": "general_settings"})
fields = {item["field_name"]: item for item in resp.json()}
assert fields["enable_anthropic_prompt_caching"]["stored_in_db"] is None
finally:
app.dependency_overrides.clear()
@pytest.mark.parametrize(
"field_name, field_value",
[
("enable_anthropic_prompt_caching", True),
("enable_anthropic_prompt_caching", False),
("anthropic_prompt_caching_ttl", "5m"),
("anthropic_prompt_caching_ttl", "1h"),
],
)
@pytest.mark.asyncio
async def test_update_config_field_prompt_caching_persists_to_litellm_settings(monkeypatch, field_name, field_value):
"""Toggling either row must set litellm.<attr> live and persist under litellm_settings,
so the running proxy caches immediately and still does after a restart."""
from unittest.mock import MagicMock
import litellm.proxy.proxy_server as ps
from litellm.proxy._types import (
ConfigFieldUpdate,
LitellmUserRoles,
UserAPIKeyAuth,
)
from litellm.proxy.proxy_server import update_config_general_settings
saved: dict = {}
async def fake_get_config():
return {"litellm_settings": {}}
async def fake_save_config(new_config=None):
saved.update(new_config or {})
monkeypatch.setattr(ps.proxy_config, "get_config", fake_get_config)
monkeypatch.setattr(ps.proxy_config, "save_config", fake_save_config)
monkeypatch.setattr(ps, "prisma_client", MagicMock())
monkeypatch.setattr(litellm, "store_audit_logs", False)
monkeypatch.setattr(litellm, field_name, None)
admin = UserAPIKeyAuth(api_key="k", user_id="a", user_role=LitellmUserRoles.PROXY_ADMIN)
await update_config_general_settings(
data=ConfigFieldUpdate(field_name=field_name, field_value=field_value, config_type="general_settings"),
user_api_key_dict=admin,
)
assert getattr(litellm, field_name) == field_value
assert saved["litellm_settings"][field_name] == field_value
@pytest.mark.parametrize(
"field_name, bad_value",
[
("enable_anthropic_prompt_caching", "yes"),
("enable_anthropic_prompt_caching", 1),
("anthropic_prompt_caching_ttl", "10m"),
("anthropic_prompt_caching_ttl", "1H"),
("anthropic_prompt_caching_ttl", 3600),
],
)
@pytest.mark.asyncio
async def test_update_config_field_prompt_caching_rejects_invalid(monkeypatch, field_name, bad_value):
"""An unsupported ttl must be refused here rather than reaching Anthropic verbatim."""
from unittest.mock import MagicMock
from fastapi import HTTPException
import litellm.proxy.proxy_server as ps
from litellm.proxy._types import (
ConfigFieldUpdate,
LitellmUserRoles,
UserAPIKeyAuth,
)
from litellm.proxy.proxy_server import update_config_general_settings
async def fake_get_config():
return {"litellm_settings": {}}
monkeypatch.setattr(ps.proxy_config, "get_config", fake_get_config)
monkeypatch.setattr(ps, "prisma_client", MagicMock())
monkeypatch.setattr(litellm, field_name, None)
admin = UserAPIKeyAuth(api_key="k", user_id="a", user_role=LitellmUserRoles.PROXY_ADMIN)
with pytest.raises(HTTPException) as exc:
await update_config_general_settings(
data=ConfigFieldUpdate(field_name=field_name, field_value=bad_value, config_type="general_settings"),
user_api_key_dict=admin,
)
assert exc.value.status_code == 400
assert getattr(litellm, field_name) is None
@pytest.mark.parametrize(
"field_name, expected_default",
[
("enable_anthropic_prompt_caching", False),
("anthropic_prompt_caching_ttl", None),
("budget_exceeded_throttle_percentage", None),
],
)
@pytest.mark.asyncio
async def test_reset_config_field_restores_type_default(monkeypatch, field_name, expected_default):
"""Reset must restore each field's own default. Blanket None would leave the boolean flag
set to None, which is not a bool and would read as neither on nor off."""
from unittest.mock import MagicMock
import litellm.proxy.proxy_server as ps
from litellm.proxy._types import (
ConfigFieldDelete,
LitellmUserRoles,
UserAPIKeyAuth,
)
from litellm.proxy.proxy_server import delete_config_general_settings
saved: dict = {}
async def fake_get_config():
return {"litellm_settings": {field_name: "stale"}}
async def fake_save_config(new_config=None):
saved.update(new_config or {})
monkeypatch.setattr(ps.proxy_config, "get_config", fake_get_config)
monkeypatch.setattr(ps.proxy_config, "save_config", fake_save_config)
monkeypatch.setattr(ps, "prisma_client", MagicMock())
monkeypatch.setattr(litellm, "store_audit_logs", False)
monkeypatch.setattr(litellm, field_name, "stale")
admin = UserAPIKeyAuth(api_key="k", user_id="a", user_role=LitellmUserRoles.PROXY_ADMIN)
await delete_config_general_settings(
data=ConfigFieldDelete(field_name=field_name, config_type="general_settings"),
user_api_key_dict=admin,
)
assert getattr(litellm, field_name) is expected_default
assert field_name not in saved["litellm_settings"]
@pytest.mark.parametrize("bad_value", [0, -0.1, 1.5, True])
@pytest.mark.asyncio
async def test_update_config_field_throttle_rejects_invalid(monkeypatch, bad_value):

15
ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx
View file

@ -14,7 +14,7 @@ import {
} from "@tremor/react";
import { TabPanel, TabPanels, TabGroup, TabList, Tab } from "@tremor/react";
import { getGeneralSettingsCall, updateConfigFieldSetting, deleteConfigFieldSetting } from "@/components/networking";
import { InputNumber } from "antd";
import { InputNumber, Select as AntdSelect } from "antd";
import { TrashIcon } from "@heroicons/react/outline";
import { StatusBadge } from "@/components/shared/table_cells";
@ -33,6 +33,7 @@ interface generalSettingsItem {
field_value: any;
field_description: string;
stored_in_db: boolean | null;
field_options?: string[] | null;
}
const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({ accessToken, userRole, userID }) => {
@ -169,6 +170,18 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({ accessToken, user
value={value.field_value}
onChange={(newValue) => handleInputChange(value.field_name, newValue)}
/>
) : value.field_type == "Select" ? (
<AntdSelect
allowClear
style={{ minWidth: "8rem" }}
placeholder="Default"
value={value.field_value || undefined}
options={(value.field_options ?? []).map((option) => ({
label: option,
value: option,
}))}
onChange={(newValue) => handleInputChange(value.field_name, newValue ?? "")}
/>
) : null}
</TableCell>
<TableCell>

2
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -22711,6 +22711,8 @@ export interface components {
field_description: string;
/** Field Name */
field_name: string;
/** Field Options */
field_options?: string[] | null;
/** Field Type */
field_type: string;
/** Field Value */