mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-06 02:48:13 +00:00
Merge pull request #34146 from BerriAI/litellm_lit4662_autorouter_budget
fix(proxy): raise dashboard session budget default to $1 and make it configurable in config and Admin UI
This commit is contained in:
tin-berri
GitHub
commit
d514497979
6 changed files with 297 additions and 13 deletions
|
|
@ -427,7 +427,9 @@ default_team_settings: Optional[List] = None
|
|||
max_user_budget: Optional[float] = None
|
||||
default_max_internal_user_budget: Optional[float] = None
|
||||
max_internal_user_budget: Optional[float] = None
|
||||
max_ui_session_budget: Optional[float] = 0.25 # $0.25 USD budgets for UI Chat sessions
|
||||
max_ui_session_budget: Optional[float] = (
|
||||
1.0 # USD budget for each dashboard login session (playground, test connection)
|
||||
)
|
||||
internal_user_budget_duration: Optional[str] = None
|
||||
tag_budget_config: Optional[Dict[str, "BudgetConfig"]] = None
|
||||
max_end_user_budget: Optional[float] = None
|
||||
|
|
|
|||
|
|
@ -1528,6 +1528,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [
|
|||
# test_general_settings_ui_fields_are_db_overridable enforces that pairing.
|
||||
"enable_anthropic_prompt_caching",
|
||||
"anthropic_prompt_caching_ttl",
|
||||
"max_ui_session_budget",
|
||||
]
|
||||
SPECIAL_LITELLM_AUTH_TOKEN = ["ui-token"]
|
||||
DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60))
|
||||
|
|
|
|||
|
|
@ -4629,6 +4629,11 @@ class ProxyConfig:
|
|||
verbose_proxy_logger.debug(
|
||||
f"{blue_color_code} Initialized polling via cache: enabled={polling_via_cache_enabled}, native_background_mode={native_background_mode}, ttl={polling_cache_ttl}{reset_color_code}"
|
||||
)
|
||||
elif key == "max_ui_session_budget":
|
||||
litellm.max_ui_session_budget = float(value) if value is not None else None
|
||||
verbose_proxy_logger.debug(
|
||||
f"{blue_color_code} setting litellm.max_ui_session_budget={litellm.max_ui_session_budget}{reset_color_code}"
|
||||
)
|
||||
elif key == "default_team_settings":
|
||||
for idx, team_setting in enumerate(value): # run through pydantic validation
|
||||
try:
|
||||
|
|
@ -14925,10 +14930,11 @@ GeneralSettingsUILiteLLMValue = Union[float, bool, str, None]
|
|||
|
||||
|
||||
class GeneralSettingsUILiteLLMFieldSpec(TypedDict):
|
||||
type: Literal["Float", "Boolean", "Select"]
|
||||
type: Literal["Float", "Dollar", "Boolean", "Select"]
|
||||
description: str
|
||||
options: NotRequired[tuple[str, ...]]
|
||||
tab: NotRequired[str] # Admin UI sub-tab this field renders under; None groups it with the rest
|
||||
default: NotRequired[float] # reset/clear restores this instead of None; fields whose None means fail-open set it
|
||||
|
||||
|
||||
_GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, GeneralSettingsUILiteLLMFieldSpec] = {
|
||||
|
|
@ -14954,21 +14960,32 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, GeneralSettingsUILiteLLMFieldSpec
|
|||
"tab": "prompt_caching",
|
||||
"description": "Empty uses Anthropic's 5m default. 1h suits long sessions but doubles the cache write cost.",
|
||||
},
|
||||
"max_ui_session_budget": {
|
||||
"type": "Dollar",
|
||||
"default": 1.0,
|
||||
"description": (
|
||||
"USD spend cap for each dashboard login session; covers LLM calls made from the dashboard "
|
||||
"such as the playground and auto router Test Connection. Each login starts a fresh session "
|
||||
"with this budget. Clearing restores the $1 default."
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _general_settings_ui_litellm_default(
|
||||
field_type: Literal["Float", "Boolean", "Select"],
|
||||
spec: GeneralSettingsUILiteLLMFieldSpec,
|
||||
) -> GeneralSettingsUILiteLLMValue:
|
||||
"""The value a field falls back to when it is cleared or reset."""
|
||||
return False if field_type == "Boolean" else None
|
||||
if "default" in spec:
|
||||
return spec["default"]
|
||||
return False if spec["type"] == "Boolean" else None
|
||||
|
||||
|
||||
def _validate_general_settings_ui_litellm_value(field_name: str, value: Any) -> GeneralSettingsUILiteLLMValue:
|
||||
spec = _GENERAL_SETTINGS_UI_LITELLM_FIELDS[field_name]
|
||||
field_type = spec["type"]
|
||||
if value is None or value == "":
|
||||
return _general_settings_ui_litellm_default(field_type)
|
||||
return _general_settings_ui_litellm_default(spec)
|
||||
match field_type:
|
||||
case "Boolean":
|
||||
if not isinstance(value, bool):
|
||||
|
|
@ -14992,6 +15009,13 @@ def _validate_general_settings_ui_litellm_value(field_name: str, value: Any) ->
|
|||
detail={"error": f"{field_name} must be a number in (0, 1] or empty"},
|
||||
)
|
||||
return float(value)
|
||||
case "Dollar":
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or float(value) <= 0:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"error": f"{field_name} must be a positive dollar amount or empty"},
|
||||
)
|
||||
return float(value)
|
||||
case _:
|
||||
assert_never(field_type)
|
||||
|
||||
|
|
@ -15014,7 +15038,7 @@ async def _persist_general_settings_ui_litellm_field(
|
|||
async def _reset_general_settings_ui_litellm_field(field_name: str, user_api_key_dict: UserAPIKeyAuth) -> dict:
|
||||
config = await proxy_config.get_config()
|
||||
before_value = config.get("litellm_settings", {}).get(field_name)
|
||||
default_value = _general_settings_ui_litellm_default(_GENERAL_SETTINGS_UI_LITELLM_FIELDS[field_name]["type"])
|
||||
default_value = _general_settings_ui_litellm_default(_GENERAL_SETTINGS_UI_LITELLM_FIELDS[field_name])
|
||||
setattr(litellm, field_name, default_value)
|
||||
if "litellm_settings" in config:
|
||||
config["litellm_settings"].pop(field_name, None)
|
||||
|
|
@ -15189,7 +15213,7 @@ async def get_config_list(
|
|||
)
|
||||
for litellm_field_name, spec in _GENERAL_SETTINGS_UI_LITELLM_FIELDS.items():
|
||||
current_value: GeneralSettingsUILiteLLMValue = getattr(litellm, litellm_field_name, None)
|
||||
default_value = _general_settings_ui_litellm_default(spec["type"])
|
||||
default_value = _general_settings_ui_litellm_default(spec)
|
||||
stored_in_db_litellm: Optional[bool]
|
||||
if litellm_field_name in db_litellm_settings:
|
||||
stored_in_db_litellm = True
|
||||
|
|
|
|||
|
|
@ -2775,6 +2775,69 @@ async def test_load_config_max_budget_env_var_coerced_to_float(tmp_path, monkeyp
|
|||
litellm.max_budget = original_max_budget
|
||||
|
||||
|
||||
def test_max_ui_session_budget_default_is_one_dollar():
|
||||
"""LIT-4662: the dashboard session budget default is a product decision; the
|
||||
old 0.25 default locked admins out of auto router Test Connection and the
|
||||
playground mid-session with an error that looked like a hardcoded cap."""
|
||||
assert litellm.max_ui_session_budget == 1.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_load_config_max_ui_session_budget_applied_and_coerced(tmp_path, monkeypatch):
|
||||
"""
|
||||
max_ui_session_budget configured via os.environ resolves to a string;
|
||||
load_config must coerce it to float so every dashboard session key is
|
||||
minted with a numeric max_budget.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
|
||||
monkeypatch.setenv("UI_SESSION_BUDGET", "2.5")
|
||||
test_config = {
|
||||
"model_list": [],
|
||||
"litellm_settings": {"max_ui_session_budget": "os.environ/UI_SESSION_BUDGET"},
|
||||
}
|
||||
config_file = tmp_path / "config.yaml"
|
||||
config_file.write_text(yaml.dump(test_config))
|
||||
|
||||
original_budget = litellm.max_ui_session_budget
|
||||
try:
|
||||
proxy_config = ProxyConfig()
|
||||
await proxy_config.load_config(
|
||||
router=MagicMock(), config_file_path=str(config_file)
|
||||
)
|
||||
assert isinstance(litellm.max_ui_session_budget, float)
|
||||
assert litellm.max_ui_session_budget == 2.5
|
||||
finally:
|
||||
litellm.max_ui_session_budget = original_budget
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_load_config_max_ui_session_budget_none_disables_cap(tmp_path):
|
||||
"""
|
||||
max_ui_session_budget: null in config disables the dashboard session cap
|
||||
entirely (session keys minted with no max_budget); load_config must pass
|
||||
None through instead of raising on float(None).
|
||||
"""
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
|
||||
test_config = {
|
||||
"model_list": [],
|
||||
"litellm_settings": {"max_ui_session_budget": None},
|
||||
}
|
||||
config_file = tmp_path / "config.yaml"
|
||||
config_file.write_text(yaml.dump(test_config))
|
||||
|
||||
original_budget = litellm.max_ui_session_budget
|
||||
try:
|
||||
proxy_config = ProxyConfig()
|
||||
await proxy_config.load_config(
|
||||
router=MagicMock(), config_file_path=str(config_file)
|
||||
)
|
||||
assert litellm.max_ui_session_budget is None
|
||||
finally:
|
||||
litellm.max_ui_session_budget = original_budget
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_load_config_default_internal_user_params_max_budget_scientific_notation(tmp_path):
|
||||
"""
|
||||
|
|
@ -9235,6 +9298,85 @@ def test_general_settings_ui_fields_are_db_overridable():
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_config_field_max_ui_session_budget_sets_live_value(monkeypatch):
|
||||
"""LIT-4662: the dashboard session budget is editable from the Admin UI General tab.
|
||||
A Dollar field must accept values above 1 (the old Float type capped at 1, which cannot
|
||||
express a dollar budget), apply live via setattr, and persist under litellm_settings."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy._types import (
|
||||
ConfigFieldUpdate,
|
||||
LitellmUserRoles,
|
||||
UserAPIKeyAuth,
|
||||
)
|
||||
from litellm.proxy.proxy_server import update_config_general_settings
|
||||
|
||||
saved: dict = {}
|
||||
|
||||
async def fake_get_config():
|
||||
return {"litellm_settings": {}}
|
||||
|
||||
async def fake_save_config(new_config=None):
|
||||
saved.update(new_config or {})
|
||||
|
||||
monkeypatch.setattr(ps.proxy_config, "get_config", fake_get_config)
|
||||
monkeypatch.setattr(ps.proxy_config, "save_config", fake_save_config)
|
||||
monkeypatch.setattr(ps, "prisma_client", MagicMock())
|
||||
monkeypatch.setattr(litellm, "store_audit_logs", False)
|
||||
monkeypatch.setattr(litellm, "max_ui_session_budget", 1.0)
|
||||
|
||||
admin = UserAPIKeyAuth(api_key="k", user_id="a", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
await update_config_general_settings(
|
||||
data=ConfigFieldUpdate(
|
||||
field_name="max_ui_session_budget",
|
||||
field_value=25.0,
|
||||
config_type="general_settings",
|
||||
),
|
||||
user_api_key_dict=admin,
|
||||
)
|
||||
|
||||
assert litellm.max_ui_session_budget == 25.0
|
||||
assert saved["litellm_settings"]["max_ui_session_budget"] == 25.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad_value", [True, "abc", -1, 0, [2.5]])
|
||||
def test_validate_max_ui_session_budget_rejects_malformed(bad_value):
|
||||
"""A Dollar field accepts only positive numbers; zero would block every dashboard
|
||||
LLM call at mint and non-numerics would break session key generation."""
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy.proxy_server import _validate_general_settings_ui_litellm_value
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
_validate_general_settings_ui_litellm_value("max_ui_session_budget", bad_value)
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.parametrize("empty_value", [None, ""])
|
||||
def test_validate_max_ui_session_budget_empty_restores_default(empty_value):
|
||||
"""Clearing the field in the UI restores the shipped $1 default rather than None;
|
||||
None would silently remove the session spend guardrail (unlimited budget), which
|
||||
must stay a deliberate config.yaml act (max_ui_session_budget: null)."""
|
||||
from litellm.proxy.proxy_server import _validate_general_settings_ui_litellm_value
|
||||
|
||||
assert _validate_general_settings_ui_litellm_value("max_ui_session_budget", empty_value) == 1.0
|
||||
|
||||
|
||||
def test_general_settings_ui_defaults_unchanged_for_existing_fields():
|
||||
"""The spec-default mechanism added for max_ui_session_budget must not change what
|
||||
clearing the pre-existing fields restores (None for Float/Select, False for Boolean)."""
|
||||
from litellm.proxy.proxy_server import (
|
||||
_GENERAL_SETTINGS_UI_LITELLM_FIELDS,
|
||||
_general_settings_ui_litellm_default,
|
||||
)
|
||||
|
||||
assert _general_settings_ui_litellm_default(_GENERAL_SETTINGS_UI_LITELLM_FIELDS["budget_exceeded_throttle_percentage"]) is None
|
||||
assert _general_settings_ui_litellm_default(_GENERAL_SETTINGS_UI_LITELLM_FIELDS["enable_anthropic_prompt_caching"]) is False
|
||||
assert _general_settings_ui_litellm_default(_GENERAL_SETTINGS_UI_LITELLM_FIELDS["anthropic_prompt_caching_ttl"]) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field_name, db_value",
|
||||
[
|
||||
|
|
|
|||
|
|
@ -0,0 +1,101 @@
|
|||
import { renderWithProviders, screen, within } from "../../../../../tests/test-utils";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { vi } from "vitest";
|
||||
import GeneralSettings from "./general_settings";
|
||||
import { deleteConfigFieldSetting, getGeneralSettingsCall, updateConfigFieldSetting } from "@/components/networking";
|
||||
|
||||
vi.mock("@/components/networking", () => ({
|
||||
getGeneralSettingsCall: vi.fn(),
|
||||
updateConfigFieldSetting: vi.fn().mockResolvedValue({}),
|
||||
deleteConfigFieldSetting: vi.fn().mockResolvedValue({}),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/router_settings", () => ({ default: () => null }));
|
||||
vi.mock("@/components/Settings/RouterSettings/Fallbacks/Fallbacks", () => ({ default: () => null }));
|
||||
vi.mock("@/components/routing_groups", () => ({ default: () => null }));
|
||||
|
||||
// Mirrors the /config/list ordering: the two prompt-caching rows sit between the
|
||||
// General-tab rows in the unfiltered response but are filtered out of the General
|
||||
// tab's table, so any index-based lookup into the unfiltered array reads the wrong
|
||||
// row for every field rendered after them.
|
||||
const SETTINGS_FIXTURE = [
|
||||
{
|
||||
field_name: "budget_exceeded_throttle_percentage",
|
||||
field_type: "Float",
|
||||
field_value: null,
|
||||
field_description: "throttle fraction",
|
||||
stored_in_db: null,
|
||||
field_default_value: null,
|
||||
},
|
||||
{
|
||||
field_name: "enable_anthropic_prompt_caching",
|
||||
field_type: "Boolean",
|
||||
field_value: true,
|
||||
field_description: "prompt caching toggle",
|
||||
stored_in_db: true,
|
||||
field_tab: "prompt_caching",
|
||||
field_default_value: false,
|
||||
},
|
||||
{
|
||||
field_name: "anthropic_prompt_caching_ttl",
|
||||
field_type: "Select",
|
||||
field_value: "5m",
|
||||
field_description: "prompt caching ttl",
|
||||
stored_in_db: true,
|
||||
field_options: ["5m", "1h"],
|
||||
field_tab: "prompt_caching",
|
||||
field_default_value: null,
|
||||
},
|
||||
{
|
||||
field_name: "max_ui_session_budget",
|
||||
field_type: "Dollar",
|
||||
field_value: 7.5,
|
||||
field_description: "dashboard session budget",
|
||||
stored_in_db: true,
|
||||
field_default_value: 1.0,
|
||||
},
|
||||
];
|
||||
|
||||
const settingsRow = async (fieldName: string) => {
|
||||
const cell = await screen.findByText(fieldName);
|
||||
const row = cell.closest("tr");
|
||||
expect(row).not.toBeNull();
|
||||
return row as HTMLElement;
|
||||
};
|
||||
|
||||
describe("GeneralSettings General tab", () => {
|
||||
beforeEach(() => {
|
||||
vi.mocked(getGeneralSettingsCall).mockResolvedValue([...SETTINGS_FIXTURE.map((s) => ({ ...s }))]);
|
||||
vi.mocked(updateConfigFieldSetting).mockClear();
|
||||
vi.mocked(deleteConfigFieldSetting).mockClear();
|
||||
});
|
||||
|
||||
it("updates max_ui_session_budget with its own value, not the value at its filtered index", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<GeneralSettings accessToken="token" userRole="Admin" userID="user" />);
|
||||
|
||||
await user.click(screen.getByText("General"));
|
||||
const row = await settingsRow("max_ui_session_budget");
|
||||
|
||||
await user.click(within(row).getByRole("button", { name: /update/i }));
|
||||
|
||||
expect(updateConfigFieldSetting).toHaveBeenCalledWith("token", "max_ui_session_budget", 7.5);
|
||||
});
|
||||
|
||||
it("reset shows the field's default value instead of an empty input", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<GeneralSettings accessToken="token" userRole="Admin" userID="user" />);
|
||||
|
||||
await user.click(screen.getByText("General"));
|
||||
const row = await settingsRow("max_ui_session_budget");
|
||||
expect(within(row).getByRole("spinbutton")).toHaveValue("7.50");
|
||||
|
||||
const actionCell = row.querySelectorAll("td")[3];
|
||||
const resetIcon = actionCell.querySelector("svg");
|
||||
expect(resetIcon).not.toBeNull();
|
||||
await user.click(resetIcon as unknown as Element);
|
||||
|
||||
expect(deleteConfigFieldSetting).toHaveBeenCalledWith("token", "max_ui_session_budget");
|
||||
expect(within(row).getByRole("spinbutton")).toHaveValue("1.00");
|
||||
});
|
||||
});
|
||||
|
|
@ -41,6 +41,7 @@ export interface generalSettingsItem {
|
|||
stored_in_db: boolean | null;
|
||||
field_options?: string[] | null;
|
||||
field_tab?: string | null;
|
||||
field_default_value?: any;
|
||||
}
|
||||
|
||||
const SettingValueEditor: React.FC<{
|
||||
|
|
@ -75,6 +76,17 @@ const SettingValueEditor: React.FC<{
|
|||
/>
|
||||
);
|
||||
}
|
||||
if (setting.field_type === "Dollar") {
|
||||
return (
|
||||
<InputNumber
|
||||
min={0.01}
|
||||
step={0.25}
|
||||
prefix="$"
|
||||
value={setting.field_value}
|
||||
onChange={(newValue) => onChange(setting.field_name, newValue)}
|
||||
/>
|
||||
);
|
||||
}
|
||||
if (setting.field_type === "Select") {
|
||||
return (
|
||||
<AntdSelect
|
||||
|
|
@ -171,12 +183,12 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({ accessToken, user
|
|||
setGeneralSettings(updatedSettings);
|
||||
};
|
||||
|
||||
const handleUpdateField = (fieldName: string, idx: number) => {
|
||||
const handleUpdateField = (fieldName: string) => {
|
||||
if (!accessToken) {
|
||||
return;
|
||||
}
|
||||
|
||||
let fieldValue = generalSettings[idx].field_value;
|
||||
let fieldValue = generalSettings.find((setting) => setting.field_name === fieldName)?.field_value;
|
||||
|
||||
if (fieldValue == null || fieldValue == undefined) {
|
||||
return;
|
||||
|
|
@ -194,7 +206,7 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({ accessToken, user
|
|||
}
|
||||
};
|
||||
|
||||
const handleResetField = (fieldName: string, idx: number) => {
|
||||
const handleResetField = (fieldName: string) => {
|
||||
if (!accessToken) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -204,7 +216,9 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({ accessToken, user
|
|||
// update value in state
|
||||
|
||||
const updatedSettings = generalSettings.map((setting) =>
|
||||
setting.field_name === fieldName ? { ...setting, stored_in_db: null, field_value: null } : setting,
|
||||
setting.field_name === fieldName
|
||||
? { ...setting, stored_in_db: null, field_value: setting.field_default_value ?? null }
|
||||
: setting,
|
||||
);
|
||||
setGeneralSettings(updatedSettings);
|
||||
} catch (error) {
|
||||
|
|
@ -281,8 +295,8 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({ accessToken, user
|
|||
)}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<Button onClick={() => handleUpdateField(value.field_name, index)}>Update</Button>
|
||||
<Icon icon={TrashIcon} color="red" onClick={() => handleResetField(value.field_name, index)}>
|
||||
<Button onClick={() => handleUpdateField(value.field_name)}>Update</Button>
|
||||
<Icon icon={TrashIcon} color="red" onClick={() => handleResetField(value.field_name)}>
|
||||
Reset
|
||||
</Icon>
|
||||
</TableCell>
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue