scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-06 08:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

fix: accept persistable router settings

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
mateo 2026-09-02 06:15:52 +00:00
parent 1d3e26fd98
commit 974b331a4d
3 changed files with 65 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
litellm/constants.py
View file

@ -31,6 +31,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
"optional_pre_call_checks",
}
)
ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset({"model_list", "search_tools"})
DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512))
DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5))
DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10))

32
litellm/proxy/proxy_server.py
View file

@ -254,6 +254,7 @@ from litellm.constants import (
PROXY_BUDGET_RESCHEDULER_MAX_TIME,
PROXY_BUDGET_RESCHEDULER_MIN_TIME,
PROXY_CONFIG_RELOAD_INTERVAL_SECONDS,
ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG,
USER_SPEND_ALERTS_JOB_ID,
WEEKLY_SPEND_REPORT_JOB_ID,
)
@ -5711,13 +5712,9 @@ class ProxyConfig:
router_settings: Final = config.get("router_settings", None)
if router_settings and isinstance(router_settings, dict):
# model list and search_tools already set
exclude_args: Final = {
"model_list",
"search_tools",
}
available_args: Final = [x for x in litellm.Router.get_valid_args() if x not in exclude_args]
available_args: Final = [
x for x in litellm.Router.get_valid_args() if x not in ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG
]
for k, v in router_settings.items():
if k in available_args:
@ -16229,14 +16226,17 @@ async def update_config(
)
raw_router_settings: Final = request_body.get("router_settings")
if isinstance(raw_router_settings, dict):
unsupported_router_settings: Final = sorted(set(raw_router_settings) - RUNTIME_UPDATABLE_ROUTER_SETTINGS)
supported_router_settings: Final = RUNTIME_UPDATABLE_ROUTER_SETTINGS | (
frozenset(litellm.Router.get_valid_args()) - ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG
)
unsupported_router_settings: Final = sorted(set(raw_router_settings) - supported_router_settings)
if unsupported_router_settings:
raise HTTPException(
status_code=400,
detail={
"error": (
f"Unsupported router settings: {', '.join(unsupported_router_settings)} "
"are not runtime-updatable router settings"
"are not valid router settings"
)
},
)
@ -16342,10 +16342,20 @@ async def update_config(
)
# router_settings: merge existing + request, request wins.
if config_info.router_settings is not None:
if isinstance(raw_router_settings, dict):
existing = await _read_section("router_settings")
before_router_settings: Final = copy.deepcopy(existing)
updates = config_info.router_settings.dict(exclude_none=True)
typed_router_settings: Final = (
config_info.router_settings.dict(exclude_none=True)
if config_info.router_settings is not None
else {}
)
raw_router_settings_without_none: Final = {
key: value
for key, value in raw_router_settings.items()
if key not in typed_router_settings and value is not None
}
updates: Final = {**typed_router_settings, **raw_router_settings_without_none}
new_router_settings: Final = {**existing, **updates}
await _upsert_section("router_settings", new_router_settings)
asyncio.create_task(

43
tests/test_litellm/proxy/proxy_server/test_routes_config.py
View file

@ -81,6 +81,49 @@ def test_config_update_persists_optional_pre_call_checks(client, auth_as, mock_p
assert persisted["optional_pre_call_checks"] == ["prompt_caching"]
def test_config_update_persists_model_group_affinity_config(client, auth_as, mock_prisma, monkeypatch):
from litellm.proxy import proxy_server as ps
from litellm.proxy._types import LitellmUserRoles
table = _install_litellm_config(mock_prisma)
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
fake_proxy_config = MagicMock()
fake_proxy_config.add_deployment = AsyncMock()
monkeypatch.setattr(ps, "proxy_config", fake_proxy_config)
model_group_affinity_config = {"gpt-4": ["session_affinity"]}
with auth_as(LitellmUserRoles.PROXY_ADMIN):
response = client.post(
"/config/update",
json={"router_settings": {"model_group_affinity_config": model_group_affinity_config}},
)
assert response.status_code == 200
persisted = json.loads(table.upsert.call_args.kwargs["data"]["create"]["param_value"])
assert persisted["model_group_affinity_config"] == model_group_affinity_config
def test_config_update_persists_disable_cooldowns(client, auth_as, mock_prisma, monkeypatch):
from litellm.proxy import proxy_server as ps
from litellm.proxy._types import LitellmUserRoles
table = _install_litellm_config(mock_prisma)
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
fake_proxy_config = MagicMock()
fake_proxy_config.add_deployment = AsyncMock()
monkeypatch.setattr(ps, "proxy_config", fake_proxy_config)
with auth_as(LitellmUserRoles.PROXY_ADMIN):
response = client.post(
"/config/update",
json={"router_settings": {"disable_cooldowns": True}},
)
assert response.status_code == 200
persisted = json.loads(table.upsert.call_args.kwargs["data"]["create"]["param_value"])
assert persisted["disable_cooldowns"] is True
def test_config_update_rejects_unknown_router_setting(client, auth_as, mock_prisma, monkeypatch):
from litellm.proxy import proxy_server as ps
from litellm.proxy._types import LitellmUserRoles