scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 129

fix: reject unknown runtime router settings

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
mateo 2026-09-02 02:17:02 +00:00
parent 62f032cca5
commit 0608f0a00f
7 changed files with 125 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
litellm/constants.py
View file

@ -38,6 +38,28 @@ DEFAULT_MAX_TOKENS: Final = int(os.getenv("DEFAULT_MAX_TOKENS", 4096))
DEFAULT_ALLOWED_FAILS: Final = int(os.getenv("DEFAULT_ALLOWED_FAILS", 3))
DEFAULT_REDIS_SYNC_INTERVAL: Final = int(os.getenv("DEFAULT_REDIS_SYNC_INTERVAL", 1))
DEFAULT_COOLDOWN_TIME_SECONDS: Final = int(os.getenv("DEFAULT_COOLDOWN_TIME_SECONDS", 5))
RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
{
"routing_strategy_args",
"routing_strategy",
"routing_groups",
"allowed_fails",
"cooldown_time",
"num_retries",
"timeout",
"max_retries",
"retry_after",
"fallbacks",
"context_window_fallbacks",
"retry_policy",
"model_group_retry_policy",
"model_group_alias",
"enable_weighted_failover",
"enable_tag_filtering",
"tag_routing_prefix",
"optional_pre_call_checks",
}
)
DEFAULT_REPLICATE_POLLING_RETRIES: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_RETRIES", 5))
DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1))
DEFAULT_IMAGE_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250))

21
litellm/proxy/proxy_server.py
View file

@ -39,7 +39,7 @@ from typing import (
import anyio
import websockets
import websockets.exceptions
from pydantic import BaseModel, Json, JsonValue, ValidationError
from pydantic import BaseModel, Json, JsonValue, TypeAdapter, ValidationError
from typing_extensions import NotRequired, ReadOnly, assert_never
from litellm._uuid import uuid
@ -60,6 +60,7 @@ from litellm.constants import (
LITELLM_SETTINGS_SAFE_DB_OVERRIDES,
LITELLM_UI_ALLOW_HEADERS,
LITELLM_UI_SESSION_DURATION,
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
)
from litellm.litellm_core_utils.litellm_logging import (
_init_custom_logger_compatible_class,
@ -16207,6 +16208,7 @@ async def invitation_delete(
)
async def update_config(
config_info: ConfigYAML,
request: Request,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
@ -16218,6 +16220,23 @@ async def update_config(
a side effect of an unrelated update.
"""
global llm_router, llm_model_list, general_settings, proxy_config, proxy_logging_obj, master_key, prisma_client
request_body: Final[Mapping[str, JsonValue]] = TypeAdapter(Mapping[str, JsonValue]).validate_python(
await request.json()
)
raw_router_settings: Final = request_body.get("router_settings")
if isinstance(raw_router_settings, dict):
unsupported_router_settings: Final = sorted(set(raw_router_settings) - RUNTIME_UPDATABLE_ROUTER_SETTINGS)
if unsupported_router_settings:
raise HTTPException(
status_code=400,
detail={
"error": (
f"Unsupported router settings: {', '.join(unsupported_router_settings)} "
"are not runtime-updatable router settings"
)
},
)
try:
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
raise HTTPException(status_code=403, detail="Only proxy admins can update config")

30
litellm/router.py
View file

@ -50,6 +50,7 @@ from litellm.constants import (
DEFAULT_HEALTH_CHECK_INTERVAL,
DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER,
DEFAULT_MAX_LRU_CACHE_SIZE,
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
)
from litellm.integrations.custom_logger import CustomLogger
@ -2072,6 +2073,10 @@ class Router:
if _callback is None:
continue
if self.optional_callbacks is not None and any(
isinstance(callback, type(_callback)) for callback in self.optional_callbacks
):
continue
if self.optional_callbacks is None:
self.optional_callbacks = []
self.optional_callbacks.append(_callback)
@ -11331,27 +11336,6 @@ class Router:
"""
Update the router settings.
"""
# only the following settings are allowed to be configured
_allowed_settings: Final = [
"routing_strategy_args",
"routing_strategy",
"routing_groups",
"allowed_fails",
"cooldown_time",
"num_retries",
"timeout",
"max_retries",
"retry_after",
"fallbacks",
"context_window_fallbacks",
"retry_policy",
"model_group_retry_policy",
"model_group_alias",
"enable_weighted_failover",
"enable_tag_filtering",
"tag_routing_prefix",
]
_int_settings: Final = [
"timeout",
"num_retries",
@ -11364,13 +11348,15 @@ class Router:
rebuild_routing_groups = False
relink_lar1_from_args = False
for var in kwargs:
if var in _allowed_settings:
if var in RUNTIME_UPDATABLE_ROUTER_SETTINGS:
if var in _int_settings:
_casted_value = int(kwargs[var])
setattr(self, var, _casted_value)
elif var == "routing_groups":
self._routing_groups_input = kwargs[var]
rebuild_routing_groups = True
elif var == "optional_pre_call_checks":
self.add_optional_pre_call_checks(kwargs[var])
elif var == "retry_policy":
value = kwargs[var]
if isinstance(value, dict):

29
litellm/types/router.py
View file

@ -106,6 +106,20 @@ class RetryPolicy(BaseModel):
InternalServerErrorRetries: int | None = None
OptionalPreCallChecks = list[
Literal[
"prompt_caching",
"router_budget_limiting",
"responses_api_deployment_check",
"deployment_affinity",
"session_affinity",
"forward_client_headers_by_model_group",
"enforce_model_rate_limits",
"encrypted_content_affinity",
]
]
class UpdateRouterConfig(BaseModel):
"""
Set of params that you can modify via `router.update_settings()`.
@ -128,6 +142,7 @@ class UpdateRouterConfig(BaseModel):
model_group_alias: dict[str, str | dict] | None = {}
enable_tag_filtering: bool | None = None
tag_routing_prefix: str | None = None
optional_pre_call_checks: OptionalPreCallChecks | None = None
model_config = ConfigDict(protected_namespaces=())
@ -869,20 +884,6 @@ class FallbackAccessCheck(Protocol):
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: "Router") -> bool: ...
OptionalPreCallChecks = list[
Literal[
"prompt_caching",
"router_budget_limiting",
"responses_api_deployment_check",
"deployment_affinity",
"session_affinity",
"forward_client_headers_by_model_group",
"enforce_model_rate_limits",
"encrypted_content_affinity",
]
]
class LiteLLM_RouterFileObject(TypedDict, total=False):
"""
Tracking the litellm params hash, used for mapping the file id to the right model

39
tests/test_litellm/proxy/proxy_server/test_routes_config.py
View file

@ -60,6 +60,45 @@ def test_config_update_happy_admin(client, auth_as, mock_prisma, monkeypatch):
assert normalize(response.json()) == {"message": "Config updated successfully"}
def test_config_update_persists_optional_pre_call_checks(client, auth_as, mock_prisma, monkeypatch):
from litellm.proxy import proxy_server as ps
from litellm.proxy._types import LitellmUserRoles
table = _install_litellm_config(mock_prisma)
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
fake_proxy_config = MagicMock()
fake_proxy_config.add_deployment = AsyncMock()
monkeypatch.setattr(ps, "proxy_config", fake_proxy_config)
with auth_as(LitellmUserRoles.PROXY_ADMIN):
response = client.post(
"/config/update",
json={"router_settings": {"optional_pre_call_checks": ["prompt_caching"]}},
)
assert response.status_code == 200
persisted = json.loads(table.upsert.call_args.kwargs["data"]["create"]["param_value"])
assert persisted["optional_pre_call_checks"] == ["prompt_caching"]
def test_config_update_rejects_unknown_router_setting(client, auth_as, mock_prisma, monkeypatch):
from litellm.proxy import proxy_server as ps
from litellm.proxy._types import LitellmUserRoles
table = _install_litellm_config(mock_prisma)
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
with auth_as(LitellmUserRoles.PROXY_ADMIN):
response = client.post(
"/config/update",
json={"router_settings": {"optional_precall_checks": ["prompt_caching"]}},
)
assert response.status_code == 400
assert "optional_precall_checks" in response.json()["detail"]["error"]
table.upsert.assert_not_called()
def test_config_update_non_admin_forbidden(client, auth_as, mock_prisma, monkeypatch):
"""POST /config/update by a non-admin caller is rejected; the error
surfaces as a ProxyException with the admin-only message."""

21
tests/test_litellm/test_router_retry_policy_update.py
View file

@ -26,8 +26,8 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from pydantic import ValidationError
import litellm
from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck
from litellm.types.router import RetryPolicy, UpdateRouterConfig
# ---------------------------------------------------------------------------
@ -100,6 +100,19 @@ def _build_router() -> litellm.Router:
)
def test_update_settings_adds_optional_pre_call_check_once():
router = _build_router()
router.update_settings(num_retries=7, optional_pre_call_checks=["prompt_caching"])
router.update_settings(optional_pre_call_checks=["prompt_caching"])
prompt_caching_callbacks = [
callback for callback in router.optional_callbacks if isinstance(callback, PromptCachingDeploymentCheck)
]
assert len(prompt_caching_callbacks) == 1
assert router.num_retries == 7
def test_update_settings_persists_retry_policy_dict():
"""When the proxy's ``_add_router_settings_from_db_config`` calls
``llm_router.update_settings(retry_policy={...})`` after reading the
@ -228,7 +241,7 @@ async def test_config_update_persists_and_reads_back_retry_policy(monkeypatch):
"""The exact global retry_policy save the UI performs must survive the
real ``/config/update`` -> DB -> apply -> ``/get/config/callbacks`` path,
not snap back to the ``num_retries`` fallback the ticket reported."""
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy import proxy_server
from litellm.proxy._types import ConfigYAML, LitellmUserRoles, UserAPIKeyAuth
router = _build_router()
@ -255,8 +268,12 @@ async def test_config_update_persists_and_reads_back_retry_policy(monkeypatch):
RateLimitErrorRetries=7,
)
)
request = MagicMock()
request.json = AsyncMock(return_value={"router_settings": {"retry_policy": posted.model_dump()}})
await proxy_server.update_config(
config_info=ConfigYAML(router_settings=posted),
request=request,
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234"),
)

2
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -37473,6 +37473,8 @@ export interface components {
} | null;
/** Num Retries */
num_retries?: number | null;
/** Optional Pre Call Checks */
optional_pre_call_checks?: ("prompt_caching" | "router_budget_limiting" | "responses_api_deployment_check" | "deployment_affinity" | "session_affinity" | "forward_client_headers_by_model_group" | "enforce_model_rate_limits" | "encrypted_content_affinity")[] | null;
/** Retry After */
retry_after?: number | null;
retry_policy?: components["schemas"]["RetryPolicy"] | null;