mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-17 23:51:30 +00:00
feat(router): expose default_litellm_params and optional_pre_call_checks in Admin UI
Router.update_settings() silently dropped default_litellm_params and optional_pre_call_checks (not in the allow-list, and optional_pre_call_checks was never even stored as a readable attribute), so the Admin UI's Router Settings page could not display or persist either setting - e.g. enabling cache_control_injection_points or prompt_caching pre-call routing required editing config.yaml directly. Router now persists optional_pre_call_checks and returns both fields from get_settings(). update_settings() merges default_litellm_params instead of replacing it (a full replace would drop the timeout/max_retries/metadata defaults Router.__init__ sets), and diffs optional_pre_call_checks against what's already applied before calling add_optional_pre_call_checks(), since that method has no dedup guard for prompt_caching/enforce_model_rate_limits and would otherwise register a duplicate callback on every re-save.
This commit is contained in:
parent
53aaabba5e
commit
498aa2997d
7 changed files with 209 additions and 16 deletions
|
|
@ -683,6 +683,7 @@ class Router:
|
|||
|
||||
self.alerting_config: Optional[AlertingConfig] = alerting_config
|
||||
|
||||
self.optional_pre_call_checks: OptionalPreCallChecks = []
|
||||
if optional_pre_call_checks is not None:
|
||||
self.add_optional_pre_call_checks(optional_pre_call_checks)
|
||||
|
||||
|
|
@ -1504,6 +1505,8 @@ class Router:
|
|||
if optional_pre_call_checks is None:
|
||||
return
|
||||
|
||||
self.optional_pre_call_checks = list(dict.fromkeys([*self.optional_pre_call_checks, *optional_pre_call_checks]))
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Unified deployment affinity (session stickiness)
|
||||
# ---------------------------------------------------------------------
|
||||
|
|
@ -9686,6 +9689,8 @@ class Router:
|
|||
"retry_policy",
|
||||
"model_group_alias",
|
||||
"enable_weighted_failover",
|
||||
"default_litellm_params",
|
||||
"optional_pre_call_checks",
|
||||
]
|
||||
|
||||
for var in vars_to_include:
|
||||
|
|
@ -9722,6 +9727,8 @@ class Router:
|
|||
"model_group_retry_policy",
|
||||
"model_group_alias",
|
||||
"enable_weighted_failover",
|
||||
"default_litellm_params",
|
||||
"optional_pre_call_checks",
|
||||
]
|
||||
|
||||
_int_settings = [
|
||||
|
|
@ -9749,6 +9756,12 @@ class Router:
|
|||
value = RetryPolicy(**value)
|
||||
if value is None or isinstance(value, RetryPolicy):
|
||||
setattr(self, var, value)
|
||||
elif var == "default_litellm_params":
|
||||
self.default_litellm_params = {**self.default_litellm_params, **kwargs[var]}
|
||||
elif var == "optional_pre_call_checks":
|
||||
new_checks = [check for check in kwargs[var] if check not in self.optional_pre_call_checks]
|
||||
if new_checks:
|
||||
self.add_optional_pre_call_checks(new_checks)
|
||||
else:
|
||||
value = kwargs[var]
|
||||
# only run routing strategy init if it has changed
|
||||
|
|
|
|||
|
|
@ -224,9 +224,37 @@ ROUTER_SETTINGS_FIELDS: List[RouterSettingsField] = [
|
|||
field_name="default_litellm_params",
|
||||
field_type="Dictionary",
|
||||
field_value=None,
|
||||
field_description="Default parameters for Router.chat.completion.create",
|
||||
field_description=(
|
||||
"Default parameters for Router.chat.completion.create. E.g. set "
|
||||
"cache_control_injection_points here to enable Anthropic/Bedrock "
|
||||
"prompt caching for every model on this proxy."
|
||||
),
|
||||
field_default=None,
|
||||
ui_field_name="Default LiteLLM Params",
|
||||
link="https://docs.litellm.ai/docs/tutorials/claude_code_prompt_cache_routing",
|
||||
),
|
||||
RouterSettingsField(
|
||||
field_name="optional_pre_call_checks",
|
||||
field_type="List",
|
||||
field_value=None,
|
||||
field_description=(
|
||||
"Extra checks the router runs before picking a deployment. Add "
|
||||
"'prompt_caching' to route repeat requests back to the deployment "
|
||||
"that cached the prompt."
|
||||
),
|
||||
field_default=[],
|
||||
options=[
|
||||
"prompt_caching",
|
||||
"router_budget_limiting",
|
||||
"responses_api_deployment_check",
|
||||
"deployment_affinity",
|
||||
"session_affinity",
|
||||
"forward_client_headers_by_model_group",
|
||||
"enforce_model_rate_limits",
|
||||
"encrypted_content_affinity",
|
||||
],
|
||||
ui_field_name="Optional Pre-call Checks",
|
||||
link="https://docs.litellm.ai/docs/tutorials/claude_code_prompt_cache_routing",
|
||||
),
|
||||
RouterSettingsField(
|
||||
field_name="set_verbose",
|
||||
|
|
|
|||
|
|
@ -26,6 +26,19 @@ class ConfigurableClientsideParamsCustomAuth(TypedDict):
|
|||
|
||||
CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = Optional[List[Union[str, ConfigurableClientsideParamsCustomAuth]]]
|
||||
|
||||
OptionalPreCallChecks = List[
|
||||
Literal[
|
||||
"prompt_caching",
|
||||
"router_budget_limiting",
|
||||
"responses_api_deployment_check",
|
||||
"deployment_affinity",
|
||||
"session_affinity",
|
||||
"forward_client_headers_by_model_group",
|
||||
"enforce_model_rate_limits",
|
||||
"encrypted_content_affinity",
|
||||
]
|
||||
]
|
||||
|
||||
|
||||
class ModelConfig(BaseModel):
|
||||
model_name: str
|
||||
|
|
@ -117,6 +130,8 @@ class UpdateRouterConfig(BaseModel):
|
|||
fallbacks: Optional[List[dict]] = None
|
||||
context_window_fallbacks: Optional[List[dict]] = None
|
||||
model_group_alias: Optional[Dict[str, Union[str, Dict]]] = {}
|
||||
default_litellm_params: Optional[Dict[str, Any]] = None
|
||||
optional_pre_call_checks: Optional[OptionalPreCallChecks] = None
|
||||
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
|
|
@ -768,20 +783,6 @@ class GenericBudgetWindowDetails(BaseModel):
|
|||
ttl_seconds: int
|
||||
|
||||
|
||||
OptionalPreCallChecks = List[
|
||||
Literal[
|
||||
"prompt_caching",
|
||||
"router_budget_limiting",
|
||||
"responses_api_deployment_check",
|
||||
"deployment_affinity",
|
||||
"session_affinity",
|
||||
"forward_client_headers_by_model_group",
|
||||
"enforce_model_rate_limits",
|
||||
"encrypted_content_affinity",
|
||||
]
|
||||
]
|
||||
|
||||
|
||||
class LiteLLM_RouterFileObject(TypedDict, total=False):
|
||||
"""
|
||||
Tracking the litellm params hash, used for mapping the file id to the right model
|
||||
|
|
|
|||
|
|
@ -78,6 +78,25 @@ class TestRouterSettingsEndpoints:
|
|||
assert isinstance(routing_strategy_field["options"], list)
|
||||
assert len(routing_strategy_field["options"]) > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_router_fields_includes_optional_pre_call_checks(self):
|
||||
"""
|
||||
Regression test: `optional_pre_call_checks` (e.g. "prompt_caching", used for
|
||||
Claude Code prompt cache routing) must be exposed as a configurable field so
|
||||
the Admin UI can render and save it, not just `default_litellm_params`.
|
||||
"""
|
||||
response = client.get(
|
||||
"/router/fields", headers={"Authorization": "Bearer sk-1234"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
fields = response.json()["fields"]
|
||||
field = next(
|
||||
(f for f in fields if f["field_name"] == "optional_pre_call_checks"), None
|
||||
)
|
||||
assert field is not None
|
||||
assert "prompt_caching" in field["options"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_router_settings_includes_routing_groups_from_live_router(
|
||||
self, monkeypatch
|
||||
|
|
|
|||
|
|
@ -5304,3 +5304,100 @@ class TestRouterRequestTimeoutPropagation:
|
|||
)
|
||||
== 60
|
||||
)
|
||||
|
||||
|
||||
def _make_router_for_settings_tests(**kwargs):
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-3.5-turbo",
|
||||
"litellm_params": {
|
||||
"model": "azure/gpt-4.1-mini",
|
||||
"api_key": "fake-key",
|
||||
"api_base": "https://fake.openai.azure.com",
|
||||
},
|
||||
}
|
||||
],
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def test_update_settings_merges_default_litellm_params_without_dropping_existing_keys():
|
||||
"""
|
||||
Regression test: `update_settings(default_litellm_params=...)` must merge into
|
||||
the existing dict, not replace it wholesale. A naive `setattr` replace would
|
||||
silently drop keys the Router set at init (e.g. `timeout`, `max_retries`,
|
||||
`metadata`) whenever an admin edits `default_litellm_params` from the UI to
|
||||
add something like `cache_control_injection_points`.
|
||||
"""
|
||||
router = _make_router_for_settings_tests(timeout=42)
|
||||
assert router.default_litellm_params.get("timeout") == 42
|
||||
|
||||
router.update_settings(
|
||||
default_litellm_params={
|
||||
"cache_control_injection_points": [
|
||||
{"location": "message", "role": "system"}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert router.default_litellm_params["timeout"] == 42
|
||||
assert router.default_litellm_params["cache_control_injection_points"] == [
|
||||
{"location": "message", "role": "system"}
|
||||
]
|
||||
|
||||
|
||||
def test_update_settings_optional_pre_call_checks_is_idempotent():
|
||||
"""
|
||||
Regression test: `add_optional_pre_call_checks` has no built-in guard against
|
||||
registering the same check twice (unlike `router_budget_limiting`, which
|
||||
checks for an existing budget limiter). `_add_router_settings_from_db_config`
|
||||
re-applies the full `optional_pre_call_checks` list on every config sync, so
|
||||
without diffing against already-applied checks in `update_settings`, saving
|
||||
the same setting twice from the UI would register a second
|
||||
`PromptCachingDeploymentCheck` callback on every save.
|
||||
"""
|
||||
from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import (
|
||||
PromptCachingDeploymentCheck,
|
||||
)
|
||||
|
||||
router = _make_router_for_settings_tests()
|
||||
assert router.optional_pre_call_checks == []
|
||||
|
||||
router.update_settings(optional_pre_call_checks=["prompt_caching"])
|
||||
assert router.optional_pre_call_checks == ["prompt_caching"]
|
||||
prompt_caching_callbacks = [
|
||||
cb
|
||||
for cb in (router.optional_callbacks or [])
|
||||
if isinstance(cb, PromptCachingDeploymentCheck)
|
||||
]
|
||||
assert len(prompt_caching_callbacks) == 1
|
||||
|
||||
# Re-applying the same setting (e.g. a second Save click, or the periodic
|
||||
# config-sync re-running update_settings with the combined config) must not
|
||||
# register a duplicate callback.
|
||||
router.update_settings(optional_pre_call_checks=["prompt_caching"])
|
||||
assert router.optional_pre_call_checks == ["prompt_caching"]
|
||||
prompt_caching_callbacks = [
|
||||
cb
|
||||
for cb in (router.optional_callbacks or [])
|
||||
if isinstance(cb, PromptCachingDeploymentCheck)
|
||||
]
|
||||
assert len(prompt_caching_callbacks) == 1
|
||||
|
||||
|
||||
def test_get_settings_includes_default_litellm_params_and_optional_pre_call_checks():
|
||||
"""
|
||||
Regression test: the Admin UI's Router Settings page reads its current
|
||||
values from `Router.get_settings()` (via `GET /get/config/callbacks`). If a
|
||||
setting isn't in `get_settings()`'s `vars_to_include`, it can never be
|
||||
displayed or edited from the UI even though the Router attribute exists.
|
||||
"""
|
||||
router = _make_router_for_settings_tests()
|
||||
router.update_settings(optional_pre_call_checks=["prompt_caching"])
|
||||
|
||||
settings = router.get_settings()
|
||||
|
||||
assert settings["optional_pre_call_checks"] == ["prompt_caching"]
|
||||
assert "default_litellm_params" in settings
|
||||
assert isinstance(settings["default_litellm_params"], dict)
|
||||
|
|
|
|||
|
|
@ -187,4 +187,38 @@ describe("RouterSettings", () => {
|
|||
});
|
||||
expect(NotificationsManager.success).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should round-trip default_litellm_params and optional_pre_call_checks as JSON on save", async () => {
|
||||
// Regression test: these two fields hold dicts/lists (e.g. cache_control_injection_points,
|
||||
// ["prompt_caching"]), not plain strings. Without listing them in the save handler's
|
||||
// jsonKeys set, they'd be persisted as raw stringified text instead of parsed JSON,
|
||||
// silently corrupting the setting the next time the router reads it.
|
||||
vi.mocked(getCallbacksCall).mockResolvedValue({
|
||||
router_settings: {
|
||||
...mockCallbacksResponse.router_settings,
|
||||
default_litellm_params: { cache_control_injection_points: [{ location: "message", role: "system" }] },
|
||||
optional_pre_call_checks: ["prompt_caching"],
|
||||
},
|
||||
});
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(<RouterSettings {...defaultProps} />);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByTestId("strategy-select")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
await user.click(screen.getByRole("button", { name: /save changes/i }));
|
||||
|
||||
await waitFor(() =>
|
||||
expect(setCallbacksCall).toHaveBeenCalledWith(
|
||||
"test-token",
|
||||
expect.objectContaining({
|
||||
router_settings: expect.objectContaining({
|
||||
default_litellm_params: { cache_control_injection_points: [{ location: "message", role: "system" }] },
|
||||
optional_pre_call_checks: ["prompt_caching"],
|
||||
}),
|
||||
}),
|
||||
),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ const RouterSettings: React.FC<RouterSettingsProps> = ({ accessToken, userRole,
|
|||
fieldsMap[field.field_name] = {
|
||||
ui_field_name: field.ui_field_name,
|
||||
field_description: field.field_description,
|
||||
field_type: field.field_type,
|
||||
options: field.options,
|
||||
link: field.link,
|
||||
};
|
||||
|
|
@ -87,7 +88,7 @@ const RouterSettings: React.FC<RouterSettingsProps> = ({ accessToken, userRole,
|
|||
const router_settings = formValue.routerSettings;
|
||||
|
||||
const numberKeys = new Set(["allowed_fails", "cooldown_time", "num_retries", "timeout", "retry_after"]);
|
||||
const jsonKeys = new Set(["model_group_alias"]);
|
||||
const jsonKeys = new Set(["model_group_alias", "default_litellm_params", "optional_pre_call_checks"]);
|
||||
// retry_policy and model_group_retry_policy are owned by the Model Retry Settings tab;
|
||||
// routing_groups is owned by the Routing Groups tab. This page must not read or write them.
|
||||
const tabOwnedKeys = new Set(["retry_policy", "model_group_retry_policy", "routing_groups"]);
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue