feat(router): expose default_litellm_params and optional_pre_call_checks in Admin UI

Router.update_settings() silently dropped default_litellm_params and
optional_pre_call_checks (not in the allow-list, and optional_pre_call_checks
was never even stored as a readable attribute), so the Admin UI's Router
Settings page could not display or persist either setting - e.g. enabling
cache_control_injection_points or prompt_caching pre-call routing required
editing config.yaml directly.

Router now persists optional_pre_call_checks and returns both fields from
get_settings(). update_settings() merges default_litellm_params instead of
replacing it (a full replace would drop the timeout/max_retries/metadata
defaults Router.__init__ sets), and diffs optional_pre_call_checks against
what's already applied before calling add_optional_pre_call_checks(), since
that method has no dedup guard for prompt_caching/enforce_model_rate_limits
and would otherwise register a duplicate callback on every re-save.
This commit is contained in:
Krrish Dholakia 2026-07-13 17:51:01 -07:00
parent 53aaabba5e
commit 498aa2997d
7 changed files with 209 additions and 16 deletions

View file

@ -683,6 +683,7 @@ class Router:
self.alerting_config: Optional[AlertingConfig] = alerting_config
self.optional_pre_call_checks: OptionalPreCallChecks = []
if optional_pre_call_checks is not None:
self.add_optional_pre_call_checks(optional_pre_call_checks)
@ -1504,6 +1505,8 @@ class Router:
if optional_pre_call_checks is None:
return
self.optional_pre_call_checks = list(dict.fromkeys([*self.optional_pre_call_checks, *optional_pre_call_checks]))
# ---------------------------------------------------------------------
# Unified deployment affinity (session stickiness)
# ---------------------------------------------------------------------
@ -9686,6 +9689,8 @@ class Router:
"retry_policy",
"model_group_alias",
"enable_weighted_failover",
"default_litellm_params",
"optional_pre_call_checks",
]
for var in vars_to_include:
@ -9722,6 +9727,8 @@ class Router:
"model_group_retry_policy",
"model_group_alias",
"enable_weighted_failover",
"default_litellm_params",
"optional_pre_call_checks",
]
_int_settings = [
@ -9749,6 +9756,12 @@ class Router:
value = RetryPolicy(**value)
if value is None or isinstance(value, RetryPolicy):
setattr(self, var, value)
elif var == "default_litellm_params":
self.default_litellm_params = {**self.default_litellm_params, **kwargs[var]}
elif var == "optional_pre_call_checks":
new_checks = [check for check in kwargs[var] if check not in self.optional_pre_call_checks]
if new_checks:
self.add_optional_pre_call_checks(new_checks)
else:
value = kwargs[var]
# only run routing strategy init if it has changed

View file

@ -224,9 +224,37 @@ ROUTER_SETTINGS_FIELDS: List[RouterSettingsField] = [
field_name="default_litellm_params",
field_type="Dictionary",
field_value=None,
field_description="Default parameters for Router.chat.completion.create",
field_description=(
"Default parameters for Router.chat.completion.create. E.g. set "
"cache_control_injection_points here to enable Anthropic/Bedrock "
"prompt caching for every model on this proxy."
),
field_default=None,
ui_field_name="Default LiteLLM Params",
link="https://docs.litellm.ai/docs/tutorials/claude_code_prompt_cache_routing",
),
RouterSettingsField(
field_name="optional_pre_call_checks",
field_type="List",
field_value=None,
field_description=(
"Extra checks the router runs before picking a deployment. Add "
"'prompt_caching' to route repeat requests back to the deployment "
"that cached the prompt."
),
field_default=[],
options=[
"prompt_caching",
"router_budget_limiting",
"responses_api_deployment_check",
"deployment_affinity",
"session_affinity",
"forward_client_headers_by_model_group",
"enforce_model_rate_limits",
"encrypted_content_affinity",
],
ui_field_name="Optional Pre-call Checks",
link="https://docs.litellm.ai/docs/tutorials/claude_code_prompt_cache_routing",
),
RouterSettingsField(
field_name="set_verbose",

View file

@ -26,6 +26,19 @@ class ConfigurableClientsideParamsCustomAuth(TypedDict):
CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = Optional[List[Union[str, ConfigurableClientsideParamsCustomAuth]]]
OptionalPreCallChecks = List[
Literal[
"prompt_caching",
"router_budget_limiting",
"responses_api_deployment_check",
"deployment_affinity",
"session_affinity",
"forward_client_headers_by_model_group",
"enforce_model_rate_limits",
"encrypted_content_affinity",
]
]
class ModelConfig(BaseModel):
model_name: str
@ -117,6 +130,8 @@ class UpdateRouterConfig(BaseModel):
fallbacks: Optional[List[dict]] = None
context_window_fallbacks: Optional[List[dict]] = None
model_group_alias: Optional[Dict[str, Union[str, Dict]]] = {}
default_litellm_params: Optional[Dict[str, Any]] = None
optional_pre_call_checks: Optional[OptionalPreCallChecks] = None
model_config = ConfigDict(protected_namespaces=())
@ -768,20 +783,6 @@ class GenericBudgetWindowDetails(BaseModel):
ttl_seconds: int
OptionalPreCallChecks = List[
Literal[
"prompt_caching",
"router_budget_limiting",
"responses_api_deployment_check",
"deployment_affinity",
"session_affinity",
"forward_client_headers_by_model_group",
"enforce_model_rate_limits",
"encrypted_content_affinity",
]
]
class LiteLLM_RouterFileObject(TypedDict, total=False):
"""
Tracking the litellm params hash, used for mapping the file id to the right model

View file

@ -78,6 +78,25 @@ class TestRouterSettingsEndpoints:
assert isinstance(routing_strategy_field["options"], list)
assert len(routing_strategy_field["options"]) > 0
@pytest.mark.asyncio
async def test_get_router_fields_includes_optional_pre_call_checks(self):
"""
Regression test: `optional_pre_call_checks` (e.g. "prompt_caching", used for
Claude Code prompt cache routing) must be exposed as a configurable field so
the Admin UI can render and save it, not just `default_litellm_params`.
"""
response = client.get(
"/router/fields", headers={"Authorization": "Bearer sk-1234"}
)
assert response.status_code == 200
fields = response.json()["fields"]
field = next(
(f for f in fields if f["field_name"] == "optional_pre_call_checks"), None
)
assert field is not None
assert "prompt_caching" in field["options"]
@pytest.mark.asyncio
async def test_get_router_settings_includes_routing_groups_from_live_router(
self, monkeypatch

View file

@ -5304,3 +5304,100 @@ class TestRouterRequestTimeoutPropagation:
)
== 60
)
def _make_router_for_settings_tests(**kwargs):
return litellm.Router(
model_list=[
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {
"model": "azure/gpt-4.1-mini",
"api_key": "fake-key",
"api_base": "https://fake.openai.azure.com",
},
}
],
**kwargs,
)
def test_update_settings_merges_default_litellm_params_without_dropping_existing_keys():
"""
Regression test: `update_settings(default_litellm_params=...)` must merge into
the existing dict, not replace it wholesale. A naive `setattr` replace would
silently drop keys the Router set at init (e.g. `timeout`, `max_retries`,
`metadata`) whenever an admin edits `default_litellm_params` from the UI to
add something like `cache_control_injection_points`.
"""
router = _make_router_for_settings_tests(timeout=42)
assert router.default_litellm_params.get("timeout") == 42
router.update_settings(
default_litellm_params={
"cache_control_injection_points": [
{"location": "message", "role": "system"}
]
}
)
assert router.default_litellm_params["timeout"] == 42
assert router.default_litellm_params["cache_control_injection_points"] == [
{"location": "message", "role": "system"}
]
def test_update_settings_optional_pre_call_checks_is_idempotent():
"""
Regression test: `add_optional_pre_call_checks` has no built-in guard against
registering the same check twice (unlike `router_budget_limiting`, which
checks for an existing budget limiter). `_add_router_settings_from_db_config`
re-applies the full `optional_pre_call_checks` list on every config sync, so
without diffing against already-applied checks in `update_settings`, saving
the same setting twice from the UI would register a second
`PromptCachingDeploymentCheck` callback on every save.
"""
from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import (
PromptCachingDeploymentCheck,
)
router = _make_router_for_settings_tests()
assert router.optional_pre_call_checks == []
router.update_settings(optional_pre_call_checks=["prompt_caching"])
assert router.optional_pre_call_checks == ["prompt_caching"]
prompt_caching_callbacks = [
cb
for cb in (router.optional_callbacks or [])
if isinstance(cb, PromptCachingDeploymentCheck)
]
assert len(prompt_caching_callbacks) == 1
# Re-applying the same setting (e.g. a second Save click, or the periodic
# config-sync re-running update_settings with the combined config) must not
# register a duplicate callback.
router.update_settings(optional_pre_call_checks=["prompt_caching"])
assert router.optional_pre_call_checks == ["prompt_caching"]
prompt_caching_callbacks = [
cb
for cb in (router.optional_callbacks or [])
if isinstance(cb, PromptCachingDeploymentCheck)
]
assert len(prompt_caching_callbacks) == 1
def test_get_settings_includes_default_litellm_params_and_optional_pre_call_checks():
"""
Regression test: the Admin UI's Router Settings page reads its current
values from `Router.get_settings()` (via `GET /get/config/callbacks`). If a
setting isn't in `get_settings()`'s `vars_to_include`, it can never be
displayed or edited from the UI even though the Router attribute exists.
"""
router = _make_router_for_settings_tests()
router.update_settings(optional_pre_call_checks=["prompt_caching"])
settings = router.get_settings()
assert settings["optional_pre_call_checks"] == ["prompt_caching"]
assert "default_litellm_params" in settings
assert isinstance(settings["default_litellm_params"], dict)

View file

@ -187,4 +187,38 @@ describe("RouterSettings", () => {
});
expect(NotificationsManager.success).not.toHaveBeenCalled();
});
it("should round-trip default_litellm_params and optional_pre_call_checks as JSON on save", async () => {
// Regression test: these two fields hold dicts/lists (e.g. cache_control_injection_points,
// ["prompt_caching"]), not plain strings. Without listing them in the save handler's
// jsonKeys set, they'd be persisted as raw stringified text instead of parsed JSON,
// silently corrupting the setting the next time the router reads it.
vi.mocked(getCallbacksCall).mockResolvedValue({
router_settings: {
...mockCallbacksResponse.router_settings,
default_litellm_params: { cache_control_injection_points: [{ location: "message", role: "system" }] },
optional_pre_call_checks: ["prompt_caching"],
},
});
const user = userEvent.setup();
renderWithProviders(<RouterSettings {...defaultProps} />);
await waitFor(() => {
expect(screen.getByTestId("strategy-select")).toBeInTheDocument();
});
await user.click(screen.getByRole("button", { name: /save changes/i }));
await waitFor(() =>
expect(setCallbacksCall).toHaveBeenCalledWith(
"test-token",
expect.objectContaining({
router_settings: expect.objectContaining({
default_litellm_params: { cache_control_injection_points: [{ location: "message", role: "system" }] },
optional_pre_call_checks: ["prompt_caching"],
}),
}),
),
);
});
});

View file

@ -50,6 +50,7 @@ const RouterSettings: React.FC<RouterSettingsProps> = ({ accessToken, userRole,
fieldsMap[field.field_name] = {
ui_field_name: field.ui_field_name,
field_description: field.field_description,
field_type: field.field_type,
options: field.options,
link: field.link,
};
@ -87,7 +88,7 @@ const RouterSettings: React.FC<RouterSettingsProps> = ({ accessToken, userRole,
const router_settings = formValue.routerSettings;
const numberKeys = new Set(["allowed_fails", "cooldown_time", "num_retries", "timeout", "retry_after"]);
const jsonKeys = new Set(["model_group_alias"]);
const jsonKeys = new Set(["model_group_alias", "default_litellm_params", "optional_pre_call_checks"]);
// retry_policy and model_group_retry_policy are owned by the Model Retry Settings tab;
// routing_groups is owned by the Routing Groups tab. This page must not read or write them.
const tabOwnedKeys = new Set(["retry_policy", "model_group_retry_policy", "routing_groups"]);