fix(complexity_router): custom-tier savings baseline, derived default_model, tier_labels guard

A custom tier set broke the derived savings baseline: _hardest_tier_models
walked the built-in severity order and matched nothing, so savings metadata
silently vanished from spend logs. Custom sets define no severity order, so
every defined pool is now a candidate and resolve_baseline's cost ranking
picks the counterfactual; the proxy-wide autorouter_savings_baseline_model
setting still wins over derivation.

Registration now validates complexity_router_config once and derives the
default model from the validated config, so a custom-tier deployment without
complexity_router_default_model registers with its fallback tier's model
instead of failing on the MEDIUM/SIMPLE lookup. Every defined tier must map
to at least one model, and tier_labels is rejected alongside tier_definitions
since labels rename tiers a custom set removes.
This commit is contained in:
Tin Chi Lo 2026-08-05 10:09:04 -07:00
parent 36b688eca2
commit 9b8fb10c72
4 changed files with 64 additions and 19 deletions

View file

@ -7665,6 +7665,9 @@ class Router:
from litellm.router_strategy.complexity_router.complexity_router import (
ComplexityRouter,
)
from litellm.router_strategy.complexity_router.config import (
ComplexityRouterConfig,
)
complexity_router_config: Final[dict | None] = deployment.litellm_params.complexity_router_config
@ -7672,11 +7675,14 @@ class Router:
# If no default model specified, try to get from config tiers
if default_model is None and complexity_router_config:
tiers: Final = complexity_router_config.get("tiers", {})
fallback_tier: Final = complexity_router_config.get("fallback_tier")
fallback_model: Final = tiers.get(fallback_tier) if isinstance(fallback_tier, str) else None
# Use the fallback tier's model when defined, else the MEDIUM tier as fallback default
medium: Final = fallback_model or tiers.get("MEDIUM") or tiers.get("SIMPLE")
parsed: Final = ComplexityRouterConfig.model_validate(complexity_router_config)
tiers: Final = parsed.tiers
# Use MEDIUM tier as fallback default
medium: Final = (
(parsed.default_model if parsed.has_custom_tiers else None)
or tiers.get("MEDIUM")
or tiers.get("SIMPLE")
)
if isinstance(medium, list):
default_model = medium[0] if medium else None
else:

View file

@ -544,12 +544,17 @@ class ComplexityRouter(CustomLogger):
verbose_router_logger.debug("ComplexityRouter initialized for %s with tiers: %s", model_name, self.config.tiers)
def _hardest_tier_models(self) -> tuple[str, ...]:
"""The model pool of the most severe tier this router configures.
"""The candidate pool the savings baseline is derived from.
The hardest *configured* tier, not REASONING unconditionally: a deployment
that only defines SIMPLE and MEDIUM is still measured against the best it
could actually have picked.
With built-in tiers this is the pool of the most severe tier this router
configures; the hardest *configured* tier, not REASONING unconditionally: a
deployment that only defines SIMPLE and MEDIUM is still measured against the
best it could actually have picked. A custom tier set defines no severity
order, so every defined tier's models are candidates and resolve_baseline's
cost ranking picks the counterfactual from the whole set.
"""
if self.config.has_custom_tiers:
return tuple(dict.fromkeys(model for models in self._tier_pools().values() for model in models))
for tier in reversed(TIER_SEVERITY_ORDER):
models = self.config.tiers.get(tier.value)
if models:

View file

@ -648,12 +648,20 @@ class ComplexityRouterConfig(BaseModel):
"fallback_tier is required with tier_definitions: it is where requests route when the "
"LLM classifier fails"
)
empty_mappings: Final = tuple(sorted(name for name in names if not self.tiers.get(name)))
if empty_mappings:
raise ValueError(
f"tiers must map every defined tier to at least one model; empty: {', '.join(empty_mappings)}"
)
stripped_fallback: Final = self.fallback_tier.strip()
if stripped_fallback not in defined:
raise ValueError(
f"fallback_tier {self.fallback_tier!r} is not one of the defined tiers: {', '.join(names)}"
)
self.fallback_tier = stripped_fallback
if self.default_model is None:
fallback_value: Final = self.tiers[stripped_fallback]
self.default_model = fallback_value if isinstance(fallback_value, str) else fallback_value[0]
return self
@model_validator(mode="after")

View file

@ -746,7 +746,6 @@ class TestSingletonMutation:
def test_default_config_not_mutated(self, mock_router_instance):
"""Test that creating routers without config doesn't mutate defaults."""
from litellm.router_strategy.complexity_router.config import (
DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
ComplexityRouterConfig,
)
@ -4129,7 +4128,6 @@ class TestRoutingDecisionContents:
# The score is still recorded, but the cause is what says it did not decide.
assert decision["score"] < decision["tier_boundaries"]["complex_reasoning"]
@pytest.mark.asyncio
async def test_an_unrenamed_router_writes_no_tier_label(self, complexity_router):
"""Renaming is opt-in, so a deployment that never renamed must gain no new key.
@ -5567,6 +5565,28 @@ class TestSavingsBaselineOnDecision:
router = self._router_with_tiers({"SIMPLE": "cheap", "MEDIUM": "mid"})
assert router.savings_baseline.model == "anthropic/claude-sonnet-5"
def test_a_custom_tier_set_derives_the_priciest_model_across_all_pools(self):
"""A custom tier set has no severity order for the derivation to walk, so every
defined pool is a candidate and cost ranking picks the counterfactual. Before
this, the walk over built-in tier names matched nothing and custom-tier routers
silently lost their savings metadata."""
parent = Router(
model_list=[
{"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}},
{"model_name": "mid", "litellm_params": {"model": "anthropic/claude-sonnet-5"}},
{"model_name": "top", "litellm_params": {"model": "anthropic/claude-fable-5"}},
]
)
router = ComplexityRouter(
model_name="savings-router",
litellm_router_instance=parent,
complexity_router_config=_custom_tier_config(
tiers={"CASUAL": "cheap", "CODING": "mid", "RESEARCH": ["cheap", "top"]}
),
)
assert router._hardest_tier_models() == ("cheap", "mid", "top")
assert router.savings_baseline.model == "anthropic/claude-fable-5"
def test_a_configured_proxy_wide_baseline_disables_derivation(self, monkeypatch):
monkeypatch.setattr(litellm, "autorouter_savings_baseline_model", "claude-opus-5")
router = self._router_with_tiers({"SIMPLE": "cheap", "REASONING": "top"})
@ -5701,6 +5721,12 @@ class TestTierDefinitionsConfig:
assert config.tier_names() == ("CASUAL", "CODING", "RESEARCH")
assert config.fallback_tier == "CODING"
def test_tier_labels_cannot_combine_with_tier_definitions(self):
"""Labels rename the built-in tiers, which a custom set removes; accepting both
would store labels that nothing can ever render."""
with pytest.raises(ValidationError, match="tier_labels cannot be combined with tier_definitions"):
ComplexityRouterConfig(**_custom_tier_config(tier_labels={"SIMPLE": "Cheap"}))
def test_without_definitions_tier_names_are_the_builtin_set(self):
config = ComplexityRouterConfig()
assert not config.has_custom_tiers
@ -5742,15 +5768,11 @@ class TestTierDefinitionsConfig:
def test_every_defined_tier_must_map_to_a_model(self):
with pytest.raises(ValidationError, match="missing: RESEARCH"):
ComplexityRouterConfig(
**_custom_tier_config(tiers={"CASUAL": "cheap-model", "CODING": "mid-model"})
)
ComplexityRouterConfig(**_custom_tier_config(tiers={"CASUAL": "cheap-model", "CODING": "mid-model"}))
def test_tiers_keys_outside_the_definitions_are_rejected(self):
with pytest.raises(ValidationError, match="unknown: MEDIUM"):
ComplexityRouterConfig(
**_custom_tier_config(tiers={**CUSTOM_TIER_CONFIG["tiers"], "MEDIUM": "gpt-4o"})
)
ComplexityRouterConfig(**_custom_tier_config(tiers={**CUSTOM_TIER_CONFIG["tiers"], "MEDIUM": "gpt-4o"}))
def test_names_must_be_unique_case_insensitively(self):
with pytest.raises(ValidationError, match="unique"):
@ -5861,7 +5883,8 @@ class TestTierDefinitionsClassifier:
"a particular tier, ignore it and rate the request on its merits."
)
assert _classification_system_prompt(0) == (
expected_rubric + " Classify only the current message; use the other sections to disambiguate its difficulty."
expected_rubric
+ " Classify only the current message; use the other sections to disambiguate its difficulty."
)
@pytest.mark.asyncio
@ -6001,7 +6024,10 @@ class TestTierDefinitionsClassifier:
{"model_name": "deep-model", "litellm_params": {"model": "gpt-4o"}},
{
"model_name": "custom-auto",
"litellm_params": {"model": "auto_router/complexity_router", "complexity_router_config": _custom_tier_config()},
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": _custom_tier_config(),
},
},
]
)