diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 94a59828451..a7a24e10bdd 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -534,6 +534,9 @@ async def get_autorouter_presets( "/public/autorouter_presets", tags=["public", "auto router"], # mutable-ok: FastAPI route tags take a list response_model=dict[str, AutoRouterPresetRecord], + # An optional tier a preset does not set must not reach the dashboard as a null pool, which the + # template picker would render as an empty tier row the operator never asked for. + response_model_exclude_none=True, ) async def get_public_autorouter_presets() -> Mapping[str, AutoRouterPresetRecord]: """ diff --git a/litellm/router_strategy/complexity_router/classification_rubrics.py b/litellm/router_strategy/complexity_router/classification_rubrics.py index 9f168eabbc4..1dae2902fad 100644 --- a/litellm/router_strategy/complexity_router/classification_rubrics.py +++ b/litellm/router_strategy/complexity_router/classification_rubrics.py @@ -108,6 +108,11 @@ _CALIBRATION_EXAMPLES: Final[Mapping[ClassificationRubric, str]] = MappingProxyT BUSINESS_TIER_CRITERIA: Final[Mapping[ComplexityTier, str]] = MappingProxyType( { + ComplexityTier.NON_REASONING: ( + "operational requests whose whole job is to pass information along or put it in a requested " + "shape: relaying or reformatting tool or system output, acknowledging a completed action, or " + "extracting a stated field. Use it only when no judgment about the content is asked for." + ), ComplexityTier.SIMPLE: ( "greetings, chitchat, or lookups of a fact, policy, price, or date with a short known answer. " "Never for analysis, strategy, or non-trivial work, even if the request is only one sentence." diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index c8644f52c57..fd0d53e0902 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -100,7 +100,12 @@ else: class TierClassification(BaseModel): - """Structured response schema for the LLM-based complexity classifier.""" + """Structured response schema for the LLM-based complexity classifier. + + The four-tier ladder, which is what a router that did not opt into NON_REASONING sends. The + enum actually put on the wire is rebuilt per router from `classifier_wire_labels`, so a + five-tier or renamed ladder widens it there rather than here. + """ tier: Literal["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"] @@ -116,8 +121,20 @@ def _tier_name(tier: ComplexityTier | str) -> str: return tier.value if isinstance(tier, ComplexityTier) else tier +def _built_in_tier_or_none(tier_name: str) -> ComplexityTier | None: + """The built-in tier a `tiers` key names, or None when the key is an operator-defined name.""" + return ComplexityTier.__members__.get(tier_name) + + _CLASSIFICATION_TIER_CRITERIA: Final[Mapping[ComplexityTier, str]] = MappingProxyType( { + ComplexityTier.NON_REASONING: ( + "operational requests whose whole job is to pass information along or put it in a " + "requested shape: relaying or reformatting tool output, acknowledging a completed action, " + "or extracting a stated value. Use it only when no judgment about the content is asked for; " + "the moment the request is to summarize, compare, explain, debug, or decide, it belongs " + "in a higher tier however short it is." + ), ComplexityTier.SIMPLE: ( "greetings, chitchat, or factual lookups with a short known answer. Do not use this tier for " "unsolved problems, proofs, deep theory, multi-step analysis, or non-trivial code, even if the " @@ -1228,7 +1245,7 @@ class ComplexityRouter(CustomLogger): """ if self.config.has_custom_tiers: return tuple(dict.fromkeys(model for models in self._tier_pools().values() for model in models)) - for tier in reversed(TIER_SEVERITY_ORDER): + for tier in reversed(self.config.active_tier_severity_order()): models = self.config.tiers.get(tier.value) if models: return tuple(models) if isinstance(models, list) else (models,) @@ -1863,7 +1880,11 @@ class ComplexityRouter(CustomLogger): default_model: Final = self.config.default_model pools: Final = self._tier_pools() tier: Final = next( - (candidate for candidate in TIER_SEVERITY_ORDER if default_model in pools.get(candidate.value, ())), + ( + candidate + for candidate in self.config.active_tier_severity_order() + if default_model in pools.get(candidate.value, ()) + ), ComplexityTier.MEDIUM, ) return ClassificationOutcome( @@ -2248,7 +2269,8 @@ class ComplexityRouter(CustomLogger): return self._fitting_tier_fallback(classified_tier, fit_filter) request_type: Final = classify_prompt(user_message) - classified_idx: Final = TIER_SEVERITY_ORDER.index(classified_tier) + severity_order: Final = self.config.active_tier_severity_order() + classified_idx: Final = severity_order.index(classified_tier) pools: Final = self._tier_pools() classified_candidates: Final = _allowed(tuple(pools.get(_tier_name(classified_tier), ())), fit_filter) cold_start_candidates: Final = tuple( @@ -2313,7 +2335,7 @@ class ComplexityRouter(CustomLogger): else: model_tiers = self._model_tiers.get(model, (classified_tier,)) distance = min( - abs(TIER_SEVERITY_ORDER.index(model_tier) - classified_idx) for model_tier in model_tiers + abs(severity_order.index(model_tier) - classified_idx) for model_tier in model_tiers ) score = quality_weight * quality_sample + cost_weight * cost_score - penalty_weight * distance candidate_scores.append( @@ -2610,10 +2632,15 @@ class ComplexityRouter(CustomLogger): def _tier_for_model(self, model: str) -> ComplexityTier | None: """Return the most-severe configured tier whose pool contains this model.""" pools: Final = self._tier_pools() - matched: Final = tuple(ComplexityTier(tier_name) for tier_name, models in pools.items() if model in models) + order: Final = self.config.active_tier_severity_order() + matched: Final = tuple( + tier + for tier_name, models in pools.items() + if model in models and (tier := _built_in_tier_or_none(tier_name)) is not None and tier in order + ) if not matched: return None - return max(matched, key=TIER_SEVERITY_ORDER.index) + return max(matched, key=order.index) def _escalate_tier(self, tier: ComplexityTier | str) -> ComplexityTier | str: """Bump a tier one step up to the next-higher configured tier. @@ -2628,9 +2655,10 @@ class ComplexityRouter(CustomLogger): if self.config.has_custom_tiers: return tier configured: Final = frozenset(self.config.tiers) - current_index: Final = TIER_SEVERITY_ORDER.index(tier) + order: Final = self.config.active_tier_severity_order() + current_index: Final = order.index(tier) higher_tiers: Final = tuple( - candidate for candidate in TIER_SEVERITY_ORDER[current_index + 1 :] if candidate.value in configured + candidate for candidate in order[current_index + 1 :] if candidate.value in configured ) return higher_tiers[0] if higher_tiers else tier diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 3ec9f9b5394..bdfa0c9c389 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -29,6 +29,7 @@ from .tier_predictor import TrainedTierArtifact class ComplexityTier(str, Enum): """Complexity tiers for routing decisions.""" + NON_REASONING = "NON_REASONING" SIMPLE = "SIMPLE" MEDIUM = "MEDIUM" COMPLEX = "COMPLEX" @@ -55,6 +56,11 @@ DEFAULT_CLASSIFICATION_RUBRIC: Final[ClassificationRubric] = ClassificationRubri LLM_CLASSIFIER_TYPES: Final[frozenset[str]] = frozenset({"llm", "heuristic_first", "hybrid"}) +# The ladder as it has always shipped. NON_REASONING is absent because it is opt-in: an existing +# router must not gain a rubric bullet, a wire label, or a rung it never configured, and the +# heuristic_v2 artifact is trained on exactly these four classes. Read the active ladder off the +# config (`tier_names`, `active_tier_severity_order`) rather than this constant wherever the +# operator's `enable_non_reasoning_tier` can reach. TIER_SEVERITY_ORDER: Final[tuple[ComplexityTier, ...]] = ( ComplexityTier.SIMPLE, ComplexityTier.MEDIUM, @@ -62,6 +68,16 @@ TIER_SEVERITY_ORDER: Final[tuple[ComplexityTier, ...]] = ( ComplexityTier.REASONING, ) +NON_REASONING_TIER_SEVERITY_ORDER: Final[tuple[ComplexityTier, ...]] = ( + ComplexityTier.NON_REASONING, + *TIER_SEVERITY_ORDER, +) + + +def tier_severity_order(non_reasoning_enabled: bool) -> tuple[ComplexityTier, ...]: + """The built-in ladder for one router, tier 0 included only when it opted in.""" + return NON_REASONING_TIER_SEVERITY_ORDER if non_reasoning_enabled else TIER_SEVERITY_ORDER + DEFAULT_TIER_DISTANCE_PENALTY: Final[float] = 0.5 DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE: Final[int] = 3 @@ -142,6 +158,9 @@ def normalize_classification_examples(value: str | None) -> str | None: return _normalize_operator_section(value, "classification_examples", MAX_CLASSIFICATION_EXAMPLES_CHARS) +_BUILT_IN_TIER_NAMES: Final[str] = ", ".join(ComplexityTier.__members__) + + class TierDefinition(BaseModel): """An operator-defined tier: the name the LLM classifier must return and its rubric description.""" @@ -152,7 +171,7 @@ class TierDefinition(BaseModel): default=None, description=( "What belongs in this tier; rendered as this tier's bullet in the classifier rubric. " - "Required unless the name is a built-in tier (SIMPLE/MEDIUM/COMPLEX/REASONING), which " + f"Required unless the name is a built-in tier ({_BUILT_IN_TIER_NAMES}), which " "inherits the built-in criteria when omitted" ), ) @@ -174,7 +193,7 @@ class TierDefinition(BaseModel): if description is None and name.upper() not in ComplexityTier.__members__: raise ValueError( f"tier_definitions entry {name!r} must have a description: only the built-in tiers " - "(SIMPLE, MEDIUM, COMPLEX, REASONING) carry one the rubric can inherit" + f"({_BUILT_IN_TIER_NAMES}) carry one the rubric can inherit" ) rendered_on_one_line: Final = (name, description or "") if any("\n" in part or "\r" in part for part in rendered_on_one_line): @@ -703,6 +722,20 @@ class ComplexityRouterConfig(BaseModel): default_factory=dict, ) + enable_non_reasoning_tier: bool = Field( + default=False, + description=( + "Add NON_REASONING as a fifth built-in tier below SIMPLE, for operational agent traffic " + "that relays or reformats information rather than reasoning about it. Off by default: " + "turning it on adds a rung to this router's ladder, a bullet to the LLM classifier's " + "rubric, and a value the classifier may return, all of which move tier decisions and " + "spend on an already-deployed router. Requires an LLM classifier or a custom classifier " + "plugin, since the heuristic scorers cannot produce the tier, and a model in `tiers` " + "under the NON_REASONING key. Escalation still walks up from it, and it is never the " + "savings baseline or a `heuristic_v2` prediction." + ), + ) + tier_definitions: tuple[TierDefinition, ...] | None = Field( default=None, description=( @@ -1491,11 +1524,16 @@ class ComplexityRouterConfig(BaseModel): which still makes it a dependency on every one of those requests.""" return self.classifier_type in LLM_CLASSIFIER_TYPES + def active_tier_severity_order(self) -> tuple[ComplexityTier, ...]: + """This router's built-in ladder, ascending. Meaningless for a custom tier set, whose + severity order is tier_definitions list order over names that are not enum members.""" + return tier_severity_order(self.enable_non_reasoning_tier) + def tier_names(self) -> tuple[str, ...]: """The active tier names: the defined names, or the built-in set in severity order.""" if self.tier_definitions is not None: return tuple(definition.name for definition in self.tier_definitions) - return tuple(tier.value for tier in TIER_SEVERITY_ORDER) + return tuple(tier.value for tier in self.active_tier_severity_order()) def classifier_wire_labels(self) -> tuple[str, ...]: """The tier names the classifier is told to emit: defined names, or the display labels.""" @@ -1587,6 +1625,41 @@ class ComplexityRouterConfig(BaseModel): if present ) + @model_validator(mode="after") + def _validate_non_reasoning_tier(self) -> "ComplexityRouterConfig": + """Gate the opt-in fifth tier on the two things that make it reachable and routable. + + The heuristic scorers cannot emit it (the v1 score ladder has no rung below simple_medium + and the v2 artifact is trained on four classes), so a router whose classifier can never + return the tier would pay for a rubric bullet and a configured pool that no request reaches. + """ + non_reasoning_key: Final = ComplexityTier.NON_REASONING.value + if not self.enable_non_reasoning_tier: + if not self.has_custom_tiers and non_reasoning_key in self.tiers: + raise ValueError( + f"tiers names {non_reasoning_key} but enable_non_reasoning_tier is False, so no request " + "can route there; set enable_non_reasoning_tier: true or drop the tier" + ) + return self + if self.has_custom_tiers: + raise ValueError( + "enable_non_reasoning_tier cannot be combined with tier_definitions: a custom tier set " + f"replaces the built-in ladder, so name a tier {non_reasoning_key} in tier_definitions instead" + ) + if self.classifier_type not in ("llm", "custom"): + raise ValueError( + f"enable_non_reasoning_tier requires classifier_type 'llm' or 'custom', got " + f"{self.classifier_type!r}: the heuristic scorers only produce the four tiers from SIMPLE up, " + f"so nothing would ever classify as {non_reasoning_key}" + ) + if not self.tiers.get(non_reasoning_key): + raise ValueError( + f"enable_non_reasoning_tier requires tiers to map {non_reasoning_key} to at least one model: " + "the tier exists to send operational traffic somewhere cheaper, and an unconfigured tier " + "would fall through to the default model" + ) + return self + @model_validator(mode="after") def _validate_tier_definitions(self) -> "ComplexityRouterConfig": if self.tier_definitions is None: @@ -1609,7 +1682,7 @@ class ComplexityRouterConfig(BaseModel): if self.classifier_type in ("heuristic", "heuristic_v2", "heuristic_first", "hybrid"): raise ValueError( "tier_definitions requires classifier_type 'llm' or 'custom': the heuristic scorer only " - "produces the four built-in tiers, as does heuristic_v2" + "produces the built-in tiers from SIMPLE up, as does heuristic_v2" ) conflicts: Final = self._tier_definition_conflicts() if conflicts: @@ -1762,16 +1835,18 @@ class ComplexityRouterConfig(BaseModel): return self.tier_labels.get(tier, "").strip() or tier.value def labeled_tiers(self) -> tuple[tuple[ComplexityTier, str], ...]: - """Every tier paired with its display name, in ascending severity order.""" - return tuple((tier, self.tier_label(tier)) for tier in TIER_SEVERITY_ORDER) + """Every active tier paired with its display name, in ascending severity order.""" + return tuple((tier, self.tier_label(tier)) for tier in self.active_tier_severity_order()) def tier_for_label(self, label: str) -> ComplexityTier | None: - """Resolve a display name back to its tier, case-insensitively, then canonical names.""" + """Resolve a display name back to its active tier, case-insensitively, then canonical + names. A tier this router did not opt into resolves to None, so a classifier naming + NON_REASONING on a four-tier router is an unparseable reply rather than a fifth rung.""" folded: Final = label.strip().casefold() labeled: Final = self.labeled_tiers() return next( (tier for tier, tier_label in labeled if tier_label.casefold() == folded), - next((tier for tier in TIER_SEVERITY_ORDER if tier.value.casefold() == folded), None), + next((tier for tier, _ in labeled if tier.value.casefold() == folded), None), ) diff --git a/litellm/types/proxy/public_endpoints/public_endpoints.py b/litellm/types/proxy/public_endpoints/public_endpoints.py index fa73926305b..fb5a37c45fc 100644 --- a/litellm/types/proxy/public_endpoints/public_endpoints.py +++ b/litellm/types/proxy/public_endpoints/public_endpoints.py @@ -74,10 +74,14 @@ class SupportedEndpointsResponse(BaseModel): class AutoRouterPresetTiers(BaseModel): - """Exactly the four built-in tiers the dashboard's preset prefill can apply. + """The built-in tiers the dashboard's preset prefill can apply. extra="forbid" on purpose: a tier name this dashboard cannot apply would grey out or crash the - picker, so such a catalog is rejected wholesale and the bundled one serves instead. + picker, so such a catalog is rejected wholesale and the bundled one serves instead. NON_REASONING + is optional rather than required so this proxy accepts both a catalog that ships the opt-in fifth + tier and the four-tier catalogs that predate it. It stays None when unset rather than defaulting + to an empty pool, so a four-tier preset serves the tier set it was published with instead of + growing a key the dashboard would render as an empty fifth tier row. """ model_config = ConfigDict(extra="forbid") @@ -86,6 +90,7 @@ class AutoRouterPresetTiers(BaseModel): MEDIUM: Sequence[str] COMPLEX: Sequence[str] REASONING: Sequence[str] + NON_REASONING: Sequence[str] | None = None class AutoRouterPresetConfig(BaseModel): diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 5b1d8562abd..ce9cd5d3b7d 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -44,6 +44,7 @@ from litellm.router_strategy.complexity_router.config import ( DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE, DEFAULT_COMPLEXITY_CONFIG, DEFAULT_TECHNICAL_KEYWORDS, + TIER_SEVERITY_ORDER, ClassificationRubric, ClassifierLLMConfig, ComplexityRouterConfig, @@ -11044,7 +11045,7 @@ async def test_tier_model_params_reach_the_hook_response_and_override_client_val async def test_tier_params_mask_credentials_in_routing_decision(route, mock_router_instance): params = {"reasoning_effort": "xhigh", "api_key": "secret-tier-key"} config = { - "tiers": {tier.value: {"model_name": "opus", "litellm_params": params} for tier in ComplexityTier}, + "tiers": {tier.value: {"model_name": "opus", "litellm_params": params} for tier in TIER_SEVERITY_ORDER}, "keyword_tier_rules": [{"keywords": ["reason carefully"], "tier": "REASONING"}] if route == "keyword" else None, "session_affinity": route == "session", } @@ -13212,3 +13213,197 @@ class TestClassifierVision: def test_max_images_must_be_positive(self): with pytest.raises(ValidationError): ClassifierLLMConfig(model="clf", vision={"enabled": True, "max_images": 0}) + + +NON_REASONING_TIERS: Final = { + "NON_REASONING": "gpt-4o-mini", + "SIMPLE": "gpt-4o-mini", + "MEDIUM": "gpt-4o", + "COMPLEX": "claude-sonnet-4-20250514", + "REASONING": "o1-preview", +} + + +class TestNonReasoningTier: + """The opt-in fifth built-in tier below SIMPLE. + + Two properties carry the feature. A router that did not opt in must be byte-identical to one + built before the tier existed, because the tier set feeds the classifier rubric, the wire enum, + and the savings baseline, all of which move live routing decisions and spend. A router that did + opt in must be able to actually reach the tier and escalate off it. + """ + + @staticmethod + def _router(mock_router_instance, **overrides) -> ComplexityRouter: + config: Final = { + "tiers": dict(NON_REASONING_TIERS), + "enable_non_reasoning_tier": True, + "classifier_type": "llm", + "classifier_llm_config": {"model": "haiku-classifier"}, + **overrides, + } + return ComplexityRouter( + model_name="test-non-reasoning-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=config, + ) + + def test_ladder_gains_a_rung_below_simple_only_when_enabled(self): + """Tier 0 sits at the bottom. Anywhere else and escalation, the savings baseline, and + heuristic_first's 'highest tier' check would all read a different ladder.""" + enabled: Final = ComplexityRouterConfig( + tiers=dict(NON_REASONING_TIERS), + enable_non_reasoning_tier=True, + classifier_type="llm", + classifier_llm_config={"model": "clf"}, + ) + assert enabled.tier_names() == ("NON_REASONING", "SIMPLE", "MEDIUM", "COMPLEX", "REASONING") + assert ComplexityRouterConfig().tier_names() == ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING") + + def test_default_router_is_unchanged_by_the_tier_existing(self): + """The regression that matters for every already-deployed router: the enum grew a member, + and nothing a four-tier router sends or resolves may change because of it.""" + default: Final = ComplexityRouterConfig() + assert default.enable_non_reasoning_tier is False + assert "NON_REASONING" not in DEFAULT_COMPLEXITY_CONFIG.tiers + assert default.classifier_wire_labels() == ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING") + assert default.labeled_tiers() == TIER_SEVERITY_ORDER_LABELED + assert default.resolve_classified_tier("NON_REASONING") is None + + @pytest.mark.parametrize("preset", tuple(ClassificationRubric)) + def test_rubric_gains_the_bullet_only_when_enabled(self, preset): + """Every preset renders one bullet per active tier, so an unset toggle must leave all four + shipped rubrics byte-identical while an enabled one must actually describe the new tier.""" + enabled: Final = ComplexityRouterConfig( + tiers=dict(NON_REASONING_TIERS), + enable_non_reasoning_tier=True, + classifier_type="llm", + classifier_llm_config={"model": "clf"}, + ) + on: Final = classification_system_prompt(3, None, enabled.labeled_tiers(), preset) + off: Final = classification_system_prompt(3, None, ComplexityRouterConfig().labeled_tiers(), preset) + assert "- NON_REASONING:" in on + assert "- NON_REASONING" not in off + + def test_enabled_router_puts_the_tier_on_the_classifier_wire(self, mock_router_instance): + """The response schema's enum is what the classifier may return; without the new label the + tier would be unreachable no matter what the rubric says.""" + router: Final = self._router(mock_router_instance) + enum: Final = router._classifier_response_format["json_schema"]["schema"]["properties"]["tier"]["enum"] + assert enum == ["NON_REASONING", "SIMPLE", "MEDIUM", "COMPLEX", "REASONING"] + + @pytest.mark.asyncio + async def test_classifier_verdict_routes_to_the_tier_model(self, mock_router_instance): + """End to end on the LLM path: the classifier names the tier and the request lands on that + tier's model with the decision recording it.""" + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "NON_REASONING"}')) + router: Final = self._router(mock_router_instance, tiers={**NON_REASONING_TIERS, "NON_REASONING": "cheap-relay"}) + response = await router.async_pre_routing_hook( + model="test-non-reasoning-router", + request_kwargs={}, + messages=[{"role": "user", "content": "here is the file, pass it along"}], + ) + assert response.model == "cheap-relay" + assert response.routing_decision["tier"] == "NON_REASONING" + assert response.routing_decision["cause"] == "llm_classifier" + + @pytest.mark.asyncio + async def test_a_four_tier_router_ignores_a_non_reasoning_verdict(self, llm_complexity_router, mock_router_instance): + """A classifier that names the tier at a router which never opted in must be an unparseable + reply that falls back, not a silent route to a tier the operator did not configure.""" + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "NON_REASONING"}')) + outcome = await llm_complexity_router.aclassify("relay this") + assert outcome.tier != ComplexityTier.NON_REASONING + assert outcome.cause != "llm_classifier" + + def test_escalation_walks_up_off_the_tier(self, mock_router_instance): + """Escalation is a built-in-ladder feature and the issue asks for it from the new tier.""" + router: Final = self._router(mock_router_instance) + assert router._escalate_tier(ComplexityTier.NON_REASONING) == ComplexityTier.SIMPLE + assert router._escalate_tier(ComplexityTier.REASONING) == ComplexityTier.REASONING + + def test_escalation_skips_the_tier_when_unconfigured(self, mock_router_instance): + """SIMPLE must still escalate to MEDIUM rather than to the cheaper new rung, or escalation + would route below the model the caller would otherwise have received.""" + router: Final = self._router( + mock_router_instance, + tiers={"NON_REASONING": "cheap-relay", "SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o"}, + ) + assert router._escalate_tier(ComplexityTier.SIMPLE) == ComplexityTier.MEDIUM + + def test_tier_zero_is_never_the_savings_baseline(self, mock_router_instance): + """Savings are measured against the hardest configured tier. If tier 0 could win that pick, + every enabled router's reported savings would invert.""" + assert self._router(mock_router_instance)._hardest_tier_models() == ("o1-preview",) + cheap_only: Final = self._router( + mock_router_instance, tiers={"NON_REASONING": "cheap-relay", "SIMPLE": "gpt-4o-mini"} + ) + assert cheap_only._hardest_tier_models() == ("gpt-4o-mini",) + + def test_the_tier_gets_its_own_display_label(self, mock_router_instance): + """tier_labels covers the built-in tiers, so the new rung must be renameable like the rest.""" + router: Final = self._router(mock_router_instance, tier_labels={"NON_REASONING": "Relay"}) + assert router.config.classifier_wire_labels()[0] == "Relay" + assert router.config.resolve_classified_tier("relay") == ComplexityTier.NON_REASONING + + @pytest.mark.parametrize( + "overrides, expected", + ( + ({"classifier_type": "heuristic", "classifier_llm_config": None}, "requires classifier_type"), + ({"classifier_type": "heuristic_v2", "classifier_llm_config": None}, "requires classifier_type"), + ({"tiers": {"SIMPLE": "a", "MEDIUM": "b"}}, "at least one model"), + ), + ids=["heuristic", "heuristic_v2", "no_model"], + ) + def test_unreachable_or_unroutable_configs_are_rejected(self, overrides, expected): + """The toggle is refused wherever it could not do anything: the heuristic scorers cannot + emit the tier, and an unconfigured tier would fall through to the default model.""" + config: Final = { + "tiers": dict(NON_REASONING_TIERS), + "enable_non_reasoning_tier": True, + "classifier_type": "llm", + "classifier_llm_config": {"model": "clf"}, + **overrides, + } + with pytest.raises(ValidationError, match=expected): + ComplexityRouterConfig.model_validate(config) + + def test_the_tier_cannot_be_configured_without_the_toggle(self): + """Silently ignoring the key would leave an operator paying for a pool nothing routes to.""" + with pytest.raises(ValidationError, match="no request can route there"): + ComplexityRouterConfig(tiers={"NON_REASONING": "cheap", "SIMPLE": "a"}) + + def test_the_toggle_is_refused_alongside_a_custom_tier_set(self): + """A custom tier set replaces the built-in ladder, so both at once has no meaning.""" + with pytest.raises(ValidationError, match="cannot be combined with tier_definitions"): + ComplexityRouterConfig( + enable_non_reasoning_tier=True, + classifier_type="llm", + classifier_llm_config={"model": "clf"}, + tier_definitions=({"name": "lo", "description": "d"}, {"name": "hi", "description": "d"}), + tiers={"lo": "a", "hi": "b"}, + fallback_tier="lo", + ) + + def test_heuristic_v2_predictions_never_reach_the_new_tier(self, mock_router_instance): + """The bundled artifact is trained on four classes, so its 1-based tier index must keep + mapping onto SIMPLE..REASONING. Reading the enabled ladder here would shift every + prediction down a rung and make REASONING unreachable.""" + router: Final = ComplexityRouter( + model_name="v2-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "tiers": {k: v for k, v in NON_REASONING_TIERS.items() if k != "NON_REASONING"}, + "classifier_type": "heuristic_v2", + }, + ) + outcome = router._classify_with_heuristic_v2("implement a distributed rate limiter under concurrency") + assert outcome.tier in TIER_SEVERITY_ORDER + # One probability signal per trained class, named for the tier that class means. A ladder + # shifted by the new rung would relabel all four and lose REASONING off the end. + assert tuple(signal.split(":")[1].split("=")[0] for signal in outcome.signals[1:]) == ( + "simple", + "medium", + "complex", + "reasoning", + ) diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 2970e14b335..7632c8f9a70 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -102,7 +102,7 @@ describe("ComplexityRouterConfig", () => { renderWithProviders(); await user.click(screen.getByText("Advanced: Response Format")); - await user.click(screen.getByRole("switch")); + await user.click(screen.getByRole("switch", { name: "Return raw model name" })); expect(onChange).toHaveBeenCalledWith({ ...defaultValue, @@ -495,7 +495,7 @@ describe("ComplexityRouterConfig", () => { />, ); fireEvent.click(screen.getByText("Advanced: Keyword/Semantic Matching")); - await user.click(screen.getByRole("switch")); + await user.click(screen.getByRole("switch", { name: "Semantic keyword matching" })); expect(onSemanticMatchingEnabledChange).toHaveBeenCalledWith(true, expect.anything()); }); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index d7df6ce33bb..01e65914c67 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -75,11 +75,16 @@ export const DEFAULT_CLASSIFICATION_MODE: ClassificationMode = "every_request"; */ export type ClassificationFrequency = ClassificationMode | "session"; +/** + * NON_REASONING is optional because it is the opt-in fifth tier: a router that never enabled it + * stores no such key, and hydrating one in would send an empty pool the backend rejects. + */ export type ComplexityTiers = { SIMPLE: string[]; MEDIUM: string[]; COMPLEX: string[]; REASONING: string[]; + NON_REASONING?: string[]; }; export type ClassificationRubric = "legacy" | "agentic" | "chat" | "business"; @@ -378,6 +383,11 @@ export type ComplexityTierLabels = Partial export interface ComplexityRouterConfigValue { tiers: ComplexityTiers; + /** + * Opt into the NON_REASONING tier below SIMPLE. Off means the router keeps the four-tier ladder + * it has always had, so an existing router's rubric and tier decisions cannot move under it. + */ + enable_non_reasoning_tier?: boolean; custom_tier_set?: CustomTierSet; tier_labels?: ComplexityTierLabels; /** An explicit pin. Unset means the default tracks the tiers - see resolveComplexityDefaultModel. */ @@ -494,6 +504,11 @@ export const TIER_DESCRIPTIONS: Record< keyof ComplexityTiers, { label: string; description: string; examples: string } > = { + NON_REASONING: { + label: "Non-reasoning", + description: "Operational relay work: passing information along with no judgment about it", + examples: '"Reformat this tool output", "Acknowledge the write succeeded"', + }, SIMPLE: { label: "Simple", description: "Basic questions, greetings, simple factual queries", @@ -516,8 +531,16 @@ export const TIER_DESCRIPTIONS: Record< }, }; +/** Every built-in tier name, including the opt-in one, for label and membership checks. */ export const TIER_KEYS = Object.keys(TIER_DESCRIPTIONS) as Array; +/** + * The four-tier ladder in ascending severity, which is what a router sends unless it opted into + * NON_REASONING. Mirrors TIER_SEVERITY_ORDER in the backend config; use tierOrderFor to get the + * ladder one router actually renders. + */ +export const BUILT_IN_TIER_ORDER: Array = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"]; + export const effectiveTierLabel = (tier: keyof ComplexityTiers, tierLabels: ComplexityTierLabels | undefined): string => tierLabels?.[tier]?.trim() || TIER_DESCRIPTIONS[tier].label; @@ -528,9 +551,46 @@ export const DEFAULT_HYBRID_BOUNDARY_MARGIN = 0.03; /** * Tiers the heuristic_first threshold may name. The top tier is excluded because it would short - * circuit every request and leave the classifier unreachable, which the backend rejects. + * circuit every request and leave the classifier unreachable, which the backend rejects. So is + * NON_REASONING, which the backend refuses alongside heuristic_first because the local scorer + * cannot produce it. */ -export const HEURISTIC_FIRST_MAX_TIER_KEYS = TIER_KEYS.slice(0, -1); +export const HEURISTIC_FIRST_MAX_TIER_KEYS = BUILT_IN_TIER_ORDER.slice(0, -1); + +/** + * The opt-in fifth tier. Only offered on the LLM classification method, matching the backend: the + * heuristic scorers cannot produce the tier, so enabling it there would buy a rubric bullet and a + * model pool that no request ever reaches. + */ +const NonReasoningTierToggle: React.FC<{ + value: ComplexityRouterConfigValue; + onChange: (value: ComplexityRouterConfigValue) => void; + available: boolean; +}> = ({ value, onChange, available }) => ( + <> +
+ { + const { NON_REASONING: _dropped, ...keptTiers } = value.tiers; + onChange({ + ...value, + enable_non_reasoning_tier: enabled ? true : undefined, + tiers: enabled ? { ...keptTiers, NON_REASONING: value.tiers.NON_REASONING ?? [] } : keptTiers, + }); + }} + aria-label="Add a non-reasoning tier" + /> + Add a non-reasoning tier +
+ + Adds NON_REASONING below Simple, for operational agent traffic that relays or reformats information rather than + reasoning about it. Escalation still moves up out of it when a request needs more. + {!available && " Requires the LLM classification method."} + + +); const PlanModeOverrideControls: React.FC<{ value: ComplexityRouterConfigValue; @@ -742,6 +802,10 @@ const ComplexityRouterConfig: React.FC = ({ ); })} + {!customTierSet && ( + + )} + = ({ const complexityRouterConfigParams: BuildComplexityRouterConfigParams = { tiers: complexityRouterConfig.tiers, + enableNonReasoningTier: complexityRouterConfig.enable_non_reasoning_tier, customTierSet: complexityRouterConfig.custom_tier_set, defaultModel: complexityRouterConfig.default_model, planModeMinTier: complexityRouterConfig.plan_mode_min_tier, diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 7769fb832fe..2f36ec2f43d 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -119,6 +119,7 @@ const scorerKnobPayload = ({ export interface BuildComplexityRouterConfigParams { tiers: ComplexityTiers; + enableNonReasoningTier?: boolean; customTierSet?: CustomTierSet; defaultModel: string | undefined; planModeMinTier: string | undefined; @@ -180,6 +181,7 @@ export interface TierDefinitionPayload { export interface ComplexityRouterConfigPayload { tiers: ComplexityTiers | Record; + enable_non_reasoning_tier?: boolean; tier_definitions?: TierDefinitionPayload[]; fallback_tier?: string; default_model?: string; @@ -460,6 +462,7 @@ const classifierWireFields = ( export const buildComplexityRouterConfig = ({ tiers, + enableNonReasoningTier, customTierSet, defaultModel, planModeMinTier, @@ -535,6 +538,9 @@ export const buildComplexityRouterConfig = ({ const payload: ComplexityRouterConfigPayload = { tiers, + // Only written when on, and never beside a custom tier set: the backend rejects the two + // together, and an explicit false on a four-tier router would be a key it never carried. + ...(!customTierSet && enableNonReasoningTier && { enable_non_reasoning_tier: true }), ...(serializedTierModelConfigs && { tier_model_configs: serializedTierModelConfigs }), ...(defaultModel?.trim() && { default_model: defaultModel }), ...(planModeMinTier?.trim() && { plan_mode_min_tier: planModeMinTier }), diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts index eb58c94b789..3fec63518e5 100644 --- a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts +++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts @@ -1,6 +1,6 @@ import type { ComplexityTier } from "./KeywordTierRules"; import type { ModelGroup } from "@/components/llm_calls/fetch_models"; -import { TIER_ORDER } from "./tier_rows"; +import { ALL_BUILT_IN_TIERS, TIER_ORDER } from "./tier_rows"; export type TierModelParams = Record; @@ -145,13 +145,14 @@ export const pruneTierModelParams = ( }; export const DEFAULT_TIER_LABELS: Record = { + NON_REASONING: "Non-reasoning", SIMPLE: "Simple", MEDIUM: "Medium", COMPLEX: "Complex", REASONING: "Reasoning", }; -const isBuiltInTier = (tier: string): tier is ComplexityTier => (TIER_ORDER as string[]).includes(tier); +const isBuiltInTier = (tier: string): tier is ComplexityTier => (ALL_BUILT_IN_TIERS as string[]).includes(tier); const builtInTierLabel = ( tierLabels: Partial> | undefined, @@ -164,7 +165,7 @@ export const tierRowLabel = ( row: { id: string; name: string }, tierLabels?: Partial>, ): string => { - const builtIn = TIER_ORDER.find((tier) => tier === row.id); + const builtIn = ALL_BUILT_IN_TIERS.find((tier) => tier === row.id); const named = row.name.trim(); if (!builtIn || named !== builtIn) return named || "New"; return builtInTierLabel(tierLabels, builtIn); diff --git a/ui/litellm-dashboard/src/components/add_model/tier_rows.test.ts b/ui/litellm-dashboard/src/components/add_model/tier_rows.test.ts index df50f116f60..07b9a4702aa 100644 --- a/ui/litellm-dashboard/src/components/add_model/tier_rows.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/tier_rows.test.ts @@ -168,3 +168,34 @@ describe("tierParamsByRowId", () => { expect(tierParamsByRowId(undefined, rows)).toBeUndefined(); }); }); + +describe("the opt-in non-reasoning tier", () => { + const withTierZero = { SIMPLE: ["a"], MEDIUM: ["b"], COMPLEX: ["c"], REASONING: ["d"], NON_REASONING: ["cheap"] }; + + it("renders no fifth row while the toggle is off", () => { + // The regression for every existing router: the tier exists in the type, and the form must + // still show the four rows it always showed. + expect(activeTierRows({ tiers: withTierZero }).map((row) => row.id)).toEqual([ + "SIMPLE", + "MEDIUM", + "COMPLEX", + "REASONING", + ]); + }); + + it("renders it first, as tier 0, when enabled", () => { + const rows = activeTierRows({ tiers: withTierZero, enable_non_reasoning_tier: true }); + expect(rows.map((row) => row.id)).toEqual(["NON_REASONING", "SIMPLE", "MEDIUM", "COMPLEX", "REASONING"]); + expect(rows[0].models).toEqual(["cheap"]); + }); + + it("renders an enabled tier with no models as an empty row rather than crashing", () => { + const rows = activeTierRows({ tiers, enable_non_reasoning_tier: true }); + expect(rows[0]).toEqual({ id: "NON_REASONING", name: "NON_REASONING", definition: "", models: [], params: {} }); + }); + + it("counts as a built-in name either way, so a custom set cannot claim the name", () => { + expect(isBuiltInTierName("NON_REASONING")).toBe(true); + expect(isBuiltInTierName("non_reasoning")).toBe(true); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts index 5e2a32addee..289a5645b51 100644 --- a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts +++ b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts @@ -4,6 +4,16 @@ import type { TierModelParams, TierModelParamsByTier } from "./complexity_router export const TIER_ORDER: ComplexityTier[] = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"]; +/** Every built-in tier name, so a stored NON_REASONING row is recognized as built-in either way. */ +export const ALL_BUILT_IN_TIERS: ComplexityTier[] = ["NON_REASONING", ...TIER_ORDER]; + +/** + * The ladder one router renders, ascending. NON_REASONING is tier 0 and appears only when enabled, + * which is what keeps an existing four-tier router's form, payload, and rubric unchanged. + */ +export const tierOrderFor = (enableNonReasoningTier: boolean | undefined): ComplexityTier[] => + enableNonReasoningTier ? ALL_BUILT_IN_TIERS : TIER_ORDER; + export interface TierRow { id: string; name: string; @@ -27,6 +37,7 @@ export const MAX_TIER_DEFINITION_CHARS = 500; export interface ActiveTierSet { tiers: ComplexityTiers; + enable_non_reasoning_tier?: boolean; custom_tier_set?: CustomTierSet; tier_model_params?: TierModelParamsByTier; } @@ -39,7 +50,8 @@ export const activeTierName = (row: TierRow): string => row.name.trim(); export const sameTierIdentity = (left: string, right: string): boolean => left.trim().toLowerCase() === right.trim().toLowerCase(); -export const isBuiltInTierName = (name: string): boolean => TIER_ORDER.some((tier) => sameTierIdentity(tier, name)); +export const isBuiltInTierName = (name: string): boolean => + ALL_BUILT_IN_TIERS.some((tier) => sameTierIdentity(tier, name)); const builtInRow = (tier: keyof ComplexityTiers, tiers: ComplexityTiers): TierRow => ({ id: tier, @@ -51,7 +63,9 @@ const builtInRow = (tier: keyof ComplexityTiers, tiers: ComplexityTiers): TierRo // The only reader of the tier set. Built-in rows carry the canonical tier key as their id, so every // pointer into the set is a row id in both modes and nothing downstream branches on the mode. export const activeTierRows = (value: ActiveTierSet): ActiveTierRow[] => { - const rows = value.custom_tier_set?.tiers ?? TIER_ORDER.map((tier) => builtInRow(tier, value.tiers)); + const rows = + value.custom_tier_set?.tiers ?? + tierOrderFor(value.enable_non_reasoning_tier).map((tier) => builtInRow(tier, value.tiers)); return rows.map((row) => ({ ...row, params: value.tier_model_params?.[row.id] ?? {} })); }; diff --git a/ui/litellm-dashboard/src/components/add_model/tier_set_actions.ts b/ui/litellm-dashboard/src/components/add_model/tier_set_actions.ts index c8254b6bff9..f737e0fdff6 100644 --- a/ui/litellm-dashboard/src/components/add_model/tier_set_actions.ts +++ b/ui/litellm-dashboard/src/components/add_model/tier_set_actions.ts @@ -4,11 +4,12 @@ import { pruneTierModelParams } from "./complexity_router_tiers"; import { type ActiveTierRow, type TierRow, - TIER_ORDER, + ALL_BUILT_IN_TIERS, activeTierName, activeTierRows, rowParamsByTier, sameTierIdentity, + tierOrderFor, tierRowById, tierRowByName, } from "./tier_rows"; @@ -74,13 +75,13 @@ const rulesFollowingRows = ( // Models and params both come from these rows, so the two cannot be keyed differently. const exitToBuiltInTiers = (value: ComplexityRouterConfigValue, rows: readonly ActiveTierRow[]) => { const { custom_tier_set: _dropped, ...rest } = value; - const builtInRows: ActiveTierRow[] = TIER_ORDER.map( + const builtInRows: ActiveTierRow[] = tierOrderFor(value.enable_non_reasoning_tier).map( (tier) => tierRowById(rows, tier) ?? { id: tier, name: tier, definition: "", - models: value.tiers[tier], + models: value.tiers[tier] ?? [], params: value.tier_model_params?.[tier] ?? {}, }, ); @@ -121,7 +122,7 @@ const nextTierSetValue = ( case "remove": { const removed = tierRowById(rows, action.id); const snapshot = - removed && (TIER_ORDER as string[]).includes(action.id) + removed && (ALL_BUILT_IN_TIERS as string[]).includes(action.id) ? { ...value, tiers: { ...value.tiers, [action.id]: removed.models } } : value; return commitTierRows( diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index 0144dff3498..5b09dabd214 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -598,6 +598,10 @@ describe("managed keys survive an untouched open-and-save", () => { "stall_escalation_repeat_threshold", ]); + // The opt-in fifth tier requires the LLM classifier, which this heuristic_first fixture is not, + // so it gets its own round trip below. + const KEYS_ANOTHER_TIER_LADDER_OWNS = new Set(["enable_non_reasoning_tier"]); + it("carries every managed key a built-in router can hold through hydrate then save", () => { const hydrated = hydrateComplexityRouterConfig(STORED_ALL_MANAGED, undefined); const saved = buildUpdatedComplexityRouterConfig(STORED_ALL_MANAGED, hydrated); @@ -605,10 +609,55 @@ describe("managed keys survive an untouched open-and-save", () => { const dropped = [...MANAGED_COMPLEXITY_ROUTER_KEYS] .filter((key) => !KEYS_ANOTHER_CLASSIFIER_TYPE_OWNS.has(key)) .filter((key) => !KEYS_ANOTHER_CLASSIFICATION_FREQUENCY_OWNS.has(key)) + .filter((key) => !KEYS_ANOTHER_TIER_LADDER_OWNS.has(key)) .filter((key) => saved[key] === undefined); expect(dropped).toEqual([]); }); + it("carries an enabled non-reasoning tier and its models through their own round trip", () => { + // `tiers` is rewritten wholesale on save, so this is the regression that matters: opening an + // enabled router and saving an unrelated edit must not delete the tier or its pool. + const stored: Record = { + ...STORED_ALL_MANAGED, + classifier_type: "llm", + classifier_llm_config: { model: "haiku-classifier" }, + heuristic_first_max_tier: undefined, + enable_non_reasoning_tier: true, + tiers: { ...(STORED_ALL_MANAGED.tiers as object), NON_REASONING: ["gpt-4o-mini"] }, + }; + const hydrated = hydrateComplexityRouterConfig(stored, undefined); + const saved = buildUpdatedComplexityRouterConfig(stored, hydrated); + + expect(saved.enable_non_reasoning_tier).toBe(true); + expect((saved.tiers as Record).NON_REASONING).toEqual(["gpt-4o-mini"]); + }); + + it("keeps a stored non-reasoning tier when the stored config never wrote the flag", () => { + // A hand-written config that names the tier: the flag is inferred from the stored pool, so an + // edit made for an unrelated reason cannot silently turn the tier off. + const stored: Record = { + ...STORED_ALL_MANAGED, + classifier_type: "llm", + classifier_llm_config: { model: "haiku-classifier" }, + heuristic_first_max_tier: undefined, + tiers: { ...(STORED_ALL_MANAGED.tiers as object), NON_REASONING: ["gpt-4o-mini"] }, + }; + const saved = buildUpdatedComplexityRouterConfig(stored, hydrateComplexityRouterConfig(stored, undefined)); + + expect(saved.enable_non_reasoning_tier).toBe(true); + expect((saved.tiers as Record).NON_REASONING).toEqual(["gpt-4o-mini"]); + }); + + it("leaves the tier and its flag out of a saved config that never had it on", () => { + const saved = buildUpdatedComplexityRouterConfig( + STORED_ALL_MANAGED, + hydrateComplexityRouterConfig(STORED_ALL_MANAGED, undefined), + ); + + expect(saved).not.toHaveProperty("enable_non_reasoning_tier"); + expect(saved.tiers).not.toHaveProperty("NON_REASONING"); + }); + it("carries the stall-escalation keys through their own round trip", () => { const stored: Record = { ...STORED_ALL_MANAGED, diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index 3c0013e267e..283fe978790 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -91,6 +91,7 @@ interface EditAutoRouterModalProps { * hydrators validate themselves stay `unknown`; the ones assigned straight through carry their type. */ export interface StoredComplexityRouterConfig { tiers?: Partial>; + enable_non_reasoning_tier?: boolean; tier_model_configs?: unknown; default_model?: string | null; plan_mode_min_tier?: unknown; @@ -135,18 +136,27 @@ export const hydrateComplexityRouterConfig = ( parsedConfig: StoredComplexityRouterConfig, complexityRouterDefaultModel: string | null | undefined, ): ComplexityRouterConfigValue => { + // `tiers` is rewritten wholesale on save, so a stored tier this misses is deleted from the + // router by any edit at all, including one made for an unrelated reason. NON_REASONING is + // therefore read back from the stored config rather than assumed absent, and the toggle follows + // what is actually stored so the round-trip cannot silently turn the tier off. + const storedNonReasoning: string[] = normalizeTierModels(parsedConfig.tiers?.NON_REASONING); + const enable_non_reasoning_tier: boolean = + parsedConfig.enable_non_reasoning_tier === true || storedNonReasoning.length > 0; const hydratedTiers: ComplexityTiers = { SIMPLE: normalizeTierModels(parsedConfig.tiers?.SIMPLE), MEDIUM: normalizeTierModels(parsedConfig.tiers?.MEDIUM), COMPLEX: normalizeTierModels(parsedConfig.tiers?.COMPLEX), REASONING: normalizeTierModels(parsedConfig.tiers?.REASONING), + ...(enable_non_reasoning_tier && { NON_REASONING: storedNonReasoning }), }; const custom_tier_set = hydrateCustomTierSet(parsedConfig); - const activeTiers = { tiers: hydratedTiers, custom_tier_set }; + const activeTiers = { tiers: hydratedTiers, enable_non_reasoning_tier, custom_tier_set }; return { tiers: hydratedTiers, + enable_non_reasoning_tier, custom_tier_set, tier_model_params: tierParamsByRowId( hydrateTierModelParams(parsedConfig.tiers, parsedConfig.tier_model_configs), @@ -234,6 +244,7 @@ export const hydrateComplexityRouterConfig = ( export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "tiers", + "enable_non_reasoning_tier", "tier_definitions", "fallback_tier", "tier_model_configs", @@ -339,6 +350,7 @@ export const buildUpdatedComplexityRouterConfig = ( const builderParams: BuildComplexityRouterConfigParams = { tiers: value.tiers, + enableNonReasoningTier: value.enable_non_reasoning_tier, customTierSet: value.custom_tier_set, defaultModel: value.default_model, planModeMinTier: value.plan_mode_min_tier, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 232b366d8ce..6a60b0db58a 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -23573,16 +23573,22 @@ export interface components { }; /** * AutoRouterPresetTiers - * @description Exactly the four built-in tiers the dashboard's preset prefill can apply. + * @description The built-in tiers the dashboard's preset prefill can apply. * * extra="forbid" on purpose: a tier name this dashboard cannot apply would grey out or crash the - * picker, so such a catalog is rejected wholesale and the bundled one serves instead. + * picker, so such a catalog is rejected wholesale and the bundled one serves instead. NON_REASONING + * is optional rather than required so this proxy accepts both a catalog that ships the opt-in fifth + * tier and the four-tier catalogs that predate it. It stays None when unset rather than defaulting + * to an empty pool, so a four-tier preset serves the tier set it was published with instead of + * growing a key the dashboard would render as an empty fifth tier row. */ AutoRouterPresetTiers: { /** Complex */ COMPLEX: string[]; /** Medium */ MEDIUM: string[]; + /** Non Reasoning */ + NON_REASONING?: string[] | null; /** Reasoning */ REASONING: string[]; /** Simple */ @@ -25579,7 +25585,7 @@ export interface components { * @description Complexity tiers for routing decisions. * @enum {string} */ - ComplexityTier: "SIMPLE" | "MEDIUM" | "COMPLEX" | "REASONING"; + ComplexityTier: "NON_REASONING" | "SIMPLE" | "MEDIUM" | "COMPLEX" | "REASONING"; /** ComplexityTierModel */ ComplexityTierModel: { /** Litellm Params */ @@ -34947,6 +34953,12 @@ export interface components { * @default true */ enable_context_window_escalation: boolean; + /** + * Enable Non Reasoning Tier + * @description Add NON_REASONING as a fifth built-in tier below SIMPLE, for operational agent traffic that relays or reformats information rather than reasoning about it. Off by default: turning it on adds a rung to this router's ladder, a bullet to the LLM classifier's rubric, and a value the classifier may return, all of which move tier decisions and spend on an already-deployed router. Requires an LLM classifier or a custom classifier plugin, since the heuristic scorers cannot produce the tier, and a model in `tiers` under the NON_REASONING key. Escalation still walks up from it, and it is never the savings baseline or a `heuristic_v2` prediction. + * @default false + */ + enable_non_reasoning_tier: boolean; /** * Escalation Keywords * @description Case-sensitive phrases a user can include to force a bump to the next-higher complexity tier when they aren't satisfied with results (they can force a stronger model, but not choose which one). Defaults to ['LITELLM ESCALATE'] when unset; set to an empty list to disable. @@ -37212,7 +37224,7 @@ export interface components { TierDefinition: { /** * Description - * @description What belongs in this tier; rendered as this tier's bullet in the classifier rubric. Required unless the name is a built-in tier (SIMPLE/MEDIUM/COMPLEX/REASONING), which inherits the built-in criteria when omitted + * @description What belongs in this tier; rendered as this tier's bullet in the classifier rubric. Required unless the name is a built-in tier (NON_REASONING, SIMPLE, MEDIUM, COMPLEX, REASONING), which inherits the built-in criteria when omitted */ description?: string | null; /**