scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

Merge remote-tracking branch 'upstream/litellm_internal_staging' into feat-xquik-search-provider

Browse source
This commit is contained in:
kriptoburak 2026-08-27 04:27:55 +02:00
commit 95d59aa60c
20 changed files with 888 additions and 200 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/proxy/management_endpoints/auto_router_endpoints.py
View file

@ -195,7 +195,7 @@ def _models_this_test_can_call(config: RequestComplexityRouterConfig) -> tuple[s
model
for model in (
config.classifier_llm_config.model
if config.classifier_type == "llm" and config.classifier_llm_config is not None
if config.uses_llm_classifier and config.classifier_llm_config is not None
else None,
config.embedding_model if config.semantic_keyword_matching else None,
)

44
litellm/router_strategy/complexity_router/README.md
View file

@ -178,6 +178,50 @@ response = litellm.completion(
## Special Behaviors
### Heuristic-first chaining
`classifier_type: heuristic_first` runs the local scorer on every request and only calls the LLM
classifier for the ones the scorer could not place cheaply. It takes the same classifier settings as
`classifier_type: llm`, plus `heuristic_first_max_tier`:
```yaml
model_list:
- model_name: smart-router
litellm_params:
model: auto_router/complexity_router
complexity_router_config:
classifier_type: heuristic_first
heuristic_first_max_tier: SIMPLE
classifier_llm_config:
model: gpt-4o-mini
tiers:
SIMPLE: gpt-4o-mini
MEDIUM: gpt-4o
COMPLEX: claude-sonnet-4
REASONING: o1-preview
```
A request short-circuits, meaning it routes on the scorer's own tier with no classifier call, when
two things hold: the scorer landed at or below `heuristic_first_max_tier`, and it produced at least
one signal. Everything else goes to the classifier, which then decides as it normally would.
The signal requirement is what keeps this from quietly routing everything to your cheapest model.
A prompt where no dimension fires scores exactly 0.0, which is below `simple_medium`, so the score
to tier mapping calls it SIMPLE by default rather than by evidence. Around half of general traffic
scores that way. Those requests reach the classifier instead, which is the whole reason to configure
one. Note the converse too: the score is not a confidence, and a prompt that fires a single weak
signal and still lands under the boundary does short-circuit, so a lower threshold buys accuracy and
a higher one buys savings.
`heuristic_first_max_tier` names a built-in tier and may not name the highest one, since that would
short-circuit everything and leave the classifier unreachable. Operator-defined tier sets
(`tier_definitions`) are not supported here, because the scorer only produces the built-in tiers.
When the classifier call fails, the fallback works exactly as it does under `classifier_type: llm`,
except that the heuristic outcome is the one already computed rather than a second scoring pass.
Spend logs record `routing_decision.cause` as `heuristic_first_short_circuit` when the classifier
was skipped, and `llm_classifier` when it ran, so the two are told apart per request.
### Reasoning Override
If 2+ reasoning markers are detected in the user message, the request is promoted to the REASONING tier even when the weighted score maps lower, so complex reasoning tasks get the appropriate model. The promotion requires the score to reach `reasoning_override_min_score`, which tracks `tier_boundaries.simple_medium` unless set, so stock phrases on an otherwise trivial prompt cannot buy the top tier. Set it to `0` to promote on the markers alone.

74
litellm/router_strategy/complexity_router/complexity_router.py
View file

@ -719,6 +719,7 @@ class ClassificationOutcome(NamedTuple):
"heuristic_scorer",
"reasoning_override",
"llm_classifier",
"heuristic_first_short_circuit",
"classifier_plugin",
"classifier_fallback",
"default_model_fallback",
@ -859,7 +860,7 @@ class ComplexityRouter(CustomLogger):
# Both are pure functions of the config, so building them per classifier call would
# re-run create_model and the schema conversion on every request for the same result.
llm_classifier_configured: Final = self.config.classifier_type == "llm" and (
llm_classifier_configured: Final = self.config.uses_llm_classifier and (
self.config.classifier_llm_config is not None
)
self._classifier_system_prompt: str | None = (
@ -1237,17 +1238,63 @@ class ComplexityRouter(CustomLogger):
"""
Classify a prompt by complexity, using the LLM classifier when configured.
Falls back to the local heuristic scorer if classifier_type is "heuristic". If the LLM call
or the classifier plugin fails, times out, or produces no usable tier, the configured
fallback_tier wins on a custom tier set, and classifier_fallback otherwise decides between
the heuristic scorer and default_model. The outcome's `cause` reports which path actually ran.
Falls back to the local heuristic scorer if classifier_type is "heuristic". Under
"heuristic_first" the scorer runs first and the classifier is called only for requests it
could not place at or below heuristic_first_max_tier. If the LLM call or the classifier
plugin fails, times out, or produces no usable tier, the configured fallback_tier wins on a
custom tier set, and classifier_fallback otherwise decides between the heuristic scorer and
default_model. The outcome's `cause` reports which path actually ran.
"""
if self.config.classifier_type == "custom":
return await self._classify_with_plugin(prompt, system_prompt, request_kwargs, raw_messages)
if self.config.classifier_type == "heuristic_first" and self.config.classifier_llm_config is not None:
return await self._classify_heuristic_first(prompt, system_prompt, request_kwargs, messages)
if self.config.classifier_type != "llm" or self.config.classifier_llm_config is None:
tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
return ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause)
return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages)
async def _classify_heuristic_first(
self,
prompt: str,
system_prompt: str | None,
request_kwargs: dict[str, Any] | None, # mutable-ok: handed to _classify_with_llm as-is
messages: Sequence[Mapping[str, object]] | None,
) -> ClassificationOutcome:
"""Score locally, and only pay for the classifier call when the scorer did not confidently
place the request at or below heuristic_first_max_tier.
Confidence is `signals`, not `score`. A prompt where no dimension fired scores exactly 0.0,
which is below simple_medium and so lands SIMPLE by default rather than by evidence, and a
threshold check alone would hand that traffic to the cheapest model without ever consulting
the classifier. Scores also go negative when simple indicators fire, so a score threshold
would reject exactly the trivial prompts this path exists to serve.
"""
tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
scored: Final = ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause)
threshold: Final = self.config.heuristic_first_max_tier
decided_cheaply: Final = (
threshold is not None
and bool(signals)
and self._active_tier_severity(tier) <= self._active_tier_severity(threshold)
)
if decided_cheaply:
return ClassificationOutcome(tier=tier, score=score, signals=signals, cause="heuristic_first_short_circuit")
return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages, scored=scored)
async def _llm_classifier_outcome(
self,
prompt: str,
system_prompt: str | None,
request_kwargs: dict[str, Any] | None, # mutable-ok: handed to _classify_with_llm as-is
messages: Sequence[Mapping[str, object]] | None,
scored: ClassificationOutcome | None = None,
) -> ClassificationOutcome:
"""Call the LLM classifier and turn its verdict, or its failure, into an outcome.
`scored` is the heuristic outcome the caller already computed, which only "heuristic_first"
has. It is handed to the failure path so a classifier error does not re-run the scorer.
"""
try:
tier, classifier_cost = await self._classify_with_llm(prompt, system_prompt, request_kwargs, messages)
return ClassificationOutcome(
@ -1258,11 +1305,20 @@ class ComplexityRouter(CustomLogger):
classifier_cost=classifier_cost,
)
except Exception as e: # noqa: BLE001 -- external LLM call can fail in many distinct ways (timeout, provider error, validation, parse error); any failure must fall back to the configured fallback path
return self._classifier_failure_outcome(f"LLM classifier failed ({e})", prompt, system_prompt)
return self._classifier_failure_outcome(f"LLM classifier failed ({e})", prompt, system_prompt, scored)
def _classifier_failure_outcome(self, reason: str, prompt: str, system_prompt: str | None) -> ClassificationOutcome:
def _classifier_failure_outcome(
self,
reason: str,
prompt: str,
system_prompt: str | None,
scored: ClassificationOutcome | None = None,
) -> ClassificationOutcome:
"""The outcome when the LLM classifier or classifier plugin produced no usable tier:
fallback_tier on a custom tier set, classifier_fallback otherwise."""
fallback_tier on a custom tier set, classifier_fallback otherwise.
A caller that already scored the prompt passes `scored` so the heuristic arm returns that
verdict instead of running the same scan again on the request path."""
fallback_tier: Final = self.config.fallback_tier
if fallback_tier is not None:
verbose_router_logger.warning("ComplexityRouter: %s, routing to fallback_tier %s", reason, fallback_tier)
@ -1277,6 +1333,8 @@ class ComplexityRouter(CustomLogger):
)
if self.config.classifier_fallback == "default_model":
return self._default_model_fallback_outcome()
if scored is not None:
return scored
tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
return ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause)

87
litellm/router_strategy/complexity_router/config.py
View file

@ -38,6 +38,11 @@ class ClassificationRubric(str, Enum):
# routers get the calibrated rubric without changing what is already running.
DEFAULT_CLASSIFICATION_RUBRIC: Final[ClassificationRubric] = ClassificationRubric.LEGACY
# The classifier_type values that can call classifier_llm_config.model. Every consumer asking
# "is the classifier model a real dependency of this router" resolves it here, including the ones
# that only hold the raw config mapping and cannot reach ComplexityRouterConfig.uses_llm_classifier.
LLM_CLASSIFIER_TYPES: Final[frozenset[str]] = frozenset({"llm", "heuristic_first"})
TIER_SEVERITY_ORDER: Final[tuple[ComplexityTier, ...]] = (
ComplexityTier.SIMPLE,
@ -591,13 +596,30 @@ class ComplexityRouterConfig(BaseModel):
)
# Classifier strategy
classifier_type: Literal["heuristic", "llm", "custom"] = Field(
classifier_type: Literal["heuristic", "llm", "custom", "heuristic_first"] = Field(
default="heuristic",
description="Classification strategy: local regex/keyword scoring, an LLM call, or a custom classifier plugin",
description=(
"Classification strategy: local regex/keyword scoring, an LLM call, a custom classifier "
"plugin, or 'heuristic_first', which scores locally and only pays for the LLM classifier "
"when the local scorer does not confidently land a cheap tier"
),
)
classifier_llm_config: ClassifierLLMConfig | None = Field(
default=None,
description="Configuration for the LLM classifier; required when classifier_type is 'llm'",
description="Configuration for the LLM classifier; required when classifier_type is 'llm' or 'heuristic_first'",
)
heuristic_first_max_tier: str | None = Field(
default=None,
description=(
"The highest tier the local scorer may decide on its own; required when classifier_type is "
"'heuristic_first' and rejected otherwise. A request whose heuristic tier is at or below this "
"one skips the LLM classifier and routes straight to that heuristic tier, so the classifier "
"call is only paid for on traffic the scorer could not place cheaply. The scorer must also "
"have produced at least one signal: a prompt where no dimension fired scores 0.0 and would "
"otherwise land SIMPLE by default rather than by evidence, which is how a chained router "
"would silently send unclassified traffic to the cheapest model. Names a built-in tier, and "
"may not name the highest one, since that would make the LLM classifier unreachable."
),
)
classifier_plugin: ClassifierPlugin | None = Field(
default=None,
@ -626,7 +648,7 @@ class ComplexityRouterConfig(BaseModel):
"which is what a classifier on some other taxonomy wants: a prompt that grades data "
"sensitivity has no use for a complexity score, and scoring one produces a tier unrelated to "
"what the operator configured. Requires default_model when set to 'default_model'. Only "
"applies when classifier_type is 'llm' or 'custom'."
"applies when classifier_type is 'llm', 'custom', or 'heuristic_first'."
),
)
@ -936,8 +958,8 @@ class ComplexityRouterConfig(BaseModel):
@model_validator(mode="after")
def _validate_classifier_config(self) -> "ComplexityRouterConfig":
if self.classifier_type == "llm" and self.classifier_llm_config is None:
raise ValueError("classifier_llm_config is required when classifier_type is 'llm'")
if self.uses_llm_classifier and self.classifier_llm_config is None:
raise ValueError(f"classifier_llm_config is required when classifier_type is {self.classifier_type!r}")
if self.classifier_type == "custom" and self.classifier_plugin is None:
raise ValueError("classifier_plugin is required when classifier_type is 'custom'")
if self.classifier_plugin is not None and self.classifier_type != "custom":
@ -947,6 +969,49 @@ class ComplexityRouterConfig(BaseModel):
)
return self
@field_validator("heuristic_first_max_tier", mode="before")
@classmethod
def _coerce_heuristic_first_max_tier(cls, value: object) -> object:
if isinstance(value, ComplexityTier):
return value.value
if isinstance(value, str):
return value.strip()
return value
@model_validator(mode="after")
def _validate_heuristic_first_max_tier(self) -> "ComplexityRouterConfig":
if self.classifier_type != "heuristic_first":
if self.heuristic_first_max_tier is not None:
raise ValueError(
f"heuristic_first_max_tier is set but classifier_type is {self.classifier_type!r}; "
"the local scorer would never gate the classifier. Set classifier_type "
"'heuristic_first' or remove heuristic_first_max_tier"
)
return self
threshold: Final = self.heuristic_first_max_tier
if threshold is None:
raise ValueError(
"heuristic_first_max_tier is required when classifier_type is 'heuristic_first': without a "
"threshold there is nothing to decide whether a request escalates to the LLM classifier"
)
names: Final = self.tier_names()
if threshold not in names:
raise ValueError(
f"heuristic_first_max_tier {threshold!r} is not an active tier: it must name one of {', '.join(names)}"
)
if threshold == names[-1]:
raise ValueError(
f"heuristic_first_max_tier {threshold} is the highest tier, so every request would short-circuit "
"and the LLM classifier would never run; name a lower tier or use classifier_type 'heuristic'"
)
if threshold not in self.tiers:
raise ValueError(
f"heuristic_first_max_tier {threshold} has no model configured in tiers; a threshold pointing at "
"an unconfigured tier would route short-circuited requests to the default fallback instead of the "
"pool the operator intended"
)
return self
@field_validator("fallback_tier", "classification_prompt")
@classmethod
def _reject_blank_optional_text(cls, value: str | None) -> str | None:
@ -969,6 +1034,14 @@ class ComplexityRouterConfig(BaseModel):
"""True when the operator replaced the built-in tier set via tier_definitions."""
return self.tier_definitions is not None
@property
def uses_llm_classifier(self) -> bool:
"""True when this router can call classifier_llm_config.model, so the model is a real
dependency: authorized against the caller's key, counted in the health graph, and given a
prebuilt rubric. 'heuristic_first' only calls it for traffic the local scorer escalates,
which still makes it a dependency on every one of those requests."""
return self.classifier_type in LLM_CLASSIFIER_TYPES
def tier_names(self) -> tuple[str, ...]:
"""The active tier names: the defined names, or the built-in set in severity order."""
if self.tier_definitions is not None:
@ -1063,7 +1136,7 @@ class ComplexityRouterConfig(BaseModel):
)
if duplicated:
raise ValueError(f"tier_definitions names must be unique (case-insensitive): {', '.join(duplicated)}")
if self.classifier_type == "heuristic":
if self.classifier_type in ("heuristic", "heuristic_first"):
raise ValueError(
"tier_definitions requires classifier_type 'llm' or 'custom': the heuristic scorer only "
"produces the built-in tiers"

8
litellm/router_utils/auto_router_model_naming.py
View file

@ -15,6 +15,8 @@ from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
from litellm.router_strategy.complexity_router.config import LLM_CLASSIFIER_TYPES
AUTO_ROUTER_MODEL_PREFIX: Final = "auto_router/"
StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality"]
@ -144,7 +146,11 @@ def strategy_router_dependencies(
dict.fromkeys(
tuple(dep for tier in _mapping(complexity.get("tiers")).values() for dep in _pool(tier, "tier"))
+ _named(litellm_params.get("complexity_router_default_model"), "default")
+ (_named(classifier.get("model"), "classifier") if complexity.get("classifier_type") == "llm" else ())
+ (
_named(classifier.get("model"), "classifier")
if complexity.get("classifier_type") in LLM_CLASSIFIER_TYPES
else ()
)
+ (
_named(complexity.get("embedding_model"), "embedding")
if complexity.get("semantic_keyword_matching")

6
litellm/types/utils.py
View file

@ -2809,6 +2809,12 @@ RoutingDecisionCause = Literal[
# meant anything that filtered `signals` silently changed what the row claimed.
"reasoning_override",
"llm_classifier",
# classifier_type 'heuristic_first': the local scorer produced at least one signal and landed at
# or below heuristic_first_max_tier, so it decided the tier and the LLM classifier was never
# called. Distinct from "heuristic_scorer", which is a router whose only classifier IS the
# scorer, and from "classifier_fallback", which is the scorer running because a call failed:
# only this cause means an LLM classifier was configured, reachable, and deliberately skipped.
"heuristic_first_short_circuit",
# The operator's classifier plugin (classifier_type 'custom') decided the tier.
"classifier_plugin",
# The LLM classifier or classifier plugin failed on a router with an operator-defined

214
tests/test_litellm/router_strategy/test_complexity_router.py
View file

@ -1810,9 +1810,7 @@ class TestLLMClassifier:
"request_kwargs",
[
pytest.param({"metadata": {"user_api_key": "sk-abc"}}, id="metadata-bucket"),
pytest.param(
{"litellm_metadata": {"user_api_key": "sk-abc"}}, id="litellm-metadata-bucket"
),
pytest.param({"litellm_metadata": {"user_api_key": "sk-abc"}}, id="litellm-metadata-bucket"),
pytest.param({}, id="no-caller-context"),
pytest.param(None, id="no-request-kwargs"),
],
@ -6044,7 +6042,8 @@ class TestContextAwareClassifier:
turn = (
"We run a multi-region gateway and last night the eu-west pod returned 502s on the "
"streaming path only, for thirty minutes, while non-streaming stayed healthy the whole "
"window and the cooldown map was mid-failover. " + "Filler sentence to push past the cap. " * 4
"window and the cooldown map was mid-failover. "
+ "Filler sentence to push past the cap. " * 4
+ "Now rewrite the streaming retry path and prove it cannot livelock."
)
@ -8889,3 +8888,210 @@ async def test_session_pin_survives_json_list_round_trip(mock_router_instance):
assert response.model == "shared"
assert response.litellm_params == {"reasoning_effort": "low"}
assert cache.async_set_cache.call_args.kwargs["value"] == {"model": "shared", "tier": "SIMPLE"}
HEURISTIC_FIRST_TIERS: dict[str, str] = {
"SIMPLE": "gpt-4o-mini",
"MEDIUM": "gpt-4o",
"COMPLEX": "claude-sonnet-4-20250514",
"REASONING": "o1-preview",
}
# The scorer maps a weighted score to a tier against these, and PR #37910 is retuning the shipped
# defaults, so every heuristic_first test pins them rather than inheriting DEFAULT_TIER_BOUNDARIES.
HEURISTIC_FIRST_BOUNDARIES: dict[str, float] = {
"simple_medium": 0.15,
"medium_complex": 0.35,
"complex_reasoning": 0.60,
}
# Scores 0.0 with an empty signals tuple: no dimension fires, so the scorer has no opinion and the
# score-to-tier mapping lands SIMPLE purely by default. This is the population the permutation
# control measured at ~zero information, and the prompt that must always escalate.
NO_SIGNAL_PROMPT = (
"A distributed ledger must guarantee linearizability across five regions while tolerating one "
"region partition and bounded clock skew. Derive the minimum quorum configuration and prove why "
"a smaller quorum violates linearizability."
)
def _heuristic_first_router(mock_router_instance, **config_overrides):
config = {
"tiers": dict(HEURISTIC_FIRST_TIERS),
"tier_boundaries": dict(HEURISTIC_FIRST_BOUNDARIES),
"classifier_type": "heuristic_first",
"heuristic_first_max_tier": "SIMPLE",
"classifier_llm_config": {"model": "haiku-classifier", "timeout_ms": 400},
**config_overrides,
}
return ComplexityRouter(
model_name="test-complexity-router",
litellm_router_instance=mock_router_instance,
complexity_router_config=config,
)
class TestHeuristicFirstConfig:
"""Config validation for classifier_type='heuristic_first'."""
@pytest.mark.parametrize(
"overrides, expected",
[
({"classifier_llm_config": None}, "classifier_llm_config is required"),
({"heuristic_first_max_tier": None}, "heuristic_first_max_tier is required"),
({"heuristic_first_max_tier": "REASONING"}, "is the highest tier"),
({"heuristic_first_max_tier": "NOPE"}, "is not an active tier"),
(
{
"tiers": {"SIMPLE": "gpt-4o-mini", "COMPLEX": "c", "REASONING": "r"},
"heuristic_first_max_tier": "MEDIUM",
},
"has no model configured in tiers",
),
],
)
def test_rejects_incoherent_config(self, overrides, expected):
config = {
"tiers": dict(HEURISTIC_FIRST_TIERS),
"classifier_type": "heuristic_first",
"heuristic_first_max_tier": "SIMPLE",
"classifier_llm_config": {"model": "haiku-classifier"},
**overrides,
}
with pytest.raises(ValidationError, match=expected):
ComplexityRouterConfig(**config)
@pytest.mark.parametrize("classifier_type", ["heuristic", "llm", "custom"])
def test_threshold_rejected_on_every_other_classifier_type(self, classifier_type):
"""A threshold on a router with no heuristic gate is a silent no-op, so it is refused
rather than accepted and ignored."""
config: dict[str, object] = {
"tiers": dict(HEURISTIC_FIRST_TIERS),
"classifier_type": classifier_type,
"heuristic_first_max_tier": "SIMPLE",
}
if classifier_type == "llm":
config["classifier_llm_config"] = {"model": "haiku-classifier"}
if classifier_type == "custom":
config["classifier_plugin"] = _FixedTierClassifier("SIMPLE")
with pytest.raises(ValidationError, match="heuristic_first_max_tier is set but classifier_type"):
ComplexityRouterConfig(**config)
def test_custom_tier_set_is_rejected(self):
"""The scorer only emits the four built-in tiers, so it cannot gate a replaced tier set."""
with pytest.raises(ValidationError, match="tier_definitions requires classifier_type"):
ComplexityRouterConfig(
classifier_type="heuristic_first",
heuristic_first_max_tier="lo",
classifier_llm_config={"model": "haiku-classifier"},
tier_definitions=[{"name": "lo", "description": "x"}, {"name": "hi", "description": "y"}],
tiers={"lo": "gpt-4o-mini", "hi": "gpt-4o"},
)
def test_classifier_model_is_a_dependency(self):
"""uses_llm_classifier is what tells the health graph and the routing-test authorizer that
the classifier model is really called, so heuristic_first must answer True."""
config = ComplexityRouterConfig(
tiers=dict(HEURISTIC_FIRST_TIERS),
classifier_type="heuristic_first",
heuristic_first_max_tier="SIMPLE",
classifier_llm_config={"model": "haiku-classifier"},
)
assert config.uses_llm_classifier is True
assert ComplexityRouterConfig(tiers=dict(HEURISTIC_FIRST_TIERS)).uses_llm_classifier is False
class TestHeuristicFirst:
"""Behavior of the heuristic-first chain: when the classifier call is skipped, and when it is not."""
@pytest.mark.asyncio
async def test_signalled_cheap_prompt_short_circuits(self, mock_router_instance):
"""A prompt the scorer actually placed at or below the threshold must not reach the LLM."""
mock_router_instance.acompletion = AsyncMock()
router = _heuristic_first_router(mock_router_instance)
outcome = await router.aclassify("thanks so much, appreciate it")
mock_router_instance.acompletion.assert_not_called()
assert outcome.tier == ComplexityTier.SIMPLE
assert outcome.cause == "heuristic_first_short_circuit"
assert outcome.score is not None
assert outcome.signals
assert outcome.classifier_cost is None
@pytest.mark.asyncio
async def test_no_signal_prompt_escalates_even_though_it_scores_simple(self, mock_router_instance):
"""The core guard. This prompt scores 0.0 and the mapping calls it SIMPLE, which is at the
threshold, so a bare tier comparison would short-circuit it to the cheapest model. No
dimension fired, so the scorer has no opinion and the classifier must decide."""
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "COMPLEX"}'))
router = _heuristic_first_router(mock_router_instance)
tier, score, signals, _cause = router._score_and_classify(NO_SIGNAL_PROMPT)
assert (tier, score, signals) == (ComplexityTier.SIMPLE, 0.0, ())
outcome = await router.aclassify(NO_SIGNAL_PROMPT)
mock_router_instance.acompletion.assert_awaited_once()
assert outcome.tier == ComplexityTier.COMPLEX
assert outcome.cause == "llm_classifier"
@pytest.mark.asyncio
async def test_signalled_prompt_above_threshold_escalates(self, mock_router_instance):
"""The scorer had an opinion, but it was above the threshold, so the classifier decides."""
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "REASONING"}'))
router = _heuristic_first_router(mock_router_instance)
tier, _score, signals, _cause = router._score_and_classify("write a python function to reverse a string")
assert tier == ComplexityTier.MEDIUM and signals
outcome = await router.aclassify("write a python function to reverse a string")
mock_router_instance.acompletion.assert_awaited_once()
assert outcome.tier == ComplexityTier.REASONING
assert outcome.cause == "llm_classifier"
@pytest.mark.asyncio
async def test_raising_threshold_short_circuits_what_it_previously_escalated(self, mock_router_instance):
"""The threshold is the knob: the same signalled MEDIUM prompt escalates at SIMPLE and
short-circuits at MEDIUM."""
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "REASONING"}'))
router = _heuristic_first_router(mock_router_instance, heuristic_first_max_tier="MEDIUM")
outcome = await router.aclassify("write a python function to reverse a string")
mock_router_instance.acompletion.assert_not_called()
assert outcome.tier == ComplexityTier.MEDIUM
assert outcome.cause == "heuristic_first_short_circuit"
@pytest.mark.asyncio
async def test_reasoning_override_never_short_circuits(self, mock_router_instance):
"""A reasoning-override prompt lands REASONING, which outranks every legal threshold, so it
always reaches the classifier."""
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "MEDIUM"}'))
router = _heuristic_first_router(mock_router_instance, heuristic_first_max_tier="COMPLEX")
outcome = await router.aclassify(
"think step by step and analyze the tradeoffs, then reason through the consequences carefully"
)
mock_router_instance.acompletion.assert_awaited_once()
assert outcome.cause == "llm_classifier"
@pytest.mark.asyncio
async def test_classifier_failure_falls_back_to_the_scorer(self, mock_router_instance):
"""An escalated request whose classifier call fails still gets the scorer's own verdict,
the same way classifier_type='llm' does, rather than erroring out."""
mock_router_instance.acompletion = AsyncMock(side_effect=RuntimeError("classifier exploded"))
router = _heuristic_first_router(mock_router_instance)
expected_tier, expected_score, expected_signals, _cause = router._score_and_classify(NO_SIGNAL_PROMPT)
outcome = await router.aclassify(NO_SIGNAL_PROMPT)
assert outcome.tier == expected_tier
assert outcome.score == expected_score
assert outcome.signals == expected_signals
assert outcome.cause == "heuristic_scorer"
@pytest.mark.asyncio
async def test_classifier_failure_honors_default_model_fallback(self, mock_router_instance):
"""classifier_fallback='default_model' still wins over the heuristic outcome, same as it
does for classifier_type='llm'."""
mock_router_instance.acompletion = AsyncMock(side_effect=RuntimeError("classifier exploded"))
router = _heuristic_first_router(
mock_router_instance, classifier_fallback="default_model", default_model="gpt-4o"
)
outcome = await router.aclassify(NO_SIGNAL_PROMPT)
assert outcome.cause == "default_model_fallback"

8
ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts
View file

@ -54,8 +54,14 @@ const asStringArray = (value: unknown): string[] =>
const dedupe = (models: string[]): string[] => Array.from(new Set(models));
const COMPLEXITY_TYPE_LABELS: Record<string, string> = {
llm: "LLM Classifier",
heuristic_first: "Heuristic first",
custom: "Custom classifier",
};
export const complexityTypeLabel = (config: Record<string, unknown>): string =>
config.classifier_type === "llm" ? "LLM Classifier" : "Heuristic";
(typeof config.classifier_type === "string" && COMPLEXITY_TYPE_LABELS[config.classifier_type]) || "Heuristic";
interface Presentation {
typeLabel: string;

89
ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
View file

@ -27,6 +27,9 @@ import {
CLASSIFICATION_RUBRIC_KEYS,
ClassificationRubric,
effectiveTierLabel,
usesLlmClassifier,
DEFAULT_HEURISTIC_FIRST_MAX_TIER,
HEURISTIC_FIRST_MAX_TIER_KEYS,
} from "./ComplexityRouterConfig";
const DEFAULT_SCORING_EXPLANATION =
@ -49,7 +52,7 @@ const CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK =
*/
const scoringExplanation = (value: ComplexityRouterConfigValue): string => {
const usesCustomPrompt =
value.classifier_type === "llm" && Boolean(value.classifier_llm_config?.system_prompt?.trim());
usesLlmClassifier(value.classifier_type) && Boolean(value.classifier_llm_config?.system_prompt?.trim());
if (!usesCustomPrompt) return DEFAULT_SCORING_EXPLANATION;
return value.classifier_fallback === "default_model"
? CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK
@ -148,7 +151,7 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
}) => {
const hasDefaultModel = Boolean(defaultModel);
const classifierModelMissing =
showValidationErrors && value.classifier_type === "llm" && !value.classifier_llm_config?.model;
showValidationErrors && usesLlmClassifier(value.classifier_type) && !value.classifier_llm_config?.model;
const usesCustomPrompt = Boolean(value.classifier_llm_config?.system_prompt?.trim());
const contextBudget = value.classifier_context_budget_chars ?? DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS;
const contextBudgetQuotesNothing = contextBudget > 0 && contextBudget < MIN_QUOTED_CONTEXT_TURN_CHARS;
@ -158,29 +161,35 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
const nextValue: ComplexityRouterConfigValue = {
...value,
classifier_type: classifierType,
classifier_llm_config:
classifierType === "llm"
? value.classifier_llm_config ?? {
model: "",
timeout_ms: DEFAULT_CLASSIFIER_TIMEOUT_MS,
classification_rubric: NEW_CLASSIFIER_CLASSIFICATION_RUBRIC,
}
classifier_llm_config: usesLlmClassifier(classifierType)
? value.classifier_llm_config ?? {
model: "",
timeout_ms: DEFAULT_CLASSIFIER_TIMEOUT_MS,
classification_rubric: NEW_CLASSIFIER_CLASSIFICATION_RUBRIC,
}
: undefined,
classifier_context_window_size: usesLlmClassifier(classifierType)
? value.classifier_context_window_size ?? DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE
: undefined,
classifier_context_budget_chars: usesLlmClassifier(classifierType)
? value.classifier_context_budget_chars ?? DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS
: undefined,
classifier_context_include_assistant_turns: usesLlmClassifier(classifierType)
? value.classifier_context_include_assistant_turns
: undefined,
classifier_fallback: usesLlmClassifier(classifierType) ? value.classifier_fallback : undefined,
heuristic_first_max_tier:
classifierType === "heuristic_first"
? value.heuristic_first_max_tier ?? DEFAULT_HEURISTIC_FIRST_MAX_TIER
: undefined,
classifier_context_window_size:
classifierType === "llm"
? value.classifier_context_window_size ?? DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE
: undefined,
classifier_context_budget_chars:
classifierType === "llm"
? value.classifier_context_budget_chars ?? DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS
: undefined,
classifier_context_include_assistant_turns:
classifierType === "llm" ? value.classifier_context_include_assistant_turns : undefined,
classifier_fallback: classifierType === "llm" ? value.classifier_fallback : undefined,
};
onChange(nextValue);
};
const handleHeuristicFirstMaxTierChange = (tier: string) => {
onChange({ ...value, heuristic_first_max_tier: tier });
};
const handleClassifierModelChange = (model: string) => {
onChange({
...value,
@ -265,7 +274,7 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
<span>
<strong className="font-semibold">Heuristic</strong>{" "}
<span className="text-muted-foreground">
(default) — rule-based scoring, no API calls, &lt;1ms latency
(default), rule-based scoring with no API calls and &lt;1ms latency
</span>
</span>
</Label>
@ -273,13 +282,47 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
<RadioGroupItem value="llm" className="mt-0.5" />
<span>
<strong className="font-semibold">LLM Classifier</strong>{" "}
<span className="text-muted-foreground">— use a model to decide the tier (e.g. a small/fast model)</span>
<span className="text-muted-foreground">calls a model to decide the tier (e.g. a small/fast model)</span>
</span>
</Label>
<Label className="items-start font-normal leading-normal">
<RadioGroupItem value="heuristic_first" className="mt-0.5" />
<span>
<strong className="font-semibold">Heuristic first</strong>{" "}
<span className="text-muted-foreground">
scores locally, and only pays for the classifier when the score does not confidently land a cheap tier
</span>
</span>
</Label>
</div>
</RadioGroup>
{value.classifier_type === "llm" && (
{value.classifier_type === "heuristic_first" && (
<div className="mt-4 space-y-2">
<strong className="block font-semibold">Decide locally up to</strong>
<Select
value={value.heuristic_first_max_tier}
onValueChange={(tier: unknown) => handleHeuristicFirstMaxTierChange(tier as string)}
>
<SelectTrigger className="w-full">
<SelectValue />
</SelectTrigger>
<SelectContent>
{HEURISTIC_FIRST_MAX_TIER_KEYS.map((tier) => (
<SelectItem key={tier} value={tier}>
{effectiveTierLabel(tier, value.tier_labels)}
</SelectItem>
))}
</SelectContent>
</Select>
<p className="text-sm text-muted-foreground">
A request the scorer places at or below this tier routes there without a classifier call. Anything the
scorer places higher, and anything it found no signal for at all, goes to the classifier instead
</p>
</div>
)}
{usesLlmClassifier(value.classifier_type) && (
<div className="mt-4 space-y-3">
<div>
<strong className="block mb-1 font-semibold">Classifier Model</strong>

27
ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
View file

@ -95,7 +95,15 @@ export interface ClassifierLLMConfig {
system_prompt?: string;
}
export type ClassifierType = "heuristic" | "llm";
export type ClassifierType = "heuristic" | "llm" | "heuristic_first";
/**
* Whether this router can call classifier_llm_config.model. Mirrors the backend's
* ComplexityRouterConfig.uses_llm_classifier, and is the single gate for every classifier-only
* control and payload key, so a new chaining type cannot strip knobs the operator set.
*/
export const usesLlmClassifier = (classifierType: ClassifierType): boolean =>
classifierType === "llm" || classifierType === "heuristic_first";
export type ClassifierFallback = "heuristic" | "default_model";
@ -113,13 +121,14 @@ export type HeuristicScoringRole = "decides" | "fallback_only" | "never";
/**
* Whether the heuristic scorer runs on this router at all, which is what gates its knobs. An LLM
* classifier still falls back to the scorer unless the fallback is the default model, so the gate cannot be
* a plain classifier_type check.
* a plain classifier_type check. Under heuristic_first the scorer runs first on every request and
* decides outright whenever it lands at or below the threshold.
*/
export const heuristicScoringRoleFor = (
classifierType: ClassifierType,
classifierFallback: ClassifierFallback | undefined,
): HeuristicScoringRole => {
if (classifierType === "heuristic") return "decides";
if (classifierType === "heuristic" || classifierType === "heuristic_first") return "decides";
return (classifierFallback ?? DEFAULT_CLASSIFIER_FALLBACK) === "heuristic" ? "fallback_only" : "never";
};
@ -142,6 +151,8 @@ export interface ComplexityRouterConfigValue {
classifier_context_per_turn_chars?: number;
classifier_context_include_assistant_turns?: boolean;
classifier_fallback?: ClassifierFallback;
/** Highest tier the scorer may decide alone under heuristic_first. Required by that type, rejected by the others. */
heuristic_first_max_tier?: string;
session_affinity?: boolean;
deployment_affinity?: boolean;
/** Tier floor for coding-agent plan-mode requests. Unset means detection is off, matching the backend. */
@ -223,6 +234,14 @@ export const TIER_KEYS = Object.keys(TIER_DESCRIPTIONS) as Array<keyof Complexit
export const effectiveTierLabel = (tier: keyof ComplexityTiers, tierLabels: ComplexityTierLabels | undefined): string =>
tierLabels?.[tier]?.trim() || TIER_DESCRIPTIONS[tier].label;
export const DEFAULT_HEURISTIC_FIRST_MAX_TIER = "SIMPLE";
/**
* Tiers the heuristic_first threshold may name. The top tier is excluded because it would short
* circuit every request and leave the classifier unreachable, which the backend rejects.
*/
export const HEURISTIC_FIRST_MAX_TIER_KEYS = TIER_KEYS.slice(0, -1);
const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
modelInfo,
value,
@ -314,7 +333,7 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
<span className="block mb-4 text-xs text-muted-foreground">
Rename a tier to use your own vocabulary in the dashboard and your spend logs. Renaming doesn&apos;t change how
requests are classified, and callers never see these names.
{value.classifier_type === "llm" &&
{usesLlmClassifier(value.classifier_type) &&
" Your classifier model reads these names, so clearer ones can sharpen its choices."}
</span>

37
ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx
View file

@ -5,6 +5,8 @@ import AddAutoRouterTab from "./add_auto_router_tab";
import { toast } from "@/lib/toast";
import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit";
import { getMissingTiersError } from "./build_complexity_router_config";
import { getSubmitBlockedReason } from "./add_auto_router_tab";
import { buildModelAvailability } from "@/lib/autorouter_presets";
import { testAutoRouterRouting } from "../networking";
import { ModelGroup } from "@/components/llm_calls/fetch_models";
import { getAllPresets, getPresetByKey, getRequiredModelsInPreset } from "@/lib/autorouter_presets";
@ -864,3 +866,38 @@ describe("AddAutoRouterTab", () => {
});
});
});
describe("getSubmitBlockedReason", () => {
const tiers = {
SIMPLE: ["gpt-4o-mini"],
MEDIUM: ["gpt-4o-mini"],
COMPLEX: ["gpt-4o-mini"],
REASONING: ["gpt-4o-mini"],
};
const availability = buildModelAvailability(["gpt-4o-mini"], []);
const referenced = {
tiers,
classifierType: "heuristic" as const,
classifierLlmConfig: undefined,
semanticMatchingEnabled: false,
embeddingModel: undefined,
defaultModel: undefined,
};
it("lets a complete heuristic router through", () => {
expect(getSubmitBlockedReason({ tiers, classifier_type: "heuristic" }, [], referenced, availability)).toBeNull();
});
it("blocks an LLM classifier with no model, which the button previously left enabled", () => {
expect(getSubmitBlockedReason({ tiers, classifier_type: "llm" }, [], referenced, availability)).toContain(
"Please select a classifier model",
);
});
it("blocks a keyword rule aimed at a tier this router does not have", () => {
const rules = [{ id: "r1", keywords: ["audit"], tier: "AUDIT" }];
expect(getSubmitBlockedReason({ tiers, classifier_type: "heuristic" }, rules, referenced, availability)).toContain(
"no longer has",
);
});
});

62
ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
View file

@ -34,6 +34,7 @@ import {
BuildComplexityRouterConfigParams,
buildComplexityRouterConfig,
getKeywordTierRulesError,
getClassifierModelError,
getMissingTiersError,
getPlanModeTierError,
getSemanticConfigError,
@ -116,7 +117,7 @@ const tierConfigSummary = (config: ComplexityRouterConfigValue): string => {
// itself and to say what is missing, so the two can never give different answers. Checks the
// config actually being built, not which preset (if any) it came from: a preset only ever
// prefills once (handlePresetChange), and everything after that is edited exactly like Custom.
const getSubmitBlockedReason = (
export const getSubmitBlockedReason = (
config: ComplexityRouterConfigValue,
keywordTierRules: KeywordTierRule[],
referencedModelsParams: Parameters<typeof getReferencedModelsError>[0],
@ -125,7 +126,8 @@ const getSubmitBlockedReason = (
getMissingTiersError(activeTierRows(config)) ??
getTierLabelsError(config.tier_labels) ??
getPlanModeTierError(config.plan_mode_min_tier, activeTierRows(config)) ??
getKeywordTierRulesError(keywordTierRules) ??
getKeywordTierRulesError(keywordTierRules, activeTierRows(config)) ??
getClassifierModelError(config) ??
getReferencedModelsError(referencedModelsParams, availability);
const autoRouterSchema = (requiresTeamScope: boolean) =>
@ -342,6 +344,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
tiers: complexityRouterConfig.tiers,
defaultModel: complexityRouterConfig.default_model,
planModeMinTier: complexityRouterConfig.plan_mode_min_tier,
heuristicFirstMaxTier: complexityRouterConfig.heuristic_first_max_tier,
tierLabels: complexityRouterConfig.tier_labels,
classifierType: complexityRouterConfig.classifier_type,
classifierLlmConfig: complexityRouterConfig.classifier_llm_config,
@ -370,50 +373,21 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
};
const submitRecommendedRouter = async (name: string) => {
const { tiers, tierLabels, classifierType, classifierLlmConfig } = complexityRouterConfigParams;
const { tiers } = complexityRouterConfigParams;
const missingTiersError = getMissingTiersError(activeTierRows(complexityRouterConfig));
if (missingTiersError) {
// The one answer the submit button reads, so a disabled button and a refused submit cannot
// disagree about why. The handler needs it in its own right: the form fires this on Enter
// regardless of the button's disabled state.
const blockedReason =
getSubmitBlockedReason(
complexityRouterConfig,
keywordTierRules,
referencedModelsParams,
groupsOnlyAvailability,
) ?? getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules });
if (blockedReason) {
setShowValidationErrors(true);
toast.fromError(missingTiersError);
return;
}
const tierLabelsError = getTierLabelsError(tierLabels);
if (tierLabelsError) {
setShowValidationErrors(true);
toast.fromError(tierLabelsError);
return;
}
if (classifierType === "llm" && !classifierLlmConfig?.model) {
setShowValidationErrors(true);
toast.fromError("Please select a classifier model, or switch back to Heuristic");
return;
}
const keywordRulesError = getKeywordTierRulesError(keywordTierRules);
if (keywordRulesError) {
setShowValidationErrors(true);
toast.fromError(keywordRulesError);
return;
}
const semanticError = getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules });
if (semanticError) {
setShowValidationErrors(true);
toast.fromError(semanticError);
return;
}
// submitBlockedReason already disables the button for this, but the form's submit handler (wired to
// this same function) fires on Enter regardless of the button's disabled state - without this check,
// Enter in the name field could still create a router referencing a model that disappeared from
// availableModelSet after the tiers were filled in.
const referencedModelsError = getReferencedModelsError(referencedModelsParams, groupsOnlyAvailability);
if (referencedModelsError) {
setShowValidationErrors(true);
toast.fromError(referencedModelsError);
toast.fromError(blockedReason);
return;
}

101
ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
View file

@ -3,6 +3,7 @@ import {
getPlanModeTierError,
normalizeClassifierLlmConfig,
getKeywordTierRulesError,
getClassifierModelError,
getMissingTiersError,
getSemanticConfigError,
getTierLabelsError,
@ -334,21 +335,24 @@ describe("getSemanticConfigError", () => {
describe("getKeywordTierRulesError", () => {
it("returns null when every rule carries a keyword", () => {
expect(
getKeywordTierRulesError([
{ id: "r1", keywords: ["invoice"], tier: "MEDIUM" },
{ id: "r2", keywords: ["deploy to k8s"], tier: "REASONING" },
]),
getKeywordTierRulesError(
[
{ id: "r1", keywords: ["invoice"], tier: "MEDIUM" },
{ id: "r2", keywords: ["deploy to k8s"], tier: "REASONING" },
],
activeTierRows({ tiers }),
),
).toBeNull();
});
it("returns null when there are no rules at all, since the section is optional", () => {
expect(getKeywordTierRulesError([])).toBeNull();
expect(getKeywordTierRulesError([], activeTierRows({ tiers }))).toBeNull();
});
// The whole point of the ticket: the semantic toggle is off by default, and an unfilled row
// used to be discarded silently on an otherwise successful create.
it("rejects a row left empty while semantic matching is off", () => {
expect(getKeywordTierRulesError([{ id: "r1", keywords: [], tier: "COMPLEX" }])).toBe(
expect(getKeywordTierRulesError([{ id: "r1", keywords: [], tier: "COMPLEX" }], activeTierRows({ tiers }))).toBe(
"Add at least one keyword to keyword rule(s): 1",
);
});
@ -357,7 +361,9 @@ describe("getKeywordTierRulesError", () => {
["whitespace only", [" "]],
["blank strings, as an unfilled row between filled ones leaves behind", ["", " ", ""]],
])("treats %s as empty rather than as a keyword", (_label, keywords) => {
expect(getKeywordTierRulesError([{ id: "r1", keywords, tier: "SIMPLE" }])).toMatch(/keyword rule\(s\): 1/);
expect(getKeywordTierRulesError([{ id: "r1", keywords, tier: "SIMPLE" }], activeTierRows({ tiers }))).toMatch(
/keyword rule\(s\): 1/,
);
});
// Row numbers have to survive rules that are fine, or the message points at the wrong input.
@ -373,7 +379,9 @@ describe("getKeywordTierRulesError", () => {
});
it("keeps a keyword whose surrounding whitespace is the only thing trimmed", () => {
expect(getKeywordTierRulesError([{ id: "r1", keywords: [" invoice "], tier: "MEDIUM" }])).toBeNull();
expect(
getKeywordTierRulesError([{ id: "r1", keywords: [" invoice "], tier: "MEDIUM" }], activeTierRows({ tiers })),
).toBeNull();
});
});
@ -674,3 +682,80 @@ describe("buildComplexityRouterConfig tier model params", () => {
});
});
});
describe("getClassifierModelError", () => {
it("stays quiet for a heuristic router, which needs no classifier model", () => {
expect(getClassifierModelError({ classifier_type: "heuristic" })).toBeNull();
});
it("blocks an LLM classifier with no model, which the router cannot start without", () => {
expect(getClassifierModelError({ classifier_type: "llm" })).toBe(
"Please select a classifier model, or switch back to Heuristic",
);
});
it("stays quiet once a model is chosen", () => {
expect(
getClassifierModelError({ classifier_type: "llm", classifier_llm_config: { model: "m", timeout_ms: 3000 } }),
).toBeNull();
});
});
describe("getKeywordTierRulesError orphaned tiers", () => {
const rows = activeTierRows({ tiers });
it("accepts a rule naming a tier the router has", () => {
expect(getKeywordTierRulesError([{ id: "r1", keywords: ["k"], tier: "COMPLEX" }], rows)).toBeNull();
});
it("names the rule pointing at a tier this router does not have", () => {
expect(getKeywordTierRulesError([{ id: "r1", keywords: ["k"], tier: "AUDIT" }], rows)).toBe(
"Keyword rule(s) 1 route to a tier this router no longer has",
);
});
it("rejects a differently cased tier, because _validate_keyword_rule_tiers matches exactly", () => {
expect(getKeywordTierRulesError([{ id: "r1", keywords: ["k"], tier: "complex" }], rows)).toBe(
"Keyword rule(s) 1 route to a tier this router no longer has",
);
});
it("reports an empty keyword row before an orphaned tier, since that is the nearer problem", () => {
expect(getKeywordTierRulesError([{ id: "r1", keywords: [], tier: "AUDIT" }], rows)).toContain(
"Add at least one keyword",
);
});
});
describe("heuristic_first", () => {
const heuristicFirstParams: BuildComplexityRouterConfigParams = {
...baseParams,
classifierType: "heuristic_first",
heuristicFirstMaxTier: "SIMPLE",
classifierLlmConfig: { model: "gpt-4o-mini", timeout_ms: 3000 },
classifierContextWindowSize: 5,
classifierContextBudgetChars: 4000,
classifierFallback: "default_model",
};
it("emits heuristic_first_max_tier", () => {
const config = buildComplexityRouterConfig(heuristicFirstParams);
expect(config.classifier_type).toBe("heuristic_first");
expect(config.heuristic_first_max_tier).toBe("SIMPLE");
});
it("keeps every classifier key the operator set, since heuristic_first still calls the classifier", () => {
const config = buildComplexityRouterConfig(heuristicFirstParams);
expect(config.classifier_llm_config).toEqual({ model: "gpt-4o-mini", timeout_ms: 3000 });
expect(config.classifier_context_window_size).toBe(5);
expect(config.classifier_context_budget_chars).toBe(4000);
expect(config.classifier_fallback).toBe("default_model");
});
it("omits heuristic_first_max_tier on every other classifier type, which the backend rejects it on", () => {
for (const classifierType of ["heuristic", "llm"] as const) {
const config = buildComplexityRouterConfig({ ...heuristicFirstParams, classifierType });
expect(config.heuristic_first_max_tier).toBeUndefined();
}
});
});

42
ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
View file

@ -9,6 +9,7 @@ import {
ClassifierLLMConfig,
ClassifierType,
ComplexityTierLabels,
ComplexityRouterConfigValue,
ComplexityTiers,
DimensionWeights,
TIER_KEYS,
@ -17,6 +18,7 @@ import {
TokenThresholds,
effectiveTierLabel,
heuristicScoringRoleFor,
usesLlmClassifier,
} from "./ComplexityRouterConfig";
/**
@ -85,6 +87,7 @@ export interface BuildComplexityRouterConfigParams {
classifierContextBudgetChars: number | undefined;
classifierContextIncludeAssistantTurns: boolean | undefined;
classifierFallback: ClassifierFallback | undefined;
heuristicFirstMaxTier: string | undefined;
sessionAffinity: boolean;
deploymentAffinity: boolean;
customTechnicalKeywords: string[];
@ -117,6 +120,7 @@ export interface ComplexityRouterConfigPayload {
classifier_context_per_turn_chars?: number;
classifier_context_include_assistant_turns?: boolean;
classifier_fallback?: ClassifierFallback;
heuristic_first_max_tier?: string;
session_affinity: boolean;
deployment_affinity: boolean;
custom_technical_keywords?: string[];
@ -187,12 +191,30 @@ export const getPlanModeTierError = (planModeMinTier: string | undefined, rows:
return `The plan-mode minimum tier (${floor ? activeTierName(floor) : planModeMinTier}) has no models. Add one or turn the override off.`;
};
export const getKeywordTierRulesError = (keywordTierRules: KeywordTierRule[]): string | null => {
// The tier is a free string since #37413, and _validate_keyword_rule_tiers matches it EXACTLY, so a
// rule naming a tier this router does not have is a raw 400 unless the gate catches it first.
export const getKeywordTierRulesError = (
keywordTierRules: KeywordTierRule[],
rows: readonly TierRow[],
): string | null => {
const emptyRows = emptyKeywordTierRuleIndexes(keywordTierRules);
if (emptyRows.length === 0) return null;
return `Add at least one keyword to keyword rule(s): ${emptyRows.map((index) => index + 1).join(", ")}`;
if (emptyRows.length > 0)
return `Add at least one keyword to keyword rule(s): ${emptyRows.map((index) => index + 1).join(", ")}`;
const names = rows.map(activeTierName);
const orphaned = keywordTierRules.flatMap((rule, index) => (names.includes(rule.tier) ? [] : [index + 1]));
if (orphaned.length === 0) return null;
return `Keyword rule(s) ${orphaned.join(", ")} route to a tier this router no longer has`;
};
// The submit gate and the submit handler both read this, so a disabled button and a refused submit
// cannot disagree about why.
export const getClassifierModelError = (
config: Pick<ComplexityRouterConfigValue, "classifier_type" | "classifier_llm_config">,
): string | null =>
usesLlmClassifier(config.classifier_type) && !config.classifier_llm_config?.model
? "Please select a classifier model, or switch back to Heuristic"
: null;
export const getSemanticConfigError = ({
semanticMatchingEnabled,
embeddingModel,
@ -217,6 +239,7 @@ export const buildComplexityRouterConfig = ({
classifierContextBudgetChars,
classifierContextIncludeAssistantTurns,
classifierFallback,
heuristicFirstMaxTier,
sessionAffinity,
deploymentAffinity,
customTechnicalKeywords,
@ -257,18 +280,21 @@ export const buildComplexityRouterConfig = ({
...(planModeMinTier?.trim() && { plan_mode_min_tier: planModeMinTier }),
...(cleanedTierLabels && { tier_labels: cleanedTierLabels }),
classifier_type: classifierType,
...(classifierType === "llm" &&
...(usesLlmClassifier(classifierType) &&
classifierLlmConfig && { classifier_llm_config: normalizeClassifierLlmConfig(classifierLlmConfig) }),
...(classifierType === "llm" && classifierFallback !== undefined && { classifier_fallback: classifierFallback }),
...(classifierType === "llm" &&
...(usesLlmClassifier(classifierType) &&
classifierFallback !== undefined && { classifier_fallback: classifierFallback }),
...(classifierType === "heuristic_first" &&
heuristicFirstMaxTier?.trim() && { heuristic_first_max_tier: heuristicFirstMaxTier }),
...(usesLlmClassifier(classifierType) &&
classifierContextWindowSize !== undefined && {
classifier_context_window_size: classifierContextWindowSize,
}),
...(classifierType === "llm" &&
...(usesLlmClassifier(classifierType) &&
classifierContextBudgetChars !== undefined && {
classifier_context_budget_chars: classifierContextBudgetChars,
}),
...(classifierType === "llm" &&
...(usesLlmClassifier(classifierType) &&
classifierContextIncludeAssistantTurns !== undefined && {
classifier_context_include_assistant_turns: classifierContextIncludeAssistantTurns,
}),

52
ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts
View file

@ -1,6 +1,11 @@
import { describe, expect, it } from "vitest";
import { buildUpdatedComplexityRouterConfig, type KeywordMatchingState } from "./edit_auto_router_modal";
import {
MANAGED_COMPLEXITY_ROUTER_KEYS,
buildUpdatedComplexityRouterConfig,
hydrateComplexityRouterConfig,
type KeywordMatchingState,
} from "./edit_auto_router_modal";
const STORED = {
tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] },
@ -440,3 +445,48 @@ describe("buildUpdatedComplexityRouterConfig tier model params", () => {
expect(result).not.toHaveProperty("tier_model_configs");
});
});
describe("managed keys survive an untouched open-and-save", () => {
// Every managed key is rewritten from form state on save, so one the hydrator forgets is silently
// dropped from the saved config. This config sets each managed key to a value that actually
// applies, so an untouched open-and-save must return every one of them.
const STORED_ALL_MANAGED: Record<string, unknown> = {
tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: ["gpt-4o"], COMPLEX: ["opus"], REASONING: ["o1"] },
tier_model_configs: { REASONING: [{ model_name: "o1", litellm_params: { reasoning_effort: "high" } }] },
default_model: "gpt-4o",
plan_mode_min_tier: "COMPLEX",
tier_labels: { SIMPLE: "Cheap" },
classifier_type: "heuristic_first",
heuristic_first_max_tier: "SIMPLE",
classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000 },
classifier_context_window_size: 5,
classifier_context_budget_chars: 4000,
classifier_context_include_assistant_turns: true,
classifier_fallback: "default_model",
session_affinity: true,
deployment_affinity: false,
adaptive: true,
adaptive_weights: { quality: 0.4, cost: 0.6 },
tier_distance_penalty: 0.25,
adaptive_eligible: "all",
return_raw_model_name: true,
tier_boundaries: { simple_medium: 0.2, medium_complex: 0.4, complex_reasoning: 0.7 },
token_thresholds: { simple: 20, complex: 500 },
dimension_weights: { tokenCount: 0.1 },
reasoning_override_min_score: 0.3,
};
it("carries every managed key through hydrate then save", () => {
const hydrated = hydrateComplexityRouterConfig(STORED_ALL_MANAGED, undefined);
const saved = buildUpdatedComplexityRouterConfig(STORED_ALL_MANAGED, hydrated);
const dropped = [...MANAGED_COMPLEXITY_ROUTER_KEYS].filter((key) => saved[key] === undefined);
expect(dropped).toEqual([]);
});
it("round-trips the heuristic_first threshold, which save requires and the backend rejects without", () => {
const hydrated = hydrateComplexityRouterConfig(STORED_ALL_MANAGED, undefined);
expect(hydrated.heuristic_first_max_tier).toBe("SIMPLE");
expect(buildUpdatedComplexityRouterConfig(STORED_ALL_MANAGED, hydrated).heuristic_first_max_tier).toBe("SIMPLE");
});
});

173
ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
View file

@ -20,6 +20,7 @@ import { isComplexityRouter } from "../add_model/auto_router_strategies";
import {
type BuildComplexityRouterConfigParams,
buildComplexityRouterConfig,
getClassifierModelError,
getKeywordTierRulesError,
getSemanticConfigError,
getPlanModeTierError,
@ -36,6 +37,10 @@ import {
hydrateTokenThresholds,
} from "../add_model/heuristic_scoring_knobs";
import ComplexityRouterConfig, {
AdaptiveEligible,
AdaptiveRouterWeights,
ClassifierLLMConfig,
ClassifierType,
ComplexityRouterConfigValue,
ComplexityTiers,
DEFAULT_ADAPTIVE_WEIGHTS,
@ -64,7 +69,101 @@ interface EditAutoRouterModalProps {
// Keys this modal rewrites from its own form state on save. Anything absent from this set is
// carried through untouched from the stored config, so a key only belongs here once the modal
// actually renders a control that can set it.
const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
/** The complexity_router_config as it comes back from the proxy, before any hydration. Fields the
* hydrators validate themselves stay `unknown`; the ones assigned straight through carry their type. */
export interface StoredComplexityRouterConfig {
tiers?: Partial<Record<keyof ComplexityTiers, unknown>>;
tier_model_configs?: unknown;
default_model?: string | null;
plan_mode_min_tier?: unknown;
heuristic_first_max_tier?: unknown;
tier_labels?: unknown;
classifier_type?: ClassifierType;
classifier_llm_config?: ClassifierLLMConfig;
classifier_context_window_size?: unknown;
classifier_context_budget_chars?: unknown;
classifier_context_include_assistant_turns?: unknown;
classifier_fallback?: unknown;
tier_boundaries?: unknown;
token_thresholds?: unknown;
dimension_weights?: unknown;
reasoning_override_min_score?: unknown;
session_affinity?: unknown;
deployment_affinity?: unknown;
adaptive?: boolean;
adaptive_weights?: AdaptiveRouterWeights;
tier_distance_penalty?: number;
adaptive_eligible?: AdaptiveEligible;
return_raw_model_name?: boolean;
}
/**
* The stored complexity_router_config as form state. Every key in MANAGED_COMPLEXITY_ROUTER_KEYS is
* rewritten from this state on save, so a key missing here is silently dropped from the saved config.
*/
export const hydrateComplexityRouterConfig = (
parsedConfig: StoredComplexityRouterConfig,
complexityRouterDefaultModel: string | null | undefined,
): ComplexityRouterConfigValue => {
const hydratedTiers: ComplexityTiers = {
SIMPLE: normalizeTierModels(parsedConfig.tiers?.SIMPLE),
MEDIUM: normalizeTierModels(parsedConfig.tiers?.MEDIUM),
COMPLEX: normalizeTierModels(parsedConfig.tiers?.COMPLEX),
REASONING: normalizeTierModels(parsedConfig.tiers?.REASONING),
};
return {
tiers: hydratedTiers,
tier_model_params: hydrateTierModelParams(parsedConfig.tiers, parsedConfig.tier_model_configs),
default_model: hydratePinnedDefaultModel(parsedConfig.default_model, complexityRouterDefaultModel, {
tiers: hydratedTiers,
}),
plan_mode_min_tier:
typeof parsedConfig.plan_mode_min_tier === "string" && parsedConfig.plan_mode_min_tier.trim() !== ""
? parsedConfig.plan_mode_min_tier
: undefined,
tier_labels: hydrateTierLabels(parsedConfig.tier_labels),
classifier_type: parsedConfig.classifier_type || "heuristic",
classifier_llm_config: parsedConfig.classifier_llm_config,
classifier_context_window_size:
typeof parsedConfig.classifier_context_window_size === "number"
? parsedConfig.classifier_context_window_size
: undefined,
classifier_context_budget_chars:
typeof parsedConfig.classifier_context_budget_chars === "number"
? parsedConfig.classifier_context_budget_chars
: undefined,
classifier_context_include_assistant_turns:
typeof parsedConfig.classifier_context_include_assistant_turns === "boolean"
? parsedConfig.classifier_context_include_assistant_turns
: undefined,
classifier_fallback:
parsedConfig.classifier_fallback === "default_model" || parsedConfig.classifier_fallback === "heuristic"
? parsedConfig.classifier_fallback
: undefined,
heuristic_first_max_tier:
typeof parsedConfig.heuristic_first_max_tier === "string" && parsedConfig.heuristic_first_max_tier.trim() !== ""
? parsedConfig.heuristic_first_max_tier
: undefined,
tier_boundaries: hydrateTierBoundaries(parsedConfig.tier_boundaries),
token_thresholds: hydrateTokenThresholds(parsedConfig.token_thresholds),
dimension_weights: hydrateDimensionWeights(parsedConfig.dimension_weights),
reasoning_override_min_score: hydrateReasoningOverrideMinScore(parsedConfig.reasoning_override_min_score),
session_affinity:
typeof parsedConfig.session_affinity === "boolean" ? parsedConfig.session_affinity : DEFAULT_SESSION_AFFINITY,
deployment_affinity:
typeof parsedConfig.deployment_affinity === "boolean"
? parsedConfig.deployment_affinity
: DEFAULT_DEPLOYMENT_AFFINITY,
adaptive: parsedConfig.adaptive || false,
adaptive_weights: parsedConfig.adaptive_weights,
tier_distance_penalty: parsedConfig.tier_distance_penalty,
adaptive_eligible: parsedConfig.adaptive_eligible || "all",
return_raw_model_name: parsedConfig.return_raw_model_name || false,
};
};
export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
"tiers",
"tier_model_configs",
"default_model",
@ -76,6 +175,7 @@ const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
"classifier_context_budget_chars",
"classifier_context_include_assistant_turns",
"classifier_fallback",
"heuristic_first_max_tier",
"session_affinity",
"deployment_affinity",
"adaptive",
@ -149,6 +249,7 @@ export const buildUpdatedComplexityRouterConfig = (
tiers: value.tiers,
defaultModel: value.default_model,
planModeMinTier: value.plan_mode_min_tier,
heuristicFirstMaxTier: value.heuristic_first_max_tier,
tierLabels: value.tier_labels,
classifierType: value.classifier_type,
classifierLlmConfig: value.classifier_llm_config,
@ -268,7 +369,8 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
: null) ??
getTierLabelsError(complexityRouterConfig.tier_labels) ??
getPlanModeTierError(complexityRouterConfig.plan_mode_min_tier, activeTierRows(complexityRouterConfig)) ??
getKeywordTierRulesError(keywordTierRules);
getKeywordTierRulesError(keywordTierRules, activeTierRows(complexityRouterConfig)) ??
getClassifierModelError(complexityRouterConfig);
useEffect(() => {
if (isVisible && modelData) {
@ -312,62 +414,10 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
parsedConfig = JSON.parse(parsedConfig);
}
const hydratedTiers: ComplexityTiers = {
SIMPLE: normalizeTierModels(parsedConfig.tiers?.SIMPLE),
MEDIUM: normalizeTierModels(parsedConfig.tiers?.MEDIUM),
COMPLEX: normalizeTierModels(parsedConfig.tiers?.COMPLEX),
REASONING: normalizeTierModels(parsedConfig.tiers?.REASONING),
};
const hydratedComplexityRouterConfig: ComplexityRouterConfigValue = {
tiers: hydratedTiers,
tier_model_params: hydrateTierModelParams(parsedConfig.tiers, parsedConfig.tier_model_configs),
default_model: hydratePinnedDefaultModel(
parsedConfig.default_model,
modelData.litellm_params?.complexity_router_default_model,
{ tiers: hydratedTiers },
),
plan_mode_min_tier:
typeof parsedConfig.plan_mode_min_tier === "string" && parsedConfig.plan_mode_min_tier.trim() !== ""
? parsedConfig.plan_mode_min_tier
: undefined,
tier_labels: hydrateTierLabels(parsedConfig.tier_labels),
classifier_type: parsedConfig.classifier_type || "heuristic",
classifier_llm_config: parsedConfig.classifier_llm_config,
classifier_context_window_size:
typeof parsedConfig.classifier_context_window_size === "number"
? parsedConfig.classifier_context_window_size
: undefined,
classifier_context_budget_chars:
typeof parsedConfig.classifier_context_budget_chars === "number"
? parsedConfig.classifier_context_budget_chars
: undefined,
classifier_context_include_assistant_turns:
typeof parsedConfig.classifier_context_include_assistant_turns === "boolean"
? parsedConfig.classifier_context_include_assistant_turns
: undefined,
classifier_fallback:
parsedConfig.classifier_fallback === "default_model" || parsedConfig.classifier_fallback === "heuristic"
? parsedConfig.classifier_fallback
: undefined,
tier_boundaries: hydrateTierBoundaries(parsedConfig.tier_boundaries),
token_thresholds: hydrateTokenThresholds(parsedConfig.token_thresholds),
dimension_weights: hydrateDimensionWeights(parsedConfig.dimension_weights),
reasoning_override_min_score: hydrateReasoningOverrideMinScore(parsedConfig.reasoning_override_min_score),
session_affinity:
typeof parsedConfig.session_affinity === "boolean"
? parsedConfig.session_affinity
: DEFAULT_SESSION_AFFINITY,
deployment_affinity:
typeof parsedConfig.deployment_affinity === "boolean"
? parsedConfig.deployment_affinity
: DEFAULT_DEPLOYMENT_AFFINITY,
adaptive: parsedConfig.adaptive || false,
adaptive_weights: parsedConfig.adaptive_weights,
tier_distance_penalty: parsedConfig.tier_distance_penalty,
adaptive_eligible: parsedConfig.adaptive_eligible || "all",
return_raw_model_name: parsedConfig.return_raw_model_name || false,
};
const hydratedComplexityRouterConfig = hydrateComplexityRouterConfig(
parsedConfig,
modelData.litellm_params?.complexity_router_default_model,
);
setComplexityRouterConfig(hydratedComplexityRouterConfig);
setCustomTechnicalKeywords(
Array.isArray(parsedConfig.custom_technical_keywords) ? parsedConfig.custom_technical_keywords : [],
@ -428,16 +478,17 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
toast.fromError("Please select at least one model for a complexity tier");
return;
}
if (classifier_type === "llm" && !classifier_llm_config?.model) {
const classifierError = getClassifierModelError(complexityRouterConfig);
if (classifierError) {
setShowValidationErrors(true);
toast.fromError("Please select a classifier model, or switch back to Heuristic");
toast.fromError(classifierError);
return;
}
// Same guards the create form applies (add_auto_router_tab.tsx). The backend rejects a
// keyword rule with no keyword, and semantic_keyword_matching without an embedding model
// or keyword rules (complexity_router/config.py), so without these a save fails as a raw
// 400 instead of an inline message.
const keywordRulesError = getKeywordTierRulesError(keywordTierRules);
const keywordRulesError = getKeywordTierRulesError(keywordTierRules, activeTierRows(complexityRouterConfig));
if (keywordRulesError) {
setShowValidationErrors(true);
toast.fromError(keywordRulesError);

40
ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/RoutingDecisionCard.tsx
View file

@ -72,6 +72,20 @@ function describeReasoningOverride(tierLabel: string | undefined, floor: number
return `Heuristic, ${tierLabel ?? "REASONING"} override (2 or more reasoning markers, score of at least ${stated})`;
}
const CONSTANT_CAUSE_LABELS: Record<string, string> = {
heuristic_scorer: "Heuristic scorer",
heuristic_first_short_circuit: "Heuristic scorer, classifier skipped",
classifier_plugin: "Custom classifier plugin",
semantic_keyword_match: "Semantic keyword match",
session_affinity_pin: "Pinned to session",
session_affinity_escalation: "Escalated from session pin",
quality_tier: "Quality tier mapping",
bandit: "Adaptive bandit",
default_fallback: "Default model, no route matched",
classifier_fallback: "Fallback tier, LLM classifier failed",
default_model_fallback: "Default model, LLM classifier failed",
};
function describeCause(decision: RoutingDecision): string {
const {
cause,
@ -81,35 +95,19 @@ function describeCause(decision: RoutingDecision): string {
reasoning_override_min_score: overrideFloor,
} = decision;
const constant = cause ? CONSTANT_CAUSE_LABELS[cause] : undefined;
if (constant) return constant;
switch (cause) {
case "heuristic_scorer":
return "Heuristic scorer";
case "reasoning_override":
return describeReasoningOverride(tierLabel, overrideFloor);
case "llm_classifier":
return classifierModel ? `LLM classifier (${classifierModel})` : "LLM classifier";
case "literal_keyword_match":
return matchedKeyword ? `Keyword match: "${matchedKeyword}"` : "Keyword match";
case "semantic_keyword_match":
return "Semantic keyword match";
case "plan_mode":
return describePlanModeFloor(matchedKeyword);
case "session_affinity_pin":
return "Pinned to session";
case "session_affinity_escalation":
return "Escalated from session pin";
case "quality_tier":
return "Quality tier mapping";
case "keyword":
return matchedKeyword ? `Keyword match: "${matchedKeyword}"` : "Keyword match";
case "bandit":
return "Adaptive bandit";
case "default_fallback":
return "Default model, no route matched";
case "classifier_fallback":
return "Fallback tier, LLM classifier failed";
case "default_model_fallback":
return "Default model, LLM classifier failed";
case "plan_mode":
return describePlanModeFloor(matchedKeyword);
default:
return cause ?? "Unknown";
}

5
ui/litellm-dashboard/src/lib/autorouter_presets.ts
View file

@ -8,6 +8,7 @@ import {
ClassifierLLMConfig,
DEFAULT_SESSION_AFFINITY,
DEFAULT_DEPLOYMENT_AFFINITY,
usesLlmClassifier,
} from "@/components/add_model/ComplexityRouterConfig";
import { KeywordTierRule } from "@/components/add_model/KeywordTierRules";
import { hydrateKeywordTierRules } from "@/components/add_model/complexity_router_keywords";
@ -177,7 +178,7 @@ export const getMissingModelsInPreset = (preset: AutoRouterPreset, availability:
// Checks the config actually being built (whether it arrived via a preset prefill or was typed by
// hand - the two are indistinguishable once the caller has started editing), not a preset's
// original bundled model list. Only counts classifier_llm_config/embedding_model as referenced
// when buildComplexityRouterConfig would actually emit them (classifierType === "llm",
// when buildComplexityRouterConfig would actually emit them (usesLlmClassifier(classifierType),
// semanticMatchingEnabled) - otherwise a dormant selection left over from a toggle no longer in
// effect would block submit for a model that was never going to be submitted.
export const getReferencedModelsError = (
@ -195,7 +196,7 @@ export const getReferencedModelsError = (
{
tiers: params.tiers,
default_model: params.defaultModel,
classifier_llm_config: params.classifierType === "llm" ? params.classifierLlmConfig : undefined,
classifier_llm_config: usesLlmClassifier(params.classifierType) ? params.classifierLlmConfig : undefined,
embedding_model: params.semanticMatchingEnabled ? params.embeddingModel : undefined,
},
availability,

15
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -32575,12 +32575,12 @@ export interface components {
classifier_context_window_size: number;
/**
* Classifier Fallback
* @description What classifies the request when the LLM classifier errors, times out, or returns an unparseable response. 'heuristic' runs the local complexity scorer, which is right when the classifier grades complexity too. 'default_model' skips scoring and routes to default_model, which is what a classifier on some other taxonomy wants: a prompt that grades data sensitivity has no use for a complexity score, and scoring one produces a tier unrelated to what the operator configured. Requires default_model when set to 'default_model'. Only applies when classifier_type is 'llm' or 'custom'.
* @description What classifies the request when the LLM classifier errors, times out, or returns an unparseable response. 'heuristic' runs the local complexity scorer, which is right when the classifier grades complexity too. 'default_model' skips scoring and routes to default_model, which is what a classifier on some other taxonomy wants: a prompt that grades data sensitivity has no use for a complexity score, and scoring one produces a tier unrelated to what the operator configured. Requires default_model when set to 'default_model'. Only applies when classifier_type is 'llm', 'custom', or 'heuristic_first'.
* @default heuristic
* @enum {string}
*/
classifier_fallback: "heuristic" | "default_model";
/** @description Configuration for the LLM classifier; required when classifier_type is 'llm' */
/** @description Configuration for the LLM classifier; required when classifier_type is 'llm' or 'heuristic_first' */
classifier_llm_config?: components["schemas"]["ClassifierLLMConfig"] | null;
/**
* Classifier Plugin
@ -32595,11 +32595,11 @@ export interface components {
classifier_plugin_timeout_ms: number;
/**
* Classifier Type
* @description Classification strategy: local regex/keyword scoring, an LLM call, or a custom classifier plugin
* @description Classification strategy: local regex/keyword scoring, an LLM call, a custom classifier plugin, or 'heuristic_first', which scores locally and only pays for the LLM classifier when the local scorer does not confidently land a cheap tier
* @default heuristic
* @enum {string}
*/
classifier_type: "heuristic" | "llm" | "custom";
classifier_type: "heuristic" | "llm" | "custom" | "heuristic_first";
/**
* Code Keywords
* @description Keywords indicating code-related content
@ -32643,6 +32643,11 @@ export interface components {
* @description Tier routed to when the LLM classifier fails (timeout, provider error, or an unparseable reply). Required with tier_definitions and must name a defined tier; the heuristic scorer cannot produce custom tiers, so this replaces the heuristic fallback for custom tier sets.
*/
fallback_tier?: string | null;
/**
* Heuristic First Max Tier
* @description The highest tier the local scorer may decide on its own; required when classifier_type is 'heuristic_first' and rejected otherwise. A request whose heuristic tier is at or below this one skips the LLM classifier and routes straight to that heuristic tier, so the classifier call is only paid for on traffic the scorer could not place cheaply. The scorer must also have produced at least one signal: a prompt where no dimension fired scores 0.0 and would otherwise land SIMPLE by default rather than by evidence, which is how a chained router would silently send unclassified traffic to the cheapest model. Names a built-in tier, and may not name the highest one, since that would make the LLM classifier unreachable.
*/
heuristic_first_max_tier?: string | null;
/**
* Keyword Tier Rules
* @description Rules that force a specific tier when their keywords match the prompt
@ -33702,7 +33707,7 @@ export interface components {
* Cause
* @enum {string}
*/
cause?: "heuristic_scorer" | "reasoning_override" | "llm_classifier" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "session_affinity_pin" | "session_affinity_escalation" | "default_fallback" | "keyword" | "quality_tier" | "bandit";
cause?: "heuristic_scorer" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "session_affinity_pin" | "session_affinity_escalation" | "default_fallback" | "keyword" | "quality_tier" | "bandit";
/** Classifier Cost */
classifier_cost?: number;
/** Classifier Model */

2
ui/litellm-dashboard/tsconfig.tsbuildinfo
View file

File diff suppressed because one or more lines are too long