scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

fix(auto-router): separate tuning and select chained heuristic (#44928)

Browse source
This commit is contained in:
tin-berri 2026-10-06 13:43:53 -07:00 • committed by GitHub
parent abd3af422d
commit 2cecc7e33f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
35 changed files with 943 additions and 262 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
litellm/proxy/management_endpoints/model_management_endpoints.py
View file

@ -126,10 +126,11 @@ from litellm.router_strategy.complexity_router.config import resolve_complexity_
from litellm.router_utils.auto_router_model_naming import (
GATED_AUTO_ROUTER_CAPABILITIES,
STRATEGY_ROUTER_PARAM_FIELDS,
GatedAutoRouterCapability,
capability_limit_violation,
carries_complexity_router_settings,
count_capability_routers,
gated_capability_of,
gated_capabilities_of,
is_complexity_router_model,
validate_complexity_router_config_placement,
validate_complexity_router_config_write,
@ -559,6 +560,20 @@ def _raise_on_tuning_quota_violation(
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"{violation} {AUTO_ROUTER_LICENSE_REMEDY}")
async def _claimed_capability_violation(
tables: _TxModelTables,
*,
capability: GatedAutoRouterCapability,
config_rows: Sequence[Mapping[str, object]],
model_id: str | None,
limit: int | None,
) -> str | None:
rows: Final = await tables.query_raw(_CAPABILITY_DB_ROWS_SQL[capability.key], model_id or "")
db_held: Final = sum(1 for row in rows if is_complexity_router_model(_decrypted_model(row.get("model"))))
held: Final = db_held + count_capability_routers(config_rows, capability=capability)
return capability_limit_violation(capability=capability, held=held + 1, limit=limit)
@asynccontextmanager
async def _auto_router_capability_slot(
prisma_client: PrismaClient,
@ -591,11 +606,11 @@ async def _auto_router_capability_slot(
)
limit: Final = _license_check.auto_router_capability_limit()
capability: Final = gated_capability_of(effective_params)
capabilities: Final = gated_capabilities_of(effective_params)
baselines: Final = heuristic_v1_tuning_baselines
tuning_candidate: Final = _tuning_candidate(effective_params, model_id=model_id)
judges_tuning: Final = baselines is not None and is_mutable_tuned_candidate(tuning_candidate, baselines)
if member_write is None and (limit is None or (capability is None and not judges_tuning)):
if member_write is None and (limit is None or (not capabilities and not judges_tuning)):
yield _proxy_model_table(prisma_client)
return
transaction_client: Final = _ModelTransactionClient.model_validate(prisma_client.db)
@ -685,14 +700,12 @@ async def _auto_router_capability_slot(
prisma_client=pinned_client,
llm_router=llm_router,
)
if capability is not None:
rows: Sequence[Mapping[str, object]] = await tx_ctx.query_raw(
_CAPABILITY_DB_ROWS_SQL[capability.key], model_id or ""
)
db_held: Final = sum(1 for row in rows if is_complexity_router_model(_decrypted_model(row.get("model"))))
held: Final = db_held + count_capability_routers(config_rows, capability=capability)
violation: Final = capability_limit_violation(capability=capability, held=held + 1, limit=limit)
if violation is not None:
for capability in capabilities:
if (
violation := await _claimed_capability_violation(
tables, capability=capability, config_rows=config_rows, model_id=model_id, limit=limit
)
) is not None:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN, detail=f"{violation} {AUTO_ROUTER_LICENSE_REMEDY}"
)

19
litellm/proxy/management_helpers/auto_router_availability.py
View file

@ -12,7 +12,7 @@ from litellm.router_utils.auto_router_model_naming import (
capability_limit_violation,
classify_strategy_router_model,
count_capability_routers,
gated_capability_of,
gated_capabilities_of,
)
from litellm.router_utils.auto_router_tuning_baseline import (
is_mutable_tuned_candidate,
@ -91,8 +91,8 @@ def auto_router_availability(
) -> AutoRouterAvailabilityResponse:
existing_params: Final = None if existing is None else existing.get("litellm_params")
candidate_params: Final = candidate.get("litellm_params")
owned: Final = gated_capability_of(existing_params) if isinstance(existing_params, Mapping) else None
claimed: Final = gated_capability_of(candidate_params) if isinstance(candidate_params, Mapping) else None
owned: Final = gated_capabilities_of(existing_params) if isinstance(existing_params, Mapping) else ()
claimed: Final = gated_capabilities_of(candidate_params) if isinstance(candidate_params, Mapping) else ()
counts: Final = tuple(
(capability, count_capability_routers(others, capability=capability))
for capability in GATED_AUTO_ROUTER_CAPABILITIES
@ -103,15 +103,16 @@ def auto_router_availability(
key=capability.key,
limit=limit,
remaining=None if limit is None else max(0, limit - held),
used_by_this_router=owned is capability,
used_by_this_router=capability in owned,
)
for capability, held in counts
)
capability_error: Final = next(
denied_capability: Final = next(
(
capability_limit_violation(capability=capability, held=held + 1, limit=limit)
capability
for capability, held in counts
if capability is claimed
if capability in claimed
and capability_limit_violation(capability=capability, held=held + 1, limit=limit) is not None
),
None,
)
@ -140,8 +141,8 @@ def auto_router_availability(
),
),
error=(
f"{capability_labels[claimed.key]} has no available allowance. Choose another option or free an existing allowance."
if capability_error is not None and claimed is not None
f"{capability_labels[denied_capability.key]} has no available allowance. Choose another option or free an existing allowance."
if denied_capability is not None
else "These scoring rules need an available Rule-based tuning allowance. Check the weights, thresholds, keywords, and custom dimensions in Advanced settings. Model choices do not use this allowance."
if tuning_error is not None
else None

8
litellm/router.py
View file

@ -164,7 +164,7 @@ from litellm.router_utils.auto_router_model_naming import (
AUTO_ROUTER_MODEL_PREFIX,
GatedAutoRouterCapability,
capability_limit_violation,
claimed_capability,
claimed_capabilities,
classify_strategy_router_model,
count_capability_routers,
)
@ -9491,10 +9491,8 @@ class Router:
)
complexity_router_config: Final[dict | None] = deployment.litellm_params.complexity_router_config
capability: Final = claimed_capability(complexity_router_config)
if capability is not None:
limit_violation: Final = self.auto_router_capability_violation(capability)
if limit_violation is not None:
for capability in claimed_capabilities(complexity_router_config):
if (limit_violation := self.auto_router_capability_violation(capability)) is not None:
raise ValueError(limit_violation)
default_model: Final = (

17
litellm/router_strategy/complexity_router/README.md
View file

@ -381,7 +381,7 @@ Each dimension contributes its weight once when any matcher hits the current ask
The API and YAML store exactly the weights written. A `dimension_weights` map and inline custom weights are read literally, missing recognized built-in names score zero, and nothing renormalizes the vector, so a total other than 1 is legal and scores accordingly. The dashboard's heuristic scoring editor is the one place that rebalances: editing one weight there holds it and redistributes the remainder across the other active dimensions in the draft, then Save sends the resulting explicit values, which the backend stores and scores as written. Opening a router, applying a preset, editing matchers, changing `scoring_mode`, or saving unrelated fields never normalizes existing weights
Only `heuristic`, `heuristic_first` and `hybrid` accept custom dimensions. Each name must be a unique ASCII identifier starting with a letter, at most 64 characters, and cannot reuse a built-in dimension name or a key in `dimension_weights`. Set its weight inline, greater than zero and at most one
Only `heuristic`, `heuristic_first` and `hybrid` using heuristic v1 accept custom dimensions. Each name must be a unique ASCII identifier starting with a letter, at most 64 characters, and cannot reuse a built-in dimension name or a key in `dimension_weights`. Set its weight inline, greater than zero and at most one
Patterns are checked at configuration time against a grammar whose worst case stays a few milliseconds on 2048 characters. Every quantifier needs an explicit upper bound of at most 64 and must repeat a single character or character class, so `\s{1,4}` is accepted while `\s+`, `(a|aa){0,12}` and `(?:ab){0,64}` are refused. Backreferences, lookarounds, atomic groups and possessive quantifiers are refused as well. Each pattern is then costed: alternation branches and repeat lengths multiply the ways the engine can retry, and every later piece of the pattern is charged once per path that can reach it, so `a?a?a?a?a?a?a?a?` followed by a long fixed tail is refused even though each quantifier is small. The budget is 2048 work units per pattern and 8192 across the router. An invalid or over-budget pattern fails the write with a message naming the pattern and the rule it broke
@ -555,7 +555,7 @@ on by default; set `classifier_llm_config.circuit_breaker_enabled: false` to dis
fallback is the local heuristic scorer, so a classifier outage does not repeat its timeout across
every turn or session handled by the router process.
A request short-circuits, meaning it routes on the scorer's own tier with no classifier call, when
With the default heuristic v1, a request short-circuits, meaning it routes on the scorer's own tier with no classifier call, when
two things hold: the scorer landed at or below `heuristic_first_max_tier`, and it produced at least
one signal. Everything else goes to the classifier, which then decides as it normally would.
@ -576,6 +576,13 @@ except that the heuristic outcome is the one already computed rather than a seco
Spend logs record `routing_decision.cause` as `heuristic_first_short_circuit` when the classifier
was skipped, and `llm_classifier` when it ran, so the two are told apart per request.
Set `local_heuristic: heuristic_v2` to chain the trained predictor instead. Its selected tier must
meet `heuristic_v2_success_threshold` and remain at or below `heuristic_first_max_tier` to skip the judge
Omitting `local_heuristic` keeps v1. Both choices preserve the configured classifier failure fallback;
the heuristic fallback uses the selected local scorer. V2 short-circuits and fallbacks also record
`heuristic_v2_forecast` with the probabilities and success threshold used for the decision
### Hybrid
`classifier_type: hybrid` also scores locally first, but it asks a different question than
@ -601,7 +608,7 @@ model_list:
REASONING: o1-preview
```
A request routes on the scorer's own tier when its score is further than `hybrid_boundary_margin`
With the default heuristic v1, a request routes on the scorer's own tier when its score is further than `hybrid_boundary_margin`
from every active boundary. Everything else goes to the classifier: a score inside the band, where a
hair's difference would have named the adjacent tier and its model pool, and a prompt where no
dimension fired at all, which has no opinion to be confident about. `hybrid_boundary_margin` is
@ -619,6 +626,10 @@ Operator-defined tier sets (`tier_definitions`) are not supported here, for the
not supported under heuristic-first: the scorer only produces the built-in tiers. Classifier failure
behaves exactly as it does under `classifier_type: llm`.
For `local_heuristic: heuristic_v2`, the selected tier must meet `heuristic_v2_success_threshold`.
The judge decides when any probability at or below the selected tier is within `hybrid_boundary_margin`
of that threshold, or when no tier meets the threshold. A zero margin still defers exact-threshold predictions
### Reasoning Override
If 2+ reasoning markers are detected in the user message, the request is promoted to the REASONING tier even when the weighted score maps lower, so complex reasoning tasks get the appropriate model. The promotion requires the score to reach `reasoning_override_min_score`, which tracks `tier_boundaries.simple_medium` unless set, so stock phrases on an otherwise trivial prompt cannot buy the top tier. Set it to `0` to promote on the markers alone.

59
litellm/router_strategy/complexity_router/complexity_router.py
View file

@ -112,6 +112,7 @@ from .config import (
CustomDimension,
OpenSourceClassifierConfig,
TierDefinition,
configured_local_heuristic,
)
from .jev_classifier import (
DEFAULT_JEV_INSTRUCTIONS,
@ -1488,7 +1489,10 @@ class ComplexityRouter(CustomLogger):
resolve_tier_artifact(self.config.heuristic_v2_artifact),
routing_threshold=self.config.heuristic_v2_success_threshold,
)
if self.config.classifier_type == "heuristic_v2"
if configured_local_heuristic(
{"classifier_type": self.config.classifier_type, "local_heuristic": self.config.local_heuristic}
)
== "heuristic_v2"
else None
)
@ -1940,10 +1944,31 @@ class ComplexityRouter(CustomLogger):
if self.config.classifier_type == "capability" and self.config.classifier_llm_config is not None:
return await self._capability_classifier_outcome(prompt, request_kwargs, messages)
if self.config.classifier_type not in ("llm", "llm_v2") or self.config.classifier_llm_config is None:
tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
return ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause)
return self._classify_locally(prompt, system_prompt)
return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages)
def _classify_locally(self, prompt: str, system_prompt: str | None) -> ClassificationOutcome:
if self._tier_success_predictor is not None:
return self._classify_with_heuristic_v2(prompt)
tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
return ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause)
def _local_outcome_is_decided(self, outcome: ClassificationOutcome, margin: float | None = None) -> bool:
forecast: Final = outcome.heuristic_v2_forecast
if forecast is None:
return bool(outcome.signals) and (
margin is None or (outcome.score is not None and not self._is_near_tier_boundary(outcome.score, margin))
)
threshold: Final = forecast["threshold"]
if forecast["probabilities"][forecast["predicted_tier"]] < threshold:
return False
if margin is None:
return True
deciding_tiers: Final = TIER_SEVERITY_ORDER[
: TIER_SEVERITY_ORDER.index(ComplexityTier(forecast["predicted_tier"])) + 1
]
return all(abs(forecast["probabilities"][tier.value] - threshold) > margin for tier in deciding_tiers)
def _classify_with_heuristic_v2(self, prompt: str) -> ClassificationOutcome:
predictor: Final = self._tier_success_predictor
if predictor is None:
@ -1981,27 +2006,27 @@ class ComplexityRouter(CustomLogger):
"""Score locally, and only pay for the classifier call when the scorer did not confidently
place the request at or below heuristic_first_max_tier.
Confidence is `signals`, not `score`. A prompt where no dimension fired scores exactly 0.0,
For v1, confidence is `signals`, not `score`. A prompt where no dimension fired scores exactly 0.0,
which is below simple_medium and so lands SIMPLE by default rather than by evidence, and a
threshold check alone would hand that traffic to the cheapest model without ever consulting
the classifier. Scores also go negative when simple indicators fire, so a score threshold
would reject exactly the trivial prompts this path exists to serve.
would reject exactly the trivial prompts this path exists to serve. V2 requires its predicted
tier to meet the configured success threshold.
A turn carrying images the classifier would see is never decided cheaply: the scorer reads
text alone, so its confidence describes a request it has only partly seen, and a trivial
caption beside a screenshot is exactly the misrouting vision classification exists to stop.
"""
tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
scored: Final = ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause)
scored: Final = self._classify_locally(prompt, system_prompt)
threshold: Final = self.config.heuristic_first_max_tier
decided_cheaply: Final = (
threshold is not None
and bool(signals)
and self._local_outcome_is_decided(scored)
and not self._classifier_image_parts(messages)
and self._active_tier_severity(tier) <= self._active_tier_severity(threshold)
and self._active_tier_severity(scored.tier) <= self._active_tier_severity(threshold)
)
if decided_cheaply:
return ClassificationOutcome(tier=tier, score=score, signals=signals, cause="heuristic_first_short_circuit")
return scored._replace(cause="heuristic_first_short_circuit")
return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages, scored=scored)
async def _classify_hybrid(
@ -2015,21 +2040,20 @@ class ComplexityRouter(CustomLogger):
Where heuristic_first asks how CHEAP the scorer's tier is, this asks how DECIDED it is, so a
confident score keeps its tier at every tier including the most expensive one. Two things make
a score undecided: landing within hybrid_boundary_margin of an active boundary, where a
a v1 score undecided: landing within hybrid_boundary_margin of an active boundary, where a
hair's difference in score would have named the adjacent tier and its model pool, and firing
no dimension at all, which scores 0.0 and lands SIMPLE by default rather than by evidence.
V2 compares its selected and lower-tier probabilities against the success threshold.
"""
tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
scored: Final = ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause)
scored: Final = self._classify_locally(prompt, system_prompt)
margin: Final = self.config.hybrid_boundary_margin
decided: Final = (
margin is not None
and bool(signals)
and self._local_outcome_is_decided(scored, margin)
and not self._classifier_image_parts(messages)
and not self._is_near_tier_boundary(score, margin)
)
if decided:
return ClassificationOutcome(tier=tier, score=score, signals=signals, cause="hybrid_short_circuit")
return scored._replace(cause="hybrid_short_circuit")
return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages, scored=scored)
def _classifier_image_parts(
@ -2311,8 +2335,7 @@ class ComplexityRouter(CustomLogger):
return _with_signal(self._default_model_fallback_outcome(), signal)
if scored is not None:
return _with_signal(scored, signal)
tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
return _with_signal(ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause), signal)
return _with_signal(self._classify_locally(prompt, system_prompt), signal)
async def _classify_with_plugin(
self,

50
litellm/router_strategy/complexity_router/config.py
View file

@ -74,6 +74,15 @@ DEFAULT_CLASSIFICATION_RUBRIC: Final[ClassificationRubric] = ClassificationRubri
LLM_CLASSIFIER_TYPES: Final[frozenset[str]] = frozenset({"llm", "capability", "llm_v2", "heuristic_first", "hybrid"})
def configured_local_heuristic(config: Mapping[str, object]) -> Literal["heuristic", "heuristic_v2"] | None:
classifier_type: Final = config.get("classifier_type", "heuristic")
if classifier_type == "heuristic" or classifier_type == "heuristic_v2":
return classifier_type
if classifier_type in ("heuristic_first", "hybrid"):
return "heuristic_v2" if config.get("local_heuristic") == "heuristic_v2" else "heuristic"
return None
TIER_SEVERITY_ORDER: Final[tuple[ComplexityTier, ...]] = (
ComplexityTier.SIMPLE,
ComplexityTier.MEDIUM,
@ -1160,10 +1169,17 @@ class ComplexityRouterConfig(LiteLLMBaseModel):
default=None,
description="Experimental joint task-demand and solver-capability forecasting for classifier_type llm_v2.",
)
local_heuristic: Literal["heuristic", "heuristic_v2"] | None = Field(
default=None,
description=(
"Local scorer for heuristic_first or hybrid. Omitted or null keeps heuristic v1; "
"heuristic_v2 uses the trained success predictor. Rejected for other classifier types."
),
)
heuristic_v2_artifact: TrainedTierArtifact | Literal["ultrafeedback"] = Field(
default="ultrafeedback",
description=(
"Success-probability artifact used by classifier_type 'heuristic_v2'. The bundled "
"Success-probability artifact used by standalone or chained heuristic_v2. The bundled "
"UltraFeedback artifact is selected by default; an inline trained artifact may replace it"
),
)
@ -1173,10 +1189,11 @@ class ComplexityRouterConfig(LiteLLMBaseModel):
ge=0.0,
le=1.0,
description=(
"Minimum predicted success probability for classifier_type 'heuristic_v2' to select a tier. "
"The first tier meeting this threshold is selected, or REASONING if none meets it. "
"Minimum predicted success probability for standalone or chained heuristic_v2 to select a tier. "
"The first tier meeting this threshold is selected. When none meets it, standalone heuristic_v2 "
"selects REASONING and chained heuristic_v2 defers to the LLM judge. "
"When omitted or null, uses the artifact's routing_threshold (0.75 for the bundled artifact). "
"Other classifier types ignore this setting"
"Ignored when heuristic_v2 is not selected"
),
)
classifier_llm_config: ClassifierLLMConfig | None = Field(
@ -1203,8 +1220,8 @@ class ComplexityRouterConfig(LiteLLMBaseModel):
"one skips the LLM classifier and routes straight to that heuristic tier, so the classifier "
"call is only paid for on traffic the scorer could not place cheaply. The scorer must also "
"have produced at least one signal: a prompt where no dimension fired scores 0.0 and would "
"otherwise land SIMPLE by default rather than by evidence, which is how a chained router "
"would silently send unclassified traffic to the cheapest model. Names a built-in tier, and "
"otherwise land SIMPLE by default rather than by evidence. With local_heuristic 'heuristic_v2', "
"the predicted tier must meet its success threshold. Names a built-in tier, and "
"may not name the highest one, since that would make the LLM classifier unreachable."
),
)
@ -1218,7 +1235,9 @@ class ComplexityRouterConfig(LiteLLMBaseModel):
"this from every active boundary routes on the scorer's own tier with no classifier call, at any "
"tier, which is what separates 'hybrid' from 'heuristic_first' and its cheap-tier ceiling. A "
"prompt where no dimension fired still goes to the classifier, since the scorer has no opinion "
"to be near a boundary with. 0 escalates only scores sitting exactly on a boundary."
"to be near a boundary with. With local_heuristic 'heuristic_v2', a tier must meet its success "
"threshold and its probability and all lower-tier probabilities must be further than this margin "
"from that threshold. 0 escalates only scores or probabilities exactly on a boundary."
),
)
classifier_plugin: ClassifierPlugin | None = Field(
@ -1855,12 +1874,25 @@ class ComplexityRouterConfig(LiteLLMBaseModel):
raise ValueError("llm_v2 requires one distinct model group in each tier")
return self
@model_validator(mode="after")
def _validate_local_heuristic(self) -> "ComplexityRouterConfig":
if self.local_heuristic is not None and self.classifier_type not in ("heuristic_first", "hybrid"):
raise ValueError("local_heuristic requires classifier_type heuristic_first or hybrid")
return self
@model_validator(mode="after")
def _validate_custom_dimensions(self) -> "ComplexityRouterConfig":
if not self.custom_dimensions:
return self
if self.classifier_type not in ("heuristic", "heuristic_first", "hybrid"):
raise ValueError("custom_dimensions requires classifier_type heuristic, heuristic_first or hybrid")
if (
configured_local_heuristic(
{"classifier_type": self.classifier_type, "local_heuristic": self.local_heuristic}
)
!= "heuristic"
):
raise ValueError(
"custom_dimensions requires classifier_type heuristic, heuristic_first or hybrid using heuristic v1"
)
names: Final = tuple(dimension.name.casefold() for dimension in self.custom_dimensions)
reserved: Final = frozenset(name.casefold() for name in DEFAULT_DIMENSION_WEIGHTS)
weighted: Final = frozenset(name.casefold() for name in self.dimension_weights)

38
litellm/router_utils/auto_router_model_naming.py
View file

@ -19,6 +19,7 @@ from litellm.router_strategy.complexity_router.config import (
COMPLEXITY_ROUTER_CONFIG_KEYS,
DEFAULT_JEV_INSTRUCTIONS,
LLM_CLASSIFIER_TYPES,
configured_local_heuristic,
normalize_classifier_config_aliases,
)
@ -186,7 +187,7 @@ def strategy_router_dependencies(
def uses_heuristic_v2_classifier(complexity_router_config: object) -> bool:
"""Whether this complexity config classifies with the bundled heuristic_v2 model."""
return _mapping(complexity_router_config).get("classifier_type") == "heuristic_v2"
return configured_local_heuristic(_mapping(complexity_router_config)) == "heuristic_v2"
def defines_custom_tiers(complexity_router_config: object) -> bool:
@ -237,8 +238,8 @@ class GatedAutoRouterCapability:
stored ``litellm_params`` (``{config}`` is the caller's expression for the normalized
``complexity_router_config`` jsonb, substituted as many times as the predicate needs); they live
on one record so they cannot drift apart. ``subject`` and ``remedy`` build the shared refusal
message. A validated config claims at most one capability: gated classifier types cannot be
combined with operator-defined tiers or classifier prompts.
message. A router chaining heuristic_v2 with an operator-written judge prompt claims both
capabilities.
"""
key: str
@ -250,10 +251,14 @@ class GatedAutoRouterCapability:
HEURISTIC_V2_CAPABILITY: Final = GatedAutoRouterCapability(
key="heuristic_v2",
subject="with classifier_type 'heuristic_v2'",
remedy="Use classifier_type 'heuristic' for this router or remove an existing heuristic_v2 router.",
subject="using the heuristic_v2 classifier",
remedy="Choose the heuristic v1 classifier for this router or remove an existing heuristic_v2 router.",
uses=uses_heuristic_v2_classifier,
sql_config_predicate="{config} ->> 'classifier_type' = 'heuristic_v2'",
sql_config_predicate=(
"{config} ->> 'classifier_type' = 'heuristic_v2' OR "
"({config} ->> 'classifier_type' IN ('heuristic_first', 'hybrid') AND "
"{config} ->> 'local_heuristic' = 'heuristic_v2')"
),
)
CAPABILITY_CLASSIFIER_CAPABILITY: Final = GatedAutoRouterCapability(
@ -307,20 +312,19 @@ GATED_AUTO_ROUTER_CAPABILITIES: Final = (
)
def claimed_capability(complexity_router_config: object) -> GatedAutoRouterCapability | None:
"""The licensed capability this complexity config claims, or None."""
return next(
(capability for capability in GATED_AUTO_ROUTER_CAPABILITIES if capability.uses(complexity_router_config)),
None,
def claimed_capabilities(complexity_router_config: object) -> tuple[GatedAutoRouterCapability, ...]:
"""The licensed capabilities this complexity config claims."""
return tuple(
capability for capability in GATED_AUTO_ROUTER_CAPABILITIES if capability.uses(complexity_router_config)
)
def gated_capability_of(litellm_params: Mapping[str, object]) -> GatedAutoRouterCapability | None:
"""The licensed capability this deployment claims, or None unless it is a complexity router."""
def gated_capabilities_of(litellm_params: Mapping[str, object]) -> tuple[GatedAutoRouterCapability, ...]:
"""The licensed capabilities this deployment claims, or empty unless it is a complexity router."""
model: Final = litellm_params.get("model")
if not is_complexity_router_model(model if isinstance(model, str) else None):
return None
return claimed_capability(litellm_params.get("complexity_router_config"))
return ()
return claimed_capabilities(litellm_params.get("complexity_router_config"))
def count_capability_routers(
@ -328,7 +332,9 @@ def count_capability_routers(
) -> int:
"""How many of ``deployments`` (router model_list entries or config.yaml rows) claim ``capability``."""
return sum(
1 for deployment in deployments if gated_capability_of(_mapping(deployment.get("litellm_params"))) is capability
1
for deployment in deployments
if capability in gated_capabilities_of(_mapping(deployment.get("litellm_params")))
)

5
litellm/router_utils/auto_router_tuning_baseline.py
View file

@ -8,7 +8,7 @@ from typing import Final
from pydantic import ValidationError
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig, configured_local_heuristic
# v2 hashes combine models and scoring rules; a new snapshot is required to separate them.
TUNING_BASELINE_PARAM_NAME: Final = "auto_router_tuning_baseline_v3"
@ -30,7 +30,6 @@ HEURISTIC_V1_TUNING_FIELDS: Final = (
_TUNING_FIELD_SET: Final = frozenset(HEURISTIC_V1_TUNING_FIELDS)
_V1_SCORING_CLASSIFIER_TYPES: Final = frozenset({"heuristic", "heuristic_first", "hybrid"})
_AUTO_ROUTER_COMPLEXITY_PREFIX: Final = "auto_router/complexity_router"
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
_EMPTY_TAGS: Final[tuple[str, ...]] = ()
@ -71,7 +70,7 @@ DEFAULT_TUNING_FINGERPRINT: Final = tuning_fingerprint(_EMPTY)
def uses_heuristic_v1(complexity_router_config: object) -> bool:
"""Whether a config's primary classifier path is the heuristic-v1 scorer."""
return _mapping(complexity_router_config).get("classifier_type", "heuristic") in _V1_SCORING_CLASSIFIER_TYPES
return configured_local_heuristic(_mapping(complexity_router_config)) == "heuristic"
def router_identity(deployment: Mapping[str, object]) -> str | None:

63
tests/unit/proxy/management_endpoints/test_model_management_endpoints.py
View file

@ -6097,9 +6097,9 @@ class TestStrategyRouterWriteValidation:
AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY,
_auto_router_capability_slot,
)
from litellm.router_utils.auto_router_model_naming import gated_capability_of
from litellm.router_utils.auto_router_model_naming import gated_capabilities_of
capability = gated_capability_of(effective_params)
capabilities: Final = gated_capabilities_of(effective_params)
fake = self._FakeDb(db_models)
live_router = self._live_router_holding_one_capability(limit, config_config) if config_config is not None else None
@ -6116,9 +6116,9 @@ class TestStrategyRouterWriteValidation:
async with _auto_router_capability_slot(fake, effective_params=effective_params, model_id=model_id):
pass
assert exc_info.value.status_code == 403
assert capability is not None
assert len(capabilities) == 1
assert "At most 1 auto-router" in str(exc_info.value.detail)
assert capability.subject in str(exc_info.value.detail)
assert capabilities[0].subject in str(exc_info.value.detail)
assert "'auto_router' feature lifts the limit" in str(exc_info.value.detail)
return
async with _auto_router_capability_slot(fake, effective_params=effective_params, model_id=model_id) as tables:
@ -6135,8 +6135,59 @@ class TestStrategyRouterWriteValidation:
assert lock_params == (AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY,)
assert count_params == (model_id or "",)
assert "AS model" in count_sql
assert capability is not None
assert capability.sql_config_predicate.split("{config}")[-1].strip() in count_sql
assert len(capabilities) == 1
assert capabilities[0].sql_config_predicate.split("{config}")[-1].strip() in count_sql
@pytest.mark.asyncio
@pytest.mark.parametrize("classifier", ("heuristic_first", "hybrid"))
@pytest.mark.parametrize(
("other_config", "denied_subject"),
(
(None, None),
(_V2, "heuristic_v2"),
(_CUSTOM_PROMPT, "operator-written classifier prompt"),
),
)
async def test_v2_chain_reserves_both_capabilities_under_one_lock(
self, classifier: str, other_config: Mapping[str, object] | None, denied_subject: str | None
) -> None:
from fastapi import HTTPException
from litellm.proxy.management_endpoints.model_management_endpoints import _auto_router_capability_slot
candidate: Final = {
"model": "auto_router/complexity_router",
"complexity_router_config": {
**self._CUSTOM_PROMPT,
"classifier_type": classifier,
"local_heuristic": "heuristic_v2",
**(
{"heuristic_first_max_tier": "MEDIUM"}
if classifier == "heuristic_first" else {"hybrid_boundary_margin": 0.1}
),
},
}
fake: Final = self._FakeDb([])
router: Final = self._live_router_holding_one_capability(1, other_config) if other_config is not None else None
with (
patch("litellm.proxy.proxy_server._license_check.auto_router_capability_limit", return_value=1), # test-quality-ok: [TQ008] the guard reads the proxy license singleton
patch("litellm.proxy.proxy_server.llm_router", router), # test-quality-ok: [TQ008] the guard reads the proxy router singleton
patch("litellm.proxy.proxy_server.heuristic_v1_tuning_baselines", {}), # test-quality-ok: [TQ008] the guard reads the proxy baseline singleton
patch("litellm.proxy.management_endpoints.model_management_endpoints.publish_config_change", new=AsyncMock()), # test-quality-ok: [TQ008] Redis publication is outside admission under test
):
if denied_subject is not None:
with pytest.raises(HTTPException) as failure:
async with _auto_router_capability_slot(fake, effective_params=candidate, model_id="held-id"):
pytest.fail("Exhausted capability allowed a write")
assert failure.value.status_code == 403
assert denied_subject in str(failure.value.detail)
else:
async with _auto_router_capability_slot(fake, effective_params=candidate, model_id="held-id") as table:
assert table is fake.tx_obj.litellm_proxymodeltable
assert sum("pg_advisory_xact_lock" in sql for sql, _ in fake.tx_obj.raw_calls) == 1
counts: Final = tuple((sql, args) for sql, args in fake.tx_obj.raw_calls if "AS model" in sql)
assert len(counts) == (1 if denied_subject == "heuristic_v2" else 2)
assert all(args == ("held-id",) for _, args in counts)
_TUNED_A = {"classifier_type": "heuristic", "tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o"}}
_TUNED_A_EDITED = {**_TUNED_A, "dimension_weights": {"codePresence": 0.9}}

27
tests/unit/proxy/management_helpers/test_auto_router_availability.py
View file

@ -61,6 +61,33 @@ def test_edit_does_not_exempt_another_classifier_allowance() -> None:
assert next(slot for slot in result.allowances if slot.key == "llm_v2").remaining == 0
@pytest.mark.parametrize("classifier", ("heuristic_first", "hybrid"))
@pytest.mark.parametrize(
("other_classifier", "expected_label"),
(("heuristic_v2", "Heuristic v2"), ("llm", "Custom tiers or classifier instructions")),
)
def test_v2_chain_checks_both_allowances_and_keeps_both_owned_slots(
classifier: str, other_classifier: str, expected_label: str
) -> None:
prompt: Final = {"classification_prompt": "Grade by difficulty"}
candidate: Final = deployment("chain", classifier, config={"local_heuristic": "heuristic_v2", **prompt})
blocked: Final = auto_router_availability(
others=(deployment("other", other_classifier, config=prompt),),
existing=None,
candidate=candidate,
baselines={},
limit=1,
)
assert blocked.error is not None and expected_label in blocked.error
edited: Final = auto_router_availability(
others=(), existing=candidate, candidate=candidate, baselines={}, limit=1
)
assert edited.error is None
assert tuple(slot.key for slot in edited.allowances if slot.used_by_this_router) == (
"heuristic_v2", "tier_or_classifier_prompt"
)
def test_model_selection_does_not_claim_occupied_scoring_allowance() -> None:
original: Final = deployment("legacy", "heuristic")
changed: Final = deployment("other", "heuristic", tuned=True)

198
tests/unit/router_strategy/test_complexity_router.py
View file

@ -2120,6 +2120,37 @@ class TestRouterComplexityDeploymentMethods:
"model_info": {"id": model_id},
}
@pytest.mark.parametrize("classifier", ("heuristic_first", "hybrid"))
@pytest.mark.parametrize("held_capability", ("heuristic_v2", "custom_prompt"))
def test_v2_chain_registration_enforces_each_claimed_allowance(
self, classifier: str, held_capability: str
) -> None:
held: Final = (
self._router_row("existing", "existing-id", "heuristic_v2")
if held_capability == "heuristic_v2"
else self._custom_prompt_row("existing", "existing-id")
)
chain: Final = {
"model_name": "chain",
"model_info": {"id": "chain-id"},
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {
"classifier_type": classifier,
"local_heuristic": "heuristic_v2",
**(
{"heuristic_first_max_tier": "MEDIUM"}
if classifier == "heuristic_first" else {"hybrid_boundary_margin": 0.1}
),
"classifier_llm_config": {"model": "gpt-4o-mini", "system_prompt": "Grade by difficulty"},
"tiers": {"SIMPLE": "gpt-4o-mini"},
},
},
}
subject: Final = "heuristic_v2" if held_capability == "heuristic_v2" else "operator-written classifier prompt"
with pytest.raises(ValueError, match=subject):
Router(model_list=[self._POOL, held, chain], auto_router_capability_limit=lambda: 1)
@pytest.mark.parametrize("field", ["classification_prompt", "classification_examples"])
def test_operator_written_prompt_sections_claim_the_customization_slot(self, field: str) -> None:
"""The dashboard prompt editor writes opening instructions and calibration examples as their own
@ -13998,6 +14029,129 @@ class TestHybrid:
assert outcome.cause == "heuristic_scorer"
@pytest.mark.parametrize("classifier_type", ("heuristic", "heuristic_v2", "llm"))
def test_local_heuristic_is_rejected_outside_chaining(classifier_type: str) -> None:
with pytest.raises(ValidationError, match="local_heuristic requires"):
ComplexityRouterConfig.model_validate(
{
"classifier_type": classifier_type,
"local_heuristic": "heuristic_v2",
"classifier_llm_config": {"model": "judge"},
}
)
@pytest.mark.parametrize("classifier_type", ("heuristic_first", "hybrid"))
def test_v2_chain_rejects_v1_custom_dimensions(classifier_type: str) -> None:
with pytest.raises(ValidationError, match="custom_dimensions requires"):
ComplexityRouterConfig.model_validate(
{
"classifier_type": classifier_type,
"local_heuristic": "heuristic_v2",
"classifier_llm_config": {"model": "judge"},
"heuristic_first_max_tier": "SIMPLE" if classifier_type == "heuristic_first" else None,
"hybrid_boundary_margin": 0.03 if classifier_type == "hybrid" else None,
"tiers": dict(HEURISTIC_FIRST_TIERS),
"custom_dimensions": [{"name": "custom", "weight": 0.5, "patterns": ["custom"]}],
}
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"classifier_type, threshold, margin, expected_tier, judge_called",
[
("heuristic_first", 0.1, 0.03, "SIMPLE", False),
("heuristic_first", 0.8, 0.03, "REASONING", True),
("hybrid", 0.8, 0.03, "COMPLEX", False),
("hybrid", 0.89, 0.03, "REASONING", True),
("hybrid", 0.22, 0.02, "REASONING", True),
("hybrid", 21 / 102, 0.0, "REASONING", True),
("hybrid", 1.0, 0.0, "REASONING", True),
],
)
async def test_selected_v2_chain_routes_using_success_probabilities(
mock_router_instance: MagicMock,
classifier_type: str,
threshold: float,
margin: float,
expected_tier: str,
judge_called: bool,
) -> None:
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "REASONING"}'))
router: Final = ComplexityRouter(
model_name="v2-chain",
litellm_router_instance=mock_router_instance,
complexity_router_config={
"classifier_type": classifier_type,
"local_heuristic": "heuristic_v2",
"heuristic_v2_artifact": _heuristic_v2_artifact(),
"heuristic_v2_success_threshold": threshold,
"classifier_llm_config": {"model": "judge"},
"heuristic_first_max_tier": "SIMPLE" if classifier_type == "heuristic_first" else None,
"hybrid_boundary_margin": margin if classifier_type == "hybrid" else None,
"tiers": dict(HEURISTIC_FIRST_TIERS),
},
)
response: Final = await router.async_pre_routing_hook(
model="v2-chain",
request_kwargs={},
messages=[{"role": "user", "content": "write a python function to reverse a string"}],
)
assert response is not None and response.routing_decision is not None
assert response.routing_decision["tier"] == expected_tier
assert mock_router_instance.acompletion.await_count == int(judge_called)
assert response.routing_decision["cause"] == (
"llm_classifier" if judge_called else f"{classifier_type}_short_circuit"
)
assert response.routing_decision.get("heuristic_v2_forecast") == (
None
if judge_called
else {
"probabilities": dict(
zip((tier.value for tier in TIER_SEVERITY_ORDER), (11 / 102, 21 / 102, 91 / 102, 100 / 102))
),
"threshold": threshold,
"predicted_tier": expected_tier,
"request_type": "code_generation",
}
)
@pytest.mark.asyncio
@pytest.mark.parametrize("classifier_type", ("heuristic_first", "hybrid"))
@pytest.mark.parametrize("encrypted", (False, True))
@pytest.mark.parametrize("fallback", ("heuristic", "default_model"))
async def test_v2_chain_judge_failure_uses_selected_fallback(
classifier_type: str, encrypted: bool, fallback: str
) -> None:
_, dependency = _native_classifier_router(failure=RuntimeError("judge failed"))
dependency.acompletion = AsyncMock(side_effect=RuntimeError("judge failed"))
router: Final = ComplexityRouter(
model_name="v2-chain",
litellm_router_instance=dependency,
complexity_router_config={
"classifier_type": classifier_type,
"local_heuristic": "heuristic_v2",
"heuristic_v2_artifact": _heuristic_v2_artifact(),
"classifier_llm_config": {"model": "classifier"},
"heuristic_first_max_tier": "SIMPLE" if classifier_type == "heuristic_first" else None,
"hybrid_boundary_margin": 1.0 if classifier_type == "hybrid" else None,
"classifier_fallback": fallback,
"default_model": "gpt-4o",
"tiers": dict(HEURISTIC_FIRST_TIERS),
},
)
outcome: Final = await router.aclassify(
"hello", request_kwargs={"input": [_encrypted_agent_task()]} if encrypted else None
)
assert outcome.cause == ("heuristic_v2" if fallback == "heuristic" else "default_model_fallback")
assert outcome.tier == (ComplexityTier.COMPLEX if fallback == "heuristic" else ComplexityTier.MEDIUM)
assert (outcome.heuristic_v2_forecast is not None) == (fallback == "heuristic")
assert dependency.aresponses.await_count == int(encrypted)
assert dependency.acompletion.await_count == int(not encrypted)
def _windowed_router(*deployments: tuple) -> Router:
"""Real Router; each deployment is (group, provider_model, declared window or None).
None means no declared override on a model the cost map does not know: unresolvable."""
@ -16407,17 +16561,31 @@ class TestClassifierVision:
@pytest.mark.parametrize(
"classifier_type, extra, short_circuit_cause", SHORT_CIRCUIT_ARMS, ids=["heuristic_first", "hybrid"]
)
@pytest.mark.parametrize("local_heuristic", ("heuristic", "heuristic_v2"))
async def test_local_scorer_cannot_short_circuit_a_turn_it_cannot_see(
self, mock_router_instance, classifier_type, extra, short_circuit_cause
):
self,
mock_router_instance: MagicMock,
classifier_type: str,
extra: dict[str, str | float],
short_circuit_cause: str,
local_heuristic: str,
) -> None:
"""The scorer reads text alone, so its confidence is not a verdict on an image turn.
Both arms are tuned so the scorer WOULD short-circuit on this exact text, which is what
makes the image the only variable; a margin loose enough to leave the score undecided
would pass whether or not the guard exists.
"""
router = self._router(mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra)
response = await router.async_pre_routing_hook(
router: Final = self._router(
mock_router_instance,
vision={"enabled": True},
classifier_type=classifier_type,
local_heuristic=local_heuristic,
heuristic_v2_artifact=_heuristic_v2_artifact(),
heuristic_v2_success_threshold=0.0,
**extra,
)
response: Final = await router.async_pre_routing_hook(
model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, IMG_PART)
)
assert response.routing_decision["cause"] == "llm_classifier"
@ -16426,12 +16594,26 @@ class TestClassifierVision:
@pytest.mark.parametrize(
"classifier_type, extra, short_circuit_cause", SHORT_CIRCUIT_ARMS, ids=["heuristic_first", "hybrid"]
)
@pytest.mark.parametrize("local_heuristic", ("heuristic", "heuristic_v2"))
async def test_local_scorer_still_short_circuits_without_images(
self, mock_router_instance, classifier_type, extra, short_circuit_cause
):
self,
mock_router_instance: MagicMock,
classifier_type: str,
extra: dict[str, str | float],
short_circuit_cause: str,
local_heuristic: str,
) -> None:
"""The negative class: same router, same text, no image, and the scorer still decides."""
router = self._router(mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra)
response = await router.async_pre_routing_hook(
router: Final = self._router(
mock_router_instance,
vision={"enabled": True},
classifier_type=classifier_type,
local_heuristic=local_heuristic,
heuristic_v2_artifact=_heuristic_v2_artifact(),
heuristic_v2_success_threshold=0.0,
**extra,
)
response: Final = await router.async_pre_routing_hook(
model="m", request_kwargs={}, messages=[{"role": "user", "content": "what is this"}]
)
assert response.routing_decision["cause"] == short_circuit_cause

56
tests/unit/router_utils/test_auto_router_model_naming.py
View file

@ -9,10 +9,10 @@ from litellm.router_utils.auto_router_model_naming import (
GATED_AUTO_ROUTER_CAPABILITIES,
capability_limit_violation,
carries_complexity_router_settings,
claimed_capability,
claimed_capabilities,
classify_strategy_router_model,
count_capability_routers,
gated_capability_of,
gated_capabilities_of,
strategy_router_dependencies,
validate_complexity_router_config_placement,
validate_complexity_router_config_write,
@ -74,11 +74,11 @@ def test_open_source_classifier_enumerates_its_accounting_model(
def test_only_non_default_open_source_instructions_claim_the_shared_customization_slot(
instructions: str | None, classifier_type: str, config_key: str
) -> None:
capability: Final = claimed_capability(
capabilities: Final = claimed_capabilities(
{"classifier_type": classifier_type, config_key: {"instructions": instructions}}
)
assert (capability.key if capability else None) == (
"tier_or_classifier_prompt" if instructions == "Route conservatively" else None
assert tuple(capability.key for capability in capabilities) == (
("tier_or_classifier_prompt",) if instructions == "Route conservatively" else ()
)
@ -273,8 +273,8 @@ def test_fuse_write_accepts_presets_and_custom_text_with_the_same_entitlement()
)
assert validate_complexity_router_config_write(presets) is None
assert validate_complexity_router_config_write(custom) is None
assert claimed_capability(presets) is claimed_capability(custom)
assert claimed_capability(presets) is not None
assert claimed_capabilities(presets) == claimed_capabilities(custom)
assert claimed_capabilities(presets)
@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
@ -579,8 +579,8 @@ def test_custom_classifier_prompt_capability(config: Mapping[str, object], expec
A shipped rubric preset stays free, and the heuristic scorers never read system_prompt, so a
value sitting on one is inert and claims nothing (heuristic_v2 still claims its own capability).
"""
claimed = claimed_capability(config)
assert (None if claimed is None else claimed.key) == expected_key
claimed: Final = claimed_capabilities(config)
assert tuple(capability.key for capability in claimed) == (() if expected_key is None else (expected_key,))
@pytest.mark.parametrize(
@ -640,13 +640,13 @@ def test_is_complexity_router_model(model: str | None, expected: bool) -> None:
({}, None),
],
)
def test_gated_capability_of(litellm_params: Mapping[str, object], expected_key: str | None) -> None:
def test_gated_capabilities_of(litellm_params: Mapping[str, object], expected_key: str | None) -> None:
"""Only a complexity router claiming a licensed capability counts toward that capability's limit.
Renaming the built-in tiers through tier_labels is not a custom tier set, so it stays ungated.
"""
capability = gated_capability_of(litellm_params)
assert (None if capability is None else capability.key) == expected_key
capabilities: Final = gated_capabilities_of(litellm_params)
assert tuple(capability.key for capability in capabilities) == (() if expected_key is None else (expected_key,))
@pytest.mark.parametrize("capability", GATED_AUTO_ROUTER_CAPABILITIES, ids=lambda c: c.key)
@ -729,11 +729,29 @@ def test_every_gated_capability_has_a_distinct_predicate_and_sql_spelling() -> N
},
],
)
def test_capabilities_are_mutually_exclusive_on_one_config(config: Mapping[str, object]) -> None:
"""No config claims two capabilities, which is what lets one lock and one count serve them all.
The config validator is what makes this true and is pinned separately in test_complexity_router:
tier_definitions rejects every heuristic classifier_type and rejects the classifier system_prompt,
and system_prompt only counts for the classifier types heuristic_v2 is not one of.
"""
def test_standalone_classifiers_claim_at_most_one_capability(config: Mapping[str, object]) -> None:
assert sum(1 for capability in GATED_AUTO_ROUTER_CAPABILITIES if capability.uses(config)) <= 1
@pytest.mark.parametrize("classifier", ("heuristic_first", "hybrid"))
@pytest.mark.parametrize("local_heuristic", (None, "heuristic", "heuristic_v2"))
@pytest.mark.parametrize("custom_prompt", (False, True))
def test_chained_heuristic_and_judge_prompt_claim_each_active_capability(
classifier: str, local_heuristic: str | None, custom_prompt: bool
) -> None:
config: Final = {
"classifier_type": classifier,
**({"local_heuristic": local_heuristic} if local_heuristic is not None else {}),
**({"classification_prompt": "Grade by difficulty"} if custom_prompt else {}),
}
params: Final = {"model": "auto_router/complexity_router", "complexity_router_config": config}
expected: Final = (
*(("heuristic_v2",) if local_heuristic == "heuristic_v2" else ()),
*(("tier_or_classifier_prompt",) if custom_prompt else ()),
)
assert tuple(capability.key for capability in claimed_capabilities(config)) == expected
assert tuple(capability.key for capability in gated_capabilities_of(params)) == expected
for capability in GATED_AUTO_ROUTER_CAPABILITIES:
assert count_capability_routers(({"litellm_params": params},), capability=capability) == (
1 if capability.key in expected else 0
)

22
tests/unit/router_utils/test_auto_router_tuning_baseline.py
View file

@ -183,6 +183,28 @@ class TestRouterIdentity:
class TestHeuristicV1Scope:
@pytest.mark.parametrize("classifier", ("heuristic_first", "hybrid"))
def test_v2_chains_ignore_retained_v1_tuning_until_v1_is_selected(self, classifier: str) -> None:
config: Final = {
"classifier_type": classifier,
"classifier_llm_config": {"model": "judge"},
**(
{"heuristic_first_max_tier": "MEDIUM"}
if classifier == "heuristic_first" else {"hybrid_boundary_margin": 0.1}
),
"tiers": _TIERS,
"code_keywords": ["internal-api"],
}
v1: Final = _router("chain", config)
v2: Final = _router("chain", {**config, "local_heuristic": "heuristic_v2"})
other: Final = _router("other", {"code_keywords": ["internal-api"]})
baselines: Final = snapshot_tuning_baselines((_router("chain", {}),))
assert heuristic_v1_router_fingerprint(v2) is None
assert not snapshot_tuning_baselines((v2,))
assert not mutable_tuned_identities((v2,), baselines)
assert tuning_quota_violation(candidate=v2, others=(other,), baselines=baselines, limit=1) is None
assert tuning_quota_violation(candidate=v1, others=(other,), baselines=baselines, limit=1) is not None
@pytest.mark.parametrize(
"config,in_scope",
[

37
ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx
View file

@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest";
import { fireEvent, renderWithProviders, screen, waitFor, within } from "../../../tests/test-utils";
import { selectAutoRouterOption } from "../../../tests/autoRouterSetup";
import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs";
import ClassificationMethodConfig from "./ClassificationMethodConfig";
import { AutoRouterAllowanceNote, AutoRouterAvailabilityContext } from "./AutoRouterAvailability";
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
@ -53,6 +54,42 @@ function Form({
}
describe("Auto-router classifier selection", () => {
it.each([false, true])("applies the v2 allowance to chain selection, owned: %s", (owned) => {
const onChange = vi.fn();
renderWithProviders(
<AutoRouterAvailabilityContext.Provider
value={{
isPending: false,
isError: false,
data: {
allowances: [{ key: "heuristic_v2", limit: 1, remaining: 0, available: true, used_by_this_router: owned }],
error: null,
},
}}
>
<ClassificationMethodConfig
advancedOnly
section="selection"
value={{ ...initial, classifier_type: "hybrid", hybrid_boundary_margin: 0.1 }}
onChange={onChange}
modelOptions={[]}
effortOptionsByModel={{}}
/>
</AutoRouterAvailabilityContext.Provider>,
);
fireEvent.click(screen.getByRole("button", { name: "Heuristic before the judge" }));
const option = screen.getByRole("menuitemradio", { name: /^Heuristic v2/ });
if (owned) {
expect(option).not.toHaveAttribute("aria-disabled", "true");
fireEvent.click(option);
expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ local_heuristic: "heuristic_v2" }));
} else {
expect(option).toHaveAttribute("aria-disabled", "true");
fireEvent.click(option);
expect(onChange).not.toHaveBeenCalled();
}
});
it.each(["heuristic", "heuristic_v2", "llm", "heuristic_first", "hybrid", "jev"] as const)(
"shows saved %s without changing its configuration",
async (classifier_type) => {

4
ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx
View file

@ -28,7 +28,7 @@ import {
AUTO_ROUTER_CONTACT_URL,
} from "./AutoRouterAvailability";
function ClassifierOption({
export function ClassifierOption({
value,
label,
description,
@ -75,7 +75,7 @@ function ClassifierOption({
);
}
function ClassifierMenu({
export function ClassifierMenu({
id,
label,
value,

140
ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
View file

@ -23,6 +23,9 @@ import ClassifierVisionConfig from "./ClassifierVisionConfig";
import { getHeuristicV2SuccessThresholdError } from "./build_complexity_router_config";
import ClassifierPluginTimeoutField from "./ClassifierPluginTimeoutField";
import ClassifierTypeRadios from "./ClassifierTypeRadios";
import { ClassifierMenu, ClassifierOption } from "./AutoRouterClassifierTabs";
import { isHeuristicChain } from "./classifier_types";
import HeuristicKeywordOverrides from "./HeuristicKeywordOverrides";
import type { ReasoningEffort } from "./complexity_router_tiers";
import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults";
import {
@ -39,6 +42,8 @@ import {
ClassificationRubric,
effectiveTierLabel,
heuristicScoringRole,
heuristicTuningType,
usesHeuristicV2,
usesLlmClassifier,
usesClassifierContext,
DEFAULT_HYBRID_BOUNDARY_MARGIN,
@ -53,8 +58,7 @@ const DEFAULT_SCORING_EXPLANATION =
const HEURISTIC_V2_EXPLANATION =
"The router estimates success probability for all four tiers with its calibrated model, then selects " +
"the first tier that meets the success threshold. If none qualify, it selects Reasoning. " +
"It runs locally with no classifier API call.";
"the first tier that meets the success threshold.";
const CLASSIFIER_TIMEOUT_ID = "classifier-timeout-ms";
const CLASSIFIER_CONTEXT_WINDOW_SIZE_ID = "classifier-context-window-size";
@ -62,30 +66,6 @@ const CLASSIFIER_CONTEXT_BUDGET_CHARS_ID = "classifier-context-budget-chars";
const HYBRID_BOUNDARY_MARGIN_ID = "hybrid-boundary-margin";
const HEURISTIC_V2_SUCCESS_THRESHOLD_ID = "heuristic-v2-success-threshold";
const CUSTOM_PROMPT_WITH_HEURISTIC_FALLBACK =
"This router classifies with your own prompt, so the tier comes from whatever rubric it states. The four tier " +
"names stay fixed. The scoring below is the heuristic, which now runs only when the classifier call fails:";
const CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK =
"This router classifies with your own prompt, so the tier comes from whatever rubric it states. The four tier " +
"names stay fixed. The scoring below no longer runs at all, since a failed classifier routes to the default " +
"model instead:";
/**
* What the scoring breakdown below it actually describes. A custom prompt means the score no longer
* decides the tier, and pairing one with the default-model fallback means the heuristic never runs
* at all, so the panel must not keep implying a score is involved on either router.
*/
const scoringExplanation = (value: ComplexityRouterConfigValue): string => {
if (value.classifier_type === "heuristic_v2") return HEURISTIC_V2_EXPLANATION;
const usesCustomPrompt =
usesLlmClassifier(value.classifier_type) && Boolean(value.classifier_llm_config?.system_prompt?.trim());
if (!usesCustomPrompt) return DEFAULT_SCORING_EXPLANATION;
return value.classifier_fallback === "default_model"
? CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK
: CUSTOM_PROMPT_WITH_HEURISTIC_FALLBACK;
};
/**
* The three boundaries this card states, as displayed strings, or null until the proxy's shipped defaults
* have arrived. Kept out of the component so the card cannot state a range the router stopped using, and
@ -121,6 +101,7 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> =
value.tier_boundaries,
value.reasoning_override_min_score,
);
const scorerDefaultsUnavailable = scorerRuns && !ranges && isError;
if (value.custom_tier_set) return null;
@ -128,7 +109,13 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> =
<Card className="bg-muted mt-4">
<CardContent>
<strong className="block mb-2 font-semibold">How Classification Works</strong>
<span className="text-[13px] text-muted-foreground">{scoringExplanation(value)}</span>
<span className="text-[13px] text-muted-foreground">
{usesHeuristicV2(value) ? HEURISTIC_V2_EXPLANATION : DEFAULT_SCORING_EXPLANATION}
{usesHeuristicV2(value) &&
(isHeuristicChain(value.classifier_type)
? " If no tier qualifies, or the local-check policy requires a judge, the judge chooses the tier."
: " If none qualify, it selects Reasoning. It runs locally with no classifier API call.")}
</span>
{scorerRuns && ranges && (
<ul className="mt-2 pl-5 text-[13px] text-muted-foreground">
<li>
@ -149,7 +136,7 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> =
</li>
</ul>
)}
{!ranges && isError && (
{scorerDefaultsUnavailable && (
<span className="text-[13px] block mt-2 text-muted-foreground">
The tier score ranges could not be loaded from the proxy.
</span>
@ -170,6 +157,7 @@ interface ClassificationMethodConfigProps {
/** The resolved default model - see resolveComplexityDefaultModel. Names and gates the radio. */
defaultModel?: string;
advancedOnly?: boolean;
section?: "selection" | "classifier" | "heuristic";
}
export const InactiveHeuristicV2Threshold: React.FC<Pick<ClassificationMethodConfigProps, "value" | "onChange">> = ({
@ -177,8 +165,9 @@ export const InactiveHeuristicV2Threshold: React.FC<Pick<ClassificationMethodCon
onChange,
}) => {
const threshold = value.heuristic_v2_success_threshold;
if (effectiveClassifierType(value) === "heuristic_v2" || threshold === undefined) return null;
if (usesHeuristicV2(value) || threshold === undefined) return null;
const error = getHeuristicV2SuccessThresholdError(threshold);
if (effectiveClassifierType(value) === "llm" && !error) return null;
return (
<section aria-label="Inactive Heuristic v2 threshold" className="mb-4 space-y-2 rounded-md border p-3">
<p className="text-sm font-medium">
@ -215,10 +204,17 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
showValidationErrors = false,
defaultModel,
advancedOnly = false,
section,
}) => {
const [draft, setDraft] = React.useState<{ id: string; raw: string } | null>(null);
const hasDefaultModel = Boolean(defaultModel);
const classifierType = effectiveClassifierType(value);
const showSelection = section === undefined || section === "selection";
const showClassifier = section === undefined || section === "classifier";
const heuristic = heuristicTuningType(value);
const showHeuristic = (section === undefined || section === "heuristic") && heuristic !== undefined;
const showLocalChecks =
showSelection && advancedOnly && ["llm", "heuristic_first", "hybrid"].includes(classifierType);
const usesCustomPrompt = Boolean(value.classifier_llm_config?.system_prompt?.trim());
const contextBudget = value.classifier_context_budget_chars ?? DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS;
const contextBudgetQuotesNothing = contextBudget > 0 && contextBudget < MIN_QUOTED_CONTEXT_TURN_CHARS;
@ -365,7 +361,7 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
return (
<>
{!advancedOnly && (
{!advancedOnly && showSelection && (
<>
<ClassifierTypeRadios
value={value}
@ -380,7 +376,7 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
/>
</>
)}
{advancedOnly && ["llm", "heuristic_first", "hybrid"].includes(classifierType) && (
{showLocalChecks && (
<div className="space-y-2">
<Label htmlFor="auto-router-local-checks">Local checks before the judge</Label>
<Select
@ -410,11 +406,39 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
</div>
)}
{classifierType === "custom" && (
{showSelection && isHeuristicChain(classifierType) && (
<div className="mt-4 space-y-2">
<Label htmlFor="auto-router-local-heuristic">Heuristic before the judge</Label>
<ClassifierMenu
id="auto-router-local-heuristic"
label="Heuristic before the judge"
selectedLabel={value.local_heuristic === "heuristic_v2" ? "Heuristic v2" : "Heuristic v1 (rule-based)"}
feature={value.local_heuristic === "heuristic_v2" ? "heuristic_v2" : undefined}
value={value.local_heuristic ?? "heuristic"}
onValueChange={(next) => {
if (next === "heuristic" || next === "heuristic_v2") onChange({ ...value, local_heuristic: next });
}}
>
<ClassifierOption
value="heuristic"
label="Heuristic v1 (rule-based)"
description="Score requests using configurable weights and keywords"
/>
<ClassifierOption
value="heuristic_v2"
label="Heuristic v2"
description="Choose a tier using calibrated success probabilities"
feature="heuristic_v2"
/>
</ClassifierMenu>
</div>
)}
{showClassifier && classifierType === "custom" && (
<ClassifierPluginTimeoutField value={value} onChange={onChange} showValidationErrors={showValidationErrors} />
)}
{classifierType === "heuristic_v2" && (
{showHeuristic && heuristic === "heuristic_v2" && (
<div className="mt-4 space-y-2">
<Label htmlFor={HEURISTIC_V2_SUCCESS_THRESHOLD_ID} className="block font-semibold">
Success threshold
@ -444,14 +468,16 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
</div>
)}
{classifierType === "heuristic_first" && (
{showSelection && classifierType === "heuristic_first" && (
<div className="mt-4 space-y-2">
<strong className="block font-semibold">Decide locally up to</strong>
<Label htmlFor="auto-router-local-max-tier" className="block font-semibold">
Decide locally up to
</Label>
<Select
value={value.heuristic_first_max_tier}
onValueChange={(tier: unknown) => handleHeuristicFirstMaxTierChange(tier as string)}
>
<SelectTrigger className="w-full">
<SelectTrigger id="auto-router-local-max-tier" className="w-full">
<SelectValue />
</SelectTrigger>
<SelectContent>
@ -463,15 +489,18 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
</SelectContent>
</Select>
<p className="text-sm text-muted-foreground">
A request the scorer places at or below this tier routes there without a classifier call. Anything the
scorer places higher, and anything it found no signal for at all, goes to the classifier instead
{usesHeuristicV2(value)
? "A tier that meets the success threshold and is at or below this limit routes locally. Higher tiers, or requests where no tier qualifies, go to the judge"
: "A request the scorer places at or below this tier routes locally. Higher tiers, or requests with no heuristic signal, go to the judge"}
</p>
</div>
)}
{classifierType === "hybrid" && (
{showSelection && classifierType === "hybrid" && (
<div className="mt-4 space-y-2">
<strong className="block font-semibold">Boundary margin</strong>
<Label htmlFor={HYBRID_BOUNDARY_MARGIN_ID} className="block font-semibold">
{usesHeuristicV2(value) ? "Success threshold margin" : "Boundary margin"}
</Label>
<Input
id={HYBRID_BOUNDARY_MARGIN_ID}
type="text"
@ -486,15 +515,15 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
className="w-full"
/>
<p className="text-sm text-muted-foreground">
A score further than this from every tier boundary routes on the scorer&apos;s own tier, however expensive
that tier is. A score closer than this, and anything the scorer found no signal for at all, goes to the
classifier to break the tie
{usesHeuristicV2(value)
? "Use the judge when the selected tier or a lower tier has a success probability within this margin of the success threshold, or no tier meets the threshold"
: "A score further than this from every tier boundary routes on the scorer's own tier. A score closer than this, or a request with no heuristic signal, goes to the judge"}
</p>
</div>
)}
{classifierType === "jev" && <JevClassifierConfig value={value} onChange={onChange} />}
{usesLlmClassifier(classifierType) && (
{showClassifier && classifierType === "jev" && <JevClassifierConfig value={value} onChange={onChange} />}
{showClassifier && usesLlmClassifier(classifierType) && (
<div className="mt-4 space-y-3">
<ClassifierReasoningEffortSelect
model={classifierModel}
@ -578,7 +607,7 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
</div>
</div>
)}
{usesClassifierContext(classifierType) && (
{showClassifier && usesClassifierContext(classifierType) && (
<div className="mt-4 space-y-3">
<RestrictedSection heading="If the classifier fails" by={restrictedBy(value, "classifierFallback")}>
<RadioGroup
@ -705,7 +734,7 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
</div>
)}
{heuristicScoringRole(value) !== "never" && (
{showHeuristic && heuristic === "heuristic" && (
<div className="mt-4">
<div className="flex items-center gap-2 mb-1">
<strong className="font-semibold">Custom Technical Keywords</strong>
@ -735,12 +764,19 @@ const ClassificationMethodConfig: React.FC<ClassificationMethodConfigProps> = ({
</div>
)}
{["heuristic", "heuristic_first", "hybrid"].includes(classifierType) && (
<AutoRouterAllowanceNote feature="heuristic_tuning" label="Custom scoring rules" />
{showHeuristic && heuristic === "heuristic" && (
<>
{heuristicScoringRole(value) === "decides" && (
<AutoRouterAllowanceNote feature="heuristic_tuning" label="Custom scoring rules" />
)}
<HeuristicScoringConfig value={value} onChange={onChange} />
<section className="mt-4 space-y-3">
<h4 className="font-semibold">Heuristic Keyword Overrides</h4>
<HeuristicKeywordOverrides value={value} onChange={onChange} />
</section>
</>
)}
<HeuristicScoringConfig value={value} onChange={onChange} />
<HowClassificationWorks value={value} />
{showHeuristic && <HowClassificationWorks value={value} />}
</>
);
};

58
ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx
View file

@ -3,7 +3,13 @@ import { ChevronRight } from "lucide-react";
import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
import { Separator } from "@/components/ui/separator";
import type { ModelGroup } from "@/components/llm_calls/fetch_models";
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
import {
effectiveClassifierType,
heuristicTuningType,
usesClassifierContext,
usesLlmClassifier,
type ComplexityRouterConfigValue,
} from "./ComplexityRouterConfig";
import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig";
import CacheAwareRoutingConfig from "./CacheAwareRoutingConfig";
import ClassificationMethodConfig from "./ClassificationMethodConfig";
@ -19,7 +25,6 @@ import CompressionControls from "./CompressionControls";
import PlanModeOverrideControls from "./PlanModeOverrideControls";
import { AffinityControls } from "./AffinityControls";
import { ModalityRoutingControls } from "./ModalityRoutingControls";
import HeuristicKeywordOverrides from "./HeuristicKeywordOverrides";
import HousekeepingRoutingControls from "./HousekeepingRoutingControls";
import ReminderMarkers from "./ReminderMarkers";
import type { AutoRouterCompressionState } from "./buildAutoRouterCompression";
@ -80,6 +85,18 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
tierRows,
customTierSet,
}) => {
const classifierType = effectiveClassifierType(value);
const classificationProps = {
advancedOnly: true,
value,
onChange,
modelOptions,
effortOptionsByModel: classifierEffortOptionsByModel,
customTechnicalKeywords,
onCustomTechnicalKeywordsChange,
showValidationErrors,
defaultModel,
};
const sections = [
...(forecast
? [
@ -98,33 +115,21 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
},
]
: []),
...(!forecast
...(!forecast && (usesClassifierContext(classifierType) || classifierType === "custom")
? [
{
key: "classifier",
label: <strong className="text-foreground font-semibold">Classification Method</strong>,
children: (
<ClassificationMethodConfig
advancedOnly
value={value}
onChange={onChange}
modelOptions={modelOptions}
effortOptionsByModel={classifierEffortOptionsByModel}
customTechnicalKeywords={customTechnicalKeywords}
onCustomTechnicalKeywordsChange={onCustomTechnicalKeywordsChange}
showValidationErrors={showValidationErrors}
defaultModel={defaultModel}
/>
),
children: <ClassificationMethodConfig {...classificationProps} section="classifier" />,
},
]
: []),
...(!forecast
...(heuristicTuningType(value)
? [
{
key: "keyword-overrides",
label: <strong className="text-foreground font-semibold">Heuristic Keyword Overrides</strong>,
children: <HeuristicKeywordOverrides value={value} onChange={onChange} />,
key: "heuristic",
label: <strong className="text-foreground font-semibold">Heuristic tuning</strong>,
children: <ClassificationMethodConfig {...classificationProps} section="heuristic" />,
},
]
: []),
@ -246,7 +251,9 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
];
const groups = [
{ label: "Classifier tuning", keys: ["classifier", "keyword-overrides", "reminder-markers"] },
{ label: "Heuristic tuning", keys: ["heuristic"] },
{ label: usesLlmClassifier(classifierType) ? "LLM tuning" : "Classifier tuning", keys: ["classifier"] },
{ label: "Request preprocessing", keys: ["reminder-markers"] },
{
label: "Routing rules and recovery",
keys: [
@ -261,7 +268,7 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
},
{ label: "Sessions and efficiency", keys: ["affinity", "adaptive", "cache-aware", "compression"] },
{ label: "Compatibility", keys: ["response"] },
];
].filter((group) => sections.some(({ key }) => group.keys.includes(key)));
const [openGroups, setOpenGroups] = React.useState<string[]>(() =>
showValidationErrors ? groups.map((group) => group.label) : [],
);
@ -272,6 +279,11 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
}
return (
<div>
{["llm", "heuristic_first", "hybrid"].includes(classifierType) && (
<div className="px-4 py-3">
<ClassificationMethodConfig {...classificationProps} section="selection" />
</div>
)}
{groups.map((group) => (
<Collapsible
key={group.label}
@ -296,7 +308,7 @@ const ComplexityRouterAdvancedSections: React.FC<ComplexityRouterAdvancedSection
)
.map(({ key, label, children }) => (
<section key={key} className="space-y-3">
{key !== "classifier" && <h4>{label}</h4>}
{key !== "classifier" && key !== "heuristic" && <h4>{label}</h4>}
{children}
</section>
))}

65
ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.integration.test.tsx
View file

@ -52,6 +52,53 @@ const baseProps = {
};
describe("ComplexityRouterConfig", () => {
it.each(["Heuristic first", "Hybrid"])(
"shows only the selected heuristic tuning when chaining with %s",
async (mode) => {
const initialValue: ComplexityRouterConfigValue = {
...defaultValue,
classifier_type: "llm",
classifier_llm_config: { model: "gpt-4", timeout_ms: 750 },
token_thresholds: { simple: 12, complex: 800 },
heuristic_v2_success_threshold: 0.9,
};
function Editor() {
const [value, setValue] = React.useState(initialValue);
return <ComplexityRouterConfig {...baseProps} value={value} onChange={setValue} />;
}
renderWithProviders(<Editor />);
fireEvent.click(screen.getByRole("button", { name: /^Advanced settings/ }));
expect(screen.getByRole("button", { name: "LLM tuning" })).toHaveAttribute("aria-expanded", "false");
expect(screen.queryByRole("button", { name: "Heuristic tuning" })).not.toBeInTheDocument();
expect(screen.queryByText("Advanced scoring")).not.toBeInTheDocument();
expect(screen.queryByText("Custom Technical Keywords")).not.toBeInTheDocument();
expect(screen.queryByText("Heuristic Keyword Overrides")).not.toBeInTheDocument();
expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument();
await selectAutoRouterOption("Local checks before the judge", mode);
expect(screen.getByRole("button", { name: "Heuristic before the judge" })).toHaveTextContent("Heuristic v1");
fireEvent.click(screen.getByRole("button", { name: "Heuristic tuning" }));
fireEvent.click(screen.getByRole("button", { name: /Advanced scoring/ }));
expect(screen.getByLabelText("Short below")).toHaveValue("12");
await selectAutoRouterOption("Heuristic before the judge", "Heuristic v2");
expect(screen.getByLabelText("Success threshold")).toHaveValue("0.9");
expect(screen.queryByText("Advanced scoring")).not.toBeInTheDocument();
expect(screen.queryByText("Custom Technical Keywords")).not.toBeInTheDocument();
expect(screen.queryByText("Heuristic Keyword Overrides")).not.toBeInTheDocument();
expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument();
if (mode === "Hybrid") expect(screen.getByLabelText("Success threshold margin")).toBeInTheDocument();
await selectAutoRouterOption("Heuristic before the judge", "Heuristic v1");
fireEvent.click(screen.getByRole("button", { name: /Advanced scoring/ }));
expect(screen.getByLabelText("Short below")).toHaveValue("12");
fireEvent.click(screen.getByRole("button", { name: "LLM tuning" }));
expect(screen.getByLabelText("Timeout (ms)")).toHaveValue("750");
await selectAutoRouterOption("Local checks before the judge", "Always use the judge");
expect(screen.queryByRole("button", { name: "Heuristic tuning" })).not.toBeInTheDocument();
expect(screen.queryByRole("button", { name: "Heuristic before the judge" })).not.toBeInTheDocument();
expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument();
expect(screen.getByLabelText("Timeout (ms)")).toHaveValue("750");
},
);
it("should render", async () => {
renderWithProviders(<ComplexityRouterConfig {...baseProps} />);
expect(screen.getByText("Models by tier")).toBeInTheDocument();
@ -100,7 +147,7 @@ describe("ComplexityRouterConfig", () => {
it("should default to heuristic and hide classifier model/timeout fields", async () => {
renderWithProviders(<ComplexityRouterConfig modelInfo={mockModelInfo} value={defaultValue} onChange={vi.fn()} />);
openAutoRouterAdvanced("Classification Method");
expect(screen.getByText("Classifier tuning")).toBeInTheDocument();
expect(screen.getByText("Heuristic tuning")).toBeInTheDocument();
expect(screen.queryByText("Judge model")).not.toBeInTheDocument();
});
@ -730,10 +777,10 @@ describe("ComplexityRouterConfig classifier fallback", () => {
/>,
);
openAutoRouterAdvanced("Classification Method");
expect(screen.getByText(/no longer runs at all/)).toBeInTheDocument();
expect(screen.queryByText("How Classification Works")).not.toBeInTheDocument();
});
it("still describes the heuristic as the fallback when a custom prompt keeps heuristic fallback", async () => {
it("keeps heuristic fallback selectable without showing heuristic tuning for a custom judge prompt", async () => {
renderWithProviders(
<ComplexityRouterConfig
modelInfo={mockModelInfo}
@ -745,7 +792,8 @@ describe("ComplexityRouterConfig classifier fallback", () => {
/>,
);
openAutoRouterAdvanced("Classification Method");
expect(screen.getByText(/only when the classifier call fails/)).toBeInTheDocument();
expect(screen.getByRole("radio", { name: /Score with the heuristic/ })).toBeChecked();
expect(screen.queryByText("How Classification Works")).not.toBeInTheDocument();
});
it("clears a stored fallback when switching back to the heuristic classifier", async () => {
@ -1505,15 +1553,6 @@ describe("ComplexityRouterConfig custom technical keywords", () => {
classifier_llm_config: llmConfig,
},
],
[
"llm falling back to the scorer",
{
...defaultValue,
classifier_type: "llm" as const,
classifier_llm_config: llmConfig,
classifier_fallback: "heuristic" as const,
},
],
])("offers the keywords on a router whose scorer runs: %s", (_label, value) => {
openClassificationPanel(value);
expect(screen.getByText("Custom Technical Keywords")).toBeInTheDocument();

29
ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
View file

@ -2,7 +2,7 @@ import RoutingOptions from "./RoutingOptions";
import ClassifierPrimarySettings from "./ClassifierPrimarySettings";
import { AutoRouterAllowanceNote } from "./AutoRouterAvailability";
import type { JevClassifierConfig } from "./jev_classifier_config";
import { type ClassifierType } from "./classifier_types";
import { isHeuristicChain, type ClassifierType, type LocalHeuristic } from "./classifier_types";
export { type ClassifierType, usesLlmClassifier, usesClassifierContext } from "./classifier_types";
import ForecastClassifierConfig, { ForecastSolverModels } from "./ForecastClassifierConfig";
import { isForecastClassifier, type CapabilitySettings, type FuseSettings } from "./forecast_classifier_config";
@ -156,16 +156,12 @@ export const DEFAULT_ADAPTIVE_WEIGHTS: AdaptiveRouterWeights = { quality: 0.3, c
export type HeuristicScoringRole = "decides" | "fallback_only" | "never";
/**
* Whether the heuristic scorer runs on this router at all, which is what gates its knobs. An LLM
* classifier still falls back to the scorer unless the fallback is the default model, so the gate cannot be
* a plain classifier_type check. Under heuristic_first the scorer runs first on every request and
* decides outright whenever it lands at or below the threshold.
*/
export const heuristicScoringRoleFor = (
classifierType: ClassifierType,
classifierFallback: ClassifierFallback | undefined,
localHeuristic?: LocalHeuristic,
): HeuristicScoringRole => {
if (isHeuristicChain(classifierType) && localHeuristic === "heuristic_v2") return "never";
if (classifierType === "heuristic_v2" || classifierType === "capability" || classifierType === "llm_v2")
return "never";
if (classifierType === "heuristic" || classifierType === "heuristic_first" || classifierType === "hybrid")
@ -174,7 +170,23 @@ export const heuristicScoringRoleFor = (
};
export const heuristicScoringRole = (value: ComplexityRouterConfigValue): HeuristicScoringRole =>
value.custom_tier_set ? "never" : heuristicScoringRoleFor(value.classifier_type, value.classifier_fallback);
value.custom_tier_set
? "never"
: heuristicScoringRoleFor(value.classifier_type, value.classifier_fallback, value.local_heuristic);
export const usesHeuristicV2 = (value: ComplexityRouterConfigValue): boolean => {
if (value.custom_tier_set) return false;
return (
value.classifier_type === "heuristic_v2" ||
(isHeuristicChain(value.classifier_type) && value.local_heuristic === "heuristic_v2")
);
};
export const heuristicTuningType = (value: ComplexityRouterConfigValue): LocalHeuristic | undefined => {
if (value.custom_tier_set || value.classifier_type === "llm") return undefined;
if (usesHeuristicV2(value)) return "heuristic_v2";
return heuristicScoringRole(value) === "never" ? undefined : "heuristic";
};
// Derived, never written into the value, so undoing a tier edit reverts the form with nothing left behind.
export const effectiveClassifierType = (
@ -359,6 +371,7 @@ export interface ComplexityRouterConfigValue {
/** An explicit pin. Unset means the default tracks the tiers - see resolveComplexityDefaultModel. */
default_model?: string;
classifier_type: ClassifierType;
local_heuristic?: LocalHeuristic;
heuristic_v2_success_threshold?: number;
capability_classifier_config?: CapabilitySettings;
llm_v2_config?: FuseSettings;

3
ui/litellm-dashboard/src/components/add_model/HeuristicKeywordOverrides.tsx
View file

@ -16,8 +16,7 @@ const HeuristicKeywordOverrides: React.FC<{
<div className="space-y-3">
<p className="text-sm text-muted-foreground">
Each list replaces the built-in keyword list of the same name for the heuristic scorer. Leave a list empty to keep
the built-in one. To add technical terms without replacing the list, use custom technical keywords under
Classification Method.
the built-in one. To add technical terms without replacing the list, use custom technical keywords above.
</p>
{fields.map(([key, label]) => {
const keywords = value[key] ?? [];

7
ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.test.tsx
View file

@ -169,7 +169,12 @@ describe("ClassificationMethodConfig scorer gating", () => {
it.each([
["heuristic decides the tier", "heuristic" as ClassifierType, undefined, true],
["heuristic v2 decides without the weighted scorer", "heuristic_v2" as ClassifierType, undefined, false],
["an LLM classifier falls back to the heuristic", "llm" as ClassifierType, "heuristic" as ClassifierFallback, true],
[
"an LLM classifier falls back to the heuristic",
"llm" as ClassifierType,
"heuristic" as ClassifierFallback,
false,
],
[
"an LLM classifier falls back to the default model",
"llm" as ClassifierType,

33
ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx
View file

@ -931,6 +931,39 @@ describe("AddAutoRouterTab", () => {
});
});
it("creates the selected v2 chain after editing its threshold and local tier ceiling", async () => {
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
const actualSubmit = await vi.importActual<typeof import("./handle_add_auto_router_submit")>(
"./handle_add_auto_router_submit",
);
vi.mocked(handleAddAutoRouterSubmit).mockImplementationOnce(actualSubmit.handleAddAutoRouterSubmit);
renderWithProviders(<Harness />);
const setup = await screen.findByRole("button", { name: "Choose models for me" });
await waitFor(() => expect(setup).toBeEnabled());
await userEvent.click(setup);
fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "chained-router" } });
await userEvent.click(screen.getByRole("radio", { name: "LLM" }));
await selectAutoRouterOption("Judge model", ALL_FAMILY_MODELS[0].model_group);
openAutoRouterAdvanced("Classification Method");
await selectAutoRouterOption("Local checks before the judge", "Heuristic first");
await selectAutoRouterOption("Heuristic before the judge", "Heuristic v2");
fireEvent.click(screen.getByRole("button", { name: "Heuristic tuning" }));
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "0.6" } });
await selectAutoRouterOption("Decide locally up to", "Medium");
await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled());
await userEvent.click(screen.getByRole("button", { name: "Add Auto Router" }));
await waitFor(() => expect(modelCreateCall).toHaveBeenCalledOnce());
const expectedConfig = {
classifier_type: "heuristic_first",
local_heuristic: "heuristic_v2",
heuristic_v2_success_threshold: 0.6,
heuristic_first_max_tier: "MEDIUM",
};
expect(vi.mocked(modelCreateCall).mock.calls.at(-1)?.[1].litellm_params.complexity_router_config).toMatchObject(
expectedConfig,
);
});
it("preserves classifier tuning when choosing models automatically", async () => {
const user = userEvent.setup();
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);

45
ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
View file

@ -57,6 +57,46 @@ const baseParams: BuildComplexityRouterConfigParams = {
};
describe("buildComplexityRouterConfig", () => {
it.each(["heuristic_first", "hybrid"] as const)(
"persists the selected local heuristic for %s and preserves inactive v1 settings",
(classifierType) => {
const params = {
...baseParams,
classifierType,
heuristicFirstMaxTier: "SIMPLE",
hybridBoundaryMargin: 0.1,
localHeuristic: "heuristic_v2" as const,
heuristicV2SuccessThreshold: 0,
tokenThresholds: { simple: 12, complex: 800 },
customDimensions: [{ id: "domain", name: "domain", weight: 0.2, keywords: ["billing"] }],
};
const expected = {
classifier_type: classifierType,
local_heuristic: "heuristic_v2",
heuristic_v2_success_threshold: 0,
token_thresholds: params.tokenThresholds,
};
expect(buildComplexityRouterConfig(params)).toMatchObject(expected);
expect(buildComplexityRouterConfig(params)).not.toHaveProperty("custom_dimensions");
expect(buildComplexityRouterConfig({ ...params, localHeuristic: "heuristic" })).toMatchObject({
local_heuristic: "heuristic",
custom_dimensions: [{ name: "domain", weight: 0.2, keywords: ["billing"] }],
});
expect(buildComplexityRouterConfig({ ...params, localHeuristic: undefined })).not.toHaveProperty(
"local_heuristic",
);
},
);
it.each(["heuristic", "heuristic_v2", "llm", "jev", "custom", "capability", "llm_v2"] as const)(
"omits the inactive chain selector under %s",
(classifierType) => {
expect(
buildComplexityRouterConfig({ ...baseParams, classifierType, localHeuristic: "heuristic_v2" }),
).not.toHaveProperty("local_heuristic");
},
);
it.each([undefined, false, true])(
"keeps cache routing opt-in and uses eligible single-model tiers only when enabled: %s",
(enabled) => {
@ -1362,13 +1402,16 @@ describe("buildComplexityRouterConfig with an edited tier set", () => {
reasoningOverrideMinScore: 0.5,
heuristicFirstMaxTier: "SIMPLE",
hybridBoundaryMargin: 0.03,
localHeuristic: "heuristic",
customTechnicalKeywords: ["kubernetes"],
stallEscalationEnabled: true,
stallEscalationWindow: 6,
stallEscalationRepeatThreshold: 3,
};
// custom_dimensions only ever ship when the scorer decides, so "llm" cannot prove it emits.
const emittingType = key === "heuristic_first_max_tier" || key === "custom_dimensions" ? "heuristic_first" : "llm";
const emittingType = ["heuristic_first_max_tier", "custom_dimensions", "local_heuristic"].includes(key)
? "heuristic_first"
: "llm";
const typeForKey = key === "hybrid_boundary_margin" ? "hybrid" : emittingType;
expect(buildComplexityRouterConfig({ ...baseParams, ...loaded, classifierType: typeForKey })).toHaveProperty(key);
expect(build(loaded)).not.toHaveProperty(key);

50
ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
View file

@ -5,6 +5,7 @@ import {
type FuseSettings,
} from "./forecast_classifier_config";
import type { ModelGroup } from "../llm_calls/fetch_models";
import { isHeuristicChain, type LocalHeuristic } from "./classifier_types";
import { KeywordTierRule } from "./KeywordTierRules";
import {
type JevClassifierConfig,
@ -22,7 +23,8 @@ import {
tierRowByName,
} from "./tier_rows";
import { emptyKeywordTierRuleIndexes, serializeKeywordTierRules } from "./complexity_router_keywords";
import { type CustomDimension, type CustomDimensionRow, serializeCustomDimensions } from "./custom_dimensions";
import { type CustomDimension, type CustomDimensionRow } from "./custom_dimensions";
import { scorerKnobPayload } from "./heuristic_scoring_knobs";
import {
TierModelParams,
TierModelParamsByTier,
@ -47,7 +49,6 @@ import {
TierBoundaries,
TokenThresholds,
effectiveTierLabel,
heuristicScoringRoleFor,
usesLlmClassifier,
usesClassifierContext,
} from "./ComplexityRouterConfig";
@ -103,45 +104,6 @@ export const normalizeClassifierLlmConfig = ({
...(vision && { vision }),
};
interface ScorerKnobInputs {
classifierType: ClassifierType;
classifierFallback: ClassifierFallback | undefined;
tierBoundaries: TierBoundaries | undefined;
tokenThresholds: TokenThresholds | undefined;
dimensionWeights: DimensionWeights | undefined;
customDimensions: CustomDimensionRow[] | undefined;
reasoningOverrideMinScore: number | undefined;
}
/**
* The scorer knobs to persist, which is none of them on a router that never scores: an LLM classifier
* falling back to the default model would otherwise carry settings that can only mislead the next reader.
* Each is omitted while untouched, so the router keeps tracking the backend defaults.
*/
const scorerKnobPayload = ({
classifierType,
classifierFallback,
tierBoundaries,
tokenThresholds,
dimensionWeights,
customDimensions,
reasoningOverrideMinScore,
}: ScorerKnobInputs) => {
const role = heuristicScoringRoleFor(classifierType, classifierFallback);
return role === "never"
? {}
: {
...(tierBoundaries && { tier_boundaries: tierBoundaries }),
...(tokenThresholds && { token_thresholds: tokenThresholds }),
...(dimensionWeights && { dimension_weights: dimensionWeights }),
// Only a scorer that decides accepts these; the backend rejects them on every other
// classifier, so a fallback-only router must not carry rows a switch left behind.
...(role === "decides" &&
customDimensions !== undefined && { custom_dimensions: serializeCustomDimensions(customDimensions) }),
...(reasoningOverrideMinScore !== undefined && { reasoning_override_min_score: reasoningOverrideMinScore }),
};
};
export interface StoredComplexityRouterConfig {
tiers?: Record<string, unknown>;
enable_non_reasoning_tier?: boolean;
@ -154,6 +116,7 @@ export interface StoredComplexityRouterConfig {
hybrid_boundary_margin?: unknown;
tier_labels?: unknown;
classifier_type?: ClassifierType | "oss_classifier";
local_heuristic?: unknown;
heuristic_v2_success_threshold?: unknown;
capability_classifier_config?: unknown;
llm_v2_config?: unknown;
@ -209,6 +172,7 @@ export interface BuildComplexityRouterConfigParams {
planModeMinTier: string | undefined;
tierLabels: ComplexityTierLabels | undefined;
classifierType: ClassifierType;
localHeuristic?: LocalHeuristic;
heuristicV2SuccessThreshold?: number;
capabilityClassifierConfig?: CapabilitySettings;
llmV2Config?: FuseSettings;
@ -291,6 +255,7 @@ export interface ComplexityRouterConfigPayload {
plan_mode_min_tier?: string;
tier_labels?: ComplexityTierLabels;
classifier_type: ClassifierType | "oss_classifier";
local_heuristic?: LocalHeuristic;
heuristic_v2_success_threshold?: number;
capability_classifier_config?: CapabilitySettings;
llm_v2_config?: FuseSettings;
@ -665,6 +630,7 @@ export const buildComplexityRouterConfig = ({
planModeMinTier,
tierLabels,
classifierType,
localHeuristic,
heuristicV2SuccessThreshold,
capabilityClassifierConfig,
llmV2Config,
@ -732,6 +698,7 @@ export const buildComplexityRouterConfig = ({
const cleanedTierLabels = serializeTierLabels(tierLabels);
const scorerInputs = {
classifierType,
localHeuristic,
classifierFallback,
tierBoundaries,
tokenThresholds,
@ -795,6 +762,7 @@ export const buildComplexityRouterConfig = ({
...(planModeMinTier?.trim() && { plan_mode_min_tier: planModeMinTier }),
...(cleanedTierLabels && { tier_labels: cleanedTierLabels }),
classifier_type: classifierType === "jev" ? "oss_classifier" : classifierType,
...(isHeuristicChain(effectiveType) && localHeuristic !== undefined && { local_heuristic: localHeuristic }),
...(effectiveType === "jev" && { opensource_classifier_config: normalizeJevClassifierConfig(jevClassifierConfig) }),
...(heuristicV2SuccessThreshold !== undefined && {
heuristic_v2_success_threshold: heuristicV2SuccessThreshold,

5
ui/litellm-dashboard/src/components/add_model/classifier_types.ts
View file

@ -9,6 +9,11 @@ export type ClassifierType =
| "llm_v2"
| "custom";
export type LocalHeuristic = "heuristic" | "heuristic_v2";
export const isHeuristicChain = (classifierType: ClassifierType): boolean =>
classifierType === "heuristic_first" || classifierType === "hybrid";
export const usesLlmClassifier = (classifierType: ClassifierType): boolean =>
(["llm", "heuristic_first", "hybrid", "capability", "llm_v2"] as const).some((type) => type === classifierType);

1
ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts
View file

@ -30,6 +30,7 @@ export const builderParamsFromValue = (
classificationMode: value.classification_mode,
tierLabels: value.tier_labels,
classifierType: value.classifier_type,
localHeuristic: value.local_heuristic,
jevClassifierConfig: value.jev_classifier_config,
heuristicV2SuccessThreshold: value.heuristic_v2_success_threshold,
capabilityClassifierConfig: value.capability_classifier_config,

33
ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.ts
View file

@ -1,4 +1,6 @@
import type { CustomDimensionRow } from "./custom_dimensions";
import { serializeCustomDimensions, type CustomDimensionRow } from "./custom_dimensions";
import { heuristicScoringRoleFor, type ClassifierFallback } from "./ComplexityRouterConfig";
import { isHeuristicChain, type ClassifierType, type LocalHeuristic } from "./classifier_types";
export type TierBoundaries = Record<string, number>;
@ -6,6 +8,35 @@ export type TokenThresholds = Record<string, number>;
export type DimensionWeights = Record<string, number>;
interface ScorerKnobInputs {
classifierType: ClassifierType;
localHeuristic: LocalHeuristic | undefined;
classifierFallback: ClassifierFallback | undefined;
tierBoundaries: TierBoundaries | undefined;
tokenThresholds: TokenThresholds | undefined;
dimensionWeights: DimensionWeights | undefined;
customDimensions: CustomDimensionRow[] | undefined;
reasoningOverrideMinScore: number | undefined;
}
export const scorerKnobPayload = (knobs: ScorerKnobInputs) => {
const role = heuristicScoringRoleFor(knobs.classifierType, knobs.classifierFallback, knobs.localHeuristic);
return role === "never" && !isHeuristicChain(knobs.classifierType)
? {}
: {
...(knobs.tierBoundaries && { tier_boundaries: knobs.tierBoundaries }),
...(knobs.tokenThresholds && { token_thresholds: knobs.tokenThresholds }),
...(knobs.dimensionWeights && { dimension_weights: knobs.dimensionWeights }),
...(role === "decides" &&
knobs.customDimensions !== undefined && {
custom_dimensions: serializeCustomDimensions(knobs.customDimensions),
}),
...(knobs.reasoningOverrideMinScore !== undefined && {
reasoning_override_min_score: knobs.reasoningOverrideMinScore,
}),
};
};
/**
* Display names for the scorer's dimensions. Only the wording lives here; the dimension set and its
* shipped weights come from the proxy (GET /public/complexity_router/scorer_defaults), so a dimension

2
ui/litellm-dashboard/src/components/add_model/tier_rows.ts
View file

@ -143,7 +143,7 @@ export const CUSTOM_TIER_RESTRICTIONS = {
reason: "Session pinning escalates along the built-in tier ladder, which your tier set replaces",
},
heuristicClassifier: {
omit: ["heuristic_first_max_tier", "hybrid_boundary_margin"],
omit: ["heuristic_first_max_tier", "hybrid_boundary_margin", "local_heuristic"],
reason:
"The heuristic scorer only produces the built-in tiers, so an edited set needs the LLM or JEV classifier. " +
"Heuristic first and hybrid are out for the same reason: their local scorer decides the traffic it is sure of",

3
ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts
View file

@ -895,6 +895,7 @@ describe("managed keys survive an untouched open-and-save", () => {
plan_mode_min_tier: "COMPLEX",
tier_labels: { SIMPLE: "Cheap" },
classifier_type: "heuristic_first",
local_heuristic: "heuristic",
heuristic_v2_success_threshold: 0.89,
heuristic_first_max_tier: "SIMPLE",
classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000, reasoning_effort: "low" },
@ -968,6 +969,8 @@ describe("managed keys survive an untouched open-and-save", () => {
it("carries every managed key a built-in router can hold through hydrate then save", () => {
const hydrated = hydrateComplexityRouterConfig(STORED_ALL_MANAGED, undefined);
const saved = buildUpdatedComplexityRouterConfig(STORED_ALL_MANAGED, hydrated);
expect(hydrated.local_heuristic).toBe(STORED_ALL_MANAGED.local_heuristic);
expect(saved.local_heuristic).toBe(STORED_ALL_MANAGED.local_heuristic);
const dropped = [...MANAGED_COMPLEXITY_ROUTER_KEYS]
.filter((key) => !KEYS_ANOTHER_CLASSIFIER_TYPE_OWNS.has(key))

34
ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx
View file

@ -199,6 +199,40 @@ describe("EditAutoRouterModal keyword matching", () => {
else expect(savedConfig().heuristic_v2_success_threshold).toBe(0);
});
it("saves a legacy heuristic-first router with a v2 chain and an edited success threshold", async () => {
renderModal({
modelData: {
...MODEL_DATA,
litellm_params: {
...MODEL_DATA.litellm_params,
complexity_router_config: {
...STORED_CONFIG,
classifier_type: "heuristic_first",
heuristic_first_max_tier: "SIMPLE",
classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000 },
token_thresholds: { simple: 12, complex: 800 },
},
},
},
});
openAutoRouterAdvanced("Classification Method");
await selectAutoRouterOption("Heuristic before the judge", "Heuristic v2");
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "1.1" } });
expect(screen.getByRole("button", { name: "Save Changes" })).toBeDisabled();
fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "0.88" } });
await waitFor(() => expect(screen.getByRole("button", { name: "Save Changes" })).toBeEnabled());
await userEvent.click(screen.getByRole("button", { name: "Save Changes" }));
await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce());
const expected = {
classifier_type: "heuristic_first",
local_heuristic: "heuristic_v2",
heuristic_v2_success_threshold: 0.88,
token_thresholds: { simple: 12, complex: 800 },
classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000 },
};
expect(savedConfig()).toMatchObject(expected);
});
it("blocks an invalid threshold edit and retains a corrected value when switching classifiers", async () => {
const user = userEvent.setup();
renderModal({

24
ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts
View file

@ -79,6 +79,30 @@ const expectedAdaptiveDisabledConfig = {
};
describe("buildUpdatedComplexityRouterConfig", () => {
it.each(["heuristic_first", "hybrid"] as const)(
"round-trips %s local heuristic selection without materializing the legacy default",
(classifier_type) => {
const stored = {
...storedConfigValue,
classifier_type,
heuristic_first_max_tier: "SIMPLE",
hybrid_boundary_margin: 0.1,
};
const legacy = hydrateComplexityRouterConfig(stored, null);
expect(buildUpdatedComplexityRouterConfig(stored, legacy)).not.toHaveProperty("local_heuristic");
const selected = { ...stored, local_heuristic: "heuristic_v2", heuristic_v2_success_threshold: 0.9 };
const hydrated = hydrateComplexityRouterConfig(selected, null);
expect(hydrated.local_heuristic).toBe("heuristic_v2");
expect(buildUpdatedComplexityRouterConfig(selected, hydrated)).toMatchObject({
local_heuristic: "heuristic_v2",
heuristic_v2_success_threshold: 0.9,
});
expect(buildUpdatedComplexityRouterConfig(selected, { ...hydrated, classifier_type: "llm" })).not.toHaveProperty(
"local_heuristic",
);
},
);
it("preserves unrelated options and omits the penalty for classified-tier routing", () => {
const updatedConfig = buildUpdatedComplexityRouterConfig(storedConfig, classifiedTierValue);

1
ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
View file

@ -97,6 +97,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
"plan_mode_min_tier",
"tier_labels",
"classifier_type",
"local_heuristic",
"capability_classifier_config",
"llm_v2_config",
"classifier_llm_config",

4
ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts
View file

@ -72,6 +72,10 @@ export const hydrateComplexityRouterConfig = (
plan_mode_min_tier: hydratePlanModeMinTier(parsedConfig.plan_mode_min_tier, custom_tier_set),
tier_labels: hydrateTierLabels(parsedConfig.tier_labels),
...classifier,
local_heuristic:
parsedConfig.local_heuristic === "heuristic" || parsedConfig.local_heuristic === "heuristic_v2"
? parsedConfig.local_heuristic
: undefined,
heuristic_v2_success_threshold:
typeof parsedConfig.heuristic_v2_success_threshold === "number"
? parsedConfig.heuristic_v2_success_threshold

13
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -42201,18 +42201,18 @@ export interface components {
fallback_tier?: string | null;
/**
* Heuristic First Max Tier
* @description The highest tier the local scorer may decide on its own; required when classifier_type is 'heuristic_first' and rejected otherwise. A request whose heuristic tier is at or below this one skips the LLM classifier and routes straight to that heuristic tier, so the classifier call is only paid for on traffic the scorer could not place cheaply. The scorer must also have produced at least one signal: a prompt where no dimension fired scores 0.0 and would otherwise land SIMPLE by default rather than by evidence, which is how a chained router would silently send unclassified traffic to the cheapest model. Names a built-in tier, and may not name the highest one, since that would make the LLM classifier unreachable.
* @description The highest tier the local scorer may decide on its own; required when classifier_type is 'heuristic_first' and rejected otherwise. A request whose heuristic tier is at or below this one skips the LLM classifier and routes straight to that heuristic tier, so the classifier call is only paid for on traffic the scorer could not place cheaply. The scorer must also have produced at least one signal: a prompt where no dimension fired scores 0.0 and would otherwise land SIMPLE by default rather than by evidence. With local_heuristic 'heuristic_v2', the predicted tier must meet its success threshold. Names a built-in tier, and may not name the highest one, since that would make the LLM classifier unreachable.
*/
heuristic_first_max_tier?: string | null;
/**
* Heuristic V2 Artifact
* @description Success-probability artifact used by classifier_type 'heuristic_v2'. The bundled UltraFeedback artifact is selected by default; an inline trained artifact may replace it
* @description Success-probability artifact used by standalone or chained heuristic_v2. The bundled UltraFeedback artifact is selected by default; an inline trained artifact may replace it
* @default ultrafeedback
*/
heuristic_v2_artifact: components["schemas"]["TrainedTierArtifact"] | "ultrafeedback";
/**
* Heuristic V2 Success Threshold
* @description Minimum predicted success probability for classifier_type 'heuristic_v2' to select a tier. The first tier meeting this threshold is selected, or REASONING if none meets it. When omitted or null, uses the artifact's routing_threshold (0.75 for the bundled artifact). Other classifier types ignore this setting
* @description Minimum predicted success probability for standalone or chained heuristic_v2 to select a tier. The first tier meeting this threshold is selected. When none meets it, standalone heuristic_v2 selects REASONING and chained heuristic_v2 defers to the LLM judge. When omitted or null, uses the artifact's routing_threshold (0.75 for the bundled artifact). Ignored when heuristic_v2 is not selected
*/
heuristic_v2_success_threshold?: number | null;
/**
@ -42222,7 +42222,7 @@ export interface components {
housekeeping_patterns?: string[] | null;
/**
* Hybrid Boundary Margin
* @description How close to a tier boundary a heuristic score has to land before the LLM classifier breaks the tie; required when classifier_type is 'hybrid' and rejected otherwise. Everything further than this from every active boundary routes on the scorer's own tier with no classifier call, at any tier, which is what separates 'hybrid' from 'heuristic_first' and its cheap-tier ceiling. A prompt where no dimension fired still goes to the classifier, since the scorer has no opinion to be near a boundary with. 0 escalates only scores sitting exactly on a boundary.
* @description How close to a tier boundary a heuristic score has to land before the LLM classifier breaks the tie; required when classifier_type is 'hybrid' and rejected otherwise. Everything further than this from every active boundary routes on the scorer's own tier with no classifier call, at any tier, which is what separates 'hybrid' from 'heuristic_first' and its cheap-tier ceiling. A prompt where no dimension fired still goes to the classifier, since the scorer has no opinion to be near a boundary with. With local_heuristic 'heuristic_v2', a tier must meet its success threshold and its probability and all lower-tier probabilities must be further than this margin from that threshold. 0 escalates only scores or probabilities exactly on a boundary.
*/
hybrid_boundary_margin?: number | null;
/**
@ -42232,6 +42232,11 @@ export interface components {
keyword_tier_rules?: components["schemas"]["KeywordTierRule"][] | null;
/** @description Experimental joint task-demand and solver-capability forecasting for classifier_type llm_v2. */
llm_v2_config?: components["schemas"]["LLMV2Config"] | null;
/**
* Local Heuristic
* @description Local scorer for heuristic_first or hybrid. Omitted or null keeps heuristic v1; heuristic_v2 uses the trained success predictor. Rejected for other classifier types.
*/
local_heuristic?: ("heuristic" | "heuristic_v2") | null;
/**
* Match Threshold
* @description Minimum cosine similarity for a semantic keyword match

17
ui/litellm-dashboard/tests/autoRouterSetup.ts
View file

@ -2,9 +2,8 @@ import userEvent from "@testing-library/user-event";
import { chooseSelectOption, fireEvent, screen } from "./test-utils";
const groups: Record<string, string> = {
"Classification Method": "Classifier tuning",
"Heuristic Keyword Overrides": "Classifier tuning",
"Ignore Custom Tags": "Classifier tuning",
"Heuristic Keyword Overrides": "Heuristic tuning",
"Ignore Custom Tags": "Request preprocessing",
Affinity: "Sessions and efficiency",
"Adaptive Routing": "Sessions and efficiency",
"Cache-aware routing": "Sessions and efficiency",
@ -15,12 +14,18 @@ const groups: Record<string, string> = {
export const openAutoRouterAdvanced = (section: string) => {
const advanced = screen.getByRole("button", { name: /^Advanced settings/ });
if (advanced.getAttribute("aria-expanded") !== "true") fireEvent.click(advanced);
const group = screen.getByRole("button", { name: groups[section] ?? "Routing rules and recovery" });
if (group.getAttribute("aria-expanded") !== "true") fireEvent.click(group);
const labels =
section === "Classification Method"
? ["Heuristic tuning", "LLM tuning", "Classifier tuning"]
: [groups[section] ?? "Routing rules and recovery"];
for (const label of labels) {
const group = screen.queryByRole("button", { name: label });
if (group && group.getAttribute("aria-expanded") !== "true") fireEvent.click(group);
}
};
export const selectAutoRouterOption = async (field: string, option: string) => {
if (field === "Heuristic" || field === "Routing approach") {
if (field === "Heuristic" || field === "Routing approach" || field === "Heuristic before the judge") {
const user = userEvent.setup();
fireEvent.click(screen.getByRole("button", { name: field }));
await user.click(await screen.findByRole("menuitemradio", { name: new RegExp(`^${option}`) }));