diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 42c4a15a4ef..c7883f5cf56 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -126,10 +126,11 @@ from litellm.router_strategy.complexity_router.config import resolve_complexity_ from litellm.router_utils.auto_router_model_naming import ( GATED_AUTO_ROUTER_CAPABILITIES, STRATEGY_ROUTER_PARAM_FIELDS, + GatedAutoRouterCapability, capability_limit_violation, carries_complexity_router_settings, count_capability_routers, - gated_capability_of, + gated_capabilities_of, is_complexity_router_model, validate_complexity_router_config_placement, validate_complexity_router_config_write, @@ -559,6 +560,20 @@ def _raise_on_tuning_quota_violation( raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"{violation} {AUTO_ROUTER_LICENSE_REMEDY}") +async def _claimed_capability_violation( + tables: _TxModelTables, + *, + capability: GatedAutoRouterCapability, + config_rows: Sequence[Mapping[str, object]], + model_id: str | None, + limit: int | None, +) -> str | None: + rows: Final = await tables.query_raw(_CAPABILITY_DB_ROWS_SQL[capability.key], model_id or "") + db_held: Final = sum(1 for row in rows if is_complexity_router_model(_decrypted_model(row.get("model")))) + held: Final = db_held + count_capability_routers(config_rows, capability=capability) + return capability_limit_violation(capability=capability, held=held + 1, limit=limit) + + @asynccontextmanager async def _auto_router_capability_slot( prisma_client: PrismaClient, @@ -591,11 +606,11 @@ async def _auto_router_capability_slot( ) limit: Final = _license_check.auto_router_capability_limit() - capability: Final = gated_capability_of(effective_params) + capabilities: Final = gated_capabilities_of(effective_params) baselines: Final = heuristic_v1_tuning_baselines tuning_candidate: Final = _tuning_candidate(effective_params, model_id=model_id) judges_tuning: Final = baselines is not None and is_mutable_tuned_candidate(tuning_candidate, baselines) - if member_write is None and (limit is None or (capability is None and not judges_tuning)): + if member_write is None and (limit is None or (not capabilities and not judges_tuning)): yield _proxy_model_table(prisma_client) return transaction_client: Final = _ModelTransactionClient.model_validate(prisma_client.db) @@ -685,14 +700,12 @@ async def _auto_router_capability_slot( prisma_client=pinned_client, llm_router=llm_router, ) - if capability is not None: - rows: Sequence[Mapping[str, object]] = await tx_ctx.query_raw( - _CAPABILITY_DB_ROWS_SQL[capability.key], model_id or "" - ) - db_held: Final = sum(1 for row in rows if is_complexity_router_model(_decrypted_model(row.get("model")))) - held: Final = db_held + count_capability_routers(config_rows, capability=capability) - violation: Final = capability_limit_violation(capability=capability, held=held + 1, limit=limit) - if violation is not None: + for capability in capabilities: + if ( + violation := await _claimed_capability_violation( + tables, capability=capability, config_rows=config_rows, model_id=model_id, limit=limit + ) + ) is not None: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=f"{violation} {AUTO_ROUTER_LICENSE_REMEDY}" ) diff --git a/litellm/proxy/management_helpers/auto_router_availability.py b/litellm/proxy/management_helpers/auto_router_availability.py index 89baf738d75..9b511ce0817 100644 --- a/litellm/proxy/management_helpers/auto_router_availability.py +++ b/litellm/proxy/management_helpers/auto_router_availability.py @@ -12,7 +12,7 @@ from litellm.router_utils.auto_router_model_naming import ( capability_limit_violation, classify_strategy_router_model, count_capability_routers, - gated_capability_of, + gated_capabilities_of, ) from litellm.router_utils.auto_router_tuning_baseline import ( is_mutable_tuned_candidate, @@ -91,8 +91,8 @@ def auto_router_availability( ) -> AutoRouterAvailabilityResponse: existing_params: Final = None if existing is None else existing.get("litellm_params") candidate_params: Final = candidate.get("litellm_params") - owned: Final = gated_capability_of(existing_params) if isinstance(existing_params, Mapping) else None - claimed: Final = gated_capability_of(candidate_params) if isinstance(candidate_params, Mapping) else None + owned: Final = gated_capabilities_of(existing_params) if isinstance(existing_params, Mapping) else () + claimed: Final = gated_capabilities_of(candidate_params) if isinstance(candidate_params, Mapping) else () counts: Final = tuple( (capability, count_capability_routers(others, capability=capability)) for capability in GATED_AUTO_ROUTER_CAPABILITIES @@ -103,15 +103,16 @@ def auto_router_availability( key=capability.key, limit=limit, remaining=None if limit is None else max(0, limit - held), - used_by_this_router=owned is capability, + used_by_this_router=capability in owned, ) for capability, held in counts ) - capability_error: Final = next( + denied_capability: Final = next( ( - capability_limit_violation(capability=capability, held=held + 1, limit=limit) + capability for capability, held in counts - if capability is claimed + if capability in claimed + and capability_limit_violation(capability=capability, held=held + 1, limit=limit) is not None ), None, ) @@ -140,8 +141,8 @@ def auto_router_availability( ), ), error=( - f"{capability_labels[claimed.key]} has no available allowance. Choose another option or free an existing allowance." - if capability_error is not None and claimed is not None + f"{capability_labels[denied_capability.key]} has no available allowance. Choose another option or free an existing allowance." + if denied_capability is not None else "These scoring rules need an available Rule-based tuning allowance. Check the weights, thresholds, keywords, and custom dimensions in Advanced settings. Model choices do not use this allowance." if tuning_error is not None else None diff --git a/litellm/router.py b/litellm/router.py index 513a5dc7c25..010e5946cb8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -164,7 +164,7 @@ from litellm.router_utils.auto_router_model_naming import ( AUTO_ROUTER_MODEL_PREFIX, GatedAutoRouterCapability, capability_limit_violation, - claimed_capability, + claimed_capabilities, classify_strategy_router_model, count_capability_routers, ) @@ -9491,10 +9491,8 @@ class Router: ) complexity_router_config: Final[dict | None] = deployment.litellm_params.complexity_router_config - capability: Final = claimed_capability(complexity_router_config) - if capability is not None: - limit_violation: Final = self.auto_router_capability_violation(capability) - if limit_violation is not None: + for capability in claimed_capabilities(complexity_router_config): + if (limit_violation := self.auto_router_capability_violation(capability)) is not None: raise ValueError(limit_violation) default_model: Final = ( diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index f55362b6c41..7567a0ef37e 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -381,7 +381,7 @@ Each dimension contributes its weight once when any matcher hits the current ask The API and YAML store exactly the weights written. A `dimension_weights` map and inline custom weights are read literally, missing recognized built-in names score zero, and nothing renormalizes the vector, so a total other than 1 is legal and scores accordingly. The dashboard's heuristic scoring editor is the one place that rebalances: editing one weight there holds it and redistributes the remainder across the other active dimensions in the draft, then Save sends the resulting explicit values, which the backend stores and scores as written. Opening a router, applying a preset, editing matchers, changing `scoring_mode`, or saving unrelated fields never normalizes existing weights -Only `heuristic`, `heuristic_first` and `hybrid` accept custom dimensions. Each name must be a unique ASCII identifier starting with a letter, at most 64 characters, and cannot reuse a built-in dimension name or a key in `dimension_weights`. Set its weight inline, greater than zero and at most one +Only `heuristic`, `heuristic_first` and `hybrid` using heuristic v1 accept custom dimensions. Each name must be a unique ASCII identifier starting with a letter, at most 64 characters, and cannot reuse a built-in dimension name or a key in `dimension_weights`. Set its weight inline, greater than zero and at most one Patterns are checked at configuration time against a grammar whose worst case stays a few milliseconds on 2048 characters. Every quantifier needs an explicit upper bound of at most 64 and must repeat a single character or character class, so `\s{1,4}` is accepted while `\s+`, `(a|aa){0,12}` and `(?:ab){0,64}` are refused. Backreferences, lookarounds, atomic groups and possessive quantifiers are refused as well. Each pattern is then costed: alternation branches and repeat lengths multiply the ways the engine can retry, and every later piece of the pattern is charged once per path that can reach it, so `a?a?a?a?a?a?a?a?` followed by a long fixed tail is refused even though each quantifier is small. The budget is 2048 work units per pattern and 8192 across the router. An invalid or over-budget pattern fails the write with a message naming the pattern and the rule it broke @@ -555,7 +555,7 @@ on by default; set `classifier_llm_config.circuit_breaker_enabled: false` to dis fallback is the local heuristic scorer, so a classifier outage does not repeat its timeout across every turn or session handled by the router process. -A request short-circuits, meaning it routes on the scorer's own tier with no classifier call, when +With the default heuristic v1, a request short-circuits, meaning it routes on the scorer's own tier with no classifier call, when two things hold: the scorer landed at or below `heuristic_first_max_tier`, and it produced at least one signal. Everything else goes to the classifier, which then decides as it normally would. @@ -576,6 +576,13 @@ except that the heuristic outcome is the one already computed rather than a seco Spend logs record `routing_decision.cause` as `heuristic_first_short_circuit` when the classifier was skipped, and `llm_classifier` when it ran, so the two are told apart per request. +Set `local_heuristic: heuristic_v2` to chain the trained predictor instead. Its selected tier must +meet `heuristic_v2_success_threshold` and remain at or below `heuristic_first_max_tier` to skip the judge + +Omitting `local_heuristic` keeps v1. Both choices preserve the configured classifier failure fallback; +the heuristic fallback uses the selected local scorer. V2 short-circuits and fallbacks also record +`heuristic_v2_forecast` with the probabilities and success threshold used for the decision + ### Hybrid `classifier_type: hybrid` also scores locally first, but it asks a different question than @@ -601,7 +608,7 @@ model_list: REASONING: o1-preview ``` -A request routes on the scorer's own tier when its score is further than `hybrid_boundary_margin` +With the default heuristic v1, a request routes on the scorer's own tier when its score is further than `hybrid_boundary_margin` from every active boundary. Everything else goes to the classifier: a score inside the band, where a hair's difference would have named the adjacent tier and its model pool, and a prompt where no dimension fired at all, which has no opinion to be confident about. `hybrid_boundary_margin` is @@ -619,6 +626,10 @@ Operator-defined tier sets (`tier_definitions`) are not supported here, for the not supported under heuristic-first: the scorer only produces the built-in tiers. Classifier failure behaves exactly as it does under `classifier_type: llm`. +For `local_heuristic: heuristic_v2`, the selected tier must meet `heuristic_v2_success_threshold`. +The judge decides when any probability at or below the selected tier is within `hybrid_boundary_margin` +of that threshold, or when no tier meets the threshold. A zero margin still defers exact-threshold predictions + ### Reasoning Override If 2+ reasoning markers are detected in the user message, the request is promoted to the REASONING tier even when the weighted score maps lower, so complex reasoning tasks get the appropriate model. The promotion requires the score to reach `reasoning_override_min_score`, which tracks `tier_boundaries.simple_medium` unless set, so stock phrases on an otherwise trivial prompt cannot buy the top tier. Set it to `0` to promote on the markers alone. diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 5600f4c9354..bdb081bad9a 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -112,6 +112,7 @@ from .config import ( CustomDimension, OpenSourceClassifierConfig, TierDefinition, + configured_local_heuristic, ) from .jev_classifier import ( DEFAULT_JEV_INSTRUCTIONS, @@ -1488,7 +1489,10 @@ class ComplexityRouter(CustomLogger): resolve_tier_artifact(self.config.heuristic_v2_artifact), routing_threshold=self.config.heuristic_v2_success_threshold, ) - if self.config.classifier_type == "heuristic_v2" + if configured_local_heuristic( + {"classifier_type": self.config.classifier_type, "local_heuristic": self.config.local_heuristic} + ) + == "heuristic_v2" else None ) @@ -1940,10 +1944,31 @@ class ComplexityRouter(CustomLogger): if self.config.classifier_type == "capability" and self.config.classifier_llm_config is not None: return await self._capability_classifier_outcome(prompt, request_kwargs, messages) if self.config.classifier_type not in ("llm", "llm_v2") or self.config.classifier_llm_config is None: - tier, score, signals, cause = self._score_and_classify(prompt, system_prompt) - return ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause) + return self._classify_locally(prompt, system_prompt) return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages) + def _classify_locally(self, prompt: str, system_prompt: str | None) -> ClassificationOutcome: + if self._tier_success_predictor is not None: + return self._classify_with_heuristic_v2(prompt) + tier, score, signals, cause = self._score_and_classify(prompt, system_prompt) + return ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause) + + def _local_outcome_is_decided(self, outcome: ClassificationOutcome, margin: float | None = None) -> bool: + forecast: Final = outcome.heuristic_v2_forecast + if forecast is None: + return bool(outcome.signals) and ( + margin is None or (outcome.score is not None and not self._is_near_tier_boundary(outcome.score, margin)) + ) + threshold: Final = forecast["threshold"] + if forecast["probabilities"][forecast["predicted_tier"]] < threshold: + return False + if margin is None: + return True + deciding_tiers: Final = TIER_SEVERITY_ORDER[ + : TIER_SEVERITY_ORDER.index(ComplexityTier(forecast["predicted_tier"])) + 1 + ] + return all(abs(forecast["probabilities"][tier.value] - threshold) > margin for tier in deciding_tiers) + def _classify_with_heuristic_v2(self, prompt: str) -> ClassificationOutcome: predictor: Final = self._tier_success_predictor if predictor is None: @@ -1981,27 +2006,27 @@ class ComplexityRouter(CustomLogger): """Score locally, and only pay for the classifier call when the scorer did not confidently place the request at or below heuristic_first_max_tier. - Confidence is `signals`, not `score`. A prompt where no dimension fired scores exactly 0.0, + For v1, confidence is `signals`, not `score`. A prompt where no dimension fired scores exactly 0.0, which is below simple_medium and so lands SIMPLE by default rather than by evidence, and a threshold check alone would hand that traffic to the cheapest model without ever consulting the classifier. Scores also go negative when simple indicators fire, so a score threshold - would reject exactly the trivial prompts this path exists to serve. + would reject exactly the trivial prompts this path exists to serve. V2 requires its predicted + tier to meet the configured success threshold. A turn carrying images the classifier would see is never decided cheaply: the scorer reads text alone, so its confidence describes a request it has only partly seen, and a trivial caption beside a screenshot is exactly the misrouting vision classification exists to stop. """ - tier, score, signals, cause = self._score_and_classify(prompt, system_prompt) - scored: Final = ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause) + scored: Final = self._classify_locally(prompt, system_prompt) threshold: Final = self.config.heuristic_first_max_tier decided_cheaply: Final = ( threshold is not None - and bool(signals) + and self._local_outcome_is_decided(scored) and not self._classifier_image_parts(messages) - and self._active_tier_severity(tier) <= self._active_tier_severity(threshold) + and self._active_tier_severity(scored.tier) <= self._active_tier_severity(threshold) ) if decided_cheaply: - return ClassificationOutcome(tier=tier, score=score, signals=signals, cause="heuristic_first_short_circuit") + return scored._replace(cause="heuristic_first_short_circuit") return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages, scored=scored) async def _classify_hybrid( @@ -2015,21 +2040,20 @@ class ComplexityRouter(CustomLogger): Where heuristic_first asks how CHEAP the scorer's tier is, this asks how DECIDED it is, so a confident score keeps its tier at every tier including the most expensive one. Two things make - a score undecided: landing within hybrid_boundary_margin of an active boundary, where a + a v1 score undecided: landing within hybrid_boundary_margin of an active boundary, where a hair's difference in score would have named the adjacent tier and its model pool, and firing no dimension at all, which scores 0.0 and lands SIMPLE by default rather than by evidence. + V2 compares its selected and lower-tier probabilities against the success threshold. """ - tier, score, signals, cause = self._score_and_classify(prompt, system_prompt) - scored: Final = ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause) + scored: Final = self._classify_locally(prompt, system_prompt) margin: Final = self.config.hybrid_boundary_margin decided: Final = ( margin is not None - and bool(signals) + and self._local_outcome_is_decided(scored, margin) and not self._classifier_image_parts(messages) - and not self._is_near_tier_boundary(score, margin) ) if decided: - return ClassificationOutcome(tier=tier, score=score, signals=signals, cause="hybrid_short_circuit") + return scored._replace(cause="hybrid_short_circuit") return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages, scored=scored) def _classifier_image_parts( @@ -2311,8 +2335,7 @@ class ComplexityRouter(CustomLogger): return _with_signal(self._default_model_fallback_outcome(), signal) if scored is not None: return _with_signal(scored, signal) - tier, score, signals, cause = self._score_and_classify(prompt, system_prompt) - return _with_signal(ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause), signal) + return _with_signal(self._classify_locally(prompt, system_prompt), signal) async def _classify_with_plugin( self, diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index bd067892ddc..797db460422 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -74,6 +74,15 @@ DEFAULT_CLASSIFICATION_RUBRIC: Final[ClassificationRubric] = ClassificationRubri LLM_CLASSIFIER_TYPES: Final[frozenset[str]] = frozenset({"llm", "capability", "llm_v2", "heuristic_first", "hybrid"}) +def configured_local_heuristic(config: Mapping[str, object]) -> Literal["heuristic", "heuristic_v2"] | None: + classifier_type: Final = config.get("classifier_type", "heuristic") + if classifier_type == "heuristic" or classifier_type == "heuristic_v2": + return classifier_type + if classifier_type in ("heuristic_first", "hybrid"): + return "heuristic_v2" if config.get("local_heuristic") == "heuristic_v2" else "heuristic" + return None + + TIER_SEVERITY_ORDER: Final[tuple[ComplexityTier, ...]] = ( ComplexityTier.SIMPLE, ComplexityTier.MEDIUM, @@ -1160,10 +1169,17 @@ class ComplexityRouterConfig(LiteLLMBaseModel): default=None, description="Experimental joint task-demand and solver-capability forecasting for classifier_type llm_v2.", ) + local_heuristic: Literal["heuristic", "heuristic_v2"] | None = Field( + default=None, + description=( + "Local scorer for heuristic_first or hybrid. Omitted or null keeps heuristic v1; " + "heuristic_v2 uses the trained success predictor. Rejected for other classifier types." + ), + ) heuristic_v2_artifact: TrainedTierArtifact | Literal["ultrafeedback"] = Field( default="ultrafeedback", description=( - "Success-probability artifact used by classifier_type 'heuristic_v2'. The bundled " + "Success-probability artifact used by standalone or chained heuristic_v2. The bundled " "UltraFeedback artifact is selected by default; an inline trained artifact may replace it" ), ) @@ -1173,10 +1189,11 @@ class ComplexityRouterConfig(LiteLLMBaseModel): ge=0.0, le=1.0, description=( - "Minimum predicted success probability for classifier_type 'heuristic_v2' to select a tier. " - "The first tier meeting this threshold is selected, or REASONING if none meets it. " + "Minimum predicted success probability for standalone or chained heuristic_v2 to select a tier. " + "The first tier meeting this threshold is selected. When none meets it, standalone heuristic_v2 " + "selects REASONING and chained heuristic_v2 defers to the LLM judge. " "When omitted or null, uses the artifact's routing_threshold (0.75 for the bundled artifact). " - "Other classifier types ignore this setting" + "Ignored when heuristic_v2 is not selected" ), ) classifier_llm_config: ClassifierLLMConfig | None = Field( @@ -1203,8 +1220,8 @@ class ComplexityRouterConfig(LiteLLMBaseModel): "one skips the LLM classifier and routes straight to that heuristic tier, so the classifier " "call is only paid for on traffic the scorer could not place cheaply. The scorer must also " "have produced at least one signal: a prompt where no dimension fired scores 0.0 and would " - "otherwise land SIMPLE by default rather than by evidence, which is how a chained router " - "would silently send unclassified traffic to the cheapest model. Names a built-in tier, and " + "otherwise land SIMPLE by default rather than by evidence. With local_heuristic 'heuristic_v2', " + "the predicted tier must meet its success threshold. Names a built-in tier, and " "may not name the highest one, since that would make the LLM classifier unreachable." ), ) @@ -1218,7 +1235,9 @@ class ComplexityRouterConfig(LiteLLMBaseModel): "this from every active boundary routes on the scorer's own tier with no classifier call, at any " "tier, which is what separates 'hybrid' from 'heuristic_first' and its cheap-tier ceiling. A " "prompt where no dimension fired still goes to the classifier, since the scorer has no opinion " - "to be near a boundary with. 0 escalates only scores sitting exactly on a boundary." + "to be near a boundary with. With local_heuristic 'heuristic_v2', a tier must meet its success " + "threshold and its probability and all lower-tier probabilities must be further than this margin " + "from that threshold. 0 escalates only scores or probabilities exactly on a boundary." ), ) classifier_plugin: ClassifierPlugin | None = Field( @@ -1855,12 +1874,25 @@ class ComplexityRouterConfig(LiteLLMBaseModel): raise ValueError("llm_v2 requires one distinct model group in each tier") return self + @model_validator(mode="after") + def _validate_local_heuristic(self) -> "ComplexityRouterConfig": + if self.local_heuristic is not None and self.classifier_type not in ("heuristic_first", "hybrid"): + raise ValueError("local_heuristic requires classifier_type heuristic_first or hybrid") + return self + @model_validator(mode="after") def _validate_custom_dimensions(self) -> "ComplexityRouterConfig": if not self.custom_dimensions: return self - if self.classifier_type not in ("heuristic", "heuristic_first", "hybrid"): - raise ValueError("custom_dimensions requires classifier_type heuristic, heuristic_first or hybrid") + if ( + configured_local_heuristic( + {"classifier_type": self.classifier_type, "local_heuristic": self.local_heuristic} + ) + != "heuristic" + ): + raise ValueError( + "custom_dimensions requires classifier_type heuristic, heuristic_first or hybrid using heuristic v1" + ) names: Final = tuple(dimension.name.casefold() for dimension in self.custom_dimensions) reserved: Final = frozenset(name.casefold() for name in DEFAULT_DIMENSION_WEIGHTS) weighted: Final = frozenset(name.casefold() for name in self.dimension_weights) diff --git a/litellm/router_utils/auto_router_model_naming.py b/litellm/router_utils/auto_router_model_naming.py index 3423836e4fd..7ace4094cf0 100644 --- a/litellm/router_utils/auto_router_model_naming.py +++ b/litellm/router_utils/auto_router_model_naming.py @@ -19,6 +19,7 @@ from litellm.router_strategy.complexity_router.config import ( COMPLEXITY_ROUTER_CONFIG_KEYS, DEFAULT_JEV_INSTRUCTIONS, LLM_CLASSIFIER_TYPES, + configured_local_heuristic, normalize_classifier_config_aliases, ) @@ -186,7 +187,7 @@ def strategy_router_dependencies( def uses_heuristic_v2_classifier(complexity_router_config: object) -> bool: """Whether this complexity config classifies with the bundled heuristic_v2 model.""" - return _mapping(complexity_router_config).get("classifier_type") == "heuristic_v2" + return configured_local_heuristic(_mapping(complexity_router_config)) == "heuristic_v2" def defines_custom_tiers(complexity_router_config: object) -> bool: @@ -237,8 +238,8 @@ class GatedAutoRouterCapability: stored ``litellm_params`` (``{config}`` is the caller's expression for the normalized ``complexity_router_config`` jsonb, substituted as many times as the predicate needs); they live on one record so they cannot drift apart. ``subject`` and ``remedy`` build the shared refusal - message. A validated config claims at most one capability: gated classifier types cannot be - combined with operator-defined tiers or classifier prompts. + message. A router chaining heuristic_v2 with an operator-written judge prompt claims both + capabilities. """ key: str @@ -250,10 +251,14 @@ class GatedAutoRouterCapability: HEURISTIC_V2_CAPABILITY: Final = GatedAutoRouterCapability( key="heuristic_v2", - subject="with classifier_type 'heuristic_v2'", - remedy="Use classifier_type 'heuristic' for this router or remove an existing heuristic_v2 router.", + subject="using the heuristic_v2 classifier", + remedy="Choose the heuristic v1 classifier for this router or remove an existing heuristic_v2 router.", uses=uses_heuristic_v2_classifier, - sql_config_predicate="{config} ->> 'classifier_type' = 'heuristic_v2'", + sql_config_predicate=( + "{config} ->> 'classifier_type' = 'heuristic_v2' OR " + "({config} ->> 'classifier_type' IN ('heuristic_first', 'hybrid') AND " + "{config} ->> 'local_heuristic' = 'heuristic_v2')" + ), ) CAPABILITY_CLASSIFIER_CAPABILITY: Final = GatedAutoRouterCapability( @@ -307,20 +312,19 @@ GATED_AUTO_ROUTER_CAPABILITIES: Final = ( ) -def claimed_capability(complexity_router_config: object) -> GatedAutoRouterCapability | None: - """The licensed capability this complexity config claims, or None.""" - return next( - (capability for capability in GATED_AUTO_ROUTER_CAPABILITIES if capability.uses(complexity_router_config)), - None, +def claimed_capabilities(complexity_router_config: object) -> tuple[GatedAutoRouterCapability, ...]: + """The licensed capabilities this complexity config claims.""" + return tuple( + capability for capability in GATED_AUTO_ROUTER_CAPABILITIES if capability.uses(complexity_router_config) ) -def gated_capability_of(litellm_params: Mapping[str, object]) -> GatedAutoRouterCapability | None: - """The licensed capability this deployment claims, or None unless it is a complexity router.""" +def gated_capabilities_of(litellm_params: Mapping[str, object]) -> tuple[GatedAutoRouterCapability, ...]: + """The licensed capabilities this deployment claims, or empty unless it is a complexity router.""" model: Final = litellm_params.get("model") if not is_complexity_router_model(model if isinstance(model, str) else None): - return None - return claimed_capability(litellm_params.get("complexity_router_config")) + return () + return claimed_capabilities(litellm_params.get("complexity_router_config")) def count_capability_routers( @@ -328,7 +332,9 @@ def count_capability_routers( ) -> int: """How many of ``deployments`` (router model_list entries or config.yaml rows) claim ``capability``.""" return sum( - 1 for deployment in deployments if gated_capability_of(_mapping(deployment.get("litellm_params"))) is capability + 1 + for deployment in deployments + if capability in gated_capabilities_of(_mapping(deployment.get("litellm_params"))) ) diff --git a/litellm/router_utils/auto_router_tuning_baseline.py b/litellm/router_utils/auto_router_tuning_baseline.py index 4707a51dfb8..9d18c66c519 100644 --- a/litellm/router_utils/auto_router_tuning_baseline.py +++ b/litellm/router_utils/auto_router_tuning_baseline.py @@ -8,7 +8,7 @@ from typing import Final from pydantic import ValidationError -from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig +from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig, configured_local_heuristic # v2 hashes combine models and scoring rules; a new snapshot is required to separate them. TUNING_BASELINE_PARAM_NAME: Final = "auto_router_tuning_baseline_v3" @@ -30,7 +30,6 @@ HEURISTIC_V1_TUNING_FIELDS: Final = ( _TUNING_FIELD_SET: Final = frozenset(HEURISTIC_V1_TUNING_FIELDS) -_V1_SCORING_CLASSIFIER_TYPES: Final = frozenset({"heuristic", "heuristic_first", "hybrid"}) _AUTO_ROUTER_COMPLEXITY_PREFIX: Final = "auto_router/complexity_router" _EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) _EMPTY_TAGS: Final[tuple[str, ...]] = () @@ -71,7 +70,7 @@ DEFAULT_TUNING_FINGERPRINT: Final = tuning_fingerprint(_EMPTY) def uses_heuristic_v1(complexity_router_config: object) -> bool: """Whether a config's primary classifier path is the heuristic-v1 scorer.""" - return _mapping(complexity_router_config).get("classifier_type", "heuristic") in _V1_SCORING_CLASSIFIER_TYPES + return configured_local_heuristic(_mapping(complexity_router_config)) == "heuristic" def router_identity(deployment: Mapping[str, object]) -> str | None: diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index d68ff29f2f5..4862af63f12 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -6097,9 +6097,9 @@ class TestStrategyRouterWriteValidation: AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY, _auto_router_capability_slot, ) - from litellm.router_utils.auto_router_model_naming import gated_capability_of + from litellm.router_utils.auto_router_model_naming import gated_capabilities_of - capability = gated_capability_of(effective_params) + capabilities: Final = gated_capabilities_of(effective_params) fake = self._FakeDb(db_models) live_router = self._live_router_holding_one_capability(limit, config_config) if config_config is not None else None @@ -6116,9 +6116,9 @@ class TestStrategyRouterWriteValidation: async with _auto_router_capability_slot(fake, effective_params=effective_params, model_id=model_id): pass assert exc_info.value.status_code == 403 - assert capability is not None + assert len(capabilities) == 1 assert "At most 1 auto-router" in str(exc_info.value.detail) - assert capability.subject in str(exc_info.value.detail) + assert capabilities[0].subject in str(exc_info.value.detail) assert "'auto_router' feature lifts the limit" in str(exc_info.value.detail) return async with _auto_router_capability_slot(fake, effective_params=effective_params, model_id=model_id) as tables: @@ -6135,8 +6135,59 @@ class TestStrategyRouterWriteValidation: assert lock_params == (AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY,) assert count_params == (model_id or "",) assert "AS model" in count_sql - assert capability is not None - assert capability.sql_config_predicate.split("{config}")[-1].strip() in count_sql + assert len(capabilities) == 1 + assert capabilities[0].sql_config_predicate.split("{config}")[-1].strip() in count_sql + + @pytest.mark.asyncio + @pytest.mark.parametrize("classifier", ("heuristic_first", "hybrid")) + @pytest.mark.parametrize( + ("other_config", "denied_subject"), + ( + (None, None), + (_V2, "heuristic_v2"), + (_CUSTOM_PROMPT, "operator-written classifier prompt"), + ), + ) + async def test_v2_chain_reserves_both_capabilities_under_one_lock( + self, classifier: str, other_config: Mapping[str, object] | None, denied_subject: str | None + ) -> None: + from fastapi import HTTPException + + from litellm.proxy.management_endpoints.model_management_endpoints import _auto_router_capability_slot + + candidate: Final = { + "model": "auto_router/complexity_router", + "complexity_router_config": { + **self._CUSTOM_PROMPT, + "classifier_type": classifier, + "local_heuristic": "heuristic_v2", + **( + {"heuristic_first_max_tier": "MEDIUM"} + if classifier == "heuristic_first" else {"hybrid_boundary_margin": 0.1} + ), + }, + } + fake: Final = self._FakeDb([]) + router: Final = self._live_router_holding_one_capability(1, other_config) if other_config is not None else None + with ( + patch("litellm.proxy.proxy_server._license_check.auto_router_capability_limit", return_value=1), # test-quality-ok: [TQ008] the guard reads the proxy license singleton + patch("litellm.proxy.proxy_server.llm_router", router), # test-quality-ok: [TQ008] the guard reads the proxy router singleton + patch("litellm.proxy.proxy_server.heuristic_v1_tuning_baselines", {}), # test-quality-ok: [TQ008] the guard reads the proxy baseline singleton + patch("litellm.proxy.management_endpoints.model_management_endpoints.publish_config_change", new=AsyncMock()), # test-quality-ok: [TQ008] Redis publication is outside admission under test + ): + if denied_subject is not None: + with pytest.raises(HTTPException) as failure: + async with _auto_router_capability_slot(fake, effective_params=candidate, model_id="held-id"): + pytest.fail("Exhausted capability allowed a write") + assert failure.value.status_code == 403 + assert denied_subject in str(failure.value.detail) + else: + async with _auto_router_capability_slot(fake, effective_params=candidate, model_id="held-id") as table: + assert table is fake.tx_obj.litellm_proxymodeltable + assert sum("pg_advisory_xact_lock" in sql for sql, _ in fake.tx_obj.raw_calls) == 1 + counts: Final = tuple((sql, args) for sql, args in fake.tx_obj.raw_calls if "AS model" in sql) + assert len(counts) == (1 if denied_subject == "heuristic_v2" else 2) + assert all(args == ("held-id",) for _, args in counts) _TUNED_A = {"classifier_type": "heuristic", "tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o"}} _TUNED_A_EDITED = {**_TUNED_A, "dimension_weights": {"codePresence": 0.9}} diff --git a/tests/unit/proxy/management_helpers/test_auto_router_availability.py b/tests/unit/proxy/management_helpers/test_auto_router_availability.py index 027930d9ebb..a88dcd7f66d 100644 --- a/tests/unit/proxy/management_helpers/test_auto_router_availability.py +++ b/tests/unit/proxy/management_helpers/test_auto_router_availability.py @@ -61,6 +61,33 @@ def test_edit_does_not_exempt_another_classifier_allowance() -> None: assert next(slot for slot in result.allowances if slot.key == "llm_v2").remaining == 0 +@pytest.mark.parametrize("classifier", ("heuristic_first", "hybrid")) +@pytest.mark.parametrize( + ("other_classifier", "expected_label"), + (("heuristic_v2", "Heuristic v2"), ("llm", "Custom tiers or classifier instructions")), +) +def test_v2_chain_checks_both_allowances_and_keeps_both_owned_slots( + classifier: str, other_classifier: str, expected_label: str +) -> None: + prompt: Final = {"classification_prompt": "Grade by difficulty"} + candidate: Final = deployment("chain", classifier, config={"local_heuristic": "heuristic_v2", **prompt}) + blocked: Final = auto_router_availability( + others=(deployment("other", other_classifier, config=prompt),), + existing=None, + candidate=candidate, + baselines={}, + limit=1, + ) + assert blocked.error is not None and expected_label in blocked.error + edited: Final = auto_router_availability( + others=(), existing=candidate, candidate=candidate, baselines={}, limit=1 + ) + assert edited.error is None + assert tuple(slot.key for slot in edited.allowances if slot.used_by_this_router) == ( + "heuristic_v2", "tier_or_classifier_prompt" + ) + + def test_model_selection_does_not_claim_occupied_scoring_allowance() -> None: original: Final = deployment("legacy", "heuristic") changed: Final = deployment("other", "heuristic", tuned=True) diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index dfeb9f8d961..67e0649958b 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -2120,6 +2120,37 @@ class TestRouterComplexityDeploymentMethods: "model_info": {"id": model_id}, } + @pytest.mark.parametrize("classifier", ("heuristic_first", "hybrid")) + @pytest.mark.parametrize("held_capability", ("heuristic_v2", "custom_prompt")) + def test_v2_chain_registration_enforces_each_claimed_allowance( + self, classifier: str, held_capability: str + ) -> None: + held: Final = ( + self._router_row("existing", "existing-id", "heuristic_v2") + if held_capability == "heuristic_v2" + else self._custom_prompt_row("existing", "existing-id") + ) + chain: Final = { + "model_name": "chain", + "model_info": {"id": "chain-id"}, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "classifier_type": classifier, + "local_heuristic": "heuristic_v2", + **( + {"heuristic_first_max_tier": "MEDIUM"} + if classifier == "heuristic_first" else {"hybrid_boundary_margin": 0.1} + ), + "classifier_llm_config": {"model": "gpt-4o-mini", "system_prompt": "Grade by difficulty"}, + "tiers": {"SIMPLE": "gpt-4o-mini"}, + }, + }, + } + subject: Final = "heuristic_v2" if held_capability == "heuristic_v2" else "operator-written classifier prompt" + with pytest.raises(ValueError, match=subject): + Router(model_list=[self._POOL, held, chain], auto_router_capability_limit=lambda: 1) + @pytest.mark.parametrize("field", ["classification_prompt", "classification_examples"]) def test_operator_written_prompt_sections_claim_the_customization_slot(self, field: str) -> None: """The dashboard prompt editor writes opening instructions and calibration examples as their own @@ -13998,6 +14029,129 @@ class TestHybrid: assert outcome.cause == "heuristic_scorer" +@pytest.mark.parametrize("classifier_type", ("heuristic", "heuristic_v2", "llm")) +def test_local_heuristic_is_rejected_outside_chaining(classifier_type: str) -> None: + with pytest.raises(ValidationError, match="local_heuristic requires"): + ComplexityRouterConfig.model_validate( + { + "classifier_type": classifier_type, + "local_heuristic": "heuristic_v2", + "classifier_llm_config": {"model": "judge"}, + } + ) + + +@pytest.mark.parametrize("classifier_type", ("heuristic_first", "hybrid")) +def test_v2_chain_rejects_v1_custom_dimensions(classifier_type: str) -> None: + with pytest.raises(ValidationError, match="custom_dimensions requires"): + ComplexityRouterConfig.model_validate( + { + "classifier_type": classifier_type, + "local_heuristic": "heuristic_v2", + "classifier_llm_config": {"model": "judge"}, + "heuristic_first_max_tier": "SIMPLE" if classifier_type == "heuristic_first" else None, + "hybrid_boundary_margin": 0.03 if classifier_type == "hybrid" else None, + "tiers": dict(HEURISTIC_FIRST_TIERS), + "custom_dimensions": [{"name": "custom", "weight": 0.5, "patterns": ["custom"]}], + } + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "classifier_type, threshold, margin, expected_tier, judge_called", + [ + ("heuristic_first", 0.1, 0.03, "SIMPLE", False), + ("heuristic_first", 0.8, 0.03, "REASONING", True), + ("hybrid", 0.8, 0.03, "COMPLEX", False), + ("hybrid", 0.89, 0.03, "REASONING", True), + ("hybrid", 0.22, 0.02, "REASONING", True), + ("hybrid", 21 / 102, 0.0, "REASONING", True), + ("hybrid", 1.0, 0.0, "REASONING", True), + ], +) +async def test_selected_v2_chain_routes_using_success_probabilities( + mock_router_instance: MagicMock, + classifier_type: str, + threshold: float, + margin: float, + expected_tier: str, + judge_called: bool, +) -> None: + mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "REASONING"}')) + router: Final = ComplexityRouter( + model_name="v2-chain", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + "classifier_type": classifier_type, + "local_heuristic": "heuristic_v2", + "heuristic_v2_artifact": _heuristic_v2_artifact(), + "heuristic_v2_success_threshold": threshold, + "classifier_llm_config": {"model": "judge"}, + "heuristic_first_max_tier": "SIMPLE" if classifier_type == "heuristic_first" else None, + "hybrid_boundary_margin": margin if classifier_type == "hybrid" else None, + "tiers": dict(HEURISTIC_FIRST_TIERS), + }, + ) + response: Final = await router.async_pre_routing_hook( + model="v2-chain", + request_kwargs={}, + messages=[{"role": "user", "content": "write a python function to reverse a string"}], + ) + assert response is not None and response.routing_decision is not None + assert response.routing_decision["tier"] == expected_tier + assert mock_router_instance.acompletion.await_count == int(judge_called) + assert response.routing_decision["cause"] == ( + "llm_classifier" if judge_called else f"{classifier_type}_short_circuit" + ) + assert response.routing_decision.get("heuristic_v2_forecast") == ( + None + if judge_called + else { + "probabilities": dict( + zip((tier.value for tier in TIER_SEVERITY_ORDER), (11 / 102, 21 / 102, 91 / 102, 100 / 102)) + ), + "threshold": threshold, + "predicted_tier": expected_tier, + "request_type": "code_generation", + } + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("classifier_type", ("heuristic_first", "hybrid")) +@pytest.mark.parametrize("encrypted", (False, True)) +@pytest.mark.parametrize("fallback", ("heuristic", "default_model")) +async def test_v2_chain_judge_failure_uses_selected_fallback( + classifier_type: str, encrypted: bool, fallback: str +) -> None: + _, dependency = _native_classifier_router(failure=RuntimeError("judge failed")) + dependency.acompletion = AsyncMock(side_effect=RuntimeError("judge failed")) + router: Final = ComplexityRouter( + model_name="v2-chain", + litellm_router_instance=dependency, + complexity_router_config={ + "classifier_type": classifier_type, + "local_heuristic": "heuristic_v2", + "heuristic_v2_artifact": _heuristic_v2_artifact(), + "classifier_llm_config": {"model": "classifier"}, + "heuristic_first_max_tier": "SIMPLE" if classifier_type == "heuristic_first" else None, + "hybrid_boundary_margin": 1.0 if classifier_type == "hybrid" else None, + "classifier_fallback": fallback, + "default_model": "gpt-4o", + "tiers": dict(HEURISTIC_FIRST_TIERS), + }, + ) + outcome: Final = await router.aclassify( + "hello", request_kwargs={"input": [_encrypted_agent_task()]} if encrypted else None + ) + assert outcome.cause == ("heuristic_v2" if fallback == "heuristic" else "default_model_fallback") + assert outcome.tier == (ComplexityTier.COMPLEX if fallback == "heuristic" else ComplexityTier.MEDIUM) + assert (outcome.heuristic_v2_forecast is not None) == (fallback == "heuristic") + assert dependency.aresponses.await_count == int(encrypted) + assert dependency.acompletion.await_count == int(not encrypted) + + def _windowed_router(*deployments: tuple) -> Router: """Real Router; each deployment is (group, provider_model, declared window or None). None means no declared override on a model the cost map does not know: unresolvable.""" @@ -16407,17 +16561,31 @@ class TestClassifierVision: @pytest.mark.parametrize( "classifier_type, extra, short_circuit_cause", SHORT_CIRCUIT_ARMS, ids=["heuristic_first", "hybrid"] ) + @pytest.mark.parametrize("local_heuristic", ("heuristic", "heuristic_v2")) async def test_local_scorer_cannot_short_circuit_a_turn_it_cannot_see( - self, mock_router_instance, classifier_type, extra, short_circuit_cause - ): + self, + mock_router_instance: MagicMock, + classifier_type: str, + extra: dict[str, str | float], + short_circuit_cause: str, + local_heuristic: str, + ) -> None: """The scorer reads text alone, so its confidence is not a verdict on an image turn. Both arms are tuned so the scorer WOULD short-circuit on this exact text, which is what makes the image the only variable; a margin loose enough to leave the score undecided would pass whether or not the guard exists. """ - router = self._router(mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra) - response = await router.async_pre_routing_hook( + router: Final = self._router( + mock_router_instance, + vision={"enabled": True}, + classifier_type=classifier_type, + local_heuristic=local_heuristic, + heuristic_v2_artifact=_heuristic_v2_artifact(), + heuristic_v2_success_threshold=0.0, + **extra, + ) + response: Final = await router.async_pre_routing_hook( model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, IMG_PART) ) assert response.routing_decision["cause"] == "llm_classifier" @@ -16426,12 +16594,26 @@ class TestClassifierVision: @pytest.mark.parametrize( "classifier_type, extra, short_circuit_cause", SHORT_CIRCUIT_ARMS, ids=["heuristic_first", "hybrid"] ) + @pytest.mark.parametrize("local_heuristic", ("heuristic", "heuristic_v2")) async def test_local_scorer_still_short_circuits_without_images( - self, mock_router_instance, classifier_type, extra, short_circuit_cause - ): + self, + mock_router_instance: MagicMock, + classifier_type: str, + extra: dict[str, str | float], + short_circuit_cause: str, + local_heuristic: str, + ) -> None: """The negative class: same router, same text, no image, and the scorer still decides.""" - router = self._router(mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra) - response = await router.async_pre_routing_hook( + router: Final = self._router( + mock_router_instance, + vision={"enabled": True}, + classifier_type=classifier_type, + local_heuristic=local_heuristic, + heuristic_v2_artifact=_heuristic_v2_artifact(), + heuristic_v2_success_threshold=0.0, + **extra, + ) + response: Final = await router.async_pre_routing_hook( model="m", request_kwargs={}, messages=[{"role": "user", "content": "what is this"}] ) assert response.routing_decision["cause"] == short_circuit_cause diff --git a/tests/unit/router_utils/test_auto_router_model_naming.py b/tests/unit/router_utils/test_auto_router_model_naming.py index 87b23ce93ae..d7016cbc9ec 100644 --- a/tests/unit/router_utils/test_auto_router_model_naming.py +++ b/tests/unit/router_utils/test_auto_router_model_naming.py @@ -9,10 +9,10 @@ from litellm.router_utils.auto_router_model_naming import ( GATED_AUTO_ROUTER_CAPABILITIES, capability_limit_violation, carries_complexity_router_settings, - claimed_capability, + claimed_capabilities, classify_strategy_router_model, count_capability_routers, - gated_capability_of, + gated_capabilities_of, strategy_router_dependencies, validate_complexity_router_config_placement, validate_complexity_router_config_write, @@ -74,11 +74,11 @@ def test_open_source_classifier_enumerates_its_accounting_model( def test_only_non_default_open_source_instructions_claim_the_shared_customization_slot( instructions: str | None, classifier_type: str, config_key: str ) -> None: - capability: Final = claimed_capability( + capabilities: Final = claimed_capabilities( {"classifier_type": classifier_type, config_key: {"instructions": instructions}} ) - assert (capability.key if capability else None) == ( - "tier_or_classifier_prompt" if instructions == "Route conservatively" else None + assert tuple(capability.key for capability in capabilities) == ( + ("tier_or_classifier_prompt",) if instructions == "Route conservatively" else () ) @@ -273,8 +273,8 @@ def test_fuse_write_accepts_presets_and_custom_text_with_the_same_entitlement() ) assert validate_complexity_router_config_write(presets) is None assert validate_complexity_router_config_write(custom) is None - assert claimed_capability(presets) is claimed_capability(custom) - assert claimed_capability(presets) is not None + assert claimed_capabilities(presets) == claimed_capabilities(custom) + assert claimed_capabilities(presets) @pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness")) @@ -579,8 +579,8 @@ def test_custom_classifier_prompt_capability(config: Mapping[str, object], expec A shipped rubric preset stays free, and the heuristic scorers never read system_prompt, so a value sitting on one is inert and claims nothing (heuristic_v2 still claims its own capability). """ - claimed = claimed_capability(config) - assert (None if claimed is None else claimed.key) == expected_key + claimed: Final = claimed_capabilities(config) + assert tuple(capability.key for capability in claimed) == (() if expected_key is None else (expected_key,)) @pytest.mark.parametrize( @@ -640,13 +640,13 @@ def test_is_complexity_router_model(model: str | None, expected: bool) -> None: ({}, None), ], ) -def test_gated_capability_of(litellm_params: Mapping[str, object], expected_key: str | None) -> None: +def test_gated_capabilities_of(litellm_params: Mapping[str, object], expected_key: str | None) -> None: """Only a complexity router claiming a licensed capability counts toward that capability's limit. Renaming the built-in tiers through tier_labels is not a custom tier set, so it stays ungated. """ - capability = gated_capability_of(litellm_params) - assert (None if capability is None else capability.key) == expected_key + capabilities: Final = gated_capabilities_of(litellm_params) + assert tuple(capability.key for capability in capabilities) == (() if expected_key is None else (expected_key,)) @pytest.mark.parametrize("capability", GATED_AUTO_ROUTER_CAPABILITIES, ids=lambda c: c.key) @@ -729,11 +729,29 @@ def test_every_gated_capability_has_a_distinct_predicate_and_sql_spelling() -> N }, ], ) -def test_capabilities_are_mutually_exclusive_on_one_config(config: Mapping[str, object]) -> None: - """No config claims two capabilities, which is what lets one lock and one count serve them all. - - The config validator is what makes this true and is pinned separately in test_complexity_router: - tier_definitions rejects every heuristic classifier_type and rejects the classifier system_prompt, - and system_prompt only counts for the classifier types heuristic_v2 is not one of. - """ +def test_standalone_classifiers_claim_at_most_one_capability(config: Mapping[str, object]) -> None: assert sum(1 for capability in GATED_AUTO_ROUTER_CAPABILITIES if capability.uses(config)) <= 1 + + +@pytest.mark.parametrize("classifier", ("heuristic_first", "hybrid")) +@pytest.mark.parametrize("local_heuristic", (None, "heuristic", "heuristic_v2")) +@pytest.mark.parametrize("custom_prompt", (False, True)) +def test_chained_heuristic_and_judge_prompt_claim_each_active_capability( + classifier: str, local_heuristic: str | None, custom_prompt: bool +) -> None: + config: Final = { + "classifier_type": classifier, + **({"local_heuristic": local_heuristic} if local_heuristic is not None else {}), + **({"classification_prompt": "Grade by difficulty"} if custom_prompt else {}), + } + params: Final = {"model": "auto_router/complexity_router", "complexity_router_config": config} + expected: Final = ( + *(("heuristic_v2",) if local_heuristic == "heuristic_v2" else ()), + *(("tier_or_classifier_prompt",) if custom_prompt else ()), + ) + assert tuple(capability.key for capability in claimed_capabilities(config)) == expected + assert tuple(capability.key for capability in gated_capabilities_of(params)) == expected + for capability in GATED_AUTO_ROUTER_CAPABILITIES: + assert count_capability_routers(({"litellm_params": params},), capability=capability) == ( + 1 if capability.key in expected else 0 + ) diff --git a/tests/unit/router_utils/test_auto_router_tuning_baseline.py b/tests/unit/router_utils/test_auto_router_tuning_baseline.py index 9307668d66f..d0d98ce7042 100644 --- a/tests/unit/router_utils/test_auto_router_tuning_baseline.py +++ b/tests/unit/router_utils/test_auto_router_tuning_baseline.py @@ -183,6 +183,28 @@ class TestRouterIdentity: class TestHeuristicV1Scope: + @pytest.mark.parametrize("classifier", ("heuristic_first", "hybrid")) + def test_v2_chains_ignore_retained_v1_tuning_until_v1_is_selected(self, classifier: str) -> None: + config: Final = { + "classifier_type": classifier, + "classifier_llm_config": {"model": "judge"}, + **( + {"heuristic_first_max_tier": "MEDIUM"} + if classifier == "heuristic_first" else {"hybrid_boundary_margin": 0.1} + ), + "tiers": _TIERS, + "code_keywords": ["internal-api"], + } + v1: Final = _router("chain", config) + v2: Final = _router("chain", {**config, "local_heuristic": "heuristic_v2"}) + other: Final = _router("other", {"code_keywords": ["internal-api"]}) + baselines: Final = snapshot_tuning_baselines((_router("chain", {}),)) + assert heuristic_v1_router_fingerprint(v2) is None + assert not snapshot_tuning_baselines((v2,)) + assert not mutable_tuned_identities((v2,), baselines) + assert tuning_quota_violation(candidate=v2, others=(other,), baselines=baselines, limit=1) is None + assert tuning_quota_violation(candidate=v1, others=(other,), baselines=baselines, limit=1) is not None + @pytest.mark.parametrize( "config,in_scope", [ diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx index 08f329e4ad7..0d3e7cfe54b 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.integration.test.tsx @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest"; import { fireEvent, renderWithProviders, screen, waitFor, within } from "../../../tests/test-utils"; import { selectAutoRouterOption } from "../../../tests/autoRouterSetup"; import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs"; +import ClassificationMethodConfig from "./ClassificationMethodConfig"; import { AutoRouterAllowanceNote, AutoRouterAvailabilityContext } from "./AutoRouterAvailability"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; @@ -53,6 +54,42 @@ function Form({ } describe("Auto-router classifier selection", () => { + it.each([false, true])("applies the v2 allowance to chain selection, owned: %s", (owned) => { + const onChange = vi.fn(); + renderWithProviders( + + + , + ); + fireEvent.click(screen.getByRole("button", { name: "Heuristic before the judge" })); + const option = screen.getByRole("menuitemradio", { name: /^Heuristic v2/ }); + if (owned) { + expect(option).not.toHaveAttribute("aria-disabled", "true"); + fireEvent.click(option); + expect(onChange).toHaveBeenCalledWith(expect.objectContaining({ local_heuristic: "heuristic_v2" })); + } else { + expect(option).toHaveAttribute("aria-disabled", "true"); + fireEvent.click(option); + expect(onChange).not.toHaveBeenCalled(); + } + }); + it.each(["heuristic", "heuristic_v2", "llm", "heuristic_first", "hybrid", "jev"] as const)( "shows saved %s without changing its configuration", async (classifier_type) => { diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx index b722ff5f5ac..a8d492a6ced 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx @@ -28,7 +28,7 @@ import { AUTO_ROUTER_CONTACT_URL, } from "./AutoRouterAvailability"; -function ClassifierOption({ +export function ClassifierOption({ value, label, description, @@ -75,7 +75,7 @@ function ClassifierOption({ ); } -function ClassifierMenu({ +export function ClassifierMenu({ id, label, value, diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index e698f0bb83b..43ce45a3570 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -23,6 +23,9 @@ import ClassifierVisionConfig from "./ClassifierVisionConfig"; import { getHeuristicV2SuccessThresholdError } from "./build_complexity_router_config"; import ClassifierPluginTimeoutField from "./ClassifierPluginTimeoutField"; import ClassifierTypeRadios from "./ClassifierTypeRadios"; +import { ClassifierMenu, ClassifierOption } from "./AutoRouterClassifierTabs"; +import { isHeuristicChain } from "./classifier_types"; +import HeuristicKeywordOverrides from "./HeuristicKeywordOverrides"; import type { ReasoningEffort } from "./complexity_router_tiers"; import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults"; import { @@ -39,6 +42,8 @@ import { ClassificationRubric, effectiveTierLabel, heuristicScoringRole, + heuristicTuningType, + usesHeuristicV2, usesLlmClassifier, usesClassifierContext, DEFAULT_HYBRID_BOUNDARY_MARGIN, @@ -53,8 +58,7 @@ const DEFAULT_SCORING_EXPLANATION = const HEURISTIC_V2_EXPLANATION = "The router estimates success probability for all four tiers with its calibrated model, then selects " + - "the first tier that meets the success threshold. If none qualify, it selects Reasoning. " + - "It runs locally with no classifier API call."; + "the first tier that meets the success threshold."; const CLASSIFIER_TIMEOUT_ID = "classifier-timeout-ms"; const CLASSIFIER_CONTEXT_WINDOW_SIZE_ID = "classifier-context-window-size"; @@ -62,30 +66,6 @@ const CLASSIFIER_CONTEXT_BUDGET_CHARS_ID = "classifier-context-budget-chars"; const HYBRID_BOUNDARY_MARGIN_ID = "hybrid-boundary-margin"; const HEURISTIC_V2_SUCCESS_THRESHOLD_ID = "heuristic-v2-success-threshold"; -const CUSTOM_PROMPT_WITH_HEURISTIC_FALLBACK = - "This router classifies with your own prompt, so the tier comes from whatever rubric it states. The four tier " + - "names stay fixed. The scoring below is the heuristic, which now runs only when the classifier call fails:"; - -const CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK = - "This router classifies with your own prompt, so the tier comes from whatever rubric it states. The four tier " + - "names stay fixed. The scoring below no longer runs at all, since a failed classifier routes to the default " + - "model instead:"; - -/** - * What the scoring breakdown below it actually describes. A custom prompt means the score no longer - * decides the tier, and pairing one with the default-model fallback means the heuristic never runs - * at all, so the panel must not keep implying a score is involved on either router. - */ -const scoringExplanation = (value: ComplexityRouterConfigValue): string => { - if (value.classifier_type === "heuristic_v2") return HEURISTIC_V2_EXPLANATION; - const usesCustomPrompt = - usesLlmClassifier(value.classifier_type) && Boolean(value.classifier_llm_config?.system_prompt?.trim()); - if (!usesCustomPrompt) return DEFAULT_SCORING_EXPLANATION; - return value.classifier_fallback === "default_model" - ? CUSTOM_PROMPT_WITH_DEFAULT_MODEL_FALLBACK - : CUSTOM_PROMPT_WITH_HEURISTIC_FALLBACK; -}; - /** * The three boundaries this card states, as displayed strings, or null until the proxy's shipped defaults * have arrived. Kept out of the component so the card cannot state a range the router stopped using, and @@ -121,6 +101,7 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> = value.tier_boundaries, value.reasoning_override_min_score, ); + const scorerDefaultsUnavailable = scorerRuns && !ranges && isError; if (value.custom_tier_set) return null; @@ -128,7 +109,13 @@ const HowClassificationWorks: React.FC<{ value: ComplexityRouterConfigValue }> = How Classification Works - {scoringExplanation(value)} + + {usesHeuristicV2(value) ? HEURISTIC_V2_EXPLANATION : DEFAULT_SCORING_EXPLANATION} + {usesHeuristicV2(value) && + (isHeuristicChain(value.classifier_type) + ? " If no tier qualifies, or the local-check policy requires a judge, the judge chooses the tier." + : " If none qualify, it selects Reasoning. It runs locally with no classifier API call.")} + {scorerRuns && ranges && ( )} - {!ranges && isError && ( + {scorerDefaultsUnavailable && ( The tier score ranges could not be loaded from the proxy. @@ -170,6 +157,7 @@ interface ClassificationMethodConfigProps { /** The resolved default model - see resolveComplexityDefaultModel. Names and gates the radio. */ defaultModel?: string; advancedOnly?: boolean; + section?: "selection" | "classifier" | "heuristic"; } export const InactiveHeuristicV2Threshold: React.FC> = ({ @@ -177,8 +165,9 @@ export const InactiveHeuristicV2Threshold: React.FC { const threshold = value.heuristic_v2_success_threshold; - if (effectiveClassifierType(value) === "heuristic_v2" || threshold === undefined) return null; + if (usesHeuristicV2(value) || threshold === undefined) return null; const error = getHeuristicV2SuccessThresholdError(threshold); + if (effectiveClassifierType(value) === "llm" && !error) return null; return (

@@ -215,10 +204,17 @@ const ClassificationMethodConfig: React.FC = ({ showValidationErrors = false, defaultModel, advancedOnly = false, + section, }) => { const [draft, setDraft] = React.useState<{ id: string; raw: string } | null>(null); const hasDefaultModel = Boolean(defaultModel); const classifierType = effectiveClassifierType(value); + const showSelection = section === undefined || section === "selection"; + const showClassifier = section === undefined || section === "classifier"; + const heuristic = heuristicTuningType(value); + const showHeuristic = (section === undefined || section === "heuristic") && heuristic !== undefined; + const showLocalChecks = + showSelection && advancedOnly && ["llm", "heuristic_first", "hybrid"].includes(classifierType); const usesCustomPrompt = Boolean(value.classifier_llm_config?.system_prompt?.trim()); const contextBudget = value.classifier_context_budget_chars ?? DEFAULT_CLASSIFIER_CONTEXT_BUDGET_CHARS; const contextBudgetQuotesNothing = contextBudget > 0 && contextBudget < MIN_QUOTED_CONTEXT_TURN_CHARS; @@ -365,7 +361,7 @@ const ClassificationMethodConfig: React.FC = ({ return ( <> - {!advancedOnly && ( + {!advancedOnly && showSelection && ( <> = ({ /> )} - {advancedOnly && ["llm", "heuristic_first", "hybrid"].includes(classifierType) && ( + {showLocalChecks && (

handleHeuristicFirstMaxTierChange(tier as string)} > - + @@ -463,15 +489,18 @@ const ClassificationMethodConfig: React.FC = ({

- A request the scorer places at or below this tier routes there without a classifier call. Anything the - scorer places higher, and anything it found no signal for at all, goes to the classifier instead + {usesHeuristicV2(value) + ? "A tier that meets the success threshold and is at or below this limit routes locally. Higher tiers, or requests where no tier qualifies, go to the judge" + : "A request the scorer places at or below this tier routes locally. Higher tiers, or requests with no heuristic signal, go to the judge"}

)} - {classifierType === "hybrid" && ( + {showSelection && classifierType === "hybrid" && (
- Boundary margin + = ({ className="w-full" />

- A score further than this from every tier boundary routes on the scorer's own tier, however expensive - that tier is. A score closer than this, and anything the scorer found no signal for at all, goes to the - classifier to break the tie + {usesHeuristicV2(value) + ? "Use the judge when the selected tier or a lower tier has a success probability within this margin of the success threshold, or no tier meets the threshold" + : "A score further than this from every tier boundary routes on the scorer's own tier. A score closer than this, or a request with no heuristic signal, goes to the judge"}

)} - {classifierType === "jev" && } - {usesLlmClassifier(classifierType) && ( + {showClassifier && classifierType === "jev" && } + {showClassifier && usesLlmClassifier(classifierType) && (
= ({
)} - {usesClassifierContext(classifierType) && ( + {showClassifier && usesClassifierContext(classifierType) && (
= ({
)} - {heuristicScoringRole(value) !== "never" && ( + {showHeuristic && heuristic === "heuristic" && (
Custom Technical Keywords @@ -735,12 +764,19 @@ const ClassificationMethodConfig: React.FC = ({
)} - {["heuristic", "heuristic_first", "hybrid"].includes(classifierType) && ( - + {showHeuristic && heuristic === "heuristic" && ( + <> + {heuristicScoringRole(value) === "decides" && ( + + )} + +
+

Heuristic Keyword Overrides

+ +
+ )} - - - + {showHeuristic && } ); }; diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx index 98960897bb5..443077718fd 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterAdvancedSections.tsx @@ -3,7 +3,13 @@ import { ChevronRight } from "lucide-react"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import { Separator } from "@/components/ui/separator"; import type { ModelGroup } from "@/components/llm_calls/fetch_models"; -import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import { + effectiveClassifierType, + heuristicTuningType, + usesClassifierContext, + usesLlmClassifier, + type ComplexityRouterConfigValue, +} from "./ComplexityRouterConfig"; import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig"; import CacheAwareRoutingConfig from "./CacheAwareRoutingConfig"; import ClassificationMethodConfig from "./ClassificationMethodConfig"; @@ -19,7 +25,6 @@ import CompressionControls from "./CompressionControls"; import PlanModeOverrideControls from "./PlanModeOverrideControls"; import { AffinityControls } from "./AffinityControls"; import { ModalityRoutingControls } from "./ModalityRoutingControls"; -import HeuristicKeywordOverrides from "./HeuristicKeywordOverrides"; import HousekeepingRoutingControls from "./HousekeepingRoutingControls"; import ReminderMarkers from "./ReminderMarkers"; import type { AutoRouterCompressionState } from "./buildAutoRouterCompression"; @@ -80,6 +85,18 @@ const ComplexityRouterAdvancedSections: React.FC { + const classifierType = effectiveClassifierType(value); + const classificationProps = { + advancedOnly: true, + value, + onChange, + modelOptions, + effortOptionsByModel: classifierEffortOptionsByModel, + customTechnicalKeywords, + onCustomTechnicalKeywordsChange, + showValidationErrors, + defaultModel, + }; const sections = [ ...(forecast ? [ @@ -98,33 +115,21 @@ const ComplexityRouterAdvancedSections: React.FCClassification Method, - children: ( - - ), + children: , }, ] : []), - ...(!forecast + ...(heuristicTuningType(value) ? [ { - key: "keyword-overrides", - label: Heuristic Keyword Overrides, - children: , + key: "heuristic", + label: Heuristic tuning, + children: , }, ] : []), @@ -246,7 +251,9 @@ const ComplexityRouterAdvancedSections: React.FC sections.some(({ key }) => group.keys.includes(key))); const [openGroups, setOpenGroups] = React.useState(() => showValidationErrors ? groups.map((group) => group.label) : [], ); @@ -272,6 +279,11 @@ const ComplexityRouterAdvancedSections: React.FC + {["llm", "heuristic_first", "hybrid"].includes(classifierType) && ( +
+ +
+ )} {groups.map((group) => ( (
- {key !== "classifier" &&

{label}

} + {key !== "classifier" && key !== "heuristic" &&

{label}

} {children}
))} diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.integration.test.tsx index 4a00a469f97..acbece64904 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.integration.test.tsx @@ -52,6 +52,53 @@ const baseProps = { }; describe("ComplexityRouterConfig", () => { + it.each(["Heuristic first", "Hybrid"])( + "shows only the selected heuristic tuning when chaining with %s", + async (mode) => { + const initialValue: ComplexityRouterConfigValue = { + ...defaultValue, + classifier_type: "llm", + classifier_llm_config: { model: "gpt-4", timeout_ms: 750 }, + token_thresholds: { simple: 12, complex: 800 }, + heuristic_v2_success_threshold: 0.9, + }; + function Editor() { + const [value, setValue] = React.useState(initialValue); + return ; + } + renderWithProviders(); + fireEvent.click(screen.getByRole("button", { name: /^Advanced settings/ })); + expect(screen.getByRole("button", { name: "LLM tuning" })).toHaveAttribute("aria-expanded", "false"); + expect(screen.queryByRole("button", { name: "Heuristic tuning" })).not.toBeInTheDocument(); + expect(screen.queryByText("Advanced scoring")).not.toBeInTheDocument(); + expect(screen.queryByText("Custom Technical Keywords")).not.toBeInTheDocument(); + expect(screen.queryByText("Heuristic Keyword Overrides")).not.toBeInTheDocument(); + expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument(); + await selectAutoRouterOption("Local checks before the judge", mode); + expect(screen.getByRole("button", { name: "Heuristic before the judge" })).toHaveTextContent("Heuristic v1"); + fireEvent.click(screen.getByRole("button", { name: "Heuristic tuning" })); + fireEvent.click(screen.getByRole("button", { name: /Advanced scoring/ })); + expect(screen.getByLabelText("Short below")).toHaveValue("12"); + await selectAutoRouterOption("Heuristic before the judge", "Heuristic v2"); + expect(screen.getByLabelText("Success threshold")).toHaveValue("0.9"); + expect(screen.queryByText("Advanced scoring")).not.toBeInTheDocument(); + expect(screen.queryByText("Custom Technical Keywords")).not.toBeInTheDocument(); + expect(screen.queryByText("Heuristic Keyword Overrides")).not.toBeInTheDocument(); + expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument(); + if (mode === "Hybrid") expect(screen.getByLabelText("Success threshold margin")).toBeInTheDocument(); + await selectAutoRouterOption("Heuristic before the judge", "Heuristic v1"); + fireEvent.click(screen.getByRole("button", { name: /Advanced scoring/ })); + expect(screen.getByLabelText("Short below")).toHaveValue("12"); + fireEvent.click(screen.getByRole("button", { name: "LLM tuning" })); + expect(screen.getByLabelText("Timeout (ms)")).toHaveValue("750"); + await selectAutoRouterOption("Local checks before the judge", "Always use the judge"); + expect(screen.queryByRole("button", { name: "Heuristic tuning" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Heuristic before the judge" })).not.toBeInTheDocument(); + expect(screen.queryByRole("region", { name: "Inactive Heuristic v2 threshold" })).not.toBeInTheDocument(); + expect(screen.getByLabelText("Timeout (ms)")).toHaveValue("750"); + }, + ); + it("should render", async () => { renderWithProviders(); expect(screen.getByText("Models by tier")).toBeInTheDocument(); @@ -100,7 +147,7 @@ describe("ComplexityRouterConfig", () => { it("should default to heuristic and hide classifier model/timeout fields", async () => { renderWithProviders(); openAutoRouterAdvanced("Classification Method"); - expect(screen.getByText("Classifier tuning")).toBeInTheDocument(); + expect(screen.getByText("Heuristic tuning")).toBeInTheDocument(); expect(screen.queryByText("Judge model")).not.toBeInTheDocument(); }); @@ -730,10 +777,10 @@ describe("ComplexityRouterConfig classifier fallback", () => { />, ); openAutoRouterAdvanced("Classification Method"); - expect(screen.getByText(/no longer runs at all/)).toBeInTheDocument(); + expect(screen.queryByText("How Classification Works")).not.toBeInTheDocument(); }); - it("still describes the heuristic as the fallback when a custom prompt keeps heuristic fallback", async () => { + it("keeps heuristic fallback selectable without showing heuristic tuning for a custom judge prompt", async () => { renderWithProviders( { />, ); openAutoRouterAdvanced("Classification Method"); - expect(screen.getByText(/only when the classifier call fails/)).toBeInTheDocument(); + expect(screen.getByRole("radio", { name: /Score with the heuristic/ })).toBeChecked(); + expect(screen.queryByText("How Classification Works")).not.toBeInTheDocument(); }); it("clears a stored fallback when switching back to the heuristic classifier", async () => { @@ -1505,15 +1553,6 @@ describe("ComplexityRouterConfig custom technical keywords", () => { classifier_llm_config: llmConfig, }, ], - [ - "llm falling back to the scorer", - { - ...defaultValue, - classifier_type: "llm" as const, - classifier_llm_config: llmConfig, - classifier_fallback: "heuristic" as const, - }, - ], ])("offers the keywords on a router whose scorer runs: %s", (_label, value) => { openClassificationPanel(value); expect(screen.getByText("Custom Technical Keywords")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 1b036f26632..4e8e981fd4a 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -2,7 +2,7 @@ import RoutingOptions from "./RoutingOptions"; import ClassifierPrimarySettings from "./ClassifierPrimarySettings"; import { AutoRouterAllowanceNote } from "./AutoRouterAvailability"; import type { JevClassifierConfig } from "./jev_classifier_config"; -import { type ClassifierType } from "./classifier_types"; +import { isHeuristicChain, type ClassifierType, type LocalHeuristic } from "./classifier_types"; export { type ClassifierType, usesLlmClassifier, usesClassifierContext } from "./classifier_types"; import ForecastClassifierConfig, { ForecastSolverModels } from "./ForecastClassifierConfig"; import { isForecastClassifier, type CapabilitySettings, type FuseSettings } from "./forecast_classifier_config"; @@ -156,16 +156,12 @@ export const DEFAULT_ADAPTIVE_WEIGHTS: AdaptiveRouterWeights = { quality: 0.3, c export type HeuristicScoringRole = "decides" | "fallback_only" | "never"; -/** - * Whether the heuristic scorer runs on this router at all, which is what gates its knobs. An LLM - * classifier still falls back to the scorer unless the fallback is the default model, so the gate cannot be - * a plain classifier_type check. Under heuristic_first the scorer runs first on every request and - * decides outright whenever it lands at or below the threshold. - */ export const heuristicScoringRoleFor = ( classifierType: ClassifierType, classifierFallback: ClassifierFallback | undefined, + localHeuristic?: LocalHeuristic, ): HeuristicScoringRole => { + if (isHeuristicChain(classifierType) && localHeuristic === "heuristic_v2") return "never"; if (classifierType === "heuristic_v2" || classifierType === "capability" || classifierType === "llm_v2") return "never"; if (classifierType === "heuristic" || classifierType === "heuristic_first" || classifierType === "hybrid") @@ -174,7 +170,23 @@ export const heuristicScoringRoleFor = ( }; export const heuristicScoringRole = (value: ComplexityRouterConfigValue): HeuristicScoringRole => - value.custom_tier_set ? "never" : heuristicScoringRoleFor(value.classifier_type, value.classifier_fallback); + value.custom_tier_set + ? "never" + : heuristicScoringRoleFor(value.classifier_type, value.classifier_fallback, value.local_heuristic); + +export const usesHeuristicV2 = (value: ComplexityRouterConfigValue): boolean => { + if (value.custom_tier_set) return false; + return ( + value.classifier_type === "heuristic_v2" || + (isHeuristicChain(value.classifier_type) && value.local_heuristic === "heuristic_v2") + ); +}; + +export const heuristicTuningType = (value: ComplexityRouterConfigValue): LocalHeuristic | undefined => { + if (value.custom_tier_set || value.classifier_type === "llm") return undefined; + if (usesHeuristicV2(value)) return "heuristic_v2"; + return heuristicScoringRole(value) === "never" ? undefined : "heuristic"; +}; // Derived, never written into the value, so undoing a tier edit reverts the form with nothing left behind. export const effectiveClassifierType = ( @@ -359,6 +371,7 @@ export interface ComplexityRouterConfigValue { /** An explicit pin. Unset means the default tracks the tiers - see resolveComplexityDefaultModel. */ default_model?: string; classifier_type: ClassifierType; + local_heuristic?: LocalHeuristic; heuristic_v2_success_threshold?: number; capability_classifier_config?: CapabilitySettings; llm_v2_config?: FuseSettings; diff --git a/ui/litellm-dashboard/src/components/add_model/HeuristicKeywordOverrides.tsx b/ui/litellm-dashboard/src/components/add_model/HeuristicKeywordOverrides.tsx index 185f187bbfc..9e13aebc250 100644 --- a/ui/litellm-dashboard/src/components/add_model/HeuristicKeywordOverrides.tsx +++ b/ui/litellm-dashboard/src/components/add_model/HeuristicKeywordOverrides.tsx @@ -16,8 +16,7 @@ const HeuristicKeywordOverrides: React.FC<{

Each list replaces the built-in keyword list of the same name for the heuristic scorer. Leave a list empty to keep - the built-in one. To add technical terms without replacing the list, use custom technical keywords under - Classification Method. + the built-in one. To add technical terms without replacing the list, use custom technical keywords above.

{fields.map(([key, label]) => { const keywords = value[key] ?? []; diff --git a/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.test.tsx index 61779318fff..172cf660bc7 100644 --- a/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.test.tsx @@ -169,7 +169,12 @@ describe("ClassificationMethodConfig scorer gating", () => { it.each([ ["heuristic decides the tier", "heuristic" as ClassifierType, undefined, true], ["heuristic v2 decides without the weighted scorer", "heuristic_v2" as ClassifierType, undefined, false], - ["an LLM classifier falls back to the heuristic", "llm" as ClassifierType, "heuristic" as ClassifierFallback, true], + [ + "an LLM classifier falls back to the heuristic", + "llm" as ClassifierType, + "heuristic" as ClassifierFallback, + false, + ], [ "an LLM classifier falls back to the default model", "llm" as ClassifierType, diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx index cf5e8432089..5e9ff3543b8 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx @@ -931,6 +931,39 @@ describe("AddAutoRouterTab", () => { }); }); + it("creates the selected v2 chain after editing its threshold and local tier ceiling", async () => { + mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); + const actualSubmit = await vi.importActual( + "./handle_add_auto_router_submit", + ); + vi.mocked(handleAddAutoRouterSubmit).mockImplementationOnce(actualSubmit.handleAddAutoRouterSubmit); + renderWithProviders(); + const setup = await screen.findByRole("button", { name: "Choose models for me" }); + await waitFor(() => expect(setup).toBeEnabled()); + await userEvent.click(setup); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "chained-router" } }); + await userEvent.click(screen.getByRole("radio", { name: "LLM" })); + await selectAutoRouterOption("Judge model", ALL_FAMILY_MODELS[0].model_group); + openAutoRouterAdvanced("Classification Method"); + await selectAutoRouterOption("Local checks before the judge", "Heuristic first"); + await selectAutoRouterOption("Heuristic before the judge", "Heuristic v2"); + fireEvent.click(screen.getByRole("button", { name: "Heuristic tuning" })); + fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "0.6" } }); + await selectAutoRouterOption("Decide locally up to", "Medium"); + await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled()); + await userEvent.click(screen.getByRole("button", { name: "Add Auto Router" })); + await waitFor(() => expect(modelCreateCall).toHaveBeenCalledOnce()); + const expectedConfig = { + classifier_type: "heuristic_first", + local_heuristic: "heuristic_v2", + heuristic_v2_success_threshold: 0.6, + heuristic_first_max_tier: "MEDIUM", + }; + expect(vi.mocked(modelCreateCall).mock.calls.at(-1)?.[1].litellm_params.complexity_router_config).toMatchObject( + expectedConfig, + ); + }); + it("preserves classifier tuning when choosing models automatically", async () => { const user = userEvent.setup(); mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index fdf5a9d9d1b..e1153e34bc8 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -57,6 +57,46 @@ const baseParams: BuildComplexityRouterConfigParams = { }; describe("buildComplexityRouterConfig", () => { + it.each(["heuristic_first", "hybrid"] as const)( + "persists the selected local heuristic for %s and preserves inactive v1 settings", + (classifierType) => { + const params = { + ...baseParams, + classifierType, + heuristicFirstMaxTier: "SIMPLE", + hybridBoundaryMargin: 0.1, + localHeuristic: "heuristic_v2" as const, + heuristicV2SuccessThreshold: 0, + tokenThresholds: { simple: 12, complex: 800 }, + customDimensions: [{ id: "domain", name: "domain", weight: 0.2, keywords: ["billing"] }], + }; + const expected = { + classifier_type: classifierType, + local_heuristic: "heuristic_v2", + heuristic_v2_success_threshold: 0, + token_thresholds: params.tokenThresholds, + }; + expect(buildComplexityRouterConfig(params)).toMatchObject(expected); + expect(buildComplexityRouterConfig(params)).not.toHaveProperty("custom_dimensions"); + expect(buildComplexityRouterConfig({ ...params, localHeuristic: "heuristic" })).toMatchObject({ + local_heuristic: "heuristic", + custom_dimensions: [{ name: "domain", weight: 0.2, keywords: ["billing"] }], + }); + expect(buildComplexityRouterConfig({ ...params, localHeuristic: undefined })).not.toHaveProperty( + "local_heuristic", + ); + }, + ); + + it.each(["heuristic", "heuristic_v2", "llm", "jev", "custom", "capability", "llm_v2"] as const)( + "omits the inactive chain selector under %s", + (classifierType) => { + expect( + buildComplexityRouterConfig({ ...baseParams, classifierType, localHeuristic: "heuristic_v2" }), + ).not.toHaveProperty("local_heuristic"); + }, + ); + it.each([undefined, false, true])( "keeps cache routing opt-in and uses eligible single-model tiers only when enabled: %s", (enabled) => { @@ -1362,13 +1402,16 @@ describe("buildComplexityRouterConfig with an edited tier set", () => { reasoningOverrideMinScore: 0.5, heuristicFirstMaxTier: "SIMPLE", hybridBoundaryMargin: 0.03, + localHeuristic: "heuristic", customTechnicalKeywords: ["kubernetes"], stallEscalationEnabled: true, stallEscalationWindow: 6, stallEscalationRepeatThreshold: 3, }; // custom_dimensions only ever ship when the scorer decides, so "llm" cannot prove it emits. - const emittingType = key === "heuristic_first_max_tier" || key === "custom_dimensions" ? "heuristic_first" : "llm"; + const emittingType = ["heuristic_first_max_tier", "custom_dimensions", "local_heuristic"].includes(key) + ? "heuristic_first" + : "llm"; const typeForKey = key === "hybrid_boundary_margin" ? "hybrid" : emittingType; expect(buildComplexityRouterConfig({ ...baseParams, ...loaded, classifierType: typeForKey })).toHaveProperty(key); expect(build(loaded)).not.toHaveProperty(key); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index d86cab6d6d8..09bf0ea41a9 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -5,6 +5,7 @@ import { type FuseSettings, } from "./forecast_classifier_config"; import type { ModelGroup } from "../llm_calls/fetch_models"; +import { isHeuristicChain, type LocalHeuristic } from "./classifier_types"; import { KeywordTierRule } from "./KeywordTierRules"; import { type JevClassifierConfig, @@ -22,7 +23,8 @@ import { tierRowByName, } from "./tier_rows"; import { emptyKeywordTierRuleIndexes, serializeKeywordTierRules } from "./complexity_router_keywords"; -import { type CustomDimension, type CustomDimensionRow, serializeCustomDimensions } from "./custom_dimensions"; +import { type CustomDimension, type CustomDimensionRow } from "./custom_dimensions"; +import { scorerKnobPayload } from "./heuristic_scoring_knobs"; import { TierModelParams, TierModelParamsByTier, @@ -47,7 +49,6 @@ import { TierBoundaries, TokenThresholds, effectiveTierLabel, - heuristicScoringRoleFor, usesLlmClassifier, usesClassifierContext, } from "./ComplexityRouterConfig"; @@ -103,45 +104,6 @@ export const normalizeClassifierLlmConfig = ({ ...(vision && { vision }), }; -interface ScorerKnobInputs { - classifierType: ClassifierType; - classifierFallback: ClassifierFallback | undefined; - tierBoundaries: TierBoundaries | undefined; - tokenThresholds: TokenThresholds | undefined; - dimensionWeights: DimensionWeights | undefined; - customDimensions: CustomDimensionRow[] | undefined; - reasoningOverrideMinScore: number | undefined; -} - -/** - * The scorer knobs to persist, which is none of them on a router that never scores: an LLM classifier - * falling back to the default model would otherwise carry settings that can only mislead the next reader. - * Each is omitted while untouched, so the router keeps tracking the backend defaults. - */ -const scorerKnobPayload = ({ - classifierType, - classifierFallback, - tierBoundaries, - tokenThresholds, - dimensionWeights, - customDimensions, - reasoningOverrideMinScore, -}: ScorerKnobInputs) => { - const role = heuristicScoringRoleFor(classifierType, classifierFallback); - return role === "never" - ? {} - : { - ...(tierBoundaries && { tier_boundaries: tierBoundaries }), - ...(tokenThresholds && { token_thresholds: tokenThresholds }), - ...(dimensionWeights && { dimension_weights: dimensionWeights }), - // Only a scorer that decides accepts these; the backend rejects them on every other - // classifier, so a fallback-only router must not carry rows a switch left behind. - ...(role === "decides" && - customDimensions !== undefined && { custom_dimensions: serializeCustomDimensions(customDimensions) }), - ...(reasoningOverrideMinScore !== undefined && { reasoning_override_min_score: reasoningOverrideMinScore }), - }; -}; - export interface StoredComplexityRouterConfig { tiers?: Record; enable_non_reasoning_tier?: boolean; @@ -154,6 +116,7 @@ export interface StoredComplexityRouterConfig { hybrid_boundary_margin?: unknown; tier_labels?: unknown; classifier_type?: ClassifierType | "oss_classifier"; + local_heuristic?: unknown; heuristic_v2_success_threshold?: unknown; capability_classifier_config?: unknown; llm_v2_config?: unknown; @@ -209,6 +172,7 @@ export interface BuildComplexityRouterConfigParams { planModeMinTier: string | undefined; tierLabels: ComplexityTierLabels | undefined; classifierType: ClassifierType; + localHeuristic?: LocalHeuristic; heuristicV2SuccessThreshold?: number; capabilityClassifierConfig?: CapabilitySettings; llmV2Config?: FuseSettings; @@ -291,6 +255,7 @@ export interface ComplexityRouterConfigPayload { plan_mode_min_tier?: string; tier_labels?: ComplexityTierLabels; classifier_type: ClassifierType | "oss_classifier"; + local_heuristic?: LocalHeuristic; heuristic_v2_success_threshold?: number; capability_classifier_config?: CapabilitySettings; llm_v2_config?: FuseSettings; @@ -665,6 +630,7 @@ export const buildComplexityRouterConfig = ({ planModeMinTier, tierLabels, classifierType, + localHeuristic, heuristicV2SuccessThreshold, capabilityClassifierConfig, llmV2Config, @@ -732,6 +698,7 @@ export const buildComplexityRouterConfig = ({ const cleanedTierLabels = serializeTierLabels(tierLabels); const scorerInputs = { classifierType, + localHeuristic, classifierFallback, tierBoundaries, tokenThresholds, @@ -795,6 +762,7 @@ export const buildComplexityRouterConfig = ({ ...(planModeMinTier?.trim() && { plan_mode_min_tier: planModeMinTier }), ...(cleanedTierLabels && { tier_labels: cleanedTierLabels }), classifier_type: classifierType === "jev" ? "oss_classifier" : classifierType, + ...(isHeuristicChain(effectiveType) && localHeuristic !== undefined && { local_heuristic: localHeuristic }), ...(effectiveType === "jev" && { opensource_classifier_config: normalizeJevClassifierConfig(jevClassifierConfig) }), ...(heuristicV2SuccessThreshold !== undefined && { heuristic_v2_success_threshold: heuristicV2SuccessThreshold, diff --git a/ui/litellm-dashboard/src/components/add_model/classifier_types.ts b/ui/litellm-dashboard/src/components/add_model/classifier_types.ts index aa9d5619052..c2c3bb1a0c3 100644 --- a/ui/litellm-dashboard/src/components/add_model/classifier_types.ts +++ b/ui/litellm-dashboard/src/components/add_model/classifier_types.ts @@ -9,6 +9,11 @@ export type ClassifierType = | "llm_v2" | "custom"; +export type LocalHeuristic = "heuristic" | "heuristic_v2"; + +export const isHeuristicChain = (classifierType: ClassifierType): boolean => + classifierType === "heuristic_first" || classifierType === "hybrid"; + export const usesLlmClassifier = (classifierType: ClassifierType): boolean => (["llm", "heuristic_first", "hybrid", "capability", "llm_v2"] as const).some((type) => type === classifierType); diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts index 2cfccebc119..aed6253d89b 100644 --- a/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts +++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_builder_params.ts @@ -30,6 +30,7 @@ export const builderParamsFromValue = ( classificationMode: value.classification_mode, tierLabels: value.tier_labels, classifierType: value.classifier_type, + localHeuristic: value.local_heuristic, jevClassifierConfig: value.jev_classifier_config, heuristicV2SuccessThreshold: value.heuristic_v2_success_threshold, capabilityClassifierConfig: value.capability_classifier_config, diff --git a/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.ts b/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.ts index 3de4af92b9c..79d5637a7f5 100644 --- a/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.ts +++ b/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.ts @@ -1,4 +1,6 @@ -import type { CustomDimensionRow } from "./custom_dimensions"; +import { serializeCustomDimensions, type CustomDimensionRow } from "./custom_dimensions"; +import { heuristicScoringRoleFor, type ClassifierFallback } from "./ComplexityRouterConfig"; +import { isHeuristicChain, type ClassifierType, type LocalHeuristic } from "./classifier_types"; export type TierBoundaries = Record; @@ -6,6 +8,35 @@ export type TokenThresholds = Record; export type DimensionWeights = Record; +interface ScorerKnobInputs { + classifierType: ClassifierType; + localHeuristic: LocalHeuristic | undefined; + classifierFallback: ClassifierFallback | undefined; + tierBoundaries: TierBoundaries | undefined; + tokenThresholds: TokenThresholds | undefined; + dimensionWeights: DimensionWeights | undefined; + customDimensions: CustomDimensionRow[] | undefined; + reasoningOverrideMinScore: number | undefined; +} + +export const scorerKnobPayload = (knobs: ScorerKnobInputs) => { + const role = heuristicScoringRoleFor(knobs.classifierType, knobs.classifierFallback, knobs.localHeuristic); + return role === "never" && !isHeuristicChain(knobs.classifierType) + ? {} + : { + ...(knobs.tierBoundaries && { tier_boundaries: knobs.tierBoundaries }), + ...(knobs.tokenThresholds && { token_thresholds: knobs.tokenThresholds }), + ...(knobs.dimensionWeights && { dimension_weights: knobs.dimensionWeights }), + ...(role === "decides" && + knobs.customDimensions !== undefined && { + custom_dimensions: serializeCustomDimensions(knobs.customDimensions), + }), + ...(knobs.reasoningOverrideMinScore !== undefined && { + reasoning_override_min_score: knobs.reasoningOverrideMinScore, + }), + }; +}; + /** * Display names for the scorer's dimensions. Only the wording lives here; the dimension set and its * shipped weights come from the proxy (GET /public/complexity_router/scorer_defaults), so a dimension diff --git a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts index dff051e5674..3f1de9d0834 100644 --- a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts +++ b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts @@ -143,7 +143,7 @@ export const CUSTOM_TIER_RESTRICTIONS = { reason: "Session pinning escalates along the built-in tier ladder, which your tier set replaces", }, heuristicClassifier: { - omit: ["heuristic_first_max_tier", "hybrid_boundary_margin"], + omit: ["heuristic_first_max_tier", "hybrid_boundary_margin", "local_heuristic"], reason: "The heuristic scorer only produces the built-in tiers, so an edited set needs the LLM or JEV classifier. " + "Heuristic first and hybrid are out for the same reason: their local scorer decides the traffic it is sure of", diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index 27a24b7638c..9445e18913e 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -895,6 +895,7 @@ describe("managed keys survive an untouched open-and-save", () => { plan_mode_min_tier: "COMPLEX", tier_labels: { SIMPLE: "Cheap" }, classifier_type: "heuristic_first", + local_heuristic: "heuristic", heuristic_v2_success_threshold: 0.89, heuristic_first_max_tier: "SIMPLE", classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000, reasoning_effort: "low" }, @@ -968,6 +969,8 @@ describe("managed keys survive an untouched open-and-save", () => { it("carries every managed key a built-in router can hold through hydrate then save", () => { const hydrated = hydrateComplexityRouterConfig(STORED_ALL_MANAGED, undefined); const saved = buildUpdatedComplexityRouterConfig(STORED_ALL_MANAGED, hydrated); + expect(hydrated.local_heuristic).toBe(STORED_ALL_MANAGED.local_heuristic); + expect(saved.local_heuristic).toBe(STORED_ALL_MANAGED.local_heuristic); const dropped = [...MANAGED_COMPLEXITY_ROUTER_KEYS] .filter((key) => !KEYS_ANOTHER_CLASSIFIER_TYPE_OWNS.has(key)) diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx index 15b398dae03..a60e91db48e 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx @@ -199,6 +199,40 @@ describe("EditAutoRouterModal keyword matching", () => { else expect(savedConfig().heuristic_v2_success_threshold).toBe(0); }); + it("saves a legacy heuristic-first router with a v2 chain and an edited success threshold", async () => { + renderModal({ + modelData: { + ...MODEL_DATA, + litellm_params: { + ...MODEL_DATA.litellm_params, + complexity_router_config: { + ...STORED_CONFIG, + classifier_type: "heuristic_first", + heuristic_first_max_tier: "SIMPLE", + classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000 }, + token_thresholds: { simple: 12, complex: 800 }, + }, + }, + }, + }); + openAutoRouterAdvanced("Classification Method"); + await selectAutoRouterOption("Heuristic before the judge", "Heuristic v2"); + fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "1.1" } }); + expect(screen.getByRole("button", { name: "Save Changes" })).toBeDisabled(); + fireEvent.change(screen.getByRole("textbox", { name: "Success threshold" }), { target: { value: "0.88" } }); + await waitFor(() => expect(screen.getByRole("button", { name: "Save Changes" })).toBeEnabled()); + await userEvent.click(screen.getByRole("button", { name: "Save Changes" })); + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalledOnce()); + const expected = { + classifier_type: "heuristic_first", + local_heuristic: "heuristic_v2", + heuristic_v2_success_threshold: 0.88, + token_thresholds: { simple: 12, complex: 800 }, + classifier_llm_config: { model: "gpt-4o-mini", timeout_ms: 3000 }, + }; + expect(savedConfig()).toMatchObject(expected); + }); + it("blocks an invalid threshold edit and retains a corrected value when switching classifiers", async () => { const user = userEvent.setup(); renderModal({ diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts index 651d5880db4..5a72f2e0f9a 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.ts @@ -79,6 +79,30 @@ const expectedAdaptiveDisabledConfig = { }; describe("buildUpdatedComplexityRouterConfig", () => { + it.each(["heuristic_first", "hybrid"] as const)( + "round-trips %s local heuristic selection without materializing the legacy default", + (classifier_type) => { + const stored = { + ...storedConfigValue, + classifier_type, + heuristic_first_max_tier: "SIMPLE", + hybrid_boundary_margin: 0.1, + }; + const legacy = hydrateComplexityRouterConfig(stored, null); + expect(buildUpdatedComplexityRouterConfig(stored, legacy)).not.toHaveProperty("local_heuristic"); + const selected = { ...stored, local_heuristic: "heuristic_v2", heuristic_v2_success_threshold: 0.9 }; + const hydrated = hydrateComplexityRouterConfig(selected, null); + expect(hydrated.local_heuristic).toBe("heuristic_v2"); + expect(buildUpdatedComplexityRouterConfig(selected, hydrated)).toMatchObject({ + local_heuristic: "heuristic_v2", + heuristic_v2_success_threshold: 0.9, + }); + expect(buildUpdatedComplexityRouterConfig(selected, { ...hydrated, classifier_type: "llm" })).not.toHaveProperty( + "local_heuristic", + ); + }, + ); + it("preserves unrelated options and omits the penalty for classified-tier routing", () => { const updatedConfig = buildUpdatedComplexityRouterConfig(storedConfig, classifiedTierValue); diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index f12364f7d05..00800e2abd2 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -97,6 +97,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "plan_mode_min_tier", "tier_labels", "classifier_type", + "local_heuristic", "capability_classifier_config", "llm_v2_config", "classifier_llm_config", diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts b/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts index 78f1bba065e..233b5aed922 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/hydrate_complexity_router_config.ts @@ -72,6 +72,10 @@ export const hydrateComplexityRouterConfig = ( plan_mode_min_tier: hydratePlanModeMinTier(parsedConfig.plan_mode_min_tier, custom_tier_set), tier_labels: hydrateTierLabels(parsedConfig.tier_labels), ...classifier, + local_heuristic: + parsedConfig.local_heuristic === "heuristic" || parsedConfig.local_heuristic === "heuristic_v2" + ? parsedConfig.local_heuristic + : undefined, heuristic_v2_success_threshold: typeof parsedConfig.heuristic_v2_success_threshold === "number" ? parsedConfig.heuristic_v2_success_threshold diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 1645cc3df82..027bf68ce19 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -42201,18 +42201,18 @@ export interface components { fallback_tier?: string | null; /** * Heuristic First Max Tier - * @description The highest tier the local scorer may decide on its own; required when classifier_type is 'heuristic_first' and rejected otherwise. A request whose heuristic tier is at or below this one skips the LLM classifier and routes straight to that heuristic tier, so the classifier call is only paid for on traffic the scorer could not place cheaply. The scorer must also have produced at least one signal: a prompt where no dimension fired scores 0.0 and would otherwise land SIMPLE by default rather than by evidence, which is how a chained router would silently send unclassified traffic to the cheapest model. Names a built-in tier, and may not name the highest one, since that would make the LLM classifier unreachable. + * @description The highest tier the local scorer may decide on its own; required when classifier_type is 'heuristic_first' and rejected otherwise. A request whose heuristic tier is at or below this one skips the LLM classifier and routes straight to that heuristic tier, so the classifier call is only paid for on traffic the scorer could not place cheaply. The scorer must also have produced at least one signal: a prompt where no dimension fired scores 0.0 and would otherwise land SIMPLE by default rather than by evidence. With local_heuristic 'heuristic_v2', the predicted tier must meet its success threshold. Names a built-in tier, and may not name the highest one, since that would make the LLM classifier unreachable. */ heuristic_first_max_tier?: string | null; /** * Heuristic V2 Artifact - * @description Success-probability artifact used by classifier_type 'heuristic_v2'. The bundled UltraFeedback artifact is selected by default; an inline trained artifact may replace it + * @description Success-probability artifact used by standalone or chained heuristic_v2. The bundled UltraFeedback artifact is selected by default; an inline trained artifact may replace it * @default ultrafeedback */ heuristic_v2_artifact: components["schemas"]["TrainedTierArtifact"] | "ultrafeedback"; /** * Heuristic V2 Success Threshold - * @description Minimum predicted success probability for classifier_type 'heuristic_v2' to select a tier. The first tier meeting this threshold is selected, or REASONING if none meets it. When omitted or null, uses the artifact's routing_threshold (0.75 for the bundled artifact). Other classifier types ignore this setting + * @description Minimum predicted success probability for standalone or chained heuristic_v2 to select a tier. The first tier meeting this threshold is selected. When none meets it, standalone heuristic_v2 selects REASONING and chained heuristic_v2 defers to the LLM judge. When omitted or null, uses the artifact's routing_threshold (0.75 for the bundled artifact). Ignored when heuristic_v2 is not selected */ heuristic_v2_success_threshold?: number | null; /** @@ -42222,7 +42222,7 @@ export interface components { housekeeping_patterns?: string[] | null; /** * Hybrid Boundary Margin - * @description How close to a tier boundary a heuristic score has to land before the LLM classifier breaks the tie; required when classifier_type is 'hybrid' and rejected otherwise. Everything further than this from every active boundary routes on the scorer's own tier with no classifier call, at any tier, which is what separates 'hybrid' from 'heuristic_first' and its cheap-tier ceiling. A prompt where no dimension fired still goes to the classifier, since the scorer has no opinion to be near a boundary with. 0 escalates only scores sitting exactly on a boundary. + * @description How close to a tier boundary a heuristic score has to land before the LLM classifier breaks the tie; required when classifier_type is 'hybrid' and rejected otherwise. Everything further than this from every active boundary routes on the scorer's own tier with no classifier call, at any tier, which is what separates 'hybrid' from 'heuristic_first' and its cheap-tier ceiling. A prompt where no dimension fired still goes to the classifier, since the scorer has no opinion to be near a boundary with. With local_heuristic 'heuristic_v2', a tier must meet its success threshold and its probability and all lower-tier probabilities must be further than this margin from that threshold. 0 escalates only scores or probabilities exactly on a boundary. */ hybrid_boundary_margin?: number | null; /** @@ -42232,6 +42232,11 @@ export interface components { keyword_tier_rules?: components["schemas"]["KeywordTierRule"][] | null; /** @description Experimental joint task-demand and solver-capability forecasting for classifier_type llm_v2. */ llm_v2_config?: components["schemas"]["LLMV2Config"] | null; + /** + * Local Heuristic + * @description Local scorer for heuristic_first or hybrid. Omitted or null keeps heuristic v1; heuristic_v2 uses the trained success predictor. Rejected for other classifier types. + */ + local_heuristic?: ("heuristic" | "heuristic_v2") | null; /** * Match Threshold * @description Minimum cosine similarity for a semantic keyword match diff --git a/ui/litellm-dashboard/tests/autoRouterSetup.ts b/ui/litellm-dashboard/tests/autoRouterSetup.ts index f574af31c5b..322e3df60f6 100644 --- a/ui/litellm-dashboard/tests/autoRouterSetup.ts +++ b/ui/litellm-dashboard/tests/autoRouterSetup.ts @@ -2,9 +2,8 @@ import userEvent from "@testing-library/user-event"; import { chooseSelectOption, fireEvent, screen } from "./test-utils"; const groups: Record = { - "Classification Method": "Classifier tuning", - "Heuristic Keyword Overrides": "Classifier tuning", - "Ignore Custom Tags": "Classifier tuning", + "Heuristic Keyword Overrides": "Heuristic tuning", + "Ignore Custom Tags": "Request preprocessing", Affinity: "Sessions and efficiency", "Adaptive Routing": "Sessions and efficiency", "Cache-aware routing": "Sessions and efficiency", @@ -15,12 +14,18 @@ const groups: Record = { export const openAutoRouterAdvanced = (section: string) => { const advanced = screen.getByRole("button", { name: /^Advanced settings/ }); if (advanced.getAttribute("aria-expanded") !== "true") fireEvent.click(advanced); - const group = screen.getByRole("button", { name: groups[section] ?? "Routing rules and recovery" }); - if (group.getAttribute("aria-expanded") !== "true") fireEvent.click(group); + const labels = + section === "Classification Method" + ? ["Heuristic tuning", "LLM tuning", "Classifier tuning"] + : [groups[section] ?? "Routing rules and recovery"]; + for (const label of labels) { + const group = screen.queryByRole("button", { name: label }); + if (group && group.getAttribute("aria-expanded") !== "true") fireEvent.click(group); + } }; export const selectAutoRouterOption = async (field: string, option: string) => { - if (field === "Heuristic" || field === "Routing approach") { + if (field === "Heuristic" || field === "Routing approach" || field === "Heuristic before the judge") { const user = userEvent.setup(); fireEvent.click(screen.getByRole("button", { name: field })); await user.click(await screen.findByRole("menuitemradio", { name: new RegExp(`^${option}`) }));