feat(complexity_router): gate Jev route-downs on per-type confidence thresholds

One confidence threshold behaves very differently for choice and yes/no
questions because Jev clips scores to 0.01-0.99: confidence spreads across
every option of a wide choice but only two of a binary decision. The gate now
follows the decision question's shape - a question with exactly two options
(built-in tiers never produce one; a two-tier custom tier_definitions set
does) is yes/no-shaped and uses confidence_threshold_yes_no, a wider one is
choice-shaped and uses confidence_threshold_choice. Each per-type value falls
back to the single confidence_threshold when absent, so existing single-
threshold configs keep gating every shape exactly as before (pydecide 0.2.0
adopted the same per-type split).

The option count comes from the criteria the router already builds for the
tier question; the member-tunable whitelist and the dashboard schema are
updated with the two new fields.

litellm/proxy/management_helpers/auto_router_permissions.py

@@ -86,6 +86,8 @@ class _MemberOpenSourceClassifierConfig(LiteLLMBaseModel):
     circuit_breaker_enabled: bool
     circuit_breaker_cooldown_seconds: float
     confidence_threshold: float | None = Field(default=None, ge=0.0, le=1.0)
+    confidence_threshold_choice: float | None = Field(default=None, ge=0.0, le=1.0)
+    confidence_threshold_yes_no: float | None = Field(default=None, ge=0.0, le=1.0)
     complexity_max: float | None = Field(default=None, ge=0.0, le=1.0)
     complexity_confidence_min: float | None = Field(default=None, ge=0.0, le=1.0)
 

litellm/router_strategy/complexity_router/complexity_router.py

@@ -2344,6 +2344,7 @@ class ComplexityRouter(CustomLogger):
                 chosen_tier=tier_name,
                 strongest_tier=self.config.tier_names()[-1],
                 confidence=answer.confidence,
+                choice_count=len(criteria),
             )
             if refusal_reason is not None:
                 # A confident-enough classifier is a healthy classifier: a gate refusal is a
@@ -2403,19 +2404,27 @@ class ComplexityRouter(CustomLogger):
         confidence: float,
         complexity_score: float | None = None,
         complexity_confidence: float | None = None,
+        choice_count: int | None = None,
     ) -> str | None:
         """Why a Jev verdict that routes below the strongest tier must not be applied, or None.
 
         A verdict naming the strongest tier is not a route-down and is never gated. Every
         configured gate must pass: the verdict's own confidence, plus the complexity answer's
-        score and confidence when the protocol carries one. Missing complexity data is hard
+        score and confidence when the protocol carries one. The confidence gate follows the
+        decision question's shape: a question with exactly two options is yes/no-shaped and
+        uses confidence_threshold_yes_no, a wider one is choice-shaped and uses
+        confidence_threshold_choice, and each per-type value falls back to the single
+        confidence_threshold. Missing complexity data is hard
         (score 1.0, confidence 0.0), so a configured complexity gate refuses every route-down
         until the wire carries a complexity answer; complexity_max=1.0 and
         complexity_confidence_min=0.0 are the values at which missing data passes.
         """
         if chosen_tier == strongest_tier:
             return None
-        if config.confidence_threshold is not None and confidence < config.confidence_threshold:
+        threshold: Final = (
+            config.confidence_threshold if choice_count is None else config.confidence_threshold_for(choice_count)
+        )
+        if threshold is not None and confidence < threshold:
             return "low-confidence"
         score: Final = 1.0 if complexity_score is None else complexity_score  # unknown complexity is hard
         if config.complexity_max is not None and score > config.complexity_max:

litellm/router_strategy/complexity_router/config.py

@@ -745,6 +745,22 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel):
             "pool with a large capability gap."
         ),
     )
+    confidence_threshold_choice: float | None = Field(
+        default=None,
+        ge=0.0,
+        le=1.0,
+        description=(
+            "Per-type confidence gate for choice-shaped decisions: a question with three or more options. Takes precedence over confidence_threshold for those questions; None falls back to it. Jev clips scores to 0.01-0.99, so a single number gates a wide choice and a binary decision unequally: confidence spreads across every option of a choice but only two of a yes/no."
+        ),
+    )
+    confidence_threshold_yes_no: float | None = Field(
+        default=None,
+        ge=0.0,
+        le=1.0,
+        description=(
+            "Per-type confidence gate for yes/no-shaped decisions: a question with exactly two options, like the built-in tier question on a two-tier custom tier set. Takes precedence over confidence_threshold for those questions; None falls back to it."
+        ),
+    )
     complexity_max: float | None = Field(
         default=None,
         ge=0.0,
@@ -775,6 +791,17 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel):
     def _normalize_provider_alias(cls, value: object) -> object:
         return "jev" if value == "typesafe" else value
 
+    def confidence_threshold_for(self, choice_count: int) -> float | None:
+        """The confidence gate for a decision question offering choice_count options.
+
+        Per-type thresholds take precedence over the single threshold; an unset per-type value
+        falls back to it, so a config that only sets confidence_threshold keeps gating every
+        shape exactly as before. A question with exactly two options is yes/no-shaped; every
+        other question is choice-shaped.
+        """
+        per_type: Final = self.confidence_threshold_yes_no if choice_count == 2 else self.confidence_threshold_choice
+        return per_type if per_type is not None else self.confidence_threshold
+
     @field_validator("instructions")
     @classmethod
     def _reject_blank_instructions(cls, value: str | None) -> str | None:

tests/test_litellm/router_strategy/complexity_router/test_jev_confidence_gate.py

@@ -30,6 +30,7 @@ _TIERS: Final[Mapping[str, str]] = {
     "COMPLEX": "strong",
     "REASONING": "fallback-strong",
 }
+_YES_NO_TIERS: Final[Mapping[str, str]] = {"TINY": "cheap", "HEAVY": "fallback-strong"}
 
 
 def _tier_answer(choice: str, confidence: float) -> JevChoiceAnswer:
@@ -79,6 +80,27 @@ class _WireJevClient:
         return TypeAdapter(JevSystemOneResponse).validate_python(self.payload)
 
 
+def _two_tier_router(client: Any, jev_overrides: Mapping[str, object] | None = None) -> ComplexityRouter:
+    """A router whose decision question offers exactly two options: the yes/no shape."""
+    jev: Final[dict[str, object]] = {"api_key": "test", "timeout_ms": 100, **(dict(jev_overrides or {}))}
+    return ComplexityRouter(
+        "test-router",
+        MagicMock(),
+        {
+            "classifier_type": "jev",
+            "jev_classifier_config": jev,
+            "tier_definitions": [
+                {"name": "TINY", "description": "short factual replies"},
+                {"name": "HEAVY", "description": "everything that needs real work"},
+            ],
+            "fallback_tier": "HEAVY",
+            "tiers": dict(_YES_NO_TIERS),
+        },
+        derive_savings_baseline=False,
+        jev_client=client,
+    )
+
+
 def _router(client: Any, jev_overrides: Mapping[str, object] | None = None) -> ComplexityRouter:
     jev: Final[dict[str, object]] = {"api_key": "test", "timeout_ms": 100, **(dict(jev_overrides or {}))}
     return ComplexityRouter(
@@ -248,6 +270,79 @@ async def test_all_three_gates_configured_refuse_with_the_first_failure_reported
     assert "jev-refused:low-confidence" in outcome.signals
 
 
+@pytest.mark.asyncio
+async def test_yes_no_threshold_gates_a_route_down_on_a_two_option_question() -> None:
+    router = _two_tier_router(_StaticJevClient(_response("TINY", 0.6)), {"confidence_threshold_yes_no": 0.9})
+
+    outcome = await router.aclassify("Explain this")
+
+    assert outcome.tier == "HEAVY"
+    assert outcome.cause == "classifier_fallback"
+    assert "jev-refused:low-confidence" in outcome.signals
+
+
+@pytest.mark.asyncio
+async def test_yes_no_threshold_passes_when_confidence_meets_it() -> None:
+    router = _two_tier_router(_StaticJevClient(_response("TINY", 0.9)), {"confidence_threshold_yes_no": 0.9})
+
+    outcome = await router.aclassify("Explain this")
+
+    assert outcome.tier == "TINY"
+    assert outcome.cause == "jev_classifier"
+
+
+@pytest.mark.asyncio
+async def test_choice_threshold_gates_a_route_down_on_a_wide_question() -> None:
+    router = _router(_StaticJevClient(_response("MEDIUM", 0.8)), {"confidence_threshold_choice": 0.9})
+
+    outcome = await router.aclassify("Explain this")
+
+    assert outcome.tier == ComplexityTier.REASONING
+    assert outcome.cause == "default_model_fallback"
+    assert "jev-refused:low-confidence" in outcome.signals
+
+
+@pytest.mark.asyncio
+async def test_per_type_threshold_for_the_other_shape_does_not_gate() -> None:
+    yes_no_only = _router(_StaticJevClient(_response("MEDIUM", 0.3)), {"confidence_threshold_yes_no": 0.99})
+
+    yes_no_outcome = await yes_no_only.aclassify("Explain this")
+
+    # The choice shape falls back to the single threshold, which is unset: no gate.
+    assert yes_no_outcome.tier == ComplexityTier.MEDIUM
+    assert yes_no_outcome.cause == "jev_classifier"
+
+    choice_only = _two_tier_router(_StaticJevClient(_response("TINY", 0.3)), {"confidence_threshold_choice": 0.99})
+
+    choice_outcome = await choice_only.aclassify("Explain this")
+
+    # The yes/no shape falls back to the single threshold, which is unset: no gate.
+    assert choice_outcome.tier == "TINY"
+    assert choice_outcome.cause == "jev_classifier"
+
+
+@pytest.mark.asyncio
+async def test_per_type_thresholds_fall_back_to_the_single_threshold_when_absent() -> None:
+    yes_no = _two_tier_router(_StaticJevClient(_response("TINY", 0.6)), {"confidence_threshold": 0.7})
+
+    yes_no_outcome = await yes_no.aclassify("Explain this")
+
+    assert yes_no_outcome.tier == "HEAVY"
+    assert yes_no_outcome.cause == "classifier_fallback"
+    assert "jev-refused:low-confidence" in yes_no_outcome.signals
+
+    wide = _router(
+        _StaticJevClient(_response("MEDIUM", 0.8)),
+        {"confidence_threshold": 0.7, "confidence_threshold_yes_no": 0.95},
+    )
+
+    wide_outcome = await wide.aclassify("Explain this")
+
+    # The choice shape stays on the single threshold: a yes/no bar does not leak into it.
+    assert wide_outcome.tier == ComplexityTier.MEDIUM
+    assert wide_outcome.cause == "jev_classifier"
+
+
 def test_gate_refusal_reasons_cover_every_gate_independently() -> None:
     config = JevClassifierConfig(
         api_key="test", confidence_threshold=0.7, complexity_max=0.5, complexity_confidence_min=0.5
@@ -266,14 +361,41 @@ def test_gate_refusal_reasons_cover_every_gate_independently() -> None:
     assert reason(config, "MEDIUM", "REASONING", 0.9, complexity_score=0.3) == "low-complexity-confidence"
 
 
+def test_per_type_confidence_thresholds_resolve_per_question_shape() -> None:
+    config = JevClassifierConfig(
+        api_key="test", confidence_threshold=0.7, confidence_threshold_choice=0.85, confidence_threshold_yes_no=0.55
+    )
+    reason = ComplexityRouter._jev_route_down_refusal_reason
+
+    # A two-option question is yes/no-shaped and uses its own bar.
+    assert reason(config, "MEDIUM", "REASONING", 0.54, choice_count=2) == "low-confidence"
+    assert reason(config, "MEDIUM", "REASONING", 0.55, choice_count=2) is None
+    # A wider question is choice-shaped and uses its own bar.
+    assert reason(config, "MEDIUM", "REASONING", 0.84, choice_count=4) == "low-confidence"
+    assert reason(config, "MEDIUM", "REASONING", 0.85, choice_count=4) is None
+    # A shape without a per-type value falls back to the single threshold.
+    fallback = JevClassifierConfig(api_key="test", confidence_threshold=0.7, confidence_threshold_yes_no=0.55)
+    assert reason(fallback, "MEDIUM", "REASONING", 0.69, choice_count=4) == "low-confidence"
+    assert reason(fallback, "MEDIUM", "REASONING", 0.7, choice_count=4) is None
+    # No shape supplied: the single threshold gates every verdict, as before.
+    assert reason(fallback, "MEDIUM", "REASONING", 0.69) == "low-confidence"
+    assert reason(fallback, "MEDIUM", "REASONING", 0.7) is None
+
+
 def test_threshold_fields_default_to_off_and_reject_out_of_range_values() -> None:
     defaults = JevClassifierConfig(api_key="test")
     assert defaults.confidence_threshold is None
+    assert defaults.confidence_threshold_choice is None
+    assert defaults.confidence_threshold_yes_no is None
     assert defaults.complexity_max is None
     assert defaults.complexity_confidence_min is None
 
     with pytest.raises(ValidationError):
         JevClassifierConfig(api_key="test", confidence_threshold=1.5)
+    with pytest.raises(ValidationError):
+        JevClassifierConfig(api_key="test", confidence_threshold_choice=1.5)
+    with pytest.raises(ValidationError):
+        JevClassifierConfig(api_key="test", confidence_threshold_yes_no=-0.1)
     with pytest.raises(ValidationError):
         JevClassifierConfig(api_key="test", complexity_max=-0.1)
     with pytest.raises(ValidationError):

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -33205,6 +33205,16 @@ export interface components {
              * @description Minimum classifier confidence (0-1) required before a verdict that routes below the strongest tier is applied. None keeps the current behavior of applying every valid verdict. A refused route-down falls back like any other unusable verdict (fallback_tier, classifier_fallback, or the llm_v2 capable tier), so pair a threshold with an expensive fallback to fail expensive. Pick per pool: 0.55-0.65 suits a pool whose cheap tier is nearly as capable as its top tier; 0.85 suits a pool with a large capability gap.
              */
             confidence_threshold?: number | null;
+            /**
+             * Confidence Threshold Choice
+             * @description Per-type confidence gate for choice-shaped decisions: a question with three or more options. Takes precedence over confidence_threshold for those questions; None falls back to it. Jev clips scores to 0.01-0.99, so a single number gates a wide choice and a binary decision unequally: confidence spreads across every option of a choice but only two of a yes/no.
+             */
+            confidence_threshold_choice?: number | null;
+            /**
+             * Confidence Threshold Yes No
+             * @description Per-type confidence gate for yes/no-shaped decisions: a question with exactly two options, like the built-in tier question on a two-tier custom tier set. Takes precedence over confidence_threshold for those questions; None falls back to it.
+             */
+            confidence_threshold_yes_no?: number | null;
             /**
              * Instructions
              * @description Replaces the built-in Jev question instructions