From abbf8dc88d3ba92d1d08564497101d1910f05d3b Mon Sep 17 00:00:00 2001 From: Nguyen Quang Minh <135627235+n24q02m@users.noreply.github.com> Date: Sat, 26 Sep 2026 03:36:10 +0700 Subject: [PATCH] feat(complexity_router): gate Jev route-downs on per-type confidence thresholds One confidence threshold behaves very differently for choice and yes/no questions because Jev clips scores to 0.01-0.99: confidence spreads across every option of a wide choice but only two of a binary decision. The gate now follows the decision question's shape - a question with exactly two options (built-in tiers never produce one; a two-tier custom tier_definitions set does) is yes/no-shaped and uses confidence_threshold_yes_no, a wider one is choice-shaped and uses confidence_threshold_choice. Each per-type value falls back to the single confidence_threshold when absent, so existing single- threshold configs keep gating every shape exactly as before (pydecide 0.2.0 adopted the same per-type split). The option count comes from the criteria the router already builds for the tier question; the member-tunable whitelist and the dashboard schema are updated with the two new fields. --- .../auto_router_permissions.py | 2 + .../complexity_router/complexity_router.py | 13 +- .../complexity_router/config.py | 27 ++++ .../test_jev_confidence_gate.py | 122 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 10 ++ 5 files changed, 172 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_helpers/auto_router_permissions.py b/litellm/proxy/management_helpers/auto_router_permissions.py index f2251a06e0b..bd7dd565403 100644 --- a/litellm/proxy/management_helpers/auto_router_permissions.py +++ b/litellm/proxy/management_helpers/auto_router_permissions.py @@ -86,6 +86,8 @@ class _MemberOpenSourceClassifierConfig(LiteLLMBaseModel): circuit_breaker_enabled: bool circuit_breaker_cooldown_seconds: float confidence_threshold: float | None = Field(default=None, ge=0.0, le=1.0) + confidence_threshold_choice: float | None = Field(default=None, ge=0.0, le=1.0) + confidence_threshold_yes_no: float | None = Field(default=None, ge=0.0, le=1.0) complexity_max: float | None = Field(default=None, ge=0.0, le=1.0) complexity_confidence_min: float | None = Field(default=None, ge=0.0, le=1.0) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 876be643562..ea37bbf26ca 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -2344,6 +2344,7 @@ class ComplexityRouter(CustomLogger): chosen_tier=tier_name, strongest_tier=self.config.tier_names()[-1], confidence=answer.confidence, + choice_count=len(criteria), ) if refusal_reason is not None: # A confident-enough classifier is a healthy classifier: a gate refusal is a @@ -2403,19 +2404,27 @@ class ComplexityRouter(CustomLogger): confidence: float, complexity_score: float | None = None, complexity_confidence: float | None = None, + choice_count: int | None = None, ) -> str | None: """Why a Jev verdict that routes below the strongest tier must not be applied, or None. A verdict naming the strongest tier is not a route-down and is never gated. Every configured gate must pass: the verdict's own confidence, plus the complexity answer's - score and confidence when the protocol carries one. Missing complexity data is hard + score and confidence when the protocol carries one. The confidence gate follows the + decision question's shape: a question with exactly two options is yes/no-shaped and + uses confidence_threshold_yes_no, a wider one is choice-shaped and uses + confidence_threshold_choice, and each per-type value falls back to the single + confidence_threshold. Missing complexity data is hard (score 1.0, confidence 0.0), so a configured complexity gate refuses every route-down until the wire carries a complexity answer; complexity_max=1.0 and complexity_confidence_min=0.0 are the values at which missing data passes. """ if chosen_tier == strongest_tier: return None - if config.confidence_threshold is not None and confidence < config.confidence_threshold: + threshold: Final = ( + config.confidence_threshold if choice_count is None else config.confidence_threshold_for(choice_count) + ) + if threshold is not None and confidence < threshold: return "low-confidence" score: Final = 1.0 if complexity_score is None else complexity_score # unknown complexity is hard if config.complexity_max is not None and score > config.complexity_max: diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 69d72be7af6..4bd5e4043d0 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -745,6 +745,22 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel): "pool with a large capability gap." ), ) + confidence_threshold_choice: float | None = Field( + default=None, + ge=0.0, + le=1.0, + description=( + "Per-type confidence gate for choice-shaped decisions: a question with three or more options. Takes precedence over confidence_threshold for those questions; None falls back to it. Jev clips scores to 0.01-0.99, so a single number gates a wide choice and a binary decision unequally: confidence spreads across every option of a choice but only two of a yes/no." + ), + ) + confidence_threshold_yes_no: float | None = Field( + default=None, + ge=0.0, + le=1.0, + description=( + "Per-type confidence gate for yes/no-shaped decisions: a question with exactly two options, like the built-in tier question on a two-tier custom tier set. Takes precedence over confidence_threshold for those questions; None falls back to it." + ), + ) complexity_max: float | None = Field( default=None, ge=0.0, @@ -775,6 +791,17 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel): def _normalize_provider_alias(cls, value: object) -> object: return "jev" if value == "typesafe" else value + def confidence_threshold_for(self, choice_count: int) -> float | None: + """The confidence gate for a decision question offering choice_count options. + + Per-type thresholds take precedence over the single threshold; an unset per-type value + falls back to it, so a config that only sets confidence_threshold keeps gating every + shape exactly as before. A question with exactly two options is yes/no-shaped; every + other question is choice-shaped. + """ + per_type: Final = self.confidence_threshold_yes_no if choice_count == 2 else self.confidence_threshold_choice + return per_type if per_type is not None else self.confidence_threshold + @field_validator("instructions") @classmethod def _reject_blank_instructions(cls, value: str | None) -> str | None: diff --git a/tests/test_litellm/router_strategy/complexity_router/test_jev_confidence_gate.py b/tests/test_litellm/router_strategy/complexity_router/test_jev_confidence_gate.py index 6e31e9e0a43..305e91d6019 100644 --- a/tests/test_litellm/router_strategy/complexity_router/test_jev_confidence_gate.py +++ b/tests/test_litellm/router_strategy/complexity_router/test_jev_confidence_gate.py @@ -30,6 +30,7 @@ _TIERS: Final[Mapping[str, str]] = { "COMPLEX": "strong", "REASONING": "fallback-strong", } +_YES_NO_TIERS: Final[Mapping[str, str]] = {"TINY": "cheap", "HEAVY": "fallback-strong"} def _tier_answer(choice: str, confidence: float) -> JevChoiceAnswer: @@ -79,6 +80,27 @@ class _WireJevClient: return TypeAdapter(JevSystemOneResponse).validate_python(self.payload) +def _two_tier_router(client: Any, jev_overrides: Mapping[str, object] | None = None) -> ComplexityRouter: + """A router whose decision question offers exactly two options: the yes/no shape.""" + jev: Final[dict[str, object]] = {"api_key": "test", "timeout_ms": 100, **(dict(jev_overrides or {}))} + return ComplexityRouter( + "test-router", + MagicMock(), + { + "classifier_type": "jev", + "jev_classifier_config": jev, + "tier_definitions": [ + {"name": "TINY", "description": "short factual replies"}, + {"name": "HEAVY", "description": "everything that needs real work"}, + ], + "fallback_tier": "HEAVY", + "tiers": dict(_YES_NO_TIERS), + }, + derive_savings_baseline=False, + jev_client=client, + ) + + def _router(client: Any, jev_overrides: Mapping[str, object] | None = None) -> ComplexityRouter: jev: Final[dict[str, object]] = {"api_key": "test", "timeout_ms": 100, **(dict(jev_overrides or {}))} return ComplexityRouter( @@ -248,6 +270,79 @@ async def test_all_three_gates_configured_refuse_with_the_first_failure_reported assert "jev-refused:low-confidence" in outcome.signals +@pytest.mark.asyncio +async def test_yes_no_threshold_gates_a_route_down_on_a_two_option_question() -> None: + router = _two_tier_router(_StaticJevClient(_response("TINY", 0.6)), {"confidence_threshold_yes_no": 0.9}) + + outcome = await router.aclassify("Explain this") + + assert outcome.tier == "HEAVY" + assert outcome.cause == "classifier_fallback" + assert "jev-refused:low-confidence" in outcome.signals + + +@pytest.mark.asyncio +async def test_yes_no_threshold_passes_when_confidence_meets_it() -> None: + router = _two_tier_router(_StaticJevClient(_response("TINY", 0.9)), {"confidence_threshold_yes_no": 0.9}) + + outcome = await router.aclassify("Explain this") + + assert outcome.tier == "TINY" + assert outcome.cause == "jev_classifier" + + +@pytest.mark.asyncio +async def test_choice_threshold_gates_a_route_down_on_a_wide_question() -> None: + router = _router(_StaticJevClient(_response("MEDIUM", 0.8)), {"confidence_threshold_choice": 0.9}) + + outcome = await router.aclassify("Explain this") + + assert outcome.tier == ComplexityTier.REASONING + assert outcome.cause == "default_model_fallback" + assert "jev-refused:low-confidence" in outcome.signals + + +@pytest.mark.asyncio +async def test_per_type_threshold_for_the_other_shape_does_not_gate() -> None: + yes_no_only = _router(_StaticJevClient(_response("MEDIUM", 0.3)), {"confidence_threshold_yes_no": 0.99}) + + yes_no_outcome = await yes_no_only.aclassify("Explain this") + + # The choice shape falls back to the single threshold, which is unset: no gate. + assert yes_no_outcome.tier == ComplexityTier.MEDIUM + assert yes_no_outcome.cause == "jev_classifier" + + choice_only = _two_tier_router(_StaticJevClient(_response("TINY", 0.3)), {"confidence_threshold_choice": 0.99}) + + choice_outcome = await choice_only.aclassify("Explain this") + + # The yes/no shape falls back to the single threshold, which is unset: no gate. + assert choice_outcome.tier == "TINY" + assert choice_outcome.cause == "jev_classifier" + + +@pytest.mark.asyncio +async def test_per_type_thresholds_fall_back_to_the_single_threshold_when_absent() -> None: + yes_no = _two_tier_router(_StaticJevClient(_response("TINY", 0.6)), {"confidence_threshold": 0.7}) + + yes_no_outcome = await yes_no.aclassify("Explain this") + + assert yes_no_outcome.tier == "HEAVY" + assert yes_no_outcome.cause == "classifier_fallback" + assert "jev-refused:low-confidence" in yes_no_outcome.signals + + wide = _router( + _StaticJevClient(_response("MEDIUM", 0.8)), + {"confidence_threshold": 0.7, "confidence_threshold_yes_no": 0.95}, + ) + + wide_outcome = await wide.aclassify("Explain this") + + # The choice shape stays on the single threshold: a yes/no bar does not leak into it. + assert wide_outcome.tier == ComplexityTier.MEDIUM + assert wide_outcome.cause == "jev_classifier" + + def test_gate_refusal_reasons_cover_every_gate_independently() -> None: config = JevClassifierConfig( api_key="test", confidence_threshold=0.7, complexity_max=0.5, complexity_confidence_min=0.5 @@ -266,14 +361,41 @@ def test_gate_refusal_reasons_cover_every_gate_independently() -> None: assert reason(config, "MEDIUM", "REASONING", 0.9, complexity_score=0.3) == "low-complexity-confidence" +def test_per_type_confidence_thresholds_resolve_per_question_shape() -> None: + config = JevClassifierConfig( + api_key="test", confidence_threshold=0.7, confidence_threshold_choice=0.85, confidence_threshold_yes_no=0.55 + ) + reason = ComplexityRouter._jev_route_down_refusal_reason + + # A two-option question is yes/no-shaped and uses its own bar. + assert reason(config, "MEDIUM", "REASONING", 0.54, choice_count=2) == "low-confidence" + assert reason(config, "MEDIUM", "REASONING", 0.55, choice_count=2) is None + # A wider question is choice-shaped and uses its own bar. + assert reason(config, "MEDIUM", "REASONING", 0.84, choice_count=4) == "low-confidence" + assert reason(config, "MEDIUM", "REASONING", 0.85, choice_count=4) is None + # A shape without a per-type value falls back to the single threshold. + fallback = JevClassifierConfig(api_key="test", confidence_threshold=0.7, confidence_threshold_yes_no=0.55) + assert reason(fallback, "MEDIUM", "REASONING", 0.69, choice_count=4) == "low-confidence" + assert reason(fallback, "MEDIUM", "REASONING", 0.7, choice_count=4) is None + # No shape supplied: the single threshold gates every verdict, as before. + assert reason(fallback, "MEDIUM", "REASONING", 0.69) == "low-confidence" + assert reason(fallback, "MEDIUM", "REASONING", 0.7) is None + + def test_threshold_fields_default_to_off_and_reject_out_of_range_values() -> None: defaults = JevClassifierConfig(api_key="test") assert defaults.confidence_threshold is None + assert defaults.confidence_threshold_choice is None + assert defaults.confidence_threshold_yes_no is None assert defaults.complexity_max is None assert defaults.complexity_confidence_min is None with pytest.raises(ValidationError): JevClassifierConfig(api_key="test", confidence_threshold=1.5) + with pytest.raises(ValidationError): + JevClassifierConfig(api_key="test", confidence_threshold_choice=1.5) + with pytest.raises(ValidationError): + JevClassifierConfig(api_key="test", confidence_threshold_yes_no=-0.1) with pytest.raises(ValidationError): JevClassifierConfig(api_key="test", complexity_max=-0.1) with pytest.raises(ValidationError): diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 06ae99f70db..f41c2917ead 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -33205,6 +33205,16 @@ export interface components { * @description Minimum classifier confidence (0-1) required before a verdict that routes below the strongest tier is applied. None keeps the current behavior of applying every valid verdict. A refused route-down falls back like any other unusable verdict (fallback_tier, classifier_fallback, or the llm_v2 capable tier), so pair a threshold with an expensive fallback to fail expensive. Pick per pool: 0.55-0.65 suits a pool whose cheap tier is nearly as capable as its top tier; 0.85 suits a pool with a large capability gap. */ confidence_threshold?: number | null; + /** + * Confidence Threshold Choice + * @description Per-type confidence gate for choice-shaped decisions: a question with three or more options. Takes precedence over confidence_threshold for those questions; None falls back to it. Jev clips scores to 0.01-0.99, so a single number gates a wide choice and a binary decision unequally: confidence spreads across every option of a choice but only two of a yes/no. + */ + confidence_threshold_choice?: number | null; + /** + * Confidence Threshold Yes No + * @description Per-type confidence gate for yes/no-shaped decisions: a question with exactly two options, like the built-in tier question on a two-tier custom tier set. Takes precedence over confidence_threshold for those questions; None falls back to it. + */ + confidence_threshold_yes_no?: number | null; /** * Instructions * @description Replaces the built-in Jev question instructions