From 858b0f9186249642c43a5728371ee516308f7a2b Mon Sep 17 00:00:00 2001 From: Nguyen Quang Minh <135627235+n24q02m@users.noreply.github.com> Date: Sun, 20 Sep 2026 23:48:47 +0700 Subject: [PATCH 1/8] feat(complexity_router): gate Jev route-downs on classifier confidence A Jev verdict that routes below the strongest tier is applied only when every configured gate passes. Three gates join JevClassifierConfig: confidence_threshold (verdict confidence), complexity_max and complexity_confidence_min (complexity-answer score and confidence). All default to None, which keeps today's behavior of applying every valid verdict; thresholds are pool-specific choices, so none are hard-coded. A refused route-down follows the standard classifier-unusable fallback chain (llm_v2 capable tier, fallback_tier, classifier_fallback), so pairing a threshold with an expensive fallback makes the router fail expensive. The refusal is a policy decision, not a health failure: the circuit breaker records success, and the verdict plus its cost stay on the outcome so provenance logging is unchanged. The built-in Jev request asks only the tier question today, so missing complexity data is treated as hard (score 1.0, confidence 0.0): a configured complexity gate refuses route-downs until the wire carries a complexity answer. Refs: BerriAI/litellm#42139 --- .../complexity_router/complexity_router.py | 54 ++++ .../complexity_router/config.py | 36 +++ .../test_jev_confidence_gate.py | 270 ++++++++++++++++++ 3 files changed, 360 insertions(+) create mode 100644 tests/unit/router_strategy/complexity_router/test_jev_confidence_gate.py diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 7ef31f7a0a2..876be643562 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -2339,6 +2339,30 @@ class ComplexityRouter(CustomLogger): ), provider=accounting_provider, ) + refusal_reason: Final = self._jev_route_down_refusal_reason( + config=config, + chosen_tier=tier_name, + strongest_tier=self.config.tier_names()[-1], + confidence=answer.confidence, + ) + if refusal_reason is not None: + # A confident-enough classifier is a healthy classifier: a gate refusal is a + # policy decision about the verdict, not a health failure, so the breaker closes. + if breaker is not None and permit is not None: + breaker.record_success(permit) + refused: Final = _with_signal( + self._classifier_failure_outcome( + f"jev classifier route-down refused ({refusal_reason}, verdict {tier_name}, " + f"confidence {answer.confidence:.4f})", + prompt, + system_prompt, + signal=f"jev-refused:{refusal_reason}", + ), + f"jev-classifier:{tier_name}", + ) + # Keep the verdict and its cost on the refusal for provenance: the request paid + # for the Jev call, and the routing decision still logs what it answered. + return refused._replace(classifier_cost=verdict.cost, jev_verdict=verdict) if breaker is not None and permit is not None: breaker.record_success(permit) return ClassificationOutcome( @@ -2371,6 +2395,36 @@ class ComplexityRouter(CustomLogger): error_type=type(e).__name__, ) + @staticmethod + def _jev_route_down_refusal_reason( + config: JevClassifierConfig, + chosen_tier: str, + strongest_tier: str, + confidence: float, + complexity_score: float | None = None, + complexity_confidence: float | None = None, + ) -> str | None: + """Why a Jev verdict that routes below the strongest tier must not be applied, or None. + + A verdict naming the strongest tier is not a route-down and is never gated. Every + configured gate must pass: the verdict's own confidence, plus the complexity answer's + score and confidence when the protocol carries one. Missing complexity data is hard + (score 1.0, confidence 0.0), so a configured complexity gate refuses every route-down + until the wire carries a complexity answer; complexity_max=1.0 and + complexity_confidence_min=0.0 are the values at which missing data passes. + """ + if chosen_tier == strongest_tier: + return None + if config.confidence_threshold is not None and confidence < config.confidence_threshold: + return "low-confidence" + score: Final = 1.0 if complexity_score is None else complexity_score # unknown complexity is hard + if config.complexity_max is not None and score > config.complexity_max: + return "high-complexity" + verdict_confidence: Final = 0.0 if complexity_confidence is None else complexity_confidence + if config.complexity_confidence_min is not None and verdict_confidence < config.complexity_confidence_min: + return "low-complexity-confidence" + return None + def _classifier_failure_outcome( self, reason: str, diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 4a2c56daf00..4f3c2ea87d1 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -731,6 +731,42 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel): ) circuit_breaker_enabled: bool = True circuit_breaker_cooldown_seconds: float = Field(default=30.0, gt=0.0) + confidence_threshold: float | None = Field( + default=None, + ge=0.0, + le=1.0, + description=( + "Minimum classifier confidence (0-1) required before a verdict that routes below the " + "strongest tier is applied. None keeps the current behavior of applying every valid " + "verdict. A refused route-down falls back like any other unusable verdict " + "(fallback_tier, classifier_fallback, or the llm_v2 capable tier), so pair a " + "threshold with an expensive fallback to fail expensive. Pick per pool: 0.55-0.65 " + "suits a pool whose cheap tier is nearly as capable as its top tier; 0.85 suits a " + "pool with a large capability gap." + ), + ) + complexity_max: float | None = Field( + default=None, + ge=0.0, + le=1.0, + description=( + "Maximum request complexity (0-1) a route-down verdict may carry. The built-in Jev " + "request asks only the tier question today, so an absent complexity answer counts as " + "the hardest score (1.0): while this is set below 1.0 every route-down is refused " + "until the classifier protocol carries a complexity answer. None disables the gate." + ), + ) + complexity_confidence_min: float | None = Field( + default=None, + ge=0.0, + le=1.0, + description=( + "Minimum confidence (0-1) required on the complexity answer before a route-down is " + "applied. The built-in Jev request asks only the tier question today, so an absent " + "complexity answer counts as confidence 0.0: while this is set above 0.0 every " + "route-down is refused. None disables the gate." + ), + ) @field_validator("provider", mode="before") @classmethod diff --git a/tests/unit/router_strategy/complexity_router/test_jev_confidence_gate.py b/tests/unit/router_strategy/complexity_router/test_jev_confidence_gate.py new file mode 100644 index 00000000000..62a61c1db22 --- /dev/null +++ b/tests/unit/router_strategy/complexity_router/test_jev_confidence_gate.py @@ -0,0 +1,270 @@ +"""Jev route-down gate: a below-top verdict is applied only when every configured gate passes. + +The gate is fail-expensive: a refused route-down keeps the request on the configured +expensive fallback (here classifier_fallback='default_model' so the destination is +deterministic), the Jev verdict and its cost stay on the outcome for provenance, and a +refusal counts as a classifier success for the circuit breaker. All tests mock the +classifier client; no TYPESAFE_API_KEY or network is involved. +""" + +from collections.abc import Mapping +from typing import Any, Final +from unittest.mock import MagicMock + +import pytest +from pydantic import TypeAdapter, ValidationError + +import litellm +from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter +from litellm.router_strategy.complexity_router.config import ComplexityTier, JevClassifierConfig +from litellm.router_strategy.complexity_router.jev_classifier import ( + JevChoiceAnswer, + JevSystemOneRequest, + JevSystemOneResponse, + JevUsage, +) + +_TIERS: Final[Mapping[str, str]] = { + "SIMPLE": "cheap", + "MEDIUM": "mid", + "COMPLEX": "strong", + "REASONING": "fallback-strong", +} + + +def _tier_answer(choice: str, confidence: float) -> JevChoiceAnswer: + return JevChoiceAnswer( + type="choice", + choice=choice, + probabilities={choice: confidence}, + confidence=confidence, + ) + + +def _response(choice: str, confidence: float) -> JevSystemOneResponse: + return JevSystemOneResponse( + model="jev-1.13.0", + answers={"tier": _tier_answer(choice, confidence)}, + usage=JevUsage(input_tokens=3, output_tokens=4), + ) + + +class _StaticJevClient: + def __init__(self, response: JevSystemOneResponse) -> None: + self.response = response + self.calls = 0 + + async def evaluate(self, request: JevSystemOneRequest, timeout_s: float) -> JevSystemOneResponse: + self.calls += 1 + return self.response + + +class _WireJevClient: + """Validates a raw wire payload the way HttpJevClassifierClient does.""" + + def __init__(self, payload: Mapping[str, object]) -> None: + self.payload = payload + + async def evaluate(self, request: JevSystemOneRequest, timeout_s: float) -> JevSystemOneResponse: + return TypeAdapter(JevSystemOneResponse).validate_python(self.payload) + + +def _router(client: Any, jev_overrides: Mapping[str, object] | None = None) -> ComplexityRouter: + jev: Final[dict[str, object]] = {"api_key": "test", "timeout_ms": 100, **(dict(jev_overrides or {}))} + return ComplexityRouter( + "test-router", + MagicMock(), + { + "classifier_type": "jev", + "jev_classifier_config": jev, + "classifier_fallback": "default_model", + "default_model": "fallback-strong", + "tiers": dict(_TIERS), + }, + derive_savings_baseline=False, + jev_client=client, + ) + + +@pytest.mark.asyncio +async def test_below_threshold_refuses_route_down_and_fails_over_to_the_expensive_destination( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "typesafe/jev-1.13.0", + {"input_cost_per_token": 0.0001, "output_cost_per_token": 0.0002}, + ) + router = _router(_StaticJevClient(_response("MEDIUM", 0.42)), {"confidence_threshold": 0.7}) + + outcome = await router.aclassify("Explain this") + + assert outcome.tier == ComplexityTier.REASONING # the tier whose pool holds default_model + assert outcome.cause == "default_model_fallback" + assert outcome.signals == ( + "classifier-failed:default-model", + "jev-refused:low-confidence", + "jev-classifier:MEDIUM", + ) + assert outcome.jev_verdict is not None + assert outcome.jev_verdict.confidence == 0.42 + assert outcome.classifier_cost == pytest.approx(0.0011) + + +@pytest.mark.asyncio +async def test_above_threshold_applies_the_route_down() -> None: + router = _router(_StaticJevClient(_response("MEDIUM", 0.8)), {"confidence_threshold": 0.7}) + + outcome = await router.aclassify("Explain this") + + assert outcome.tier == ComplexityTier.MEDIUM + assert outcome.cause == "jev_classifier" + assert outcome.signals == ( + "jev-classifier:MEDIUM", + "jev-confidence=0.800000", + "tier-probability:MEDIUM=0.800000", + ) + + +@pytest.mark.asyncio +async def test_confidence_exactly_at_threshold_applies_the_route_down() -> None: + router = _router(_StaticJevClient(_response("MEDIUM", 0.7)), {"confidence_threshold": 0.7}) + + outcome = await router.aclassify("Explain this") + + assert outcome.tier == ComplexityTier.MEDIUM + assert outcome.cause == "jev_classifier" + + +@pytest.mark.asyncio +async def test_gate_disabled_keeps_current_behavior_of_applying_every_verdict() -> None: + router = _router(_StaticJevClient(_response("MEDIUM", 0.05))) + + outcome = await router.aclassify("Explain this") + + assert outcome.tier == ComplexityTier.MEDIUM + assert outcome.cause == "jev_classifier" + + +@pytest.mark.asyncio +async def test_verdict_naming_the_strongest_tier_is_never_gated() -> None: + router = _router(_StaticJevClient(_response("REASONING", 0.1)), {"confidence_threshold": 0.99}) + + outcome = await router.aclassify("Explain this") + + assert outcome.tier == ComplexityTier.REASONING + assert outcome.cause == "jev_classifier" + + +@pytest.mark.asyncio +async def test_wire_answer_without_confidence_falls_back_conservatively() -> None: + client = _WireJevClient( + { + "model": "jev-1.13.0", + "answers": {"tier": {"type": "choice", "choice": "MEDIUM", "probabilities": {"MEDIUM": 0.9}}}, + } + ) + router = _router(client, {"confidence_threshold": 0.7}) + + outcome = await router.aclassify("Explain this") + + assert outcome.tier == ComplexityTier.REASONING + assert outcome.cause == "default_model_fallback" + assert outcome.jev_verdict is None + + +@pytest.mark.asyncio +async def test_gate_refusal_is_a_classifier_success_for_the_circuit_breaker() -> None: + client = _StaticJevClient(_response("MEDIUM", 0.1)) + router = _router(client, {"confidence_threshold": 0.7}) + + first = await router.aclassify("Explain this") + second = await router.aclassify("Explain this") + + assert client.calls == 2 # the breaker stayed closed: refusals are policy, not health + assert first.cause == second.cause == "default_model_fallback" + + +@pytest.mark.asyncio +async def test_complexity_max_refuses_a_route_down_when_no_complexity_answer_exists() -> None: + router = _router(_StaticJevClient(_response("MEDIUM", 0.99)), {"complexity_max": 0.5}) + + outcome = await router.aclassify("Explain this") + + assert outcome.cause == "default_model_fallback" + assert "jev-refused:high-complexity" in outcome.signals + + +@pytest.mark.asyncio +async def test_complexity_max_of_one_keeps_route_down_available_without_a_complexity_answer() -> None: + router = _router(_StaticJevClient(_response("MEDIUM", 0.5)), {"complexity_max": 1.0}) + + outcome = await router.aclassify("Explain this") + + assert outcome.tier == ComplexityTier.MEDIUM + assert outcome.cause == "jev_classifier" + + +@pytest.mark.asyncio +async def test_complexity_confidence_min_refuses_a_route_down_when_no_complexity_answer_exists() -> None: + router = _router(_StaticJevClient(_response("MEDIUM", 0.99)), {"complexity_confidence_min": 0.5}) + + outcome = await router.aclassify("Explain this") + + assert outcome.cause == "default_model_fallback" + assert "jev-refused:low-complexity-confidence" in outcome.signals + + +@pytest.mark.asyncio +async def test_complexity_confidence_min_of_zero_keeps_route_down_available() -> None: + router = _router(_StaticJevClient(_response("MEDIUM", 0.5)), {"complexity_confidence_min": 0.0}) + + outcome = await router.aclassify("Explain this") + + assert outcome.tier == ComplexityTier.MEDIUM + assert outcome.cause == "jev_classifier" + + +@pytest.mark.asyncio +async def test_all_three_gates_configured_refuse_with_the_first_failure_reported() -> None: + router = _router( + _StaticJevClient(_response("MEDIUM", 0.1)), + {"confidence_threshold": 0.7, "complexity_max": 0.5, "complexity_confidence_min": 0.5}, + ) + + outcome = await router.aclassify("Explain this") + + assert outcome.cause == "default_model_fallback" + assert "jev-refused:low-confidence" in outcome.signals + + +def test_gate_refusal_reasons_cover_every_gate_independently() -> None: + config = JevClassifierConfig( + api_key="test", confidence_threshold=0.7, complexity_max=0.5, complexity_confidence_min=0.5 + ) + reason = ComplexityRouter._jev_route_down_refusal_reason + + # Not a route-down: the strongest tier is never gated, whatever the confidence. + assert reason(config, "REASONING", "REASONING", 0.0) is None + # Confidence gate: closed below the threshold, open at it (complexity gates follow). + assert reason(config, "MEDIUM", "REASONING", 0.69) == "low-confidence" + assert reason(config, "MEDIUM", "REASONING", 0.7) == "high-complexity" # missing score is 1.0 > 0.5 + # Complexity gates pass when the wire carries a qualifying answer. + assert reason(config, "MEDIUM", "REASONING", 0.7, complexity_score=0.3, complexity_confidence=0.6) is None + # Missing complexity data is hard: unknown score 1.0 refuses, unknown confidence 0.0 refuses. + assert reason(config, "MEDIUM", "REASONING", 0.9, complexity_score=0.6) == "high-complexity" + assert reason(config, "MEDIUM", "REASONING", 0.9, complexity_score=0.3) == "low-complexity-confidence" + + +def test_threshold_fields_default_to_off_and_reject_out_of_range_values() -> None: + defaults = JevClassifierConfig(api_key="test") + assert defaults.confidence_threshold is None + assert defaults.complexity_max is None + assert defaults.complexity_confidence_min is None + + with pytest.raises(ValidationError): + JevClassifierConfig(api_key="test", confidence_threshold=1.5) + with pytest.raises(ValidationError): + JevClassifierConfig(api_key="test", complexity_max=-0.1) + with pytest.raises(ValidationError): + JevClassifierConfig(api_key="test", complexity_confidence_min=1.5) From f82976776f4f27538e03ca26c203f40314238e09 Mon Sep 17 00:00:00 2001 From: Nguyen Quang Minh <135627235+n24q02m@users.noreply.github.com> Date: Tue, 22 Sep 2026 07:27:49 +0700 Subject: [PATCH 2/8] docs(complexity_router): note wire scores are weighted level indices complexity_max is validated 0-1; decision-API score answers arrive as weighted level indices (0..levels-1, routing-spec v1.1). The built-in request asks only the tier question today, so document the normalization requirement for any future score question instead of encoding a scale conversion nothing exercises yet. --- litellm/router_strategy/complexity_router/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 4f3c2ea87d1..69d72be7af6 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -753,7 +753,9 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel): "Maximum request complexity (0-1) a route-down verdict may carry. The built-in Jev " "request asks only the tier question today, so an absent complexity answer counts as " "the hardest score (1.0): while this is set below 1.0 every route-down is refused " - "until the classifier protocol carries a complexity answer. None disables the gate." + "until the classifier protocol carries a complexity answer. Wire complexity scores " + "arrive as weighted level indices (0..levels-1), so a future score question " + "must normalize to 0-1 before this gate (routing-spec v1.1). None disables the gate." ), ) complexity_confidence_min: float | None = Field( From ce9c25c0edf28cd22712de792e92e1464185f033 Mon Sep 17 00:00:00 2001 From: n24q02m Date: Wed, 23 Sep 2026 14:20:18 +0700 Subject: [PATCH 3/8] fix(complexity_router): allow member Jev tuning fields; update test clients for request_kwargs --- .../management_helpers/auto_router_permissions.py | 3 +++ .../complexity_router/test_jev_confidence_gate.py | 14 ++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_helpers/auto_router_permissions.py b/litellm/proxy/management_helpers/auto_router_permissions.py index a4428eb3a85..f2251a06e0b 100644 --- a/litellm/proxy/management_helpers/auto_router_permissions.py +++ b/litellm/proxy/management_helpers/auto_router_permissions.py @@ -85,6 +85,9 @@ class _MemberOpenSourceClassifierConfig(LiteLLMBaseModel): instructions: str | None = None circuit_breaker_enabled: bool circuit_breaker_cooldown_seconds: float + confidence_threshold: float | None = Field(default=None, ge=0.0, le=1.0) + complexity_max: float | None = Field(default=None, ge=0.0, le=1.0) + complexity_confidence_min: float | None = Field(default=None, ge=0.0, le=1.0) class _MemberComplexityRouterConfig(RequestComplexityRouterConfig): diff --git a/tests/unit/router_strategy/complexity_router/test_jev_confidence_gate.py b/tests/unit/router_strategy/complexity_router/test_jev_confidence_gate.py index 62a61c1db22..6e31e9e0a43 100644 --- a/tests/unit/router_strategy/complexity_router/test_jev_confidence_gate.py +++ b/tests/unit/router_strategy/complexity_router/test_jev_confidence_gate.py @@ -54,7 +54,12 @@ class _StaticJevClient: self.response = response self.calls = 0 - async def evaluate(self, request: JevSystemOneRequest, timeout_s: float) -> JevSystemOneResponse: + async def evaluate( + self, + request: JevSystemOneRequest, + timeout_s: float, + request_kwargs: Mapping[str, object] | None = None, + ) -> JevSystemOneResponse: self.calls += 1 return self.response @@ -65,7 +70,12 @@ class _WireJevClient: def __init__(self, payload: Mapping[str, object]) -> None: self.payload = payload - async def evaluate(self, request: JevSystemOneRequest, timeout_s: float) -> JevSystemOneResponse: + async def evaluate( + self, + request: JevSystemOneRequest, + timeout_s: float, + request_kwargs: Mapping[str, object] | None = None, + ) -> JevSystemOneResponse: return TypeAdapter(JevSystemOneResponse).validate_python(self.payload) From ff4d61ed929c694cc897c27681423df2dd7d469e Mon Sep 17 00:00:00 2001 From: Nguyen Quang Minh <135627235+n24q02m@users.noreply.github.com> Date: Thu, 24 Sep 2026 08:43:07 +0700 Subject: [PATCH 4/8] chore(ui): regen schema.d.ts for Jev tuning fields (complexity_confidence_min, complexity_max, confidence_threshold) --- ui/litellm-dashboard/src/lib/http/schema.d.ts | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index ada0bb2b8a2..06ae99f70db 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -33168,6 +33168,59 @@ export interface components { /** Updated By */ updated_by?: string | null; }; + /** JevClassifierConfig */ + JevClassifierConfig: { + /** + * Api Base + * @description TypeSafe API base, falling back to TYPESAFE_API_BASE and then https://api.typesafe.ai + */ + api_base?: string | null; + /** + * Api Key + * @description TypeSafe API key, falling back to TYPESAFE_API_KEY + */ + api_key?: string | null; + /** + * Circuit Breaker Cooldown Seconds + * @default 30 + */ + circuit_breaker_cooldown_seconds: number; + /** + * Circuit Breaker Enabled + * @default true + */ + circuit_breaker_enabled: boolean; + /** + * Complexity Confidence Min + * @description Minimum confidence (0-1) required on the complexity answer before a route-down is applied. The built-in Jev request asks only the tier question today, so an absent complexity answer counts as confidence 0.0: while this is set above 0.0 every route-down is refused. None disables the gate. + */ + complexity_confidence_min?: number | null; + /** + * Complexity Max + * @description Maximum request complexity (0-1) a route-down verdict may carry. The built-in Jev request asks only the tier question today, so an absent complexity answer counts as the hardest score (1.0): while this is set below 1.0 every route-down is refused until the classifier protocol carries a complexity answer. Wire complexity scores arrive as weighted level indices (0..levels-1), so a future score question must normalize to 0-1 before this gate (routing-spec v1.1). None disables the gate. + */ + complexity_max?: number | null; + /** + * Confidence Threshold + * @description Minimum classifier confidence (0-1) required before a verdict that routes below the strongest tier is applied. None keeps the current behavior of applying every valid verdict. A refused route-down falls back like any other unusable verdict (fallback_tier, classifier_fallback, or the llm_v2 capable tier), so pair a threshold with an expensive fallback to fail expensive. Pick per pool: 0.55-0.65 suits a pool whose cheap tier is nearly as capable as its top tier; 0.85 suits a pool with a large capability gap. + */ + confidence_threshold?: number | null; + /** + * Instructions + * @description Replaces the built-in Jev question instructions + */ + instructions?: string | null; + /** + * Model + * @default jev-latest + */ + model: string; + /** + * Timeout Ms + * @default 3000 + */ + timeout_ms: number; + }; JsonValue: unknown; /** KeyActivityRow */ KeyActivityRow: { From 42028c5e11031c5cbda4bfa01d8ab6d1ec0974c5 Mon Sep 17 00:00:00 2001 From: Nguyen Quang Minh <135627235+n24q02m@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:27:31 +0700 Subject: [PATCH 5/8] test(complexity_router): move jev confidence gate tests into the CI unit shard tests/unit/** is not referenced by any workflow shard, so the gate tests never ran in CI and codecov/patch stayed red. The enterprise-routing shard (test-unit.yml) already runs tests/test_litellm/router_strategy, so the moved file executes there and its coverage uploads. --- .../router_strategy/complexity_router/test_jev_confidence_gate.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{unit => test_litellm}/router_strategy/complexity_router/test_jev_confidence_gate.py (100%) diff --git a/tests/unit/router_strategy/complexity_router/test_jev_confidence_gate.py b/tests/test_litellm/router_strategy/complexity_router/test_jev_confidence_gate.py similarity index 100% rename from tests/unit/router_strategy/complexity_router/test_jev_confidence_gate.py rename to tests/test_litellm/router_strategy/complexity_router/test_jev_confidence_gate.py From abbf8dc88d3ba92d1d08564497101d1910f05d3b Mon Sep 17 00:00:00 2001 From: Nguyen Quang Minh <135627235+n24q02m@users.noreply.github.com> Date: Sat, 26 Sep 2026 03:36:10 +0700 Subject: [PATCH 6/8] feat(complexity_router): gate Jev route-downs on per-type confidence thresholds One confidence threshold behaves very differently for choice and yes/no questions because Jev clips scores to 0.01-0.99: confidence spreads across every option of a wide choice but only two of a binary decision. The gate now follows the decision question's shape - a question with exactly two options (built-in tiers never produce one; a two-tier custom tier_definitions set does) is yes/no-shaped and uses confidence_threshold_yes_no, a wider one is choice-shaped and uses confidence_threshold_choice. Each per-type value falls back to the single confidence_threshold when absent, so existing single- threshold configs keep gating every shape exactly as before (pydecide 0.2.0 adopted the same per-type split). The option count comes from the criteria the router already builds for the tier question; the member-tunable whitelist and the dashboard schema are updated with the two new fields. --- .../auto_router_permissions.py | 2 + .../complexity_router/complexity_router.py | 13 +- .../complexity_router/config.py | 27 ++++ .../test_jev_confidence_gate.py | 122 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 10 ++ 5 files changed, 172 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_helpers/auto_router_permissions.py b/litellm/proxy/management_helpers/auto_router_permissions.py index f2251a06e0b..bd7dd565403 100644 --- a/litellm/proxy/management_helpers/auto_router_permissions.py +++ b/litellm/proxy/management_helpers/auto_router_permissions.py @@ -86,6 +86,8 @@ class _MemberOpenSourceClassifierConfig(LiteLLMBaseModel): circuit_breaker_enabled: bool circuit_breaker_cooldown_seconds: float confidence_threshold: float | None = Field(default=None, ge=0.0, le=1.0) + confidence_threshold_choice: float | None = Field(default=None, ge=0.0, le=1.0) + confidence_threshold_yes_no: float | None = Field(default=None, ge=0.0, le=1.0) complexity_max: float | None = Field(default=None, ge=0.0, le=1.0) complexity_confidence_min: float | None = Field(default=None, ge=0.0, le=1.0) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 876be643562..ea37bbf26ca 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -2344,6 +2344,7 @@ class ComplexityRouter(CustomLogger): chosen_tier=tier_name, strongest_tier=self.config.tier_names()[-1], confidence=answer.confidence, + choice_count=len(criteria), ) if refusal_reason is not None: # A confident-enough classifier is a healthy classifier: a gate refusal is a @@ -2403,19 +2404,27 @@ class ComplexityRouter(CustomLogger): confidence: float, complexity_score: float | None = None, complexity_confidence: float | None = None, + choice_count: int | None = None, ) -> str | None: """Why a Jev verdict that routes below the strongest tier must not be applied, or None. A verdict naming the strongest tier is not a route-down and is never gated. Every configured gate must pass: the verdict's own confidence, plus the complexity answer's - score and confidence when the protocol carries one. Missing complexity data is hard + score and confidence when the protocol carries one. The confidence gate follows the + decision question's shape: a question with exactly two options is yes/no-shaped and + uses confidence_threshold_yes_no, a wider one is choice-shaped and uses + confidence_threshold_choice, and each per-type value falls back to the single + confidence_threshold. Missing complexity data is hard (score 1.0, confidence 0.0), so a configured complexity gate refuses every route-down until the wire carries a complexity answer; complexity_max=1.0 and complexity_confidence_min=0.0 are the values at which missing data passes. """ if chosen_tier == strongest_tier: return None - if config.confidence_threshold is not None and confidence < config.confidence_threshold: + threshold: Final = ( + config.confidence_threshold if choice_count is None else config.confidence_threshold_for(choice_count) + ) + if threshold is not None and confidence < threshold: return "low-confidence" score: Final = 1.0 if complexity_score is None else complexity_score # unknown complexity is hard if config.complexity_max is not None and score > config.complexity_max: diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 69d72be7af6..4bd5e4043d0 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -745,6 +745,22 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel): "pool with a large capability gap." ), ) + confidence_threshold_choice: float | None = Field( + default=None, + ge=0.0, + le=1.0, + description=( + "Per-type confidence gate for choice-shaped decisions: a question with three or more options. Takes precedence over confidence_threshold for those questions; None falls back to it. Jev clips scores to 0.01-0.99, so a single number gates a wide choice and a binary decision unequally: confidence spreads across every option of a choice but only two of a yes/no." + ), + ) + confidence_threshold_yes_no: float | None = Field( + default=None, + ge=0.0, + le=1.0, + description=( + "Per-type confidence gate for yes/no-shaped decisions: a question with exactly two options, like the built-in tier question on a two-tier custom tier set. Takes precedence over confidence_threshold for those questions; None falls back to it." + ), + ) complexity_max: float | None = Field( default=None, ge=0.0, @@ -775,6 +791,17 @@ class OpenSourceClassifierConfig(LiteLLMBaseModel): def _normalize_provider_alias(cls, value: object) -> object: return "jev" if value == "typesafe" else value + def confidence_threshold_for(self, choice_count: int) -> float | None: + """The confidence gate for a decision question offering choice_count options. + + Per-type thresholds take precedence over the single threshold; an unset per-type value + falls back to it, so a config that only sets confidence_threshold keeps gating every + shape exactly as before. A question with exactly two options is yes/no-shaped; every + other question is choice-shaped. + """ + per_type: Final = self.confidence_threshold_yes_no if choice_count == 2 else self.confidence_threshold_choice + return per_type if per_type is not None else self.confidence_threshold + @field_validator("instructions") @classmethod def _reject_blank_instructions(cls, value: str | None) -> str | None: diff --git a/tests/test_litellm/router_strategy/complexity_router/test_jev_confidence_gate.py b/tests/test_litellm/router_strategy/complexity_router/test_jev_confidence_gate.py index 6e31e9e0a43..305e91d6019 100644 --- a/tests/test_litellm/router_strategy/complexity_router/test_jev_confidence_gate.py +++ b/tests/test_litellm/router_strategy/complexity_router/test_jev_confidence_gate.py @@ -30,6 +30,7 @@ _TIERS: Final[Mapping[str, str]] = { "COMPLEX": "strong", "REASONING": "fallback-strong", } +_YES_NO_TIERS: Final[Mapping[str, str]] = {"TINY": "cheap", "HEAVY": "fallback-strong"} def _tier_answer(choice: str, confidence: float) -> JevChoiceAnswer: @@ -79,6 +80,27 @@ class _WireJevClient: return TypeAdapter(JevSystemOneResponse).validate_python(self.payload) +def _two_tier_router(client: Any, jev_overrides: Mapping[str, object] | None = None) -> ComplexityRouter: + """A router whose decision question offers exactly two options: the yes/no shape.""" + jev: Final[dict[str, object]] = {"api_key": "test", "timeout_ms": 100, **(dict(jev_overrides or {}))} + return ComplexityRouter( + "test-router", + MagicMock(), + { + "classifier_type": "jev", + "jev_classifier_config": jev, + "tier_definitions": [ + {"name": "TINY", "description": "short factual replies"}, + {"name": "HEAVY", "description": "everything that needs real work"}, + ], + "fallback_tier": "HEAVY", + "tiers": dict(_YES_NO_TIERS), + }, + derive_savings_baseline=False, + jev_client=client, + ) + + def _router(client: Any, jev_overrides: Mapping[str, object] | None = None) -> ComplexityRouter: jev: Final[dict[str, object]] = {"api_key": "test", "timeout_ms": 100, **(dict(jev_overrides or {}))} return ComplexityRouter( @@ -248,6 +270,79 @@ async def test_all_three_gates_configured_refuse_with_the_first_failure_reported assert "jev-refused:low-confidence" in outcome.signals +@pytest.mark.asyncio +async def test_yes_no_threshold_gates_a_route_down_on_a_two_option_question() -> None: + router = _two_tier_router(_StaticJevClient(_response("TINY", 0.6)), {"confidence_threshold_yes_no": 0.9}) + + outcome = await router.aclassify("Explain this") + + assert outcome.tier == "HEAVY" + assert outcome.cause == "classifier_fallback" + assert "jev-refused:low-confidence" in outcome.signals + + +@pytest.mark.asyncio +async def test_yes_no_threshold_passes_when_confidence_meets_it() -> None: + router = _two_tier_router(_StaticJevClient(_response("TINY", 0.9)), {"confidence_threshold_yes_no": 0.9}) + + outcome = await router.aclassify("Explain this") + + assert outcome.tier == "TINY" + assert outcome.cause == "jev_classifier" + + +@pytest.mark.asyncio +async def test_choice_threshold_gates_a_route_down_on_a_wide_question() -> None: + router = _router(_StaticJevClient(_response("MEDIUM", 0.8)), {"confidence_threshold_choice": 0.9}) + + outcome = await router.aclassify("Explain this") + + assert outcome.tier == ComplexityTier.REASONING + assert outcome.cause == "default_model_fallback" + assert "jev-refused:low-confidence" in outcome.signals + + +@pytest.mark.asyncio +async def test_per_type_threshold_for_the_other_shape_does_not_gate() -> None: + yes_no_only = _router(_StaticJevClient(_response("MEDIUM", 0.3)), {"confidence_threshold_yes_no": 0.99}) + + yes_no_outcome = await yes_no_only.aclassify("Explain this") + + # The choice shape falls back to the single threshold, which is unset: no gate. + assert yes_no_outcome.tier == ComplexityTier.MEDIUM + assert yes_no_outcome.cause == "jev_classifier" + + choice_only = _two_tier_router(_StaticJevClient(_response("TINY", 0.3)), {"confidence_threshold_choice": 0.99}) + + choice_outcome = await choice_only.aclassify("Explain this") + + # The yes/no shape falls back to the single threshold, which is unset: no gate. + assert choice_outcome.tier == "TINY" + assert choice_outcome.cause == "jev_classifier" + + +@pytest.mark.asyncio +async def test_per_type_thresholds_fall_back_to_the_single_threshold_when_absent() -> None: + yes_no = _two_tier_router(_StaticJevClient(_response("TINY", 0.6)), {"confidence_threshold": 0.7}) + + yes_no_outcome = await yes_no.aclassify("Explain this") + + assert yes_no_outcome.tier == "HEAVY" + assert yes_no_outcome.cause == "classifier_fallback" + assert "jev-refused:low-confidence" in yes_no_outcome.signals + + wide = _router( + _StaticJevClient(_response("MEDIUM", 0.8)), + {"confidence_threshold": 0.7, "confidence_threshold_yes_no": 0.95}, + ) + + wide_outcome = await wide.aclassify("Explain this") + + # The choice shape stays on the single threshold: a yes/no bar does not leak into it. + assert wide_outcome.tier == ComplexityTier.MEDIUM + assert wide_outcome.cause == "jev_classifier" + + def test_gate_refusal_reasons_cover_every_gate_independently() -> None: config = JevClassifierConfig( api_key="test", confidence_threshold=0.7, complexity_max=0.5, complexity_confidence_min=0.5 @@ -266,14 +361,41 @@ def test_gate_refusal_reasons_cover_every_gate_independently() -> None: assert reason(config, "MEDIUM", "REASONING", 0.9, complexity_score=0.3) == "low-complexity-confidence" +def test_per_type_confidence_thresholds_resolve_per_question_shape() -> None: + config = JevClassifierConfig( + api_key="test", confidence_threshold=0.7, confidence_threshold_choice=0.85, confidence_threshold_yes_no=0.55 + ) + reason = ComplexityRouter._jev_route_down_refusal_reason + + # A two-option question is yes/no-shaped and uses its own bar. + assert reason(config, "MEDIUM", "REASONING", 0.54, choice_count=2) == "low-confidence" + assert reason(config, "MEDIUM", "REASONING", 0.55, choice_count=2) is None + # A wider question is choice-shaped and uses its own bar. + assert reason(config, "MEDIUM", "REASONING", 0.84, choice_count=4) == "low-confidence" + assert reason(config, "MEDIUM", "REASONING", 0.85, choice_count=4) is None + # A shape without a per-type value falls back to the single threshold. + fallback = JevClassifierConfig(api_key="test", confidence_threshold=0.7, confidence_threshold_yes_no=0.55) + assert reason(fallback, "MEDIUM", "REASONING", 0.69, choice_count=4) == "low-confidence" + assert reason(fallback, "MEDIUM", "REASONING", 0.7, choice_count=4) is None + # No shape supplied: the single threshold gates every verdict, as before. + assert reason(fallback, "MEDIUM", "REASONING", 0.69) == "low-confidence" + assert reason(fallback, "MEDIUM", "REASONING", 0.7) is None + + def test_threshold_fields_default_to_off_and_reject_out_of_range_values() -> None: defaults = JevClassifierConfig(api_key="test") assert defaults.confidence_threshold is None + assert defaults.confidence_threshold_choice is None + assert defaults.confidence_threshold_yes_no is None assert defaults.complexity_max is None assert defaults.complexity_confidence_min is None with pytest.raises(ValidationError): JevClassifierConfig(api_key="test", confidence_threshold=1.5) + with pytest.raises(ValidationError): + JevClassifierConfig(api_key="test", confidence_threshold_choice=1.5) + with pytest.raises(ValidationError): + JevClassifierConfig(api_key="test", confidence_threshold_yes_no=-0.1) with pytest.raises(ValidationError): JevClassifierConfig(api_key="test", complexity_max=-0.1) with pytest.raises(ValidationError): diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 06ae99f70db..f41c2917ead 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -33205,6 +33205,16 @@ export interface components { * @description Minimum classifier confidence (0-1) required before a verdict that routes below the strongest tier is applied. None keeps the current behavior of applying every valid verdict. A refused route-down falls back like any other unusable verdict (fallback_tier, classifier_fallback, or the llm_v2 capable tier), so pair a threshold with an expensive fallback to fail expensive. Pick per pool: 0.55-0.65 suits a pool whose cheap tier is nearly as capable as its top tier; 0.85 suits a pool with a large capability gap. */ confidence_threshold?: number | null; + /** + * Confidence Threshold Choice + * @description Per-type confidence gate for choice-shaped decisions: a question with three or more options. Takes precedence over confidence_threshold for those questions; None falls back to it. Jev clips scores to 0.01-0.99, so a single number gates a wide choice and a binary decision unequally: confidence spreads across every option of a choice but only two of a yes/no. + */ + confidence_threshold_choice?: number | null; + /** + * Confidence Threshold Yes No + * @description Per-type confidence gate for yes/no-shaped decisions: a question with exactly two options, like the built-in tier question on a two-tier custom tier set. Takes precedence over confidence_threshold for those questions; None falls back to it. + */ + confidence_threshold_yes_no?: number | null; /** * Instructions * @description Replaces the built-in Jev question instructions From b3df9b012b488e679ad9c6b3553029b6cdbf7ab7 Mon Sep 17 00:00:00 2001 From: n24q02m Date: Mon, 12 Oct 2026 15:36:53 +0700 Subject: [PATCH 7/8] fix: adapt Jev confidence gate to the renamed classifier API and move its test to tests/unit Upstream replaced _with_signal with the failure_reason and signal arguments of _classifier_failure_outcome and renamed the config to OpenSourceClassifierConfig. The test now lives under tests/unit, which is the sharded root that CI assigns to a shard. --- .../complexity_router/complexity_router.py | 26 ++++++++++--------- .../test_jev_confidence_gate.py | 0 2 files changed, 14 insertions(+), 12 deletions(-) rename tests/{test_litellm => unit}/router_strategy/complexity_router/test_jev_confidence_gate.py (100%) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index ea37bbf26ca..0eea6c7147c 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -2351,19 +2351,21 @@ class ComplexityRouter(CustomLogger): # policy decision about the verdict, not a health failure, so the breaker closes. if breaker is not None and permit is not None: breaker.record_success(permit) - refused: Final = _with_signal( - self._classifier_failure_outcome( - f"jev classifier route-down refused ({refusal_reason}, verdict {tier_name}, " - f"confidence {answer.confidence:.4f})", - prompt, - system_prompt, - signal=f"jev-refused:{refusal_reason}", - ), - f"jev-classifier:{tier_name}", + refused: Final = self._classifier_failure_outcome( + f"{config.provider} classifier route-down refused ({refusal_reason}, verdict {tier_name}, " + f"confidence {answer.confidence:.4f})", + prompt, + system_prompt, + failure_reason="declined", + signal=f"{config.provider}-refused:{refusal_reason}", ) # Keep the verdict and its cost on the refusal for provenance: the request paid - # for the Jev call, and the routing decision still logs what it answered. - return refused._replace(classifier_cost=verdict.cost, jev_verdict=verdict) + # for the classifier call, and the routing decision still logs what it answered. + return refused._replace( + signals=(*refused.signals, f"{config.provider}-classifier:{tier_name}"), + classifier_cost=verdict.cost, + jev_verdict=verdict, + ) if breaker is not None and permit is not None: breaker.record_success(permit) return ClassificationOutcome( @@ -2398,7 +2400,7 @@ class ComplexityRouter(CustomLogger): @staticmethod def _jev_route_down_refusal_reason( - config: JevClassifierConfig, + config: OpenSourceClassifierConfig, chosen_tier: str, strongest_tier: str, confidence: float, diff --git a/tests/test_litellm/router_strategy/complexity_router/test_jev_confidence_gate.py b/tests/unit/router_strategy/complexity_router/test_jev_confidence_gate.py similarity index 100% rename from tests/test_litellm/router_strategy/complexity_router/test_jev_confidence_gate.py rename to tests/unit/router_strategy/complexity_router/test_jev_confidence_gate.py From 43eb46cf0fe2092de538acda74a0e2ae3c57c707 Mon Sep 17 00:00:00 2001 From: n24q02m Date: Mon, 12 Oct 2026 15:50:59 +0700 Subject: [PATCH 8/8] fix(ui): regenerate API types for the renamed open source classifier config --- ui/litellm-dashboard/src/lib/http/schema.d.ts | 88 ++++++------------- 1 file changed, 25 insertions(+), 63 deletions(-) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index f41c2917ead..8ccb15d688f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -33168,69 +33168,6 @@ export interface components { /** Updated By */ updated_by?: string | null; }; - /** JevClassifierConfig */ - JevClassifierConfig: { - /** - * Api Base - * @description TypeSafe API base, falling back to TYPESAFE_API_BASE and then https://api.typesafe.ai - */ - api_base?: string | null; - /** - * Api Key - * @description TypeSafe API key, falling back to TYPESAFE_API_KEY - */ - api_key?: string | null; - /** - * Circuit Breaker Cooldown Seconds - * @default 30 - */ - circuit_breaker_cooldown_seconds: number; - /** - * Circuit Breaker Enabled - * @default true - */ - circuit_breaker_enabled: boolean; - /** - * Complexity Confidence Min - * @description Minimum confidence (0-1) required on the complexity answer before a route-down is applied. The built-in Jev request asks only the tier question today, so an absent complexity answer counts as confidence 0.0: while this is set above 0.0 every route-down is refused. None disables the gate. - */ - complexity_confidence_min?: number | null; - /** - * Complexity Max - * @description Maximum request complexity (0-1) a route-down verdict may carry. The built-in Jev request asks only the tier question today, so an absent complexity answer counts as the hardest score (1.0): while this is set below 1.0 every route-down is refused until the classifier protocol carries a complexity answer. Wire complexity scores arrive as weighted level indices (0..levels-1), so a future score question must normalize to 0-1 before this gate (routing-spec v1.1). None disables the gate. - */ - complexity_max?: number | null; - /** - * Confidence Threshold - * @description Minimum classifier confidence (0-1) required before a verdict that routes below the strongest tier is applied. None keeps the current behavior of applying every valid verdict. A refused route-down falls back like any other unusable verdict (fallback_tier, classifier_fallback, or the llm_v2 capable tier), so pair a threshold with an expensive fallback to fail expensive. Pick per pool: 0.55-0.65 suits a pool whose cheap tier is nearly as capable as its top tier; 0.85 suits a pool with a large capability gap. - */ - confidence_threshold?: number | null; - /** - * Confidence Threshold Choice - * @description Per-type confidence gate for choice-shaped decisions: a question with three or more options. Takes precedence over confidence_threshold for those questions; None falls back to it. Jev clips scores to 0.01-0.99, so a single number gates a wide choice and a binary decision unequally: confidence spreads across every option of a choice but only two of a yes/no. - */ - confidence_threshold_choice?: number | null; - /** - * Confidence Threshold Yes No - * @description Per-type confidence gate for yes/no-shaped decisions: a question with exactly two options, like the built-in tier question on a two-tier custom tier set. Takes precedence over confidence_threshold for those questions; None falls back to it. - */ - confidence_threshold_yes_no?: number | null; - /** - * Instructions - * @description Replaces the built-in Jev question instructions - */ - instructions?: string | null; - /** - * Model - * @default jev-latest - */ - model: string; - /** - * Timeout Ms - * @default 3000 - */ - timeout_ms: number; - }; JsonValue: unknown; /** KeyActivityRow */ KeyActivityRow: { @@ -39274,6 +39211,31 @@ export interface components { * @default true */ circuit_breaker_enabled: boolean; + /** + * Complexity Confidence Min + * @description Minimum confidence (0-1) required on the complexity answer before a route-down is applied. The built-in Jev request asks only the tier question today, so an absent complexity answer counts as confidence 0.0: while this is set above 0.0 every route-down is refused. None disables the gate. + */ + complexity_confidence_min?: number | null; + /** + * Complexity Max + * @description Maximum request complexity (0-1) a route-down verdict may carry. The built-in Jev request asks only the tier question today, so an absent complexity answer counts as the hardest score (1.0): while this is set below 1.0 every route-down is refused until the classifier protocol carries a complexity answer. Wire complexity scores arrive as weighted level indices (0..levels-1), so a future score question must normalize to 0-1 before this gate (routing-spec v1.1). None disables the gate. + */ + complexity_max?: number | null; + /** + * Confidence Threshold + * @description Minimum classifier confidence (0-1) required before a verdict that routes below the strongest tier is applied. None keeps the current behavior of applying every valid verdict. A refused route-down falls back like any other unusable verdict (fallback_tier, classifier_fallback, or the llm_v2 capable tier), so pair a threshold with an expensive fallback to fail expensive. Pick per pool: 0.55-0.65 suits a pool whose cheap tier is nearly as capable as its top tier; 0.85 suits a pool with a large capability gap. + */ + confidence_threshold?: number | null; + /** + * Confidence Threshold Choice + * @description Per-type confidence gate for choice-shaped decisions: a question with three or more options. Takes precedence over confidence_threshold for those questions; None falls back to it. Jev clips scores to 0.01-0.99, so a single number gates a wide choice and a binary decision unequally: confidence spreads across every option of a choice but only two of a yes/no. + */ + confidence_threshold_choice?: number | null; + /** + * Confidence Threshold Yes No + * @description Per-type confidence gate for yes/no-shaped decisions: a question with exactly two options, like the built-in tier question on a two-tier custom tier set. Takes precedence over confidence_threshold for those questions; None falls back to it. + */ + confidence_threshold_yes_no?: number | null; /** Deployment Name */ deployment_name?: string | null; /**