fix(complexity_router): resolve custom-tier classifier replies case-insensitively

An exact membership check meant a reply like "research" or " RESEARCH " for a
defined RESEARCH tier was treated as unknown and rerouted to the fallback tier,
while the built-in path already resolves labels case-insensitively. Custom tier
names are validated unique case-insensitively at config write, so folding is
unambiguous; the decision records the canonical configured name, never the
model's spelling.
This commit is contained in:
Tin Chi Lo 2026-08-05 12:05:16 -07:00
parent 6781579107
commit 1592842473
2 changed files with 29 additions and 2 deletions

View file

@ -999,9 +999,11 @@ class ComplexityRouter(CustomLogger):
raise ValueError("LLM classifier returned empty content")
raw_tier: Final = _TierReply.model_validate_json(content).tier
if self.config.has_custom_tiers:
if raw_tier not in self.config.tier_names():
folded: Final = raw_tier.strip().casefold()
matched: Final = next((name for name in self.config.tier_names() if name.casefold() == folded), None)
if matched is None:
raise ValueError(f"LLM classifier returned an unknown tier: {raw_tier!r}")
return raw_tier
return matched
tier: Final = self.config.tier_for_label(raw_tier)
if tier is None:
raise ValueError(f"LLM classifier returned an unrecognized tier: {raw_tier!r}")

View file

@ -5940,6 +5940,31 @@ class TestTierDefinitionsClassifier:
assert "score" not in result.routing_decision
assert "tier_boundaries" not in result.routing_decision
@pytest.mark.asyncio
@pytest.mark.parametrize(
"reply",
[
pytest.param("research", id="different-case"),
pytest.param(" RESEARCH ", id="surrounding-whitespace"),
],
)
async def test_custom_tier_reply_resolves_case_insensitively_to_the_canonical_name(
self, custom_tier_router, mock_router_instance, reply
):
"""A model that answers in the wrong case or with stray whitespace still names a
defined tier, matching the built-in path's label resolution; rejecting it would
silently reroute the request to the fallback tier. The decision records the
canonical configured name, never the model's spelling."""
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response(f'{{"tier": "{reply}"}}'))
result = await custom_tier_router.async_pre_routing_hook(
model="custom-tier-router",
request_kwargs={},
messages=[{"role": "user", "content": "compare these two consensus protocols"}],
)
assert result.model == "deep-model"
assert result.routing_decision["tier"] == "RESEARCH"
assert result.routing_decision["cause"] == "llm_classifier"
@pytest.mark.asyncio
async def test_classifier_failure_routes_to_the_fallback_tier(self, custom_tier_router, mock_router_instance):
"""The heuristic scorer cannot produce a custom tier, so it must never be the