mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
fix(complexity_router): name the missing tier when a plugin config has no models for it
Some checks failed
LiteLLM Rust / rustfmt, clippy, test (push) Has been cancelled
Some checks failed
LiteLLM Rust / rustfmt, clippy, test (push) Has been cancelled
With routing plugins configured, a tier absent from `tiers` produced "No candidate models left for tier MEDIUM after routing-plugin filtering". Nothing was filtered: the pool was empty before any plugin ran, so the message sent an operator to read plugin code for what is a gap in `tiers` The two cases now raise separately. An empty pool names the tier and says why default_model is not consulted; a plugin narrowing a real pool to zero keeps its own message, since that one really is a policy decision This is the path no-signal traffic now takes, because the implicit MEDIUM default is deliberately not validated at load: doing so would turn every partial tiers map into a startup failure. The tier ladder in #35331 resolves the same shape at runtime by climbing to the next configured tier, so this is the standalone behavior
This commit is contained in:
parent
9879baeabf
commit
be7b23ce0f
2 changed files with 37 additions and 1 deletions
|
|
@ -871,11 +871,20 @@ class ComplexityRouter(CustomLogger):
|
|||
from litellm.types.router import RoutingContext
|
||||
|
||||
tier_key = tier.value
|
||||
candidates = list(self._tier_pools().get(tier_key, []))
|
||||
if not candidates:
|
||||
# Distinct from the plugin denial below: nothing was filtered out, the tier was
|
||||
# never given models. Saying "after routing-plugin filtering" would send an
|
||||
# operator to read plugin code for what is a gap in `tiers`.
|
||||
raise ValueError(
|
||||
f"Tier {tier_key} has no models configured. Routing plugins are configured, so "
|
||||
f"default_model is not consulted: a model the plugins never vetted must not serve"
|
||||
)
|
||||
metadata_key = "litellm_metadata" if "litellm_metadata" in request_kwargs else "metadata"
|
||||
context = RoutingContext(
|
||||
raw_messages=raw_messages or [],
|
||||
structured_messages=resolved_messages or [],
|
||||
candidate_models=list(self._tier_pools().get(tier_key, [])),
|
||||
candidate_models=candidates,
|
||||
metadata=request_kwargs.get(metadata_key) or {},
|
||||
)
|
||||
for plugin in self.config.plugins:
|
||||
|
|
|
|||
|
|
@ -4914,6 +4914,33 @@ class TestNoSignalDefaultTier:
|
|||
},
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_unconfigured_default_tier_under_plugins_names_the_real_gap(self, mock_router_instance):
|
||||
"""The implicit MEDIUM default is deliberately not validated at load, so a partial
|
||||
tiers map keeps loading. With plugins configured it has no fallback, so the request
|
||||
fails; the error has to name the missing tier rather than blaming the plugins for a
|
||||
filter they never applied."""
|
||||
|
||||
class NoOpPlugin:
|
||||
async def run(self, context):
|
||||
return context
|
||||
|
||||
router = ComplexityRouter(
|
||||
model_name="test-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
complexity_router_config={
|
||||
"tiers": {"SIMPLE": "simple-model", "COMPLEX": "complex-model"},
|
||||
"plugins": [NoOpPlugin()],
|
||||
"session_affinity": False,
|
||||
},
|
||||
)
|
||||
with pytest.raises(ValueError, match="Tier MEDIUM has no models configured"):
|
||||
await router.async_pre_routing_hook(
|
||||
model="test-model",
|
||||
request_kwargs={},
|
||||
messages=[{"role": "user", "content": RIVER_CROSSING}],
|
||||
)
|
||||
|
||||
def test_default_tier_outside_tiers_is_allowed_with_default_model(self, mock_router_instance):
|
||||
router = ComplexityRouter(
|
||||
model_name="test-router",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue