test(complexity_router): pin that a deployment with no id counts as live

The membership test read `deployment_id not in cooled_down`, which is True for None, so
a deployment missing model_info.id counted as live by falling out of a frozenset[str]
rather than by saying so. The behavior is right: cooldowns are keyed by id, so a
deployment that cannot be identified can never be known to be cooling, and the rule
everywhere else in the ladder is that an absent health signal means live. Excluding it
instead would let missing metadata empty a tier and push traffic to a pricier one

Now written as `deployment_id is None or deployment_id not in cooled_down`, so the rule
is on the page instead of being derived, with a test that pins it. Behavior is unchanged

Reachable only via a raw deployment dict: ModelInfo mints a uuid whenever id is None,
so anything registered through the typed path already has one
This commit is contained in:
Tin Chi Lo 2026-07-31 22:40:43 -07:00
parent 364be9306d
commit 1716d82952
2 changed files with 31 additions and 1 deletions

View file

@ -926,9 +926,17 @@ class ComplexityRouter(CustomLogger):
Reads the router's in-process deployment list, so it costs no I/O: the cooldown
set was already fetched once for the whole request.
A deployment with no id counts as live. Cooldowns are keyed by id, so one that
cannot be identified can never be known to be cooling, and the ladder's rule
everywhere else is that an absent health signal means live. Excluding it instead
would let missing metadata empty a tier and push traffic to a pricier one.
"""
deployments = self.litellm_router_instance.get_model_list(model_name=model) or ()
return any(self._deployment_id(deployment) not in cooled_down for deployment in deployments)
return any(
deployment_id is None or deployment_id not in cooled_down
for deployment_id in map(self._deployment_id, deployments)
)
@staticmethod
def _deployment_id(deployment: Mapping[str, Any]) -> str | None:

View file

@ -4933,6 +4933,28 @@ class TestTierFallbackLadder:
assert response.model == "medium-model"
assert response.routing_decision["tier_fallback_from"] == "SIMPLE"
@pytest.mark.asyncio
async def test_a_deployment_with_no_id_counts_as_live(self):
"""Cooldowns are keyed by deployment id, so one without an id can never be known to
be cooling. It counts as live, matching the rule everywhere else that an absent
health signal means live; excluding it would let missing metadata empty a tier and
push traffic to a pricier one."""
class IdlessRouter(StubRouter):
def get_model_list(self, model_name=None, team_id=None):
self.listed.append(model_name)
return [{"model_info": {}}] if model_name in self.registered else []
router = ComplexityRouter(
model_name="test-ladder-router",
litellm_router_instance=IdlessRouter(live=set()),
complexity_router_config={"tiers": dict(LADDER_TIERS), "session_affinity": False},
)
response = await _route(router, "What is 2+2?")
assert response.model in {"simple-a", "simple-b"}
assert "tier_fallback_from" not in response.routing_decision
assert "resolved_by" not in response.routing_decision
@pytest.mark.asyncio
async def test_health_lookup_failure_keeps_the_classified_tier(self):
"""A health view that cannot be read must not empty every tier and push traffic up