From 671551b039f14848b67ab9df6a9a972d4d6ce29c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 17 Apr 2026 19:08:27 -0700 Subject: [PATCH] feat(quality_router): reorder tiebreak to (quality, order, price) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes the tiebreak ordering so quality_tier always wins first, then explicit `order` is used to break ties within the same tier, then price breaks the rest: 1. quality_tier DESC ← best model wins first 2. order ASC ← explicit priority within a tier 3. input_cost_per_token ASC 4. model_name ASC Previously `order` was the primary key — that meant a tier-2 model with `order=1` would beat a tier-3 model with no `order`, which is the wrong default. Now `order` only resolves collisions among same-tier candidates. Tier resolution (within a single tier) keeps the same key minus quality: (order ASC, cost ASC, name). Test renames + flips: - test_explicit_order_overrides_quality_tier → test_quality_wins_over_explicit_order - new: test_order_breaks_tie_within_same_quality_tier Co-Authored-By: Claude Opus 4 (1M context) --- .../router_strategy/quality_router/config.py | 13 +++--- .../quality_router/quality_router.py | 28 +++++++----- .../router_strategy/test_quality_router.py | 43 +++++++++++++++---- 3 files changed, 59 insertions(+), 25 deletions(-) diff --git a/litellm/router_strategy/quality_router/config.py b/litellm/router_strategy/quality_router/config.py index 44bfde467d8..125ecd5bb9b 100644 --- a/litellm/router_strategy/quality_router/config.py +++ b/litellm/router_strategy/quality_router/config.py @@ -61,12 +61,13 @@ class RoutingPreferences(BaseModel): order: Optional[int] = Field( default=None, description=( - "Explicit priority used to break ties between deployments. Lower " - "values win. Applies both to keyword collisions and to picking " - "between multiple deployments at the same quality tier. A " - "deployment with `order` set always wins over one without; among " - "deployments sharing the same `order` (or both unset), ties fall " - "back to (quality_tier DESC, input_cost_per_token ASC, model_name)." + "Explicit priority used to break ties between deployments at the " + "same quality tier. Lower values win. Applies both to keyword " + "collisions and to picking between multiple deployments at the " + "same quality_tier. Tiebreak order is " + "(quality_tier DESC, order ASC, input_cost_per_token ASC, " + "model_name ASC) — quality always wins first, then explicit " + "order, then price." ), ) diff --git a/litellm/router_strategy/quality_router/quality_router.py b/litellm/router_strategy/quality_router/quality_router.py index 55b42414f1d..7cf35dcad6e 100644 --- a/litellm/router_strategy/quality_router/quality_router.py +++ b/litellm/router_strategy/quality_router/quality_router.py @@ -197,12 +197,12 @@ class QualityRouter(CustomLogger): f"the router's model_list (or are missing routing preferences): {missing}" ) - # Sort each tier's model list by (order ASC, model_name ASC) so that - # `_resolve_model_for_quality_tier` (which picks index [0]) honors the - # admin's explicit priority. Unset order treated as +inf so explicit - # always wins over implicit. + # Sort each tier's model list so `_resolve_model_for_quality_tier` + # (which picks index [0]) honors (order ASC, cost ASC, name ASC). + # Quality is moot within a single tier; keep parity with the keyword + # tiebreak by ordering on (order, cost, name) here. for models in tier_to_models.values(): - models.sort(key=lambda n: (self._order_key(n), n)) + models.sort(key=lambda n: (self._order_key(n), self._cost_key(n), n)) return tier_to_models @@ -211,15 +211,21 @@ class QualityRouter(CustomLogger): order = self._model_order.get(model_name) return float(order) if order is not None else math.inf + def _cost_key(self, model_name: str) -> float: + """`input_cost_per_token` as a float — unset becomes +inf.""" + cost = self._model_cost.get(model_name) + return float(cost) if cost is not None else math.inf + def _keyword_override(self, user_message: str) -> Optional[Tuple[str, str]]: """ Find a deployment whose declared keywords appear in `user_message`. Returns (model_name, matched_keyword) or None when no keyword matches. When multiple deployments match, sorts by: - 1. `order` ASC (explicit priority — unset = +inf so explicit wins) - 2. quality_tier DESC - 3. input_cost_per_token ASC (unpriced = +inf) + 1. quality_tier DESC (best quality always wins first) + 2. `order` ASC (explicit priority — unset = +inf so explicit wins + within the same tier) + 3. input_cost_per_token ASC (unpriced = +inf so priced wins) 4. model_name ASC (deterministic stability) """ text = user_message.lower() @@ -234,14 +240,14 @@ class QualityRouter(CustomLogger): if not matches: return None - def sort_key(match: Tuple[str, str]) -> Tuple[float, int, float, str]: + def sort_key(match: Tuple[str, str]) -> Tuple[int, float, float, str]: name = match[0] - order_val = self._order_key(name) quality = self._model_quality.get(name, 0) + order_val = self._order_key(name) cost = self._model_cost.get(name) cost_val = cost if cost is not None else math.inf # Negate quality so higher tier sorts first under ASC sort. - return (order_val, -quality, cost_val, name) + return (-quality, order_val, cost_val, name) matches.sort(key=sort_key) return matches[0] diff --git a/tests/test_litellm/router_strategy/test_quality_router.py b/tests/test_litellm/router_strategy/test_quality_router.py index a2ba9b327de..bb8f3dfa486 100644 --- a/tests/test_litellm/router_strategy/test_quality_router.py +++ b/tests/test_litellm/router_strategy/test_quality_router.py @@ -411,13 +411,12 @@ class TestKeywordOverride: assert decision["matched_keyword"] == "code" assert decision["complexity_tier"] is None # short-circuited - def test_explicit_order_overrides_quality_tier(self): - # Both models declare "code". By implicit rules (quality DESC), the - # tier-3 model would win. With an explicit `order=1` on the tier-2 - # model, it must win regardless. + def test_quality_wins_over_explicit_order(self): + # Quality always beats order. A tier-3 model with no `order` wins over + # a tier-2 model with `order=1`. spec = [ { - "model_name": "preferred-tier2", + "model_name": "ordered-tier2", "quality_tier": 2, "keywords": ["code"], "order": 1, @@ -435,13 +434,41 @@ class TestKeywordOverride: qr = QualityRouter( model_name="qr", litellm_router_instance=router, - default_model="preferred-tier2", + default_model="ordered-tier2", quality_router_config={ - "available_models": ["preferred-tier2", "implicit-tier3"] + "available_models": ["ordered-tier2", "implicit-tier3"] }, ) match = qr._keyword_override("write some code") - assert match == ("preferred-tier2", "code") + assert match == ("implicit-tier3", "code") + + def test_order_breaks_tie_within_same_quality_tier(self): + # Two tier-3 models, both match "code". Lower `order` wins. + spec = [ + { + "model_name": "preferred", + "quality_tier": 3, + "keywords": ["code"], + "order": 1, + "input_cost_per_token": 0.000050, # more expensive + }, + { + "model_name": "default-tier3", + "quality_tier": 3, + "keywords": ["code"], + "input_cost_per_token": 0.000005, # cheaper + }, + ] + router = MagicMock() + router.model_list = _make_model_list(spec) + qr = QualityRouter( + model_name="qr", + litellm_router_instance=router, + default_model="default-tier3", + quality_router_config={"available_models": ["preferred", "default-tier3"]}, + ) + match = qr._keyword_override("write some code") + assert match == ("preferred", "code") def test_explicit_order_overrides_price(self): # Same tier, but the more expensive one has a lower `order` and wins.