diff --git a/litellm/router_strategy/quality_router/config.py b/litellm/router_strategy/quality_router/config.py index 5ee5c26baf6..44bfde467d8 100644 --- a/litellm/router_strategy/quality_router/config.py +++ b/litellm/router_strategy/quality_router/config.py @@ -52,9 +52,21 @@ class RoutingPreferences(BaseModel): default_factory=list, description=( "Substring keywords (case-insensitive) that, when present in the " - "user message, route the request to this deployment. When multiple " - "deployments match, ties are broken by (highest quality_tier, " - "then cheapest model_info.input_cost_per_token)." + "user message, route the request to this deployment. See `order` " + "for explicit collision handling, otherwise ties fall through to " + "(highest quality_tier, then cheapest model_info.input_cost_per_token)." + ), + ) + + order: Optional[int] = Field( + default=None, + description=( + "Explicit priority used to break ties between deployments. Lower " + "values win. Applies both to keyword collisions and to picking " + "between multiple deployments at the same quality tier. A " + "deployment with `order` set always wins over one without; among " + "deployments sharing the same `order` (or both unset), ties fall " + "back to (quality_tier DESC, input_cost_per_token ASC, model_name)." ), ) diff --git a/litellm/router_strategy/quality_router/quality_router.py b/litellm/router_strategy/quality_router/quality_router.py index 9dc6f5909b7..55b42414f1d 100644 --- a/litellm/router_strategy/quality_router/quality_router.py +++ b/litellm/router_strategy/quality_router/quality_router.py @@ -67,11 +67,13 @@ class QualityRouter(CustomLogger): # Per-model indices populated alongside the tier index. `_model_keywords` # stores keywords lowercased so we can substring-match against the - # lowercased user message in O(total-keyword-count). `_model_quality` - # and `_model_cost` are needed for keyword-match tiebreaking. + # lowercased user message in O(total-keyword-count). `_model_quality`, + # `_model_cost`, and `_model_order` drive tiebreaking — `_model_order` + # is the explicit priority (lower wins, unset = +inf). self._model_keywords: Dict[str, List[str]] = {} self._model_quality: Dict[str, int] = {} self._model_cost: Dict[str, Optional[float]] = {} + self._model_order: Dict[str, Optional[int]] = {} # Pre-built tier → models index for O(1) tier resolution. self._tier_to_models: Dict[int, List[str]] = self._build_tier_index() @@ -165,9 +167,11 @@ class QualityRouter(CustomLogger): if isinstance(prefs, dict): tier = prefs.get("quality_tier") keywords = prefs.get("keywords") or [] + order = prefs.get("order") else: tier = getattr(prefs, "quality_tier", None) keywords = getattr(prefs, "keywords", None) or [] + order = getattr(prefs, "order", None) if tier is None: raise ValueError( @@ -180,6 +184,10 @@ class QualityRouter(CustomLogger): self._model_keywords[name] = [str(k).lower() for k in keywords if k] self._model_quality[name] = tier_int self._model_cost[name] = self._get_deployment_input_cost(deployment) + try: + self._model_order[name] = int(order) if order is not None else None + except (TypeError, ValueError): + self._model_order[name] = None seen[name] = True missing = [name for name, found in seen.items() if not found] @@ -189,16 +197,30 @@ class QualityRouter(CustomLogger): f"the router's model_list (or are missing routing preferences): {missing}" ) + # Sort each tier's model list by (order ASC, model_name ASC) so that + # `_resolve_model_for_quality_tier` (which picks index [0]) honors the + # admin's explicit priority. Unset order treated as +inf so explicit + # always wins over implicit. + for models in tier_to_models.values(): + models.sort(key=lambda n: (self._order_key(n), n)) + return tier_to_models + def _order_key(self, model_name: str) -> float: + """`order` lookup as a float — unset becomes +inf so explicit wins.""" + order = self._model_order.get(model_name) + return float(order) if order is not None else math.inf + def _keyword_override(self, user_message: str) -> Optional[Tuple[str, str]]: """ Find a deployment whose declared keywords appear in `user_message`. Returns (model_name, matched_keyword) or None when no keyword matches. - When multiple deployments match, sorts by (quality_tier DESC, - input_cost_per_token ASC, model_name ASC) and returns the winner. - Unpriced models are treated as `+inf` so priced models win on price. + When multiple deployments match, sorts by: + 1. `order` ASC (explicit priority — unset = +inf so explicit wins) + 2. quality_tier DESC + 3. input_cost_per_token ASC (unpriced = +inf) + 4. model_name ASC (deterministic stability) """ text = user_message.lower() @@ -212,13 +234,14 @@ class QualityRouter(CustomLogger): if not matches: return None - def sort_key(match: Tuple[str, str]) -> Tuple[int, float, str]: + def sort_key(match: Tuple[str, str]) -> Tuple[float, int, float, str]: name = match[0] + order_val = self._order_key(name) quality = self._model_quality.get(name, 0) cost = self._model_cost.get(name) cost_val = cost if cost is not None else math.inf # Negate quality so higher tier sorts first under ASC sort. - return (-quality, cost_val, name) + return (order_val, -quality, cost_val, name) matches.sort(key=sort_key) return matches[0] diff --git a/tests/test_litellm/router_strategy/test_quality_router.py b/tests/test_litellm/router_strategy/test_quality_router.py index ebee62f1db8..a2ba9b327de 100644 --- a/tests/test_litellm/router_strategy/test_quality_router.py +++ b/tests/test_litellm/router_strategy/test_quality_router.py @@ -32,6 +32,7 @@ def _make_model_list(spec: List[Dict[str, Any]]) -> List[Dict[str, Any]]: "model_name": str, "quality_tier": Optional[int], "keywords": Optional[List[str]], + "order": Optional[int], "input_cost_per_token": Optional[float], } If quality_tier is None, the deployment is created without @@ -44,6 +45,8 @@ def _make_model_list(spec: List[Dict[str, Any]]) -> List[Dict[str, Any]]: prefs: Dict[str, Any] = {"quality_tier": entry["quality_tier"]} if "keywords" in entry: prefs["keywords"] = entry["keywords"] + if "order" in entry: + prefs["order"] = entry["order"] model_info["litellm_routing_preferences"] = prefs if "input_cost_per_token" in entry: model_info["input_cost_per_token"] = entry["input_cost_per_token"] @@ -408,6 +411,163 @@ class TestKeywordOverride: assert decision["matched_keyword"] == "code" assert decision["complexity_tier"] is None # short-circuited + def test_explicit_order_overrides_quality_tier(self): + # Both models declare "code". By implicit rules (quality DESC), the + # tier-3 model would win. With an explicit `order=1` on the tier-2 + # model, it must win regardless. + spec = [ + { + "model_name": "preferred-tier2", + "quality_tier": 2, + "keywords": ["code"], + "order": 1, + "input_cost_per_token": 0.000010, + }, + { + "model_name": "implicit-tier3", + "quality_tier": 3, + "keywords": ["code"], + "input_cost_per_token": 0.000005, + }, + ] + router = MagicMock() + router.model_list = _make_model_list(spec) + qr = QualityRouter( + model_name="qr", + litellm_router_instance=router, + default_model="preferred-tier2", + quality_router_config={ + "available_models": ["preferred-tier2", "implicit-tier3"] + }, + ) + match = qr._keyword_override("write some code") + assert match == ("preferred-tier2", "code") + + def test_explicit_order_overrides_price(self): + # Same tier, but the more expensive one has a lower `order` and wins. + spec = [ + { + "model_name": "expensive-but-preferred", + "quality_tier": 2, + "keywords": ["data"], + "order": 1, + "input_cost_per_token": 0.000050, + }, + { + "model_name": "cheap-default", + "quality_tier": 2, + "keywords": ["data"], + "input_cost_per_token": 0.000005, + }, + ] + router = MagicMock() + router.model_list = _make_model_list(spec) + qr = QualityRouter( + model_name="qr", + litellm_router_instance=router, + default_model="cheap-default", + quality_router_config={ + "available_models": ["expensive-but-preferred", "cheap-default"] + }, + ) + match = qr._keyword_override("show me the data") + assert match == ("expensive-but-preferred", "data") + + def test_lower_order_wins_between_two_explicitly_ordered(self): + spec = [ + { + "model_name": "second", + "quality_tier": 2, + "keywords": ["data"], + "order": 5, + }, + { + "model_name": "first", + "quality_tier": 2, + "keywords": ["data"], + "order": 1, + }, + ] + router = MagicMock() + router.model_list = _make_model_list(spec) + qr = QualityRouter( + model_name="qr", + litellm_router_instance=router, + default_model="first", + quality_router_config={"available_models": ["first", "second"]}, + ) + match = qr._keyword_override("show me the data") + assert match == ("first", "data") + + def test_same_order_falls_through_to_quality_then_price(self): + # All three models share order=1 → tiebreak falls through to + # (quality DESC, cost ASC). + spec = [ + { + "model_name": "low-tier", + "quality_tier": 1, + "keywords": ["data"], + "order": 1, + "input_cost_per_token": 0.000001, + }, + { + "model_name": "high-tier-cheap", + "quality_tier": 3, + "keywords": ["data"], + "order": 1, + "input_cost_per_token": 0.000005, + }, + { + "model_name": "high-tier-expensive", + "quality_tier": 3, + "keywords": ["data"], + "order": 1, + "input_cost_per_token": 0.000050, + }, + ] + router = MagicMock() + router.model_list = _make_model_list(spec) + qr = QualityRouter( + model_name="qr", + litellm_router_instance=router, + default_model="low-tier", + quality_router_config={ + "available_models": [ + "low-tier", + "high-tier-cheap", + "high-tier-expensive", + ] + }, + ) + match = qr._keyword_override("show me the data") + assert match == ("high-tier-cheap", "data") + + def test_order_is_used_in_tier_resolution_too(self): + # Two models at the same tier. Explicit `order=1` on the second one + # should make _resolve_model_for_quality_tier(2) pick it. + spec = [ + { + "model_name": "default-pick", + "quality_tier": 2, + }, + { + "model_name": "preferred-pick", + "quality_tier": 2, + "order": 1, + }, + ] + router = MagicMock() + router.model_list = _make_model_list(spec) + qr = QualityRouter( + model_name="qr", + litellm_router_instance=router, + default_model="default-pick", + quality_router_config={ + "available_models": ["default-pick", "preferred-pick"] + }, + ) + assert qr._resolve_model_for_quality_tier(2) == "preferred-pick" + @pytest.mark.asyncio async def test_hook_falls_back_to_complexity_when_no_keyword(self, keyword_router): # No declared keyword in the message → complexity-based routing.