scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

feat(quality_router): add explicit order to RoutingPreferences

Browse source 
Adds an explicit priority field to RoutingPreferences for resolving
collisions deterministically:

  RoutingPreferences.order: Optional[int]   # lower wins; unset = +inf

Used as the PRIMARY tiebreaker in two places:

1. Keyword overlap: when multiple deployments declare the same matching
   keyword, sort by (order ASC, quality_tier DESC, input_cost_per_token
   ASC, model_name ASC). Explicit always beats implicit.

2. Tier resolution: when multiple deployments share a quality tier,
   `_resolve_model_for_quality_tier` picks the one with the lowest
   order. The tier list is now sorted at index-build time.

This lets admins make routing decisions explicit when the natural
quality-and-price ordering would pick the wrong model.

Co-Authored-By: Claude Opus 4 (1M context) <noreply@anthropic.com>
This commit is contained in:
Krrish Dholakia 2026-04-17 19:04:23 -07:00
parent 1d8828665f
commit b65c39317c
3 changed files with 205 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
litellm/router_strategy/quality_router/config.py
View file

@ -52,9 +52,21 @@ class RoutingPreferences(BaseModel):
default_factory=list,
description=(
"Substring keywords (case-insensitive) that, when present in the "
"user message, route the request to this deployment. When multiple "
"deployments match, ties are broken by (highest quality_tier, "
"then cheapest model_info.input_cost_per_token)."
"user message, route the request to this deployment. See `order` "
"for explicit collision handling, otherwise ties fall through to "
"(highest quality_tier, then cheapest model_info.input_cost_per_token)."
),
)
order: Optional[int] = Field(
default=None,
description=(
"Explicit priority used to break ties between deployments. Lower "
"values win. Applies both to keyword collisions and to picking "
"between multiple deployments at the same quality tier. A "
"deployment with `order` set always wins over one without; among "
"deployments sharing the same `order` (or both unset), ties fall "
"back to (quality_tier DESC, input_cost_per_token ASC, model_name)."
),
)

37
litellm/router_strategy/quality_router/quality_router.py
View file

@ -67,11 +67,13 @@ class QualityRouter(CustomLogger):
# Per-model indices populated alongside the tier index. `_model_keywords`
# stores keywords lowercased so we can substring-match against the
# lowercased user message in O(total-keyword-count). `_model_quality`
# and `_model_cost` are needed for keyword-match tiebreaking.
# lowercased user message in O(total-keyword-count). `_model_quality`,
# `_model_cost`, and `_model_order` drive tiebreaking — `_model_order`
# is the explicit priority (lower wins, unset = +inf).
self._model_keywords: Dict[str, List[str]] = {}
self._model_quality: Dict[str, int] = {}
self._model_cost: Dict[str, Optional[float]] = {}
self._model_order: Dict[str, Optional[int]] = {}
# Pre-built tier → models index for O(1) tier resolution.
self._tier_to_models: Dict[int, List[str]] = self._build_tier_index()
@ -165,9 +167,11 @@ class QualityRouter(CustomLogger):
if isinstance(prefs, dict):
tier = prefs.get("quality_tier")
keywords = prefs.get("keywords") or []
order = prefs.get("order")
else:
tier = getattr(prefs, "quality_tier", None)
keywords = getattr(prefs, "keywords", None) or []
order = getattr(prefs, "order", None)
if tier is None:
raise ValueError(
@ -180,6 +184,10 @@ class QualityRouter(CustomLogger):
self._model_keywords[name] = [str(k).lower() for k in keywords if k]
self._model_quality[name] = tier_int
self._model_cost[name] = self._get_deployment_input_cost(deployment)
try:
self._model_order[name] = int(order) if order is not None else None
except (TypeError, ValueError):
self._model_order[name] = None
seen[name] = True
missing = [name for name, found in seen.items() if not found]
@ -189,16 +197,30 @@ class QualityRouter(CustomLogger):
f"the router's model_list (or are missing routing preferences): {missing}"
)
# Sort each tier's model list by (order ASC, model_name ASC) so that
# `_resolve_model_for_quality_tier` (which picks index [0]) honors the
# admin's explicit priority. Unset order treated as +inf so explicit
# always wins over implicit.
for models in tier_to_models.values():
models.sort(key=lambda n: (self._order_key(n), n))
return tier_to_models
def _order_key(self, model_name: str) -> float:
"""`order` lookup as a float — unset becomes +inf so explicit wins."""
order = self._model_order.get(model_name)
return float(order) if order is not None else math.inf
def _keyword_override(self, user_message: str) -> Optional[Tuple[str, str]]:
"""
Find a deployment whose declared keywords appear in `user_message`.
Returns (model_name, matched_keyword) or None when no keyword matches.
When multiple deployments match, sorts by (quality_tier DESC,
input_cost_per_token ASC, model_name ASC) and returns the winner.
Unpriced models are treated as `+inf` so priced models win on price.
When multiple deployments match, sorts by:
1. `order` ASC (explicit priority — unset = +inf so explicit wins)
2. quality_tier DESC
3. input_cost_per_token ASC (unpriced = +inf)
4. model_name ASC (deterministic stability)
"""
text = user_message.lower()
@ -212,13 +234,14 @@ class QualityRouter(CustomLogger):
if not matches:
return None
def sort_key(match: Tuple[str, str]) -> Tuple[int, float, str]:
def sort_key(match: Tuple[str, str]) -> Tuple[float, int, float, str]:
name = match[0]
order_val = self._order_key(name)
quality = self._model_quality.get(name, 0)
cost = self._model_cost.get(name)
cost_val = cost if cost is not None else math.inf
# Negate quality so higher tier sorts first under ASC sort.
return (-quality, cost_val, name)
return (order_val, -quality, cost_val, name)
matches.sort(key=sort_key)
return matches[0]

160
tests/test_litellm/router_strategy/test_quality_router.py
View file

@ -32,6 +32,7 @@ def _make_model_list(spec: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"model_name": str,
"quality_tier": Optional[int],
"keywords": Optional[List[str]],
"order": Optional[int],
"input_cost_per_token": Optional[float],
}
If quality_tier is None, the deployment is created without
@ -44,6 +45,8 @@ def _make_model_list(spec: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
prefs: Dict[str, Any] = {"quality_tier": entry["quality_tier"]}
if "keywords" in entry:
prefs["keywords"] = entry["keywords"]
if "order" in entry:
prefs["order"] = entry["order"]
model_info["litellm_routing_preferences"] = prefs
if "input_cost_per_token" in entry:
model_info["input_cost_per_token"] = entry["input_cost_per_token"]
@ -408,6 +411,163 @@ class TestKeywordOverride:
assert decision["matched_keyword"] == "code"
assert decision["complexity_tier"] is None # short-circuited
def test_explicit_order_overrides_quality_tier(self):
# Both models declare "code". By implicit rules (quality DESC), the
# tier-3 model would win. With an explicit `order=1` on the tier-2
# model, it must win regardless.
spec = [
{
"model_name": "preferred-tier2",
"quality_tier": 2,
"keywords": ["code"],
"order": 1,
"input_cost_per_token": 0.000010,
},
{
"model_name": "implicit-tier3",
"quality_tier": 3,
"keywords": ["code"],
"input_cost_per_token": 0.000005,
},
]
router = MagicMock()
router.model_list = _make_model_list(spec)
qr = QualityRouter(
model_name="qr",
litellm_router_instance=router,
default_model="preferred-tier2",
quality_router_config={
"available_models": ["preferred-tier2", "implicit-tier3"]
},
)
match = qr._keyword_override("write some code")
assert match == ("preferred-tier2", "code")
def test_explicit_order_overrides_price(self):
# Same tier, but the more expensive one has a lower `order` and wins.
spec = [
{
"model_name": "expensive-but-preferred",
"quality_tier": 2,
"keywords": ["data"],
"order": 1,
"input_cost_per_token": 0.000050,
},
{
"model_name": "cheap-default",
"quality_tier": 2,
"keywords": ["data"],
"input_cost_per_token": 0.000005,
},
]
router = MagicMock()
router.model_list = _make_model_list(spec)
qr = QualityRouter(
model_name="qr",
litellm_router_instance=router,
default_model="cheap-default",
quality_router_config={
"available_models": ["expensive-but-preferred", "cheap-default"]
},
)
match = qr._keyword_override("show me the data")
assert match == ("expensive-but-preferred", "data")
def test_lower_order_wins_between_two_explicitly_ordered(self):
spec = [
{
"model_name": "second",
"quality_tier": 2,
"keywords": ["data"],
"order": 5,
},
{
"model_name": "first",
"quality_tier": 2,
"keywords": ["data"],
"order": 1,
},
]
router = MagicMock()
router.model_list = _make_model_list(spec)
qr = QualityRouter(
model_name="qr",
litellm_router_instance=router,
default_model="first",
quality_router_config={"available_models": ["first", "second"]},
)
match = qr._keyword_override("show me the data")
assert match == ("first", "data")
def test_same_order_falls_through_to_quality_then_price(self):
# All three models share order=1 → tiebreak falls through to
# (quality DESC, cost ASC).
spec = [
{
"model_name": "low-tier",
"quality_tier": 1,
"keywords": ["data"],
"order": 1,
"input_cost_per_token": 0.000001,
},
{
"model_name": "high-tier-cheap",
"quality_tier": 3,
"keywords": ["data"],
"order": 1,
"input_cost_per_token": 0.000005,
},
{
"model_name": "high-tier-expensive",
"quality_tier": 3,
"keywords": ["data"],
"order": 1,
"input_cost_per_token": 0.000050,
},
]
router = MagicMock()
router.model_list = _make_model_list(spec)
qr = QualityRouter(
model_name="qr",
litellm_router_instance=router,
default_model="low-tier",
quality_router_config={
"available_models": [
"low-tier",
"high-tier-cheap",
"high-tier-expensive",
]
},
)
match = qr._keyword_override("show me the data")
assert match == ("high-tier-cheap", "data")
def test_order_is_used_in_tier_resolution_too(self):
# Two models at the same tier. Explicit `order=1` on the second one
# should make _resolve_model_for_quality_tier(2) pick it.
spec = [
{
"model_name": "default-pick",
"quality_tier": 2,
},
{
"model_name": "preferred-pick",
"quality_tier": 2,
"order": 1,
},
]
router = MagicMock()
router.model_list = _make_model_list(spec)
qr = QualityRouter(
model_name="qr",
litellm_router_instance=router,
default_model="default-pick",
quality_router_config={
"available_models": ["default-pick", "preferred-pick"]
},
)
assert qr._resolve_model_for_quality_tier(2) == "preferred-pick"
@pytest.mark.asyncio
async def test_hook_falls_back_to_complexity_when_no_keyword(self, keyword_router):
# No declared keyword in the message → complexity-based routing.