address review: trim comments, add behavioral pick_model regression tests

- Shrink the fallback comments to one line each; the fuller rationale
  was redundant per repo comment policy.
- Add test_pick_model_favors_the_cheaper_model_info_priced_deployment
  and its hybrid-router counterpart, which exercise pick_model's actual
  Thompson-sampling/scoring output instead of only asserting the
  model_to_cost dict. Both are ordered so the expensive model wins
  pick_best's insertion-order tie-break on the pre-fix code (proven by
  reverting the production diff and rerunning), so they fail before the
  fix and pass after it.
This commit is contained in:
moe-berri 2026-09-05 15:25:06 -07:00
parent a00b60933c
commit afc604d1da
4 changed files with 80 additions and 8 deletions

View file

@ -9075,10 +9075,7 @@ class Router:
if prefs_raw is not None:
model_to_prefs[name] = AdaptiveRouterPreferences(**prefs_raw)
# `input_cost_per_token` is a LiteLLM_Params field per types/router.py, but custom
# pricing is conventionally declared under model_info everywhere else in LiteLLM
# (cost_calculator.py, add_deployment's litellm.model_cost registration), so fall
# back to it here too rather than silently reading a zero cost for such deployments.
# model_info is the conventional pricing location elsewhere in LiteLLM; litellm_params wins if set.
lp = d.get("litellm_params") if isinstance(d, dict) else d.litellm_params
lp_dict: dict[str, Any] = lp if isinstance(lp, dict) else (lp.model_dump() if lp else {})
cost = lp_dict.get("input_cost_per_token")

View file

@ -2174,10 +2174,7 @@ class ComplexityRouter(CustomLogger):
else:
model_to_prefs[name] = AdaptiveRouterPreferences(quality_tier=2, strengths=[])
# `input_cost_per_token` is a LiteLLM_Params field per types/router.py, but custom
# pricing is conventionally declared under model_info everywhere else in LiteLLM
# (cost_calculator.py, add_deployment's litellm.model_cost registration), so fall
# back to it here too rather than silently costing such a deployment at 0.0.
# model_info is the conventional pricing location elsewhere in LiteLLM; litellm_params wins if set.
lp = deployment.get("litellm_params") if isinstance(deployment, dict) else deployment.litellm_params
lp_dict: dict[str, Any] = lp if isinstance(lp, dict) else (lp.model_dump() if lp else {})
cost = lp_dict.get("input_cost_per_token")

View file

@ -166,6 +166,44 @@ def test_init_adaptive_router_falls_back_to_model_info_cost():
}
@pytest.mark.asyncio
async def test_pick_model_favors_the_cheaper_model_info_priced_deployment():
"""Same fix, exercised through pick_model's actual scoring rather than the model_to_cost
dict alone: with cost as the only weight and equal quality priors, the cheaper deployment
must win every draw. `smart` (expensive) is listed first deliberately: before the fix both
models silently cost 0.0, tying every score, and pick_best's insertion-order tie-break would
hand every request to the first-listed (expensive) model instead."""
r = Router(
model_list=[
{
"model_name": "smart-cheap-router",
"litellm_params": {
"model": "auto_router/adaptive_router",
"adaptive_router_config": {
"available_models": ["smart", "fast"],
"weights": {"quality": 0.0, "cost": 1.0},
},
},
},
{
"model_name": "smart",
"litellm_params": {"model": "openai/gpt-4o"},
"model_info": {"input_cost_per_token": 0.0000050},
},
{
"model_name": "fast",
"litellm_params": {"model": "openai/gpt-4o-mini"},
"model_info": {"input_cost_per_token": 0.00000015},
},
]
)
adaptive = _adaptive(r, "smart-cheap-router")
picks = [await adaptive.pick_model(RequestType.GENERAL) for _ in range(10)]
assert picks == ["fast"] * 10
def test_init_adaptive_router_prefers_litellm_params_cost_over_model_info():
r = Router(
model_list=[

View file

@ -1548,6 +1548,46 @@ class TestRouterComplexityDeploymentMethods:
"premium": pytest.approx(0.000005),
}
@pytest.mark.asyncio
async def test_hybrid_adaptive_router_pick_model_favors_the_cheaper_model_info_priced_deployment(self):
"""Same fix, exercised through pick_model's actual scoring rather than the model_to_cost
dict alone. `premium` is listed first (SIMPLE tier) deliberately: before the fix both
models silently cost 0.0, tying every score, and pick_best's insertion-order tie-break
would hand every request to the first-listed (expensive) model instead."""
from litellm.types.router import RequestType
router = Router(
model_list=[
{
"model_name": "hybrid",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_default_model": "cheap",
"complexity_router_config": {
"adaptive": True,
"adaptive_weights": {"quality": 0.0, "cost": 1.0},
"tiers": {"SIMPLE": ["premium"], "MEDIUM": ["premium", "cheap"]},
},
},
},
{
"model_name": "premium",
"litellm_params": {"model": "openai/gpt-4o"},
"model_info": {"input_cost_per_token": 0.000005},
},
{
"model_name": "cheap",
"litellm_params": {"model": "openai/gpt-4o-mini"},
"model_info": {"input_cost_per_token": 0.00000015},
},
]
)
adaptive = router.adaptive_routers["hybrid"][0].strategy
picks = [await adaptive.pick_model(RequestType.GENERAL) for _ in range(10)]
assert picks == ["cheap"] * 10
def test_hybrid_adaptive_router_prefers_litellm_params_cost_over_model_info(self):
router = Router(
model_list=[