From afc604d1da842ef600e14589a3eccd908427639a Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 5 Sep 2026 15:25:06 -0700 Subject: [PATCH] address review: trim comments, add behavioral pick_model regression tests - Shrink the fallback comments to one line each; the fuller rationale was redundant per repo comment policy. - Add test_pick_model_favors_the_cheaper_model_info_priced_deployment and its hybrid-router counterpart, which exercise pick_model's actual Thompson-sampling/scoring output instead of only asserting the model_to_cost dict. Both are ordered so the expensive model wins pick_best's insertion-order tie-break on the pre-fix code (proven by reverting the production diff and rerunning), so they fail before the fix and pass after it. --- litellm/router.py | 5 +-- .../complexity_router/complexity_router.py | 5 +-- .../adaptive_router/test_router_dispatch.py | 38 ++++++++++++++++++ .../router_strategy/test_complexity_router.py | 40 +++++++++++++++++++ 4 files changed, 80 insertions(+), 8 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 5debdeed39a..3f450661946 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9075,10 +9075,7 @@ class Router: if prefs_raw is not None: model_to_prefs[name] = AdaptiveRouterPreferences(**prefs_raw) - # `input_cost_per_token` is a LiteLLM_Params field per types/router.py, but custom - # pricing is conventionally declared under model_info everywhere else in LiteLLM - # (cost_calculator.py, add_deployment's litellm.model_cost registration), so fall - # back to it here too rather than silently reading a zero cost for such deployments. + # model_info is the conventional pricing location elsewhere in LiteLLM; litellm_params wins if set. lp = d.get("litellm_params") if isinstance(d, dict) else d.litellm_params lp_dict: dict[str, Any] = lp if isinstance(lp, dict) else (lp.model_dump() if lp else {}) cost = lp_dict.get("input_cost_per_token") diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index e0353efaefd..d8b9b5ea8b2 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -2174,10 +2174,7 @@ class ComplexityRouter(CustomLogger): else: model_to_prefs[name] = AdaptiveRouterPreferences(quality_tier=2, strengths=[]) - # `input_cost_per_token` is a LiteLLM_Params field per types/router.py, but custom - # pricing is conventionally declared under model_info everywhere else in LiteLLM - # (cost_calculator.py, add_deployment's litellm.model_cost registration), so fall - # back to it here too rather than silently costing such a deployment at 0.0. + # model_info is the conventional pricing location elsewhere in LiteLLM; litellm_params wins if set. lp = deployment.get("litellm_params") if isinstance(deployment, dict) else deployment.litellm_params lp_dict: dict[str, Any] = lp if isinstance(lp, dict) else (lp.model_dump() if lp else {}) cost = lp_dict.get("input_cost_per_token") diff --git a/tests/test_litellm/router_strategy/adaptive_router/test_router_dispatch.py b/tests/test_litellm/router_strategy/adaptive_router/test_router_dispatch.py index 68961483ef0..6fe39599b0f 100644 --- a/tests/test_litellm/router_strategy/adaptive_router/test_router_dispatch.py +++ b/tests/test_litellm/router_strategy/adaptive_router/test_router_dispatch.py @@ -166,6 +166,44 @@ def test_init_adaptive_router_falls_back_to_model_info_cost(): } +@pytest.mark.asyncio +async def test_pick_model_favors_the_cheaper_model_info_priced_deployment(): + """Same fix, exercised through pick_model's actual scoring rather than the model_to_cost + dict alone: with cost as the only weight and equal quality priors, the cheaper deployment + must win every draw. `smart` (expensive) is listed first deliberately: before the fix both + models silently cost 0.0, tying every score, and pick_best's insertion-order tie-break would + hand every request to the first-listed (expensive) model instead.""" + r = Router( + model_list=[ + { + "model_name": "smart-cheap-router", + "litellm_params": { + "model": "auto_router/adaptive_router", + "adaptive_router_config": { + "available_models": ["smart", "fast"], + "weights": {"quality": 0.0, "cost": 1.0}, + }, + }, + }, + { + "model_name": "smart", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": {"input_cost_per_token": 0.0000050}, + }, + { + "model_name": "fast", + "litellm_params": {"model": "openai/gpt-4o-mini"}, + "model_info": {"input_cost_per_token": 0.00000015}, + }, + ] + ) + adaptive = _adaptive(r, "smart-cheap-router") + + picks = [await adaptive.pick_model(RequestType.GENERAL) for _ in range(10)] + + assert picks == ["fast"] * 10 + + def test_init_adaptive_router_prefers_litellm_params_cost_over_model_info(): r = Router( model_list=[ diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index e6186ae50db..ebfb631f93b 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1548,6 +1548,46 @@ class TestRouterComplexityDeploymentMethods: "premium": pytest.approx(0.000005), } + @pytest.mark.asyncio + async def test_hybrid_adaptive_router_pick_model_favors_the_cheaper_model_info_priced_deployment(self): + """Same fix, exercised through pick_model's actual scoring rather than the model_to_cost + dict alone. `premium` is listed first (SIMPLE tier) deliberately: before the fix both + models silently cost 0.0, tying every score, and pick_best's insertion-order tie-break + would hand every request to the first-listed (expensive) model instead.""" + from litellm.types.router import RequestType + + router = Router( + model_list=[ + { + "model_name": "hybrid", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_default_model": "cheap", + "complexity_router_config": { + "adaptive": True, + "adaptive_weights": {"quality": 0.0, "cost": 1.0}, + "tiers": {"SIMPLE": ["premium"], "MEDIUM": ["premium", "cheap"]}, + }, + }, + }, + { + "model_name": "premium", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": {"input_cost_per_token": 0.000005}, + }, + { + "model_name": "cheap", + "litellm_params": {"model": "openai/gpt-4o-mini"}, + "model_info": {"input_cost_per_token": 0.00000015}, + }, + ] + ) + adaptive = router.adaptive_routers["hybrid"][0].strategy + + picks = [await adaptive.pick_model(RequestType.GENERAL) for _ in range(10)] + + assert picks == ["cheap"] * 10 + def test_hybrid_adaptive_router_prefers_litellm_params_cost_over_model_info(self): router = Router( model_list=[