diff --git a/litellm/router.py b/litellm/router.py index 6d6efad9f42..3f450661946 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9075,10 +9075,12 @@ class Router: if prefs_raw is not None: model_to_prefs[name] = AdaptiveRouterPreferences(**prefs_raw) - # `input_cost_per_token` is a LiteLLM_Params field per types/router.py. + # model_info is the conventional pricing location elsewhere in LiteLLM; litellm_params wins if set. lp = d.get("litellm_params") if isinstance(d, dict) else d.litellm_params lp_dict: dict[str, Any] = lp if isinstance(lp, dict) else (lp.model_dump() if lp else {}) cost = lp_dict.get("input_cost_per_token") + if cost is None: + cost = mi_dict.get("input_cost_per_token") if cost is not None: model_to_cost[name] = float(cost) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 98a1eb7ac9e..d8b9b5ea8b2 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -2174,9 +2174,12 @@ class ComplexityRouter(CustomLogger): else: model_to_prefs[name] = AdaptiveRouterPreferences(quality_tier=2, strengths=[]) + # model_info is the conventional pricing location elsewhere in LiteLLM; litellm_params wins if set. lp = deployment.get("litellm_params") if isinstance(deployment, dict) else deployment.litellm_params lp_dict: dict[str, Any] = lp if isinstance(lp, dict) else (lp.model_dump() if lp else {}) cost = lp_dict.get("input_cost_per_token") + if cost is None: + cost = mi_dict.get("input_cost_per_token") model_to_cost[name] = float(cost) if cost is not None else 0.0 self.adaptive_router = AdaptiveRouter( diff --git a/tests/test_litellm/router_strategy/adaptive_router/test_router_dispatch.py b/tests/test_litellm/router_strategy/adaptive_router/test_router_dispatch.py index a4e803f59ad..6fe39599b0f 100644 --- a/tests/test_litellm/router_strategy/adaptive_router/test_router_dispatch.py +++ b/tests/test_litellm/router_strategy/adaptive_router/test_router_dispatch.py @@ -133,6 +133,102 @@ def test_init_adaptive_router_reads_cost_from_litellm_params(): } +def test_init_adaptive_router_falls_back_to_model_info_cost(): + """Custom pricing declared under model_info (the conventional location everywhere else in + LiteLLM: cost_calculator.py, add_deployment's litellm.model_cost registration) must still + feed cost-weighted routing, not silently zero it out.""" + r = Router( + model_list=[ + { + "model_name": "smart-cheap-router", + "litellm_params": { + "model": "auto_router/adaptive_router", + "adaptive_router_config": { + "available_models": ["fast", "smart"], + }, + }, + }, + { + "model_name": "fast", + "litellm_params": {"model": "openai/gpt-4o-mini"}, + "model_info": {"input_cost_per_token": 0.00000015}, + }, + { + "model_name": "smart", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": {"input_cost_per_token": 0.0000050}, + }, + ] + ) + assert _adaptive(r, "smart-cheap-router").model_to_cost == { + "fast": 0.00000015, + "smart": 0.0000050, + } + + +@pytest.mark.asyncio +async def test_pick_model_favors_the_cheaper_model_info_priced_deployment(): + """Same fix, exercised through pick_model's actual scoring rather than the model_to_cost + dict alone: with cost as the only weight and equal quality priors, the cheaper deployment + must win every draw. `smart` (expensive) is listed first deliberately: before the fix both + models silently cost 0.0, tying every score, and pick_best's insertion-order tie-break would + hand every request to the first-listed (expensive) model instead.""" + r = Router( + model_list=[ + { + "model_name": "smart-cheap-router", + "litellm_params": { + "model": "auto_router/adaptive_router", + "adaptive_router_config": { + "available_models": ["smart", "fast"], + "weights": {"quality": 0.0, "cost": 1.0}, + }, + }, + }, + { + "model_name": "smart", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": {"input_cost_per_token": 0.0000050}, + }, + { + "model_name": "fast", + "litellm_params": {"model": "openai/gpt-4o-mini"}, + "model_info": {"input_cost_per_token": 0.00000015}, + }, + ] + ) + adaptive = _adaptive(r, "smart-cheap-router") + + picks = [await adaptive.pick_model(RequestType.GENERAL) for _ in range(10)] + + assert picks == ["fast"] * 10 + + +def test_init_adaptive_router_prefers_litellm_params_cost_over_model_info(): + r = Router( + model_list=[ + { + "model_name": "smart-cheap-router", + "litellm_params": { + "model": "auto_router/adaptive_router", + "adaptive_router_config": { + "available_models": ["fast"], + }, + }, + }, + { + "model_name": "fast", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "input_cost_per_token": 0.00000015, + }, + "model_info": {"input_cost_per_token": 0.0000050}, + }, + ] + ) + assert _adaptive(r, "smart-cheap-router").model_to_cost == {"fast": 0.00000015} + + # ---- Fix 4: pre-routing dispatch --------------------------------------- diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 918ec7bc100..ebfb631f93b 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1512,6 +1512,110 @@ class TestRouterComplexityDeploymentMethods: assert adaptive.model_to_prefs["cheap"].quality_tier == 1 assert adaptive.model_to_prefs["premium"].quality_tier == 3 + def test_hybrid_adaptive_router_falls_back_to_model_info_cost(self): + """Custom pricing declared under model_info (the conventional location everywhere else + in LiteLLM) must still feed the hybrid adaptive router's cost-weighted scoring, not + silently cost the deployment at 0.0.""" + router = Router( + model_list=[ + { + "model_name": "hybrid", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_default_model": "cheap", + "complexity_router_config": { + "adaptive": True, + "tiers": {"SIMPLE": ["cheap"], "MEDIUM": ["cheap", "premium"]}, + }, + }, + }, + { + "model_name": "cheap", + "litellm_params": {"model": "openai/gpt-4o-mini"}, + "model_info": {"input_cost_per_token": 0.00000015}, + }, + { + "model_name": "premium", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": {"input_cost_per_token": 0.000005}, + }, + ] + ) + + adaptive = router.adaptive_routers["hybrid"][0].strategy + assert adaptive.model_to_cost == { + "cheap": pytest.approx(0.00000015), + "premium": pytest.approx(0.000005), + } + + @pytest.mark.asyncio + async def test_hybrid_adaptive_router_pick_model_favors_the_cheaper_model_info_priced_deployment(self): + """Same fix, exercised through pick_model's actual scoring rather than the model_to_cost + dict alone. `premium` is listed first (SIMPLE tier) deliberately: before the fix both + models silently cost 0.0, tying every score, and pick_best's insertion-order tie-break + would hand every request to the first-listed (expensive) model instead.""" + from litellm.types.router import RequestType + + router = Router( + model_list=[ + { + "model_name": "hybrid", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_default_model": "cheap", + "complexity_router_config": { + "adaptive": True, + "adaptive_weights": {"quality": 0.0, "cost": 1.0}, + "tiers": {"SIMPLE": ["premium"], "MEDIUM": ["premium", "cheap"]}, + }, + }, + }, + { + "model_name": "premium", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": {"input_cost_per_token": 0.000005}, + }, + { + "model_name": "cheap", + "litellm_params": {"model": "openai/gpt-4o-mini"}, + "model_info": {"input_cost_per_token": 0.00000015}, + }, + ] + ) + adaptive = router.adaptive_routers["hybrid"][0].strategy + + picks = [await adaptive.pick_model(RequestType.GENERAL) for _ in range(10)] + + assert picks == ["cheap"] * 10 + + def test_hybrid_adaptive_router_prefers_litellm_params_cost_over_model_info(self): + router = Router( + model_list=[ + { + "model_name": "hybrid", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_default_model": "cheap", + "complexity_router_config": { + "adaptive": True, + "tiers": {"SIMPLE": ["cheap"]}, + }, + }, + }, + { + "model_name": "cheap", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "input_cost_per_token": 0.00000015, + }, + "model_info": {"input_cost_per_token": 0.000005}, + }, + ] + ) + + adaptive = router.adaptive_routers["hybrid"][0].strategy + assert adaptive.model_to_cost == {"cheap": pytest.approx(0.00000015)} + class TestComplexityRouterTagBasedRouting: """Regression tests for https://github.com/BerriAI/litellm/issues/33655.