scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge pull request #39957 from BerriAI/litellm_fix_adaptive_router_cost_from_model_info

Browse source 
fix(adaptive_router): fall back to model_info for cost-weighted scoring
This commit is contained in:
moe-berri 2026-09-05 18:01:36 -07:00 • committed by GitHub
commit aee819c976
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 206 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
litellm/router.py
View file

@ -9075,10 +9075,12 @@ class Router:
if prefs_raw is not None:
model_to_prefs[name] = AdaptiveRouterPreferences(**prefs_raw)
# `input_cost_per_token` is a LiteLLM_Params field per types/router.py.
# model_info is the conventional pricing location elsewhere in LiteLLM; litellm_params wins if set.
lp = d.get("litellm_params") if isinstance(d, dict) else d.litellm_params
lp_dict: dict[str, Any] = lp if isinstance(lp, dict) else (lp.model_dump() if lp else {})
cost = lp_dict.get("input_cost_per_token")
if cost is None:
cost = mi_dict.get("input_cost_per_token")
if cost is not None:
model_to_cost[name] = float(cost)

3
litellm/router_strategy/complexity_router/complexity_router.py
View file

@ -2174,9 +2174,12 @@ class ComplexityRouter(CustomLogger):
else:
model_to_prefs[name] = AdaptiveRouterPreferences(quality_tier=2, strengths=[])
# model_info is the conventional pricing location elsewhere in LiteLLM; litellm_params wins if set.
lp = deployment.get("litellm_params") if isinstance(deployment, dict) else deployment.litellm_params
lp_dict: dict[str, Any] = lp if isinstance(lp, dict) else (lp.model_dump() if lp else {})
cost = lp_dict.get("input_cost_per_token")
if cost is None:
cost = mi_dict.get("input_cost_per_token")
model_to_cost[name] = float(cost) if cost is not None else 0.0
self.adaptive_router = AdaptiveRouter(

96
tests/test_litellm/router_strategy/adaptive_router/test_router_dispatch.py
View file

@ -133,6 +133,102 @@ def test_init_adaptive_router_reads_cost_from_litellm_params():
}
def test_init_adaptive_router_falls_back_to_model_info_cost():
"""Custom pricing declared under model_info (the conventional location everywhere else in
LiteLLM: cost_calculator.py, add_deployment's litellm.model_cost registration) must still
feed cost-weighted routing, not silently zero it out."""
r = Router(
model_list=[
{
"model_name": "smart-cheap-router",
"litellm_params": {
"model": "auto_router/adaptive_router",
"adaptive_router_config": {
"available_models": ["fast", "smart"],
},
},
},
{
"model_name": "fast",
"litellm_params": {"model": "openai/gpt-4o-mini"},
"model_info": {"input_cost_per_token": 0.00000015},
},
{
"model_name": "smart",
"litellm_params": {"model": "openai/gpt-4o"},
"model_info": {"input_cost_per_token": 0.0000050},
},
]
)
assert _adaptive(r, "smart-cheap-router").model_to_cost == {
"fast": 0.00000015,
"smart": 0.0000050,
}
@pytest.mark.asyncio
async def test_pick_model_favors_the_cheaper_model_info_priced_deployment():
"""Same fix, exercised through pick_model's actual scoring rather than the model_to_cost
dict alone: with cost as the only weight and equal quality priors, the cheaper deployment
must win every draw. `smart` (expensive) is listed first deliberately: before the fix both
models silently cost 0.0, tying every score, and pick_best's insertion-order tie-break would
hand every request to the first-listed (expensive) model instead."""
r = Router(
model_list=[
{
"model_name": "smart-cheap-router",
"litellm_params": {
"model": "auto_router/adaptive_router",
"adaptive_router_config": {
"available_models": ["smart", "fast"],
"weights": {"quality": 0.0, "cost": 1.0},
},
},
},
{
"model_name": "smart",
"litellm_params": {"model": "openai/gpt-4o"},
"model_info": {"input_cost_per_token": 0.0000050},
},
{
"model_name": "fast",
"litellm_params": {"model": "openai/gpt-4o-mini"},
"model_info": {"input_cost_per_token": 0.00000015},
},
]
)
adaptive = _adaptive(r, "smart-cheap-router")
picks = [await adaptive.pick_model(RequestType.GENERAL) for _ in range(10)]
assert picks == ["fast"] * 10
def test_init_adaptive_router_prefers_litellm_params_cost_over_model_info():
r = Router(
model_list=[
{
"model_name": "smart-cheap-router",
"litellm_params": {
"model": "auto_router/adaptive_router",
"adaptive_router_config": {
"available_models": ["fast"],
},
},
},
{
"model_name": "fast",
"litellm_params": {
"model": "openai/gpt-4o-mini",
"input_cost_per_token": 0.00000015,
},
"model_info": {"input_cost_per_token": 0.0000050},
},
]
)
assert _adaptive(r, "smart-cheap-router").model_to_cost == {"fast": 0.00000015}
# ---- Fix 4: pre-routing dispatch ---------------------------------------

104
tests/test_litellm/router_strategy/test_complexity_router.py
View file

@ -1512,6 +1512,110 @@ class TestRouterComplexityDeploymentMethods:
assert adaptive.model_to_prefs["cheap"].quality_tier == 1
assert adaptive.model_to_prefs["premium"].quality_tier == 3
def test_hybrid_adaptive_router_falls_back_to_model_info_cost(self):
"""Custom pricing declared under model_info (the conventional location everywhere else
in LiteLLM) must still feed the hybrid adaptive router's cost-weighted scoring, not
silently cost the deployment at 0.0."""
router = Router(
model_list=[
{
"model_name": "hybrid",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_default_model": "cheap",
"complexity_router_config": {
"adaptive": True,
"tiers": {"SIMPLE": ["cheap"], "MEDIUM": ["cheap", "premium"]},
},
},
},
{
"model_name": "cheap",
"litellm_params": {"model": "openai/gpt-4o-mini"},
"model_info": {"input_cost_per_token": 0.00000015},
},
{
"model_name": "premium",
"litellm_params": {"model": "openai/gpt-4o"},
"model_info": {"input_cost_per_token": 0.000005},
},
]
)
adaptive = router.adaptive_routers["hybrid"][0].strategy
assert adaptive.model_to_cost == {
"cheap": pytest.approx(0.00000015),
"premium": pytest.approx(0.000005),
}
@pytest.mark.asyncio
async def test_hybrid_adaptive_router_pick_model_favors_the_cheaper_model_info_priced_deployment(self):
"""Same fix, exercised through pick_model's actual scoring rather than the model_to_cost
dict alone. `premium` is listed first (SIMPLE tier) deliberately: before the fix both
models silently cost 0.0, tying every score, and pick_best's insertion-order tie-break
would hand every request to the first-listed (expensive) model instead."""
from litellm.types.router import RequestType
router = Router(
model_list=[
{
"model_name": "hybrid",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_default_model": "cheap",
"complexity_router_config": {
"adaptive": True,
"adaptive_weights": {"quality": 0.0, "cost": 1.0},
"tiers": {"SIMPLE": ["premium"], "MEDIUM": ["premium", "cheap"]},
},
},
},
{
"model_name": "premium",
"litellm_params": {"model": "openai/gpt-4o"},
"model_info": {"input_cost_per_token": 0.000005},
},
{
"model_name": "cheap",
"litellm_params": {"model": "openai/gpt-4o-mini"},
"model_info": {"input_cost_per_token": 0.00000015},
},
]
)
adaptive = router.adaptive_routers["hybrid"][0].strategy
picks = [await adaptive.pick_model(RequestType.GENERAL) for _ in range(10)]
assert picks == ["cheap"] * 10
def test_hybrid_adaptive_router_prefers_litellm_params_cost_over_model_info(self):
router = Router(
model_list=[
{
"model_name": "hybrid",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_default_model": "cheap",
"complexity_router_config": {
"adaptive": True,
"tiers": {"SIMPLE": ["cheap"]},
},
},
},
{
"model_name": "cheap",
"litellm_params": {
"model": "openai/gpt-4o-mini",
"input_cost_per_token": 0.00000015,
},
"model_info": {"input_cost_per_token": 0.000005},
},
]
)
adaptive = router.adaptive_routers["hybrid"][0].strategy
assert adaptive.model_to_cost == {"cheap": pytest.approx(0.00000015)}
class TestComplexityRouterTagBasedRouting:
"""Regression tests for https://github.com/BerriAI/litellm/issues/33655.