mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-08 22:21:35 +00:00
address review: trim comments, add behavioral pick_model regression tests
- Shrink the fallback comments to one line each; the fuller rationale was redundant per repo comment policy. - Add test_pick_model_favors_the_cheaper_model_info_priced_deployment and its hybrid-router counterpart, which exercise pick_model's actual Thompson-sampling/scoring output instead of only asserting the model_to_cost dict. Both are ordered so the expensive model wins pick_best's insertion-order tie-break on the pre-fix code (proven by reverting the production diff and rerunning), so they fail before the fix and pass after it.
This commit is contained in:
parent
a00b60933c
commit
afc604d1da
4 changed files with 80 additions and 8 deletions
|
|
@ -9075,10 +9075,7 @@ class Router:
|
|||
if prefs_raw is not None:
|
||||
model_to_prefs[name] = AdaptiveRouterPreferences(**prefs_raw)
|
||||
|
||||
# `input_cost_per_token` is a LiteLLM_Params field per types/router.py, but custom
|
||||
# pricing is conventionally declared under model_info everywhere else in LiteLLM
|
||||
# (cost_calculator.py, add_deployment's litellm.model_cost registration), so fall
|
||||
# back to it here too rather than silently reading a zero cost for such deployments.
|
||||
# model_info is the conventional pricing location elsewhere in LiteLLM; litellm_params wins if set.
|
||||
lp = d.get("litellm_params") if isinstance(d, dict) else d.litellm_params
|
||||
lp_dict: dict[str, Any] = lp if isinstance(lp, dict) else (lp.model_dump() if lp else {})
|
||||
cost = lp_dict.get("input_cost_per_token")
|
||||
|
|
|
|||
|
|
@ -2174,10 +2174,7 @@ class ComplexityRouter(CustomLogger):
|
|||
else:
|
||||
model_to_prefs[name] = AdaptiveRouterPreferences(quality_tier=2, strengths=[])
|
||||
|
||||
# `input_cost_per_token` is a LiteLLM_Params field per types/router.py, but custom
|
||||
# pricing is conventionally declared under model_info everywhere else in LiteLLM
|
||||
# (cost_calculator.py, add_deployment's litellm.model_cost registration), so fall
|
||||
# back to it here too rather than silently costing such a deployment at 0.0.
|
||||
# model_info is the conventional pricing location elsewhere in LiteLLM; litellm_params wins if set.
|
||||
lp = deployment.get("litellm_params") if isinstance(deployment, dict) else deployment.litellm_params
|
||||
lp_dict: dict[str, Any] = lp if isinstance(lp, dict) else (lp.model_dump() if lp else {})
|
||||
cost = lp_dict.get("input_cost_per_token")
|
||||
|
|
|
|||
|
|
@ -166,6 +166,44 @@ def test_init_adaptive_router_falls_back_to_model_info_cost():
|
|||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pick_model_favors_the_cheaper_model_info_priced_deployment():
|
||||
"""Same fix, exercised through pick_model's actual scoring rather than the model_to_cost
|
||||
dict alone: with cost as the only weight and equal quality priors, the cheaper deployment
|
||||
must win every draw. `smart` (expensive) is listed first deliberately: before the fix both
|
||||
models silently cost 0.0, tying every score, and pick_best's insertion-order tie-break would
|
||||
hand every request to the first-listed (expensive) model instead."""
|
||||
r = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "smart-cheap-router",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/adaptive_router",
|
||||
"adaptive_router_config": {
|
||||
"available_models": ["smart", "fast"],
|
||||
"weights": {"quality": 0.0, "cost": 1.0},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "smart",
|
||||
"litellm_params": {"model": "openai/gpt-4o"},
|
||||
"model_info": {"input_cost_per_token": 0.0000050},
|
||||
},
|
||||
{
|
||||
"model_name": "fast",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini"},
|
||||
"model_info": {"input_cost_per_token": 0.00000015},
|
||||
},
|
||||
]
|
||||
)
|
||||
adaptive = _adaptive(r, "smart-cheap-router")
|
||||
|
||||
picks = [await adaptive.pick_model(RequestType.GENERAL) for _ in range(10)]
|
||||
|
||||
assert picks == ["fast"] * 10
|
||||
|
||||
|
||||
def test_init_adaptive_router_prefers_litellm_params_cost_over_model_info():
|
||||
r = Router(
|
||||
model_list=[
|
||||
|
|
|
|||
|
|
@ -1548,6 +1548,46 @@ class TestRouterComplexityDeploymentMethods:
|
|||
"premium": pytest.approx(0.000005),
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hybrid_adaptive_router_pick_model_favors_the_cheaper_model_info_priced_deployment(self):
|
||||
"""Same fix, exercised through pick_model's actual scoring rather than the model_to_cost
|
||||
dict alone. `premium` is listed first (SIMPLE tier) deliberately: before the fix both
|
||||
models silently cost 0.0, tying every score, and pick_best's insertion-order tie-break
|
||||
would hand every request to the first-listed (expensive) model instead."""
|
||||
from litellm.types.router import RequestType
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "hybrid",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_default_model": "cheap",
|
||||
"complexity_router_config": {
|
||||
"adaptive": True,
|
||||
"adaptive_weights": {"quality": 0.0, "cost": 1.0},
|
||||
"tiers": {"SIMPLE": ["premium"], "MEDIUM": ["premium", "cheap"]},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "premium",
|
||||
"litellm_params": {"model": "openai/gpt-4o"},
|
||||
"model_info": {"input_cost_per_token": 0.000005},
|
||||
},
|
||||
{
|
||||
"model_name": "cheap",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini"},
|
||||
"model_info": {"input_cost_per_token": 0.00000015},
|
||||
},
|
||||
]
|
||||
)
|
||||
adaptive = router.adaptive_routers["hybrid"][0].strategy
|
||||
|
||||
picks = [await adaptive.pick_model(RequestType.GENERAL) for _ in range(10)]
|
||||
|
||||
assert picks == ["cheap"] * 10
|
||||
|
||||
def test_hybrid_adaptive_router_prefers_litellm_params_cost_over_model_info(self):
|
||||
router = Router(
|
||||
model_list=[
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue