scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 82

fix(router_strategy): read custom pricing from model_info in cost-based-routing

Browse source 
LowestCostLoggingHandler only checked litellm_params and
model_cost[model_name] for pricing, missing deployments where
input_cost_per_token/output_cost_per_token are stored in model_info
(e.g. DB-loaded models via meicloud). This caused both members of a
group to fall back to the default 5.0 cost, making cost-based-routing
behave as if all targets were equally priced.

Add model_info as a pricing fallback, matching the existing tpm/rpm
fallback pattern already used in the same function.
This commit is contained in:
waani 2026-06-29 11:08:26 +08:00
parent b76a858826
commit c55924ad65
1 changed files with 12 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
litellm/router_strategy/lowest_cost.py
View file

@ -248,10 +248,6 @@ class LowestCostLoggingHandler(CustomLogger):
or _deployment.get("model_info", {}).get("rpm", None)
or float("inf")
)
item_litellm_model_name = _deployment.get("litellm_params", {}).get("model")
item_litellm_model_cost_map = litellm.model_cost.get(item_litellm_model_name, {})
# check if user provided input_cost_per_token and output_cost_per_token in litellm_params
item_input_cost = None
item_output_cost = None
if _deployment.get("litellm_params", {}).get("input_cost_per_token", None):
@ -260,11 +256,20 @@ class LowestCostLoggingHandler(CustomLogger):
if _deployment.get("litellm_params", {}).get("output_cost_per_token", None):
item_output_cost = _deployment.get("litellm_params", {}).get("output_cost_per_token")
# Fallback: check model_info (mirrors tpm/rpm fallback)
if item_input_cost is None:
item_input_cost = item_litellm_model_cost_map.get("input_cost_per_token", 5.0)
item_input_cost = _deployment.get("model_info", {}).get("input_cost_per_token")
if item_output_cost is None:
item_output_cost = item_litellm_model_cost_map.get("output_cost_per_token", 5.0)
item_output_cost = _deployment.get("model_info", {}).get("output_cost_per_token")
# Fallback: model_cost map (built-in pricing, default 5.0)
if item_input_cost is None or item_output_cost is None:
item_litellm_model_name = _deployment.get("litellm_params", {}).get("model")
_model_cost_entry = litellm.model_cost.get(item_litellm_model_name, {})
if item_input_cost is None:
item_input_cost = _model_cost_entry.get("input_cost_per_token", 5.0)
if item_output_cost is None:
item_output_cost = _model_cost_entry.get("output_cost_per_token", 5.0)
# if litellm["model"] is not in model_cost map -> use item_cost = $10