Fix zero cost for claude-opus-4-6 when proxy DB has stale 0.0 pricing defaults

The old proxy ModelInfo class defaulted input_cost_per_token and
output_cost_per_token to 0.0 instead of None. These stale values in
existing DB records overwrote the base model's real pricing in the UUID
cost entry, causing cost=0 for models like claude-opus-4-6.

Changes:
- Change proxy ModelInfo defaults from 0.0 to None to prevent new stale entries
- When building UUID cost entries, start from base model pricing and skip
  0.0 pricing fields from model_info (stale DB defaults), while preserving
  intentional zero-cost set via litellm_params
- Add regression test for the stale DB defaults scenario

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
yuneng-jiang 2026-03-06 22:38:49 -08:00
parent 4eee2b752d
commit 3715981199
3 changed files with 143 additions and 13 deletions

View file

@ -782,8 +782,8 @@ class ModelInfoDelete(LiteLLMPydanticObjectBase):
class ModelInfo(LiteLLMPydanticObjectBase):
id: Optional[str]
mode: Optional[Literal["embedding", "chat", "completion"]]
input_cost_per_token: Optional[float] = 0.0
output_cost_per_token: Optional[float] = 0.0
input_cost_per_token: Optional[float] = None
output_cost_per_token: Optional[float] = None
max_tokens: Optional[int] = 2048 # assume 2048 if not set
# for azure models we need users to specify the base model, one azure you can call deployments - azure/my-random-model

View file

@ -207,6 +207,24 @@ class RoutingArgs(enum.Enum):
ttl = 60 # 1min (RPM/TPM expire key)
def _get_base_model_cost_info(model_name: str) -> dict:
"""
Get the base model's pricing info from litellm.model_cost.
Used when registering deployment-specific pricing under UUID to ensure
the UUID entry inherits complete pricing (e.g. cache costs) from the
base model, even if the deployment's model_info only has a subset of
pricing fields.
Returns an empty dict if the model is not found.
"""
try:
info = litellm.get_model_info(model=model_name)
return dict(info)
except Exception:
return {}
class Router:
model_names: set = set()
cache_responses: Optional[bool] = False
@ -6390,14 +6408,6 @@ class Router:
## REGISTER MODEL INFO IN LITELLM MODEL COST MAP
model_id = deployment.model_info.id
if model_id is not None:
litellm.register_model(
model_cost={
model_id: _model_info,
}
)
## OLD MODEL REGISTRATION ## Kept to prevent breaking changes
_model_name = deployment.litellm_params.model
if deployment.litellm_params.custom_llm_provider is not None:
_model_name = (
@ -6419,6 +6429,44 @@ class Router:
}
)
# Register deployment-specific pricing under UUID.
# Start with the base model's full pricing so that the UUID
# entry inherits all pricing fields (e.g. cache costs).
# Then overlay the deployment's model_info on top (which
# includes any custom pricing from both model_info config
# and litellm_params).
# Without this, UUID entries miss pricing fields not
# explicitly set on the deployment, causing $0 cost when
# custom_pricing=True triggers UUID-based cost lookup.
if model_id is not None:
_model_info_for_id = _get_base_model_cost_info(
model_name=_model_name
)
# Overlay model_info but skip 0.0 pricing fields — these are
# stale DB defaults from the old proxy ModelInfo class that
# defaulted pricing to 0.0 instead of None. Intentional
# zero-cost pricing set via litellm_params is already in
# _model_info (copied above) and will be re-applied below.
_pricing_fields = set(
CustomPricingLiteLLMParams.model_fields.keys()
)
for k, v in _model_info.items():
if v is None:
continue
if k in _pricing_fields and v == 0.0:
continue
_model_info_for_id[k] = v
# Re-apply litellm_params pricing (intentional, even if 0.0)
for field in _pricing_fields:
field_value = deployment.litellm_params.get(field)
if field_value is not None:
_model_info_for_id[field] = field_value
litellm.register_model(
model_cost={
model_id: _model_info_for_id,
}
)
## Check if LLM Deployment is allowed for this deployment
if (
self.deployment_is_active_for_environment(deployment=deployment)
@ -6870,11 +6918,32 @@ class Router:
_model_info_dict: dict = deployment.model_info.model_dump(
exclude_none=True
)
for field in CustomPricingLiteLLMParams.model_fields.keys():
# Start with the base model's full pricing so that the UUID
# entry inherits all pricing fields (e.g. cache costs).
_model_name = deployment.litellm_params.model
if deployment.litellm_params.custom_llm_provider is not None:
_model_name = (
deployment.litellm_params.custom_llm_provider
+ "/"
+ _model_name
)
_base_info = _get_base_model_cost_info(model_name=_model_name)
# Skip 0.0 pricing fields — stale DB defaults (see _create_deployment)
_pricing_fields = set(
CustomPricingLiteLLMParams.model_fields.keys()
)
for k, v in _model_info_dict.items():
if v is None:
continue
if k in _pricing_fields and v == 0.0:
continue
_base_info[k] = v
for field in _pricing_fields:
field_value = deployment.litellm_params.get(field)
if field_value is not None:
_model_info_dict[field] = field_value
litellm.register_model(model_cost={_model_id: _model_info_dict})
_base_info[field] = field_value
litellm.register_model(model_cost={_model_id: _base_info})
# add to model names
self._add_model_to_list_and_index_map(

View file

@ -332,6 +332,67 @@ def test_custom_pricing_with_router_model_id():
assert model_info["cache_read_input_token_cost"] == 0.0000006
def test_stale_db_zero_pricing_not_override_base_model():
"""
Regression test: old proxy DB records have input_cost_per_token=0.0 and
output_cost_per_token=0.0 as defaults (from the old proxy ModelInfo class).
These stale 0.0 values should NOT override the base model's real pricing
in the UUID cost entry. Intentional zero-cost via litellm_params must
still be respected.
"""
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
from litellm import Router
# Scenario 1: stale DB defaults — pricing should come from base model
router = Router(
model_list=[
{
"model_name": "claude-opus",
"litellm_params": {
"model": "anthropic/claude-opus-4-6",
"api_key": "fake",
},
"model_info": {
"id": "stale-uuid",
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
},
},
]
)
uuid_info = litellm.model_cost.get("stale-uuid", {})
# Base model pricing should be used, not stale 0.0
assert uuid_info["input_cost_per_token"] == 5e-06
assert uuid_info["output_cost_per_token"] == 2.5e-05
assert uuid_info.get("cache_creation_input_token_cost") is not None
assert uuid_info["cache_creation_input_token_cost"] > 0
# Scenario 2: intentional zero via litellm_params — should stay 0.0
router2 = Router(
model_list=[
{
"model_name": "free-claude",
"litellm_params": {
"model": "anthropic/claude-opus-4-6",
"api_key": "fake",
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
},
"model_info": {
"id": "free-uuid",
},
},
]
)
free_info = litellm.model_cost.get("free-uuid", {})
assert free_info["input_cost_per_token"] == 0.0
assert free_info["output_cost_per_token"] == 0.0
def test_azure_realtime_cost_calculator():
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")