mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-08 22:21:35 +00:00
Fix zero cost for claude-opus-4-6 when proxy DB has stale 0.0 pricing defaults
The old proxy ModelInfo class defaulted input_cost_per_token and output_cost_per_token to 0.0 instead of None. These stale values in existing DB records overwrote the base model's real pricing in the UUID cost entry, causing cost=0 for models like claude-opus-4-6. Changes: - Change proxy ModelInfo defaults from 0.0 to None to prevent new stale entries - When building UUID cost entries, start from base model pricing and skip 0.0 pricing fields from model_info (stale DB defaults), while preserving intentional zero-cost set via litellm_params - Add regression test for the stale DB defaults scenario Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
4eee2b752d
commit
3715981199
3 changed files with 143 additions and 13 deletions
|
|
@ -782,8 +782,8 @@ class ModelInfoDelete(LiteLLMPydanticObjectBase):
|
|||
class ModelInfo(LiteLLMPydanticObjectBase):
|
||||
id: Optional[str]
|
||||
mode: Optional[Literal["embedding", "chat", "completion"]]
|
||||
input_cost_per_token: Optional[float] = 0.0
|
||||
output_cost_per_token: Optional[float] = 0.0
|
||||
input_cost_per_token: Optional[float] = None
|
||||
output_cost_per_token: Optional[float] = None
|
||||
max_tokens: Optional[int] = 2048 # assume 2048 if not set
|
||||
|
||||
# for azure models we need users to specify the base model, one azure you can call deployments - azure/my-random-model
|
||||
|
|
|
|||
|
|
@ -207,6 +207,24 @@ class RoutingArgs(enum.Enum):
|
|||
ttl = 60 # 1min (RPM/TPM expire key)
|
||||
|
||||
|
||||
def _get_base_model_cost_info(model_name: str) -> dict:
|
||||
"""
|
||||
Get the base model's pricing info from litellm.model_cost.
|
||||
|
||||
Used when registering deployment-specific pricing under UUID to ensure
|
||||
the UUID entry inherits complete pricing (e.g. cache costs) from the
|
||||
base model, even if the deployment's model_info only has a subset of
|
||||
pricing fields.
|
||||
|
||||
Returns an empty dict if the model is not found.
|
||||
"""
|
||||
try:
|
||||
info = litellm.get_model_info(model=model_name)
|
||||
return dict(info)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
class Router:
|
||||
model_names: set = set()
|
||||
cache_responses: Optional[bool] = False
|
||||
|
|
@ -6390,14 +6408,6 @@ class Router:
|
|||
|
||||
## REGISTER MODEL INFO IN LITELLM MODEL COST MAP
|
||||
model_id = deployment.model_info.id
|
||||
if model_id is not None:
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
model_id: _model_info,
|
||||
}
|
||||
)
|
||||
|
||||
## OLD MODEL REGISTRATION ## Kept to prevent breaking changes
|
||||
_model_name = deployment.litellm_params.model
|
||||
if deployment.litellm_params.custom_llm_provider is not None:
|
||||
_model_name = (
|
||||
|
|
@ -6419,6 +6429,44 @@ class Router:
|
|||
}
|
||||
)
|
||||
|
||||
# Register deployment-specific pricing under UUID.
|
||||
# Start with the base model's full pricing so that the UUID
|
||||
# entry inherits all pricing fields (e.g. cache costs).
|
||||
# Then overlay the deployment's model_info on top (which
|
||||
# includes any custom pricing from both model_info config
|
||||
# and litellm_params).
|
||||
# Without this, UUID entries miss pricing fields not
|
||||
# explicitly set on the deployment, causing $0 cost when
|
||||
# custom_pricing=True triggers UUID-based cost lookup.
|
||||
if model_id is not None:
|
||||
_model_info_for_id = _get_base_model_cost_info(
|
||||
model_name=_model_name
|
||||
)
|
||||
# Overlay model_info but skip 0.0 pricing fields — these are
|
||||
# stale DB defaults from the old proxy ModelInfo class that
|
||||
# defaulted pricing to 0.0 instead of None. Intentional
|
||||
# zero-cost pricing set via litellm_params is already in
|
||||
# _model_info (copied above) and will be re-applied below.
|
||||
_pricing_fields = set(
|
||||
CustomPricingLiteLLMParams.model_fields.keys()
|
||||
)
|
||||
for k, v in _model_info.items():
|
||||
if v is None:
|
||||
continue
|
||||
if k in _pricing_fields and v == 0.0:
|
||||
continue
|
||||
_model_info_for_id[k] = v
|
||||
# Re-apply litellm_params pricing (intentional, even if 0.0)
|
||||
for field in _pricing_fields:
|
||||
field_value = deployment.litellm_params.get(field)
|
||||
if field_value is not None:
|
||||
_model_info_for_id[field] = field_value
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
model_id: _model_info_for_id,
|
||||
}
|
||||
)
|
||||
|
||||
## Check if LLM Deployment is allowed for this deployment
|
||||
if (
|
||||
self.deployment_is_active_for_environment(deployment=deployment)
|
||||
|
|
@ -6870,11 +6918,32 @@ class Router:
|
|||
_model_info_dict: dict = deployment.model_info.model_dump(
|
||||
exclude_none=True
|
||||
)
|
||||
for field in CustomPricingLiteLLMParams.model_fields.keys():
|
||||
|
||||
# Start with the base model's full pricing so that the UUID
|
||||
# entry inherits all pricing fields (e.g. cache costs).
|
||||
_model_name = deployment.litellm_params.model
|
||||
if deployment.litellm_params.custom_llm_provider is not None:
|
||||
_model_name = (
|
||||
deployment.litellm_params.custom_llm_provider
|
||||
+ "/"
|
||||
+ _model_name
|
||||
)
|
||||
_base_info = _get_base_model_cost_info(model_name=_model_name)
|
||||
# Skip 0.0 pricing fields — stale DB defaults (see _create_deployment)
|
||||
_pricing_fields = set(
|
||||
CustomPricingLiteLLMParams.model_fields.keys()
|
||||
)
|
||||
for k, v in _model_info_dict.items():
|
||||
if v is None:
|
||||
continue
|
||||
if k in _pricing_fields and v == 0.0:
|
||||
continue
|
||||
_base_info[k] = v
|
||||
for field in _pricing_fields:
|
||||
field_value = deployment.litellm_params.get(field)
|
||||
if field_value is not None:
|
||||
_model_info_dict[field] = field_value
|
||||
litellm.register_model(model_cost={_model_id: _model_info_dict})
|
||||
_base_info[field] = field_value
|
||||
litellm.register_model(model_cost={_model_id: _base_info})
|
||||
|
||||
# add to model names
|
||||
self._add_model_to_list_and_index_map(
|
||||
|
|
|
|||
|
|
@ -332,6 +332,67 @@ def test_custom_pricing_with_router_model_id():
|
|||
assert model_info["cache_read_input_token_cost"] == 0.0000006
|
||||
|
||||
|
||||
def test_stale_db_zero_pricing_not_override_base_model():
|
||||
"""
|
||||
Regression test: old proxy DB records have input_cost_per_token=0.0 and
|
||||
output_cost_per_token=0.0 as defaults (from the old proxy ModelInfo class).
|
||||
These stale 0.0 values should NOT override the base model's real pricing
|
||||
in the UUID cost entry. Intentional zero-cost via litellm_params must
|
||||
still be respected.
|
||||
"""
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
from litellm import Router
|
||||
|
||||
# Scenario 1: stale DB defaults — pricing should come from base model
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude-opus",
|
||||
"litellm_params": {
|
||||
"model": "anthropic/claude-opus-4-6",
|
||||
"api_key": "fake",
|
||||
},
|
||||
"model_info": {
|
||||
"id": "stale-uuid",
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
uuid_info = litellm.model_cost.get("stale-uuid", {})
|
||||
# Base model pricing should be used, not stale 0.0
|
||||
assert uuid_info["input_cost_per_token"] == 5e-06
|
||||
assert uuid_info["output_cost_per_token"] == 2.5e-05
|
||||
assert uuid_info.get("cache_creation_input_token_cost") is not None
|
||||
assert uuid_info["cache_creation_input_token_cost"] > 0
|
||||
|
||||
# Scenario 2: intentional zero via litellm_params — should stay 0.0
|
||||
router2 = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "free-claude",
|
||||
"litellm_params": {
|
||||
"model": "anthropic/claude-opus-4-6",
|
||||
"api_key": "fake",
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
},
|
||||
"model_info": {
|
||||
"id": "free-uuid",
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
free_info = litellm.model_cost.get("free-uuid", {})
|
||||
assert free_info["input_cost_per_token"] == 0.0
|
||||
assert free_info["output_cost_per_token"] == 0.0
|
||||
|
||||
|
||||
def test_azure_realtime_cost_calculator():
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue