scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-15 09:12:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 311

fix(main): price Router request overrides and flat rates over inherited tiers in the per-call entry

Browse source 
A Router call that sets its own price is priced from it over the registered deployment entry, PTU zeroing applied as at registration. Flat token rates on a call drop an inherited tiered_pricing table, which generic_cost_per_token would otherwise prefer
This commit is contained in:
devin-ai-integration[bot] 2026-11-03 18:57:16 +00:00
parent 47a5c4b8cd
commit 1005b8832e
2 changed files with 97 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
litellm/main.py
View file

@ -114,6 +114,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
messages_without_pdf_file_parts,
)
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
from litellm.litellm_core_utils.ptu_pricing import zeroed_ptu_pricing
from litellm.litellm_core_utils.reasoning_effort_capability import (
model_group_from_metadata,
reasoning_effort_needs_allowing,
@ -1426,27 +1427,34 @@ _METADATA_MODEL_INFO_ID: Final = TypeAdapter(Mapping[str, object])
_MODEL_INFO: Final = TypeAdapter(ModelInfo)
def _is_router_call(kwargs: Mapping[str, object]) -> bool:
"""A Router stamps ``model_info.id`` into the call's metadata and registers that deployment's full custom pricing
under that id when the deployment is added, so the cost calculator prices those calls from the registered entry."""
def _router_deployment_model_info(kwargs: Mapping[str, object]) -> Mapping[str, object] | None:
"""A Router stamps the deployment's ``model_info`` (with its ``id``) into the call's metadata and registers that
deployment's full custom pricing under that id when the deployment is added."""
for metadata_key in ("litellm_metadata", "metadata"):
metadata = kwargs.get(metadata_key)
if not isinstance(metadata, Mapping):
continue
model_info = _METADATA_MODEL_INFO_ID.validate_python(metadata).get("model_info")
if isinstance(model_info, Mapping) and model_info.get("id") is not None:
return True
return False
if isinstance(model_info, Mapping) and isinstance(model_info.get("id"), str):
return _METADATA_MODEL_INFO_ID.validate_python(model_info)
return None
def _per_call_model_info(
model: str, custom_llm_provider: str, kwargs: Mapping[str, object], mode: Literal["chat", "embedding"]
) -> ModelInfo | None:
"""The full price entry for one call: the cost map row for the model, if any, overlaid with every custom pricing
field the caller put on the call. It travels on the logging object and is never written into ``litellm.model_cost``,
so the next call for the same model name is priced from the cost map again."""
"""The full price entry for one call, or None when the cost map (or the Router's registered deployment entry)
already prices it. The cost map row for the model is overlaid with the registered deployment entry and then with
the custom pricing fields the request itself set, so a Router call pays request overrides over deployment prices.
The entry travels on the logging object and is never written into ``litellm.model_cost``, so the next call for the
same model name is priced from the cost map again. Flat token rates on the call replace an inherited
``tiered_pricing`` table, which would otherwise win in the cost calculator."""
deployment: Final = _router_deployment_model_info(kwargs)
registered: Final = litellm.model_cost.get(str(deployment["id"])) if deployment is not None else None
custom: Final = {
field: kwargs[field] for field in CustomPricingLiteLLMParams.model_fields if kwargs.get(field) is not None
field: kwargs[field]
for field in CustomPricingLiteLLMParams.model_fields
if kwargs.get(field) is not None and (registered is None or registered.get(field) != kwargs[field])
}
if not custom:
return None
@ -1461,8 +1469,16 @@ def _per_call_model_info(
"mode": mode,
"supported_openai_params": None,
}
catalog: Final = _catalog_row(model, custom_llm_provider) or unmapped
return _MODEL_INFO.validate_python({**catalog, **custom})
row: Final = _catalog_row(model, custom_llm_provider) or unmapped
catalog: Final = (
{**row, **{key: value for key, value in registered.items() if value is not None}} if registered else row
)
flat_rates_changed: Final = any(
key in custom and custom[key] != catalog.get(key) for key in ("input_cost_per_token", "output_cost_per_token")
)
base: Final = {key: value for key, value in catalog.items() if not (flat_rates_changed and key == "tiered_pricing")}
zeroed: Final = zeroed_ptu_pricing(deployment, custom) if deployment is not None else None
return _MODEL_INFO.validate_python({**base, **custom, **(zeroed or {}), "mode": catalog.get("mode") or mode})
def _catalog_row(model: str, custom_llm_provider: str) -> ModelInfo | None:
@ -5936,8 +5952,7 @@ def completion(
"aws_region_name", None
) # support region-based pricing for bedrock
if not _is_router_call(kwargs):
logging.set_per_call_pricing(_per_call_model_info(model, custom_llm_provider, kwargs, "chat"))
logging.set_per_call_pricing(_per_call_model_info(model, custom_llm_provider, kwargs, "chat"))
### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ###
custom_prompt_dict = {}
if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token:
@ -6991,8 +7006,7 @@ def embedding(
)
logging: Final[LiteLLMLoggingObj] = litellm_logging_obj
if not _is_router_call(kwargs):
logging.set_per_call_pricing(_per_call_model_info(model, custom_llm_provider, kwargs, "embedding"))
logging.set_per_call_pricing(_per_call_model_info(model, custom_llm_provider, kwargs, "embedding"))
logging.update_environment_variables(
model=model,
user=user,

67
tests/unit/test_register_model_custom_pricing.py
View file

@ -1560,3 +1560,70 @@ def test_direct_call_with_directional_per_second_rates_is_priced_from_them_witho
finally:
litellm.model_cost.pop(model_key, None)
_invalidate_model_cost_lowercase_map()
def test_router_call_with_request_prices_bills_them_over_the_deployment_rates():
from litellm import Router
router: Final = Router(
model_list=[
{
"model_name": "lit-request-override",
"litellm_params": {
"model": "openai/gpt-5.4-mini",
"api_key": "fake-key",
"mock_response": "ok",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 2e-06,
"input_cost_per_audio_token": 4e-05,
"output_cost_per_audio_token": 8e-05,
},
}
]
)
overridden: Final = router.completion(
model="lit-request-override",
messages=[{"role": "user", "content": "hello"}],
input_cost_per_token=5e-05,
output_cost_per_token=9e-05,
)
assert overridden._hidden_params["response_cost"] == pytest.approx(
overridden.usage.prompt_tokens * 5e-05 + overridden.usage.completion_tokens * 9e-05
)
plain: Final = router.completion(model="lit-request-override", messages=[{"role": "user", "content": "hello"}])
assert plain._hidden_params["response_cost"] == pytest.approx(
plain.usage.prompt_tokens * 1e-06 + plain.usage.completion_tokens * 2e-06
)
def test_direct_call_flat_rates_beat_the_catalogs_tiered_pricing():
model_key: Final = "openai/lit-tiered-catalog-model"
litellm.register_model(
{
model_key: {
"litellm_provider": "openai",
"mode": "chat",
"tiered_pricing": [
{"range": [0, 128000], "input_cost_per_token": 3e-07, "output_cost_per_token": 6e-07},
{"range": [128000, 1000000], "input_cost_per_token": 6e-07, "output_cost_per_token": 1.2e-06},
],
}
},
persist_across_reloads=False,
)
try:
response: Final = litellm.completion(
model=model_key,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
mock_response="ok",
input_cost_per_token=1e-05,
output_cost_per_token=2e-05,
)
assert response._hidden_params["response_cost"] == pytest.approx(
response.usage.prompt_tokens * 1e-05 + response.usage.completion_tokens * 2e-05
)
assert "input_cost_per_token" not in litellm.model_cost[model_key]
finally:
litellm.model_cost.pop(model_key, None)
_invalidate_model_cost_lowercase_map()