From 1005b8832ee4fa407d95c85f7bf7fd80d77bcf22 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 18:57:16 +0000 Subject: [PATCH] fix(main): price Router request overrides and flat rates over inherited tiers in the per-call entry A Router call that sets its own price is priced from it over the registered deployment entry, PTU zeroing applied as at registration. Flat token rates on a call drop an inherited tiered_pricing table, which generic_cost_per_token would otherwise prefer --- litellm/main.py | 46 ++++++++----- .../test_register_model_custom_pricing.py | 67 +++++++++++++++++++ 2 files changed, 97 insertions(+), 16 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 1c01c304c5f..91b12b8a65b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -114,6 +114,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( messages_without_pdf_file_parts, ) from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header +from litellm.litellm_core_utils.ptu_pricing import zeroed_ptu_pricing from litellm.litellm_core_utils.reasoning_effort_capability import ( model_group_from_metadata, reasoning_effort_needs_allowing, @@ -1426,27 +1427,34 @@ _METADATA_MODEL_INFO_ID: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO: Final = TypeAdapter(ModelInfo) -def _is_router_call(kwargs: Mapping[str, object]) -> bool: - """A Router stamps ``model_info.id`` into the call's metadata and registers that deployment's full custom pricing - under that id when the deployment is added, so the cost calculator prices those calls from the registered entry.""" +def _router_deployment_model_info(kwargs: Mapping[str, object]) -> Mapping[str, object] | None: + """A Router stamps the deployment's ``model_info`` (with its ``id``) into the call's metadata and registers that + deployment's full custom pricing under that id when the deployment is added.""" for metadata_key in ("litellm_metadata", "metadata"): metadata = kwargs.get(metadata_key) if not isinstance(metadata, Mapping): continue model_info = _METADATA_MODEL_INFO_ID.validate_python(metadata).get("model_info") - if isinstance(model_info, Mapping) and model_info.get("id") is not None: - return True - return False + if isinstance(model_info, Mapping) and isinstance(model_info.get("id"), str): + return _METADATA_MODEL_INFO_ID.validate_python(model_info) + return None def _per_call_model_info( model: str, custom_llm_provider: str, kwargs: Mapping[str, object], mode: Literal["chat", "embedding"] ) -> ModelInfo | None: - """The full price entry for one call: the cost map row for the model, if any, overlaid with every custom pricing - field the caller put on the call. It travels on the logging object and is never written into ``litellm.model_cost``, - so the next call for the same model name is priced from the cost map again.""" + """The full price entry for one call, or None when the cost map (or the Router's registered deployment entry) + already prices it. The cost map row for the model is overlaid with the registered deployment entry and then with + the custom pricing fields the request itself set, so a Router call pays request overrides over deployment prices. + The entry travels on the logging object and is never written into ``litellm.model_cost``, so the next call for the + same model name is priced from the cost map again. Flat token rates on the call replace an inherited + ``tiered_pricing`` table, which would otherwise win in the cost calculator.""" + deployment: Final = _router_deployment_model_info(kwargs) + registered: Final = litellm.model_cost.get(str(deployment["id"])) if deployment is not None else None custom: Final = { - field: kwargs[field] for field in CustomPricingLiteLLMParams.model_fields if kwargs.get(field) is not None + field: kwargs[field] + for field in CustomPricingLiteLLMParams.model_fields + if kwargs.get(field) is not None and (registered is None or registered.get(field) != kwargs[field]) } if not custom: return None @@ -1461,8 +1469,16 @@ def _per_call_model_info( "mode": mode, "supported_openai_params": None, } - catalog: Final = _catalog_row(model, custom_llm_provider) or unmapped - return _MODEL_INFO.validate_python({**catalog, **custom}) + row: Final = _catalog_row(model, custom_llm_provider) or unmapped + catalog: Final = ( + {**row, **{key: value for key, value in registered.items() if value is not None}} if registered else row + ) + flat_rates_changed: Final = any( + key in custom and custom[key] != catalog.get(key) for key in ("input_cost_per_token", "output_cost_per_token") + ) + base: Final = {key: value for key, value in catalog.items() if not (flat_rates_changed and key == "tiered_pricing")} + zeroed: Final = zeroed_ptu_pricing(deployment, custom) if deployment is not None else None + return _MODEL_INFO.validate_python({**base, **custom, **(zeroed or {}), "mode": catalog.get("mode") or mode}) def _catalog_row(model: str, custom_llm_provider: str) -> ModelInfo | None: @@ -5936,8 +5952,7 @@ def completion( "aws_region_name", None ) # support region-based pricing for bedrock - if not _is_router_call(kwargs): - logging.set_per_call_pricing(_per_call_model_info(model, custom_llm_provider, kwargs, "chat")) + logging.set_per_call_pricing(_per_call_model_info(model, custom_llm_provider, kwargs, "chat")) ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ### custom_prompt_dict = {} if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token: @@ -6991,8 +7006,7 @@ def embedding( ) logging: Final[LiteLLMLoggingObj] = litellm_logging_obj - if not _is_router_call(kwargs): - logging.set_per_call_pricing(_per_call_model_info(model, custom_llm_provider, kwargs, "embedding")) + logging.set_per_call_pricing(_per_call_model_info(model, custom_llm_provider, kwargs, "embedding")) logging.update_environment_variables( model=model, user=user, diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index be9110fb9bf..46f68cc8f62 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -1560,3 +1560,70 @@ def test_direct_call_with_directional_per_second_rates_is_priced_from_them_witho finally: litellm.model_cost.pop(model_key, None) _invalidate_model_cost_lowercase_map() + + +def test_router_call_with_request_prices_bills_them_over_the_deployment_rates(): + from litellm import Router + + router: Final = Router( + model_list=[ + { + "model_name": "lit-request-override", + "litellm_params": { + "model": "openai/gpt-5.4-mini", + "api_key": "fake-key", + "mock_response": "ok", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "input_cost_per_audio_token": 4e-05, + "output_cost_per_audio_token": 8e-05, + }, + } + ] + ) + overridden: Final = router.completion( + model="lit-request-override", + messages=[{"role": "user", "content": "hello"}], + input_cost_per_token=5e-05, + output_cost_per_token=9e-05, + ) + assert overridden._hidden_params["response_cost"] == pytest.approx( + overridden.usage.prompt_tokens * 5e-05 + overridden.usage.completion_tokens * 9e-05 + ) + plain: Final = router.completion(model="lit-request-override", messages=[{"role": "user", "content": "hello"}]) + assert plain._hidden_params["response_cost"] == pytest.approx( + plain.usage.prompt_tokens * 1e-06 + plain.usage.completion_tokens * 2e-06 + ) + + +def test_direct_call_flat_rates_beat_the_catalogs_tiered_pricing(): + model_key: Final = "openai/lit-tiered-catalog-model" + litellm.register_model( + { + model_key: { + "litellm_provider": "openai", + "mode": "chat", + "tiered_pricing": [ + {"range": [0, 128000], "input_cost_per_token": 3e-07, "output_cost_per_token": 6e-07}, + {"range": [128000, 1000000], "input_cost_per_token": 6e-07, "output_cost_per_token": 1.2e-06}, + ], + } + }, + persist_across_reloads=False, + ) + try: + response: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response="ok", + input_cost_per_token=1e-05, + output_cost_per_token=2e-05, + ) + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens * 1e-05 + response.usage.completion_tokens * 2e-05 + ) + assert "input_cost_per_token" not in litellm.model_cost[model_key] + finally: + litellm.model_cost.pop(model_key, None) + _invalidate_model_cost_lowercase_map()