From 6baa82c72220233426101e7d618ed65e3ae1faa1 Mon Sep 17 00:00:00 2001 From: Willow Lopez <100782273+Oxygen56@users.noreply.github.com> Date: Wed, 1 Jul 2026 11:28:47 +0800 Subject: [PATCH 1/3] fix: honor explicit custom pricing for BYOK deployments --- litellm/cost_calculator.py | 4 + litellm/litellm_core_utils/litellm_logging.py | 186 +++++++++- litellm/main.py | 82 ++++- .../test_custom_pricing_robustness.py | 347 ++++++++++++++++++ 4 files changed, 595 insertions(+), 24 deletions(-) create mode 100644 tests/test_litellm/test_custom_pricing_robustness.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 98e00cf5788..a543f33cd53 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1804,6 +1804,8 @@ def response_cost_calculator( cache_hit: Optional[bool] = None, base_model: Optional[str] = None, custom_pricing: Optional[bool] = None, + custom_cost_per_token: Optional[CostPerToken] = None, + custom_cost_per_second: Optional[float] = None, prompt: str = "", standard_built_in_tools_params: Optional[StandardBuiltInToolsParams] = None, litellm_model_name: Optional[str] = None, @@ -1837,6 +1839,8 @@ def response_cost_calculator( custom_llm_provider=custom_llm_provider, optional_params=optional_params, custom_pricing=custom_pricing, + custom_cost_per_token=custom_cost_per_token, + custom_cost_per_second=custom_cost_per_second, base_model=base_model, prompt=prompt, standard_built_in_tools_params=standard_built_in_tools_params, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 973048814f1..bd5d3bd6151 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1,6 +1,8 @@ # What is this? ## Common Utility file for Logging handler # Logging function -> log the exact model details + what's being sent | Non-Blocking +from __future__ import annotations + import copy import datetime import json @@ -170,6 +172,9 @@ from .specialty_caches.dynamic_logging_cache import DynamicLoggingCache if TYPE_CHECKING: from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig + from litellm.types.utils import CostPerToken +else: + CostPerToken = dict try: from litellm_enterprise.enterprise_callbacks.callback_controls import ( EnterpriseCallbackControls, @@ -1523,10 +1528,26 @@ class Logging(LiteLLMLoggingBaseClass): router_model_id = self.get_router_model_id() ## RESPONSE COST ## + _litellm_params_for_cost = ( + self.litellm_params if hasattr(self, "litellm_params") else None + ) custom_pricing = use_custom_pricing_for_model( - litellm_params=( - self.litellm_params if hasattr(self, "litellm_params") else None - ) + litellm_params=_litellm_params_for_cost + ) + # Thread the deployment's explicit per-token rates through to the cost + # calculator so pricing does not depend on the deployment-id entry + # surviving in the process-global litellm.model_cost (which is wiped on + # model-cost-map reloads). See GitHub issue #30081. + # Only trust litellm_params['model_info'] when injected by the router + # (not client-supplied), to prevent zero-cost billing bypass. + _is_router_deployment = self.get_router_model_id() is not None + custom_cost_per_token = extract_custom_cost_per_token( + litellm_params=_litellm_params_for_cost, + _model_info_from_deployment=_is_router_deployment, + ) + custom_cost_per_second = extract_custom_cost_per_second( + litellm_params=_litellm_params_for_cost, + _model_info_from_deployment=_is_router_deployment, ) prompt = "" # use for tts cost calc @@ -1551,6 +1572,8 @@ class Logging(LiteLLMLoggingBaseClass): "call_type": self.call_type, "optional_params": self.optional_params, "custom_pricing": custom_pricing, + "custom_cost_per_token": custom_cost_per_token, + "custom_cost_per_second": custom_cost_per_second, "prompt": prompt, "standard_built_in_tools_params": self.standard_built_in_tools_params, "router_model_id": router_model_id, @@ -4658,6 +4681,160 @@ def use_custom_pricing_for_model(litellm_params: Optional[dict]) -> bool: return False +def _coerce_cost_value(value: Any) -> Optional[float]: + """Coerce a configured cost value to float, returning None when absent/invalid. + + Note: ``0`` / ``0.0`` are valid (e.g. BYOK deployments priced at zero), so we + only treat ``None`` and non-numeric values as "missing". + """ + if value is None: + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _custom_cost_per_token_from_source( + source: Optional[dict], +) -> Optional[CostPerToken]: + """Build a ``CostPerToken`` from a single dict source (litellm_params or model_info). + + Only returns a value when BOTH ``input_cost_per_token`` and + ``output_cost_per_token`` are explicitly present (not ``None``). This keeps + partially-configured deployments on the standard pricing path instead of + silently treating a missing rate as zero. + """ + if not source or not isinstance(source, dict): + return None + + input_cost = _coerce_cost_value(source.get("input_cost_per_token")) + output_cost = _coerce_cost_value(source.get("output_cost_per_token")) + if input_cost is None or output_cost is None: + return None + + custom_cost_per_token: CostPerToken = { + "input_cost_per_token": input_cost, + "output_cost_per_token": output_cost, + } + + cache_read_cost = _coerce_cost_value(source.get("cache_read_input_token_cost")) + if cache_read_cost is not None: + custom_cost_per_token["cache_read_input_token_cost"] = cache_read_cost + + cache_creation_cost = _coerce_cost_value( + source.get("cache_creation_input_token_cost") + ) + if cache_creation_cost is not None: + custom_cost_per_token["cache_creation_input_token_cost"] = cache_creation_cost + + return custom_cost_per_token + + +def extract_custom_cost_per_token( + litellm_params: Optional[dict], + *, + _model_info_from_deployment: bool = False, +) -> Optional[CostPerToken]: + """Extract a deployment's explicit per-token pricing from ``litellm_params``. + + Looks for ``input_cost_per_token`` / ``output_cost_per_token`` (plus optional + cache rates) in the following sources, in priority order: + + 1. The top-level ``litellm_params`` (per-request custom pricing). + 2. ``litellm_params['model_info']`` — **only when + ``_model_info_from_deployment`` is True** (set by the router when it + injected the deployment configuration). Gated to prevent proxy clients + from supplying a client-controlled ``model_info`` dict with zero rates + that bypasses spend tracking and budget enforcement. + 3. ``litellm_params['metadata']['model_info']``. + 4. ``litellm_params['litellm_metadata']['model_info']`` (used by + ``/responses`` and ``/messages`` generic-api routes). + + The resulting ``CostPerToken`` is passed to the cost calculator so that a + deployment configured with explicit rates is always priced with those rates, + regardless of whether its entry still exists in ``litellm.model_cost`` + (which is wiped wholesale on model-cost-map reloads). See issue #30081. + """ + if litellm_params is None: + return None + + # 1. top-level litellm_params (per-request custom pricing) + result = _custom_cost_per_token_from_source(litellm_params) + if result is not None: + return result + + # 2. litellm_params['model_info'] — only when injected by the router + if _model_info_from_deployment: + result = _custom_cost_per_token_from_source(litellm_params.get("model_info")) + if result is not None: + return result + + # 3 + 4. model_info nested under metadata / litellm_metadata + for metadata_key in ("metadata", "litellm_metadata"): + metadata = litellm_params.get(metadata_key) or {} + if isinstance(metadata, dict): + result = _custom_cost_per_token_from_source(metadata.get("model_info")) + if result is not None: + return result + + return None + + +def extract_custom_cost_per_second( + litellm_params: Optional[dict], + *, + _model_info_from_deployment: bool = False, +) -> Optional[float]: + """Extract a deployment's explicit per-second pricing from ``litellm_params``. + + Looks for ``input_cost_per_second`` in the same sources as + ``extract_custom_cost_per_token``, in priority order: + + 1. The top-level ``litellm_params`` (per-request custom pricing). + 2. ``litellm_params['model_info']`` — **only when + ``_model_info_from_deployment`` is True** (set by the router when it + injected the deployment configuration). Gated to prevent proxy clients + from supplying a client-controlled ``model_info`` dict with zero rates + that bypasses spend tracking and budget enforcement. + 3. ``litellm_params['metadata']['model_info']``. + 4. ``litellm_params['litellm_metadata']['model_info']`` (used by + ``/responses`` and ``/messages`` generic-api routes). + + Returns the first non-None value, or ``None`` if no per-second rate is + configured. This is threaded through to ``completion_cost`` so that + per-second custom pricing survives model-cost-map reloads (same root cause + as issue #30081). + """ + if litellm_params is None: + return None + + # 1. top-level litellm_params (per-request custom pricing) + cost = _coerce_cost_value(litellm_params.get("input_cost_per_second")) + if cost is not None: + return cost + + # 2. litellm_params['model_info'] — only when injected by the router + if _model_info_from_deployment: + model_info = litellm_params.get("model_info") + if isinstance(model_info, dict): + cost = _coerce_cost_value(model_info.get("input_cost_per_second")) + if cost is not None: + return cost + + # 3 + 4. model_info nested under metadata / litellm_metadata + for metadata_key in ("metadata", "litellm_metadata"): + metadata = litellm_params.get(metadata_key) or {} + if isinstance(metadata, dict): + model_info = metadata.get("model_info") + if isinstance(model_info, dict): + cost = _coerce_cost_value(model_info.get("input_cost_per_second")) + if cost is not None: + return cost + + return None + + def is_valid_sha256_hash(value: str) -> bool: # Check if the value is a valid SHA-256 hash (64 hexadecimal characters) return bool(re.fullmatch(r"[a-fA-F0-9]{64}", value)) @@ -5770,9 +5947,6 @@ def _get_traceback_str_for_error(error_str: str) -> str: from decimal import Decimal -# used for unit testing -from typing import Any, Dict, List, Optional, Union - def create_dummy_standard_logging_payload() -> StandardLoggingPayload: # First create the nested objects with proper typing diff --git a/litellm/main.py b/litellm/main.py index e17a5ad9a48..933b76d3a3c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1080,6 +1080,34 @@ def _build_custom_pricing_entry( return entry +def _model_has_known_pricing(model: str, custom_llm_provider: Optional[str]) -> bool: + """Return True if ``model`` already resolves to a priced entry in litellm.model_cost. + + Per-request custom pricing must not overwrite the shared canonical + ``litellm.model_cost`` entry, otherwise a single request that supplies + ``input_cost_per_token`` (e.g. a BYOK deployment priced at 0) would re-price + the canonical model for the whole process and silently mis-bill all other + traffic. Known models get their per-request rates applied via the + ``custom_cost_per_token`` threading in the cost calculator instead, so the + registration is only needed for otherwise-unknown models. See issue #30081. + """ + try: + existing_model_info = litellm.get_model_info( + model=model, custom_llm_provider=custom_llm_provider + ) + except Exception: + return False + + if not existing_model_info: + return False + + return ( + existing_model_info.get("input_cost_per_token") is not None + or existing_model_info.get("output_cost_per_token") is not None + or existing_model_info.get("input_cost_per_second") is not None + ) + + @tracer.wrap() @client def completion( # type: ignore # noqa: PLR0915 @@ -1447,15 +1475,24 @@ def completion( # type: ignore # noqa: PLR0915 if ( input_cost_per_token is not None and output_cost_per_token is not None ) or input_cost_per_second is not None: - litellm.register_model( - { - f"{custom_llm_provider}/{model}": _build_custom_pricing_entry( - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) - } - ) + if _model_has_known_pricing( + model=model, custom_llm_provider=custom_llm_provider + ): + verbose_logger.warning( + f"Custom pricing for model='{model}' not registered: model already " + f"has canonical pricing in litellm.model_cost. Per-request rates " + f"will be applied via cost threading for this request only." + ) + else: + litellm.register_model( + { + f"{custom_llm_provider}/{model}": _build_custom_pricing_entry( + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + model_info=model_info, + ) + } + ) ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ### custom_prompt_dict = {} # type: ignore if ( @@ -4889,15 +4926,24 @@ def embedding( # noqa: PLR0915 if ( input_cost_per_token is not None and output_cost_per_token is not None ) or input_cost_per_second is not None: - litellm.register_model( - { - f"{custom_llm_provider}/{model}": _build_custom_pricing_entry( - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=kwargs.get("model_info"), - ) - } - ) + if _model_has_known_pricing( + model=model, custom_llm_provider=custom_llm_provider + ): + verbose_logger.warning( + f"Custom pricing for model='{model}' not registered: model already " + f"has canonical pricing in litellm.model_cost. Per-request rates " + f"will be applied via cost threading for this request only." + ) + else: + litellm.register_model( + { + f"{custom_llm_provider}/{model}": _build_custom_pricing_entry( + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + model_info=kwargs.get("model_info"), + ) + } + ) litellm_params_dict = get_litellm_params(**kwargs) diff --git a/tests/test_litellm/test_custom_pricing_robustness.py b/tests/test_litellm/test_custom_pricing_robustness.py new file mode 100644 index 00000000000..bc945978cbe --- /dev/null +++ b/tests/test_litellm/test_custom_pricing_robustness.py @@ -0,0 +1,347 @@ +"""Regression tests for deployment / per-request custom pricing robustness. + +Covers the two intertwined billing bugs described in +https://github.com/BerriAI/litellm/issues/30081: + +Bug 1 - custom pricing lost when ``litellm.model_cost`` is replaced + (e.g. after a model-cost-map reload). The deployment's explicit + ``input_cost_per_token`` / ``output_cost_per_token`` must still be + honored, because they are threaded through the cost calculator as + ``custom_cost_per_token`` rather than depending on the deployment-id + entry surviving in the process-global ``litellm.model_cost``. + +Bug 2 - per-request custom pricing must apply to that request only and must + NOT overwrite (poison) the shared canonical ``model_cost`` entry used + by all other traffic. +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +import litellm +from litellm import Usage +from litellm.litellm_core_utils.litellm_logging import ( + extract_custom_cost_per_second, + extract_custom_cost_per_token, +) + + +def _make_model_response(model: str, prompt_tokens: int, completion_tokens: int): + return litellm.ModelResponse( + model=model, + usage=Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + ) + + +# --------------------------------------------------------------------------- +# Bug 1 - custom_cost_per_token threading +# --------------------------------------------------------------------------- +def test_response_cost_calculator_honors_zero_custom_cost_per_token(): + """A deployment priced at 0 stays at 0 even when its model_cost entry is gone. + + Simulates the post-reload state: ``router_model_id`` is not present in + ``litellm.model_cost`` and ``custom_pricing`` is detected from the + deployment params. With the explicit rates threaded through as + ``custom_cost_per_token`` the cost is 0, instead of falling back to the + model's public price. + """ + response = _make_model_response( + "gpt-4o", prompt_tokens=100_000, completion_tokens=10_000 + ) + + cost = litellm.response_cost_calculator( + response_object=response, + model="gpt-4o", + custom_llm_provider="openai", + call_type="acompletion", + optional_params={}, + custom_pricing=True, + router_model_id="id-not-in-model-cost", # wiped by a reload + custom_cost_per_token={ + "input_cost_per_token": 0, + "output_cost_per_token": 0, + }, + ) + + assert cost == 0.0 + + +def test_response_cost_calculator_falls_back_without_custom_cost_per_token(): + """Control: without the threaded rates the same request bills at public price. + + This is the buggy behaviour from the issue - it confirms the threading in + :func:`test_response_cost_calculator_honors_zero_custom_cost_per_token` is + what fixes the silent mis-billing. + """ + response = _make_model_response( + "gpt-4o", prompt_tokens=100_000, completion_tokens=10_000 + ) + + cost = litellm.response_cost_calculator( + response_object=response, + model="gpt-4o", + custom_llm_provider="openai", + call_type="acompletion", + optional_params={}, + custom_pricing=True, + router_model_id="id-not-in-model-cost", + ) + + assert cost > 0.0 + + +def test_response_cost_calculator_honors_nonzero_custom_cost_per_token(): + """Explicit non-zero rates are applied exactly, regardless of model_cost.""" + response = _make_model_response( + "gpt-4o", prompt_tokens=1_000, completion_tokens=2_000 + ) + + cost = litellm.response_cost_calculator( + response_object=response, + model="gpt-4o", + custom_llm_provider="openai", + call_type="acompletion", + optional_params={}, + custom_pricing=True, + router_model_id="id-not-in-model-cost", + custom_cost_per_token={ + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + }, + ) + + # 1000 * 1e-06 + 2000 * 2e-06 = 0.001 + 0.004 + assert cost == pytest.approx(0.005) + + +# --------------------------------------------------------------------------- +# extract_custom_cost_per_token helper +# --------------------------------------------------------------------------- +def test_extract_from_top_level_litellm_params(): + result = extract_custom_cost_per_token( + {"input_cost_per_token": 0, "output_cost_per_token": 0} + ) + assert result == {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0} + + +def test_extract_from_metadata_model_info(): + litellm_params = { + "metadata": { + "model_info": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + } + } + } + result = extract_custom_cost_per_token(litellm_params) + assert result == { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + } + + +def test_extract_from_litellm_metadata_model_info(): + litellm_params = { + "litellm_metadata": { + "model_info": { + "input_cost_per_token": 0, + "output_cost_per_token": 0, + } + } + } + result = extract_custom_cost_per_token(litellm_params) + assert result == {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0} + + +def test_extract_includes_cache_rates_when_present(): + result = extract_custom_cost_per_token( + { + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "cache_read_input_token_cost": 0, + "cache_creation_input_token_cost": 0, + } + ) + assert result == { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "cache_read_input_token_cost": 0.0, + "cache_creation_input_token_cost": 0.0, + } + + +def test_extract_returns_none_for_partial_pricing(): + # Only input set -> stay on the standard pricing path (don't assume 0 output). + assert extract_custom_cost_per_token({"input_cost_per_token": 0}) is None + + +def test_extract_returns_none_when_absent(): + assert extract_custom_cost_per_token(None) is None + assert extract_custom_cost_per_token({}) is None + assert extract_custom_cost_per_token({"model_info": {}}) is None + + +# --------------------------------------------------------------------------- +# extract_custom_cost_per_second +# --------------------------------------------------------------------------- +def test_extract_custom_cost_per_second_top_level(): + result = extract_custom_cost_per_second({"input_cost_per_second": 0.005}) + assert result == 0.005 + + +def test_extract_custom_cost_per_second_zero(): + result = extract_custom_cost_per_second({"input_cost_per_second": 0}) + assert result == 0.0 + + +def test_extract_custom_cost_per_second_skips_model_info_without_deployment_flag(): + """Top-level model_info must be ignored when _model_info_from_deployment is False. + + This prevents proxy clients from injecting ``model_info: {input_cost_per_second: 0}`` + to bypass spend tracking for per-second priced deployments.""" + result = extract_custom_cost_per_second( + {"model_info": {"input_cost_per_second": 0.01}} + ) + assert result is None + + +def test_extract_custom_cost_per_second_from_model_info_with_flag(): + result = extract_custom_cost_per_second( + {"model_info": {"input_cost_per_second": 0.01}}, + _model_info_from_deployment=True, + ) + assert result == 0.01 + + +def test_extract_custom_cost_per_second_from_metadata(): + result = extract_custom_cost_per_second( + {"metadata": {"model_info": {"input_cost_per_second": 0.02}}} + ) + assert result == 0.02 + + +def test_extract_custom_cost_per_second_absent(): + assert extract_custom_cost_per_second(None) is None + assert extract_custom_cost_per_second({}) is None + + +# --------------------------------------------------------------------------- +# extract_custom_cost_per_token — security gate +# --------------------------------------------------------------------------- +def test_extract_skips_model_info_without_deployment_flag(): + """Top-level model_info must be ignored when _model_info_from_deployment is False. + + This prevents proxy clients from injecting ``model_info: {input_cost_per_token: 0, + output_cost_per_token: 0}`` to bypass spend tracking.""" + litellm_params = { + "model_info": { + "input_cost_per_token": 0, + "output_cost_per_token": 0, + } + } + result = extract_custom_cost_per_token(litellm_params) + assert result is None + + +def test_extract_uses_model_info_with_deployment_flag(): + litellm_params = { + "model_info": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + } + } + result = extract_custom_cost_per_token( + litellm_params, _model_info_from_deployment=True + ) + assert result == { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + } + + +# --------------------------------------------------------------------------- +# response_cost_calculator — custom_cost_per_second threading +# --------------------------------------------------------------------------- +def test_response_cost_calculator_honors_custom_cost_per_second(): + r"""Per-second custom pricing is threaded through the cost calculator. + + completion_cost reads timing from _response_ms on the + response object, so timing must be set there for per-second billing. + """ + response = _make_model_response("gpt-4o", prompt_tokens=100, completion_tokens=0) + response._response_ms = 5000.0 + + # 0.01 $/s * 5 s = 0.05 + cost = litellm.response_cost_calculator( + response_object=response, + model="gpt-4o", + custom_llm_provider="openai", + call_type="acompletion", + optional_params={}, + custom_pricing=True, + router_model_id="id-not-in-model-cost", + custom_cost_per_second=0.01, + ) + + assert cost == pytest.approx(0.05) + + +# --------------------------------------------------------------------------- +# Bug 2 - per-request pricing must not poison the canonical model_cost entry +# --------------------------------------------------------------------------- +def test_per_request_custom_pricing_does_not_poison_canonical_entry(): + """A single zero-priced request must not re-price the shared model entry. + + Verifies both: + - The request itself is billed at the custom (zero) rate. + - The canonical ``litellm.model_cost`` entry is left unchanged + for all other traffic. + """ + model = "gpt-4o" + assert model in litellm.model_cost + canonical_input_cost_before = litellm.model_cost[model]["input_cost_per_token"] + canonical_output_cost_before = litellm.model_cost[model]["output_cost_per_token"] + assert canonical_input_cost_before > 0 + + response = litellm.completion( + model=model, + messages=[{"role": "user", "content": "hi"}], + mock_response="ok", + input_cost_per_token=0, + output_cost_per_token=0, + ) + + # The canonical entry must not be mutated. + assert ( + litellm.model_cost[model]["input_cost_per_token"] == canonical_input_cost_before + ) + assert ( + litellm.model_cost[model]["output_cost_per_token"] + == canonical_output_cost_before + ) + + # The per-request cost must also be calculated with the custom rates. + # Verify via the cost calculator directly (mock_response bypasses the + # normal cost-calculation path, so the response object may not carry + # cost metadata). + cost = litellm.response_cost_calculator( + response_object=response, + model=model, + custom_llm_provider="openai", + call_type="completion", + optional_params={}, + custom_pricing=True, + custom_cost_per_token={ + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, + ) + assert cost == 0.0 From 4031f8339a91ae137c8444443c64a65354e92c19 Mon Sep 17 00:00:00 2001 From: Willow Lopez <100782273+Oxygen56@users.noreply.github.com> Date: Wed, 1 Jul 2026 11:40:30 +0800 Subject: [PATCH 2/3] test: update interactions OpenAPI response fields --- tests/test_litellm/interactions/test_openapi_compliance.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_litellm/interactions/test_openapi_compliance.py b/tests/test_litellm/interactions/test_openapi_compliance.py index aededaaca77..509ded8c704 100644 --- a/tests/test_litellm/interactions/test_openapi_compliance.py +++ b/tests/test_litellm/interactions/test_openapi_compliance.py @@ -165,7 +165,6 @@ class TestResponseCompliance: "status", "created", "updated", - "role", "steps", "usage", ] From 8c8fe0ee7271e4df11b37ef99927b4b385e9c1c4 Mon Sep 17 00:00:00 2001 From: Oxygen56 Date: Thu, 17 Sep 2026 22:52:23 +0800 Subject: [PATCH 3/3] fix: thread trusted deployment pricing through completion logging --- litellm/litellm_core_utils/litellm_logging.py | 117 +------- litellm/main.py | 5 +- litellm/router.py | 7 + .../test_custom_pricing_robustness.py | 260 +++++++++++------- 4 files changed, 176 insertions(+), 213 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index bd5d3bd6151..6c1a29ae9dc 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1,8 +1,6 @@ # What is this? ## Common Utility file for Logging handler # Logging function -> log the exact model details + what's being sent | Non-Blocking -from __future__ import annotations - import copy import datetime import json @@ -1534,20 +1532,9 @@ class Logging(LiteLLMLoggingBaseClass): custom_pricing = use_custom_pricing_for_model( litellm_params=_litellm_params_for_cost ) - # Thread the deployment's explicit per-token rates through to the cost - # calculator so pricing does not depend on the deployment-id entry - # surviving in the process-global litellm.model_cost (which is wiped on - # model-cost-map reloads). See GitHub issue #30081. - # Only trust litellm_params['model_info'] when injected by the router - # (not client-supplied), to prevent zero-cost billing bypass. - _is_router_deployment = self.get_router_model_id() is not None - custom_cost_per_token = extract_custom_cost_per_token( - litellm_params=_litellm_params_for_cost, - _model_info_from_deployment=_is_router_deployment, - ) + custom_cost_per_token = extract_custom_cost_per_token(_litellm_params_for_cost) custom_cost_per_second = extract_custom_cost_per_second( - litellm_params=_litellm_params_for_cost, - _model_info_from_deployment=_is_router_deployment, + _litellm_params_for_cost ) prompt = "" # use for tts cost calc @@ -4733,106 +4720,16 @@ def _custom_cost_per_token_from_source( def extract_custom_cost_per_token( litellm_params: Optional[dict], - *, - _model_info_from_deployment: bool = False, ) -> Optional[CostPerToken]: - """Extract a deployment's explicit per-token pricing from ``litellm_params``. + """Use explicit request rates; metadata is caller-controlled, not pricing configuration.""" + return _custom_cost_per_token_from_source(litellm_params) - Looks for ``input_cost_per_token`` / ``output_cost_per_token`` (plus optional - cache rates) in the following sources, in priority order: - 1. The top-level ``litellm_params`` (per-request custom pricing). - 2. ``litellm_params['model_info']`` — **only when - ``_model_info_from_deployment`` is True** (set by the router when it - injected the deployment configuration). Gated to prevent proxy clients - from supplying a client-controlled ``model_info`` dict with zero rates - that bypasses spend tracking and budget enforcement. - 3. ``litellm_params['metadata']['model_info']``. - 4. ``litellm_params['litellm_metadata']['model_info']`` (used by - ``/responses`` and ``/messages`` generic-api routes). - - The resulting ``CostPerToken`` is passed to the cost calculator so that a - deployment configured with explicit rates is always priced with those rates, - regardless of whether its entry still exists in ``litellm.model_cost`` - (which is wiped wholesale on model-cost-map reloads). See issue #30081. - """ +def extract_custom_cost_per_second(litellm_params: Optional[dict]) -> Optional[float]: + """Use the explicit request rate rather than caller-controlled model metadata.""" if litellm_params is None: return None - - # 1. top-level litellm_params (per-request custom pricing) - result = _custom_cost_per_token_from_source(litellm_params) - if result is not None: - return result - - # 2. litellm_params['model_info'] — only when injected by the router - if _model_info_from_deployment: - result = _custom_cost_per_token_from_source(litellm_params.get("model_info")) - if result is not None: - return result - - # 3 + 4. model_info nested under metadata / litellm_metadata - for metadata_key in ("metadata", "litellm_metadata"): - metadata = litellm_params.get(metadata_key) or {} - if isinstance(metadata, dict): - result = _custom_cost_per_token_from_source(metadata.get("model_info")) - if result is not None: - return result - - return None - - -def extract_custom_cost_per_second( - litellm_params: Optional[dict], - *, - _model_info_from_deployment: bool = False, -) -> Optional[float]: - """Extract a deployment's explicit per-second pricing from ``litellm_params``. - - Looks for ``input_cost_per_second`` in the same sources as - ``extract_custom_cost_per_token``, in priority order: - - 1. The top-level ``litellm_params`` (per-request custom pricing). - 2. ``litellm_params['model_info']`` — **only when - ``_model_info_from_deployment`` is True** (set by the router when it - injected the deployment configuration). Gated to prevent proxy clients - from supplying a client-controlled ``model_info`` dict with zero rates - that bypasses spend tracking and budget enforcement. - 3. ``litellm_params['metadata']['model_info']``. - 4. ``litellm_params['litellm_metadata']['model_info']`` (used by - ``/responses`` and ``/messages`` generic-api routes). - - Returns the first non-None value, or ``None`` if no per-second rate is - configured. This is threaded through to ``completion_cost`` so that - per-second custom pricing survives model-cost-map reloads (same root cause - as issue #30081). - """ - if litellm_params is None: - return None - - # 1. top-level litellm_params (per-request custom pricing) - cost = _coerce_cost_value(litellm_params.get("input_cost_per_second")) - if cost is not None: - return cost - - # 2. litellm_params['model_info'] — only when injected by the router - if _model_info_from_deployment: - model_info = litellm_params.get("model_info") - if isinstance(model_info, dict): - cost = _coerce_cost_value(model_info.get("input_cost_per_second")) - if cost is not None: - return cost - - # 3 + 4. model_info nested under metadata / litellm_metadata - for metadata_key in ("metadata", "litellm_metadata"): - metadata = litellm_params.get(metadata_key) or {} - if isinstance(metadata, dict): - model_info = metadata.get("model_info") - if isinstance(model_info, dict): - cost = _coerce_cost_value(model_info.get("input_cost_per_second")) - if cost is not None: - return cost - - return None + return _coerce_cost_value(litellm_params.get("input_cost_per_second")) def is_valid_sha256_hash(value: str) -> bool: diff --git a/litellm/main.py b/litellm/main.py index 933b76d3a3c..d46f04818b4 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1479,9 +1479,8 @@ def completion( # type: ignore # noqa: PLR0915 model=model, custom_llm_provider=custom_llm_provider ): verbose_logger.warning( - f"Custom pricing for model='{model}' not registered: model already " - f"has canonical pricing in litellm.model_cost. Per-request rates " - f"will be applied via cost threading for this request only." + "Custom pricing not registered: model already has canonical pricing " + "in litellm.model_cost. Per-request rates apply to this request only." ) else: litellm.register_model( diff --git a/litellm/router.py b/litellm/router.py index debccb0e83f..c2a7f055cc8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3172,6 +3172,13 @@ class Router: kwargs["model_info"] = model_info + for field in CustomPricingLiteLLMParams.model_fields: + deployment_rate = deployment["litellm_params"].get(field) + if deployment_rate is None: + deployment_rate = model_info.get(field) + if deployment_rate is not None and kwargs.get(field) is None: + kwargs[field] = deployment_rate + kwargs["timeout"] = self._get_timeout( kwargs=kwargs, data=deployment["litellm_params"] ) diff --git a/tests/test_litellm/test_custom_pricing_robustness.py b/tests/test_litellm/test_custom_pricing_robustness.py index bc945978cbe..6a443013468 100644 --- a/tests/test_litellm/test_custom_pricing_robustness.py +++ b/tests/test_litellm/test_custom_pricing_robustness.py @@ -15,8 +15,11 @@ Bug 2 - per-request custom pricing must apply to that request only and must by all other traffic. """ +import json import os import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Thread import pytest @@ -132,33 +135,16 @@ def test_extract_from_top_level_litellm_params(): assert result == {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0} -def test_extract_from_metadata_model_info(): - litellm_params = { - "metadata": { - "model_info": { - "input_cost_per_token": 1e-06, - "output_cost_per_token": 2e-06, - } - } +@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata", "model_info"]) +def test_extract_ignores_caller_model_info(metadata_key): + costs = { + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "input_cost_per_second": 0, } - result = extract_custom_cost_per_token(litellm_params) - assert result == { - "input_cost_per_token": 1e-06, - "output_cost_per_token": 2e-06, - } - - -def test_extract_from_litellm_metadata_model_info(): - litellm_params = { - "litellm_metadata": { - "model_info": { - "input_cost_per_token": 0, - "output_cost_per_token": 0, - } - } - } - result = extract_custom_cost_per_token(litellm_params) - assert result == {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0} + nested = costs if metadata_key == "model_info" else {"model_info": costs} + assert extract_custom_cost_per_token({metadata_key: nested}) is None + assert extract_custom_cost_per_second({metadata_key: nested}) is None def test_extract_includes_cache_rates_when_present(): @@ -202,8 +188,8 @@ def test_extract_custom_cost_per_second_zero(): assert result == 0.0 -def test_extract_custom_cost_per_second_skips_model_info_without_deployment_flag(): - """Top-level model_info must be ignored when _model_info_from_deployment is False. +def test_extract_custom_cost_per_second_ignores_untrusted_model_info(): + """Top-level model_info is caller-controlled and must not supply request prices. This prevents proxy clients from injecting ``model_info: {input_cost_per_second: 0}`` to bypass spend tracking for per-second priced deployments.""" @@ -213,21 +199,6 @@ def test_extract_custom_cost_per_second_skips_model_info_without_deployment_flag assert result is None -def test_extract_custom_cost_per_second_from_model_info_with_flag(): - result = extract_custom_cost_per_second( - {"model_info": {"input_cost_per_second": 0.01}}, - _model_info_from_deployment=True, - ) - assert result == 0.01 - - -def test_extract_custom_cost_per_second_from_metadata(): - result = extract_custom_cost_per_second( - {"metadata": {"model_info": {"input_cost_per_second": 0.02}}} - ) - assert result == 0.02 - - def test_extract_custom_cost_per_second_absent(): assert extract_custom_cost_per_second(None) is None assert extract_custom_cost_per_second({}) is None @@ -236,8 +207,8 @@ def test_extract_custom_cost_per_second_absent(): # --------------------------------------------------------------------------- # extract_custom_cost_per_token — security gate # --------------------------------------------------------------------------- -def test_extract_skips_model_info_without_deployment_flag(): - """Top-level model_info must be ignored when _model_info_from_deployment is False. +def test_extract_ignores_untrusted_model_info(): + """Top-level model_info is caller-controlled and must not supply request prices. This prevents proxy clients from injecting ``model_info: {input_cost_per_token: 0, output_cost_per_token: 0}`` to bypass spend tracking.""" @@ -251,22 +222,6 @@ def test_extract_skips_model_info_without_deployment_flag(): assert result is None -def test_extract_uses_model_info_with_deployment_flag(): - litellm_params = { - "model_info": { - "input_cost_per_token": 1e-06, - "output_cost_per_token": 2e-06, - } - } - result = extract_custom_cost_per_token( - litellm_params, _model_info_from_deployment=True - ) - assert result == { - "input_cost_per_token": 1e-06, - "output_cost_per_token": 2e-06, - } - - # --------------------------------------------------------------------------- # response_cost_calculator — custom_cost_per_second threading # --------------------------------------------------------------------------- @@ -297,51 +252,156 @@ def test_response_cost_calculator_honors_custom_cost_per_second(): # --------------------------------------------------------------------------- # Bug 2 - per-request pricing must not poison the canonical model_cost entry # --------------------------------------------------------------------------- -def test_per_request_custom_pricing_does_not_poison_canonical_entry(): - """A single zero-priced request must not re-price the shared model entry. +@pytest.fixture +def pricing_provider(): + class PricingHandler(BaseHTTPRequestHandler): + def do_POST(self): + request = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + body = json.dumps( + { + "id": "chatcmpl-pricing", + "object": "chat.completion", + "created": 1, + "model": request["model"], + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + }, + } + ).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) - Verifies both: - - The request itself is billed at the custom (zero) rate. - - The canonical ``litellm.model_cost`` entry is left unchanged - for all other traffic. - """ + def log_message(self, format, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), PricingHandler) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}/v1" + finally: + server.shutdown() + server.server_close() + thread.join() + + +def test_per_request_custom_pricing_does_not_poison_canonical_entry(pricing_provider): model = "gpt-4o" - assert model in litellm.model_cost - canonical_input_cost_before = litellm.model_cost[model]["input_cost_per_token"] - canonical_output_cost_before = litellm.model_cost[model]["output_cost_per_token"] - assert canonical_input_cost_before > 0 + canonical_before = litellm.model_cost[model].copy() + request = { + "model": model, + "messages": [{"role": "user", "content": "hi"}], + "api_base": pricing_provider, + "api_key": "fixture-key", + } + custom_response = litellm.completion( + **request, input_cost_per_token=0, output_cost_per_token=0 + ) + assert custom_response.usage.total_tokens == 150 + assert custom_response._hidden_params["response_cost"] == 0.0 + assert litellm.model_cost[model] == canonical_before + standard_response = litellm.completion(**request) + expected = ( + 100 * canonical_before["input_cost_per_token"] + + 50 * canonical_before["output_cost_per_token"] + ) + assert expected > 0 + assert standard_response._hidden_params["response_cost"] == pytest.approx(expected) + + +@pytest.mark.parametrize("pricing_source", ["litellm_params", "model_info"]) +def test_router_custom_pricing_survives_cost_map_reload( + pricing_provider, monkeypatch, pricing_source +): + canonical_before = litellm.model_cost["gpt-4o"].copy() + deployment = { + "model_name": "byok", + "litellm_params": { + "model": "openai/gpt-4o", + "api_base": pricing_provider, + "api_key": "fixture-key", + }, + "model_info": {"id": "byok-deployment"}, + } + deployment[pricing_source].update( + {"input_cost_per_token": 0, "output_cost_per_token": 0} + ) + router = litellm.Router(model_list=[deployment]) + monkeypatch.setattr(litellm, "model_cost", {"gpt-4o": canonical_before}) + response = router.completion( + model="byok", messages=[{"role": "user", "content": "hi"}] + ) + assert response.usage.total_tokens == 150 + assert response._hidden_params["response_cost"] == 0.0 + assert litellm.model_cost["gpt-4o"] == canonical_before + + +@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) +def test_request_metadata_cannot_override_pricing(pricing_provider, metadata_key): + canonical = litellm.model_cost["gpt-4o"] response = litellm.completion( - model=model, + model="gpt-4o", messages=[{"role": "user", "content": "hi"}], - mock_response="ok", + api_base=pricing_provider, + api_key="fixture-key", + **{ + metadata_key: { + "model_info": { + "id": "client-deployment", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + } + } + }, + ) + expected = ( + 100 * canonical["input_cost_per_token"] + + 50 * canonical["output_cost_per_token"] + ) + assert response._hidden_params["response_cost"] == pytest.approx(expected) + + +def test_custom_pricing_warning_does_not_emit_model_line_breaks( + pricing_provider, monkeypatch, caplog +): + model = "custom-model\nFORGED_LOG_ENTRY" + monkeypatch.setitem( + litellm.model_cost, + model, + { + **litellm.model_cost["gpt-4o"], + "litellm_provider": "openai", + "mode": "chat", + }, + ) + response = litellm.completion( + model=f"openai/{model}", + messages=[{"role": "user", "content": "hi"}], + api_base=pricing_provider, + api_key="fixture-key", input_cost_per_token=0, output_cost_per_token=0, ) - - # The canonical entry must not be mutated. - assert ( - litellm.model_cost[model]["input_cost_per_token"] == canonical_input_cost_before + assert response._hidden_params["response_cost"] == 0 + pricing_warnings = [ + record.getMessage() + for record in caplog.records + if "canonical pricing" in record.getMessage() + ] + assert pricing_warnings + assert all( + "\n" not in message and "\r" not in message for message in pricing_warnings ) - assert ( - litellm.model_cost[model]["output_cost_per_token"] - == canonical_output_cost_before - ) - - # The per-request cost must also be calculated with the custom rates. - # Verify via the cost calculator directly (mock_response bypasses the - # normal cost-calculation path, so the response object may not carry - # cost metadata). - cost = litellm.response_cost_calculator( - response_object=response, - model=model, - custom_llm_provider="openai", - call_type="completion", - optional_params={}, - custom_pricing=True, - custom_cost_per_token={ - "input_cost_per_token": 0.0, - "output_cost_per_token": 0.0, - }, - ) - assert cost == 0.0