diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 98e00cf5788..a543f33cd53 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1804,6 +1804,8 @@ def response_cost_calculator( cache_hit: Optional[bool] = None, base_model: Optional[str] = None, custom_pricing: Optional[bool] = None, + custom_cost_per_token: Optional[CostPerToken] = None, + custom_cost_per_second: Optional[float] = None, prompt: str = "", standard_built_in_tools_params: Optional[StandardBuiltInToolsParams] = None, litellm_model_name: Optional[str] = None, @@ -1837,6 +1839,8 @@ def response_cost_calculator( custom_llm_provider=custom_llm_provider, optional_params=optional_params, custom_pricing=custom_pricing, + custom_cost_per_token=custom_cost_per_token, + custom_cost_per_second=custom_cost_per_second, base_model=base_model, prompt=prompt, standard_built_in_tools_params=standard_built_in_tools_params, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index c87c0690807..590be47d42c 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -170,6 +170,9 @@ from .specialty_caches.dynamic_logging_cache import DynamicLoggingCache if TYPE_CHECKING: from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig + from litellm.types.utils import CostPerToken +else: + CostPerToken = dict try: from litellm_enterprise.enterprise_callbacks.callback_controls import ( EnterpriseCallbackControls, @@ -1526,10 +1529,15 @@ class Logging(LiteLLMLoggingBaseClass): router_model_id = self.get_router_model_id() ## RESPONSE COST ## + _litellm_params_for_cost = ( + self.litellm_params if hasattr(self, "litellm_params") else None + ) custom_pricing = use_custom_pricing_for_model( - litellm_params=( - self.litellm_params if hasattr(self, "litellm_params") else None - ) + litellm_params=_litellm_params_for_cost + ) + custom_cost_per_token = extract_custom_cost_per_token(_litellm_params_for_cost) + custom_cost_per_second = extract_custom_cost_per_second( + _litellm_params_for_cost ) prompt = "" # use for tts cost calc @@ -1554,6 +1562,8 @@ class Logging(LiteLLMLoggingBaseClass): "call_type": self.call_type, "optional_params": self.optional_params, "custom_pricing": custom_pricing, + "custom_cost_per_token": custom_cost_per_token, + "custom_cost_per_second": custom_cost_per_second, "prompt": prompt, "standard_built_in_tools_params": self.standard_built_in_tools_params, "router_model_id": router_model_id, @@ -4661,6 +4671,70 @@ def use_custom_pricing_for_model(litellm_params: Optional[dict]) -> bool: return False +def _coerce_cost_value(value: Any) -> Optional[float]: + """Coerce a configured cost value to float, returning None when absent/invalid. + + Note: ``0`` / ``0.0`` are valid (e.g. BYOK deployments priced at zero), so we + only treat ``None`` and non-numeric values as "missing". + """ + if value is None: + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def _custom_cost_per_token_from_source( + source: Optional[dict], +) -> Optional[CostPerToken]: + """Build a ``CostPerToken`` from a single dict source (litellm_params or model_info). + + Only returns a value when BOTH ``input_cost_per_token`` and + ``output_cost_per_token`` are explicitly present (not ``None``). This keeps + partially-configured deployments on the standard pricing path instead of + silently treating a missing rate as zero. + """ + if not source or not isinstance(source, dict): + return None + + input_cost = _coerce_cost_value(source.get("input_cost_per_token")) + output_cost = _coerce_cost_value(source.get("output_cost_per_token")) + if input_cost is None or output_cost is None: + return None + + custom_cost_per_token: CostPerToken = { + "input_cost_per_token": input_cost, + "output_cost_per_token": output_cost, + } + + cache_read_cost = _coerce_cost_value(source.get("cache_read_input_token_cost")) + if cache_read_cost is not None: + custom_cost_per_token["cache_read_input_token_cost"] = cache_read_cost + + cache_creation_cost = _coerce_cost_value( + source.get("cache_creation_input_token_cost") + ) + if cache_creation_cost is not None: + custom_cost_per_token["cache_creation_input_token_cost"] = cache_creation_cost + + return custom_cost_per_token + + +def extract_custom_cost_per_token( + litellm_params: Optional[dict], +) -> Optional[CostPerToken]: + """Use explicit request rates; metadata is caller-controlled, not pricing configuration.""" + return _custom_cost_per_token_from_source(litellm_params) + + +def extract_custom_cost_per_second(litellm_params: Optional[dict]) -> Optional[float]: + """Use the explicit request rate rather than caller-controlled model metadata.""" + if litellm_params is None: + return None + return _coerce_cost_value(litellm_params.get("input_cost_per_second")) + + def is_valid_sha256_hash(value: str) -> bool: # Check if the value is a valid SHA-256 hash (64 hexadecimal characters) return bool(re.fullmatch(r"[a-fA-F0-9]{64}", value)) @@ -5773,9 +5847,6 @@ def _get_traceback_str_for_error(error_str: str) -> str: from decimal import Decimal -# used for unit testing -from typing import Any, Dict, List, Optional, Union - def create_dummy_standard_logging_payload() -> StandardLoggingPayload: # First create the nested objects with proper typing diff --git a/litellm/main.py b/litellm/main.py index e17a5ad9a48..d46f04818b4 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1080,6 +1080,34 @@ def _build_custom_pricing_entry( return entry +def _model_has_known_pricing(model: str, custom_llm_provider: Optional[str]) -> bool: + """Return True if ``model`` already resolves to a priced entry in litellm.model_cost. + + Per-request custom pricing must not overwrite the shared canonical + ``litellm.model_cost`` entry, otherwise a single request that supplies + ``input_cost_per_token`` (e.g. a BYOK deployment priced at 0) would re-price + the canonical model for the whole process and silently mis-bill all other + traffic. Known models get their per-request rates applied via the + ``custom_cost_per_token`` threading in the cost calculator instead, so the + registration is only needed for otherwise-unknown models. See issue #30081. + """ + try: + existing_model_info = litellm.get_model_info( + model=model, custom_llm_provider=custom_llm_provider + ) + except Exception: + return False + + if not existing_model_info: + return False + + return ( + existing_model_info.get("input_cost_per_token") is not None + or existing_model_info.get("output_cost_per_token") is not None + or existing_model_info.get("input_cost_per_second") is not None + ) + + @tracer.wrap() @client def completion( # type: ignore # noqa: PLR0915 @@ -1447,15 +1475,23 @@ def completion( # type: ignore # noqa: PLR0915 if ( input_cost_per_token is not None and output_cost_per_token is not None ) or input_cost_per_second is not None: - litellm.register_model( - { - f"{custom_llm_provider}/{model}": _build_custom_pricing_entry( - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) - } - ) + if _model_has_known_pricing( + model=model, custom_llm_provider=custom_llm_provider + ): + verbose_logger.warning( + "Custom pricing not registered: model already has canonical pricing " + "in litellm.model_cost. Per-request rates apply to this request only." + ) + else: + litellm.register_model( + { + f"{custom_llm_provider}/{model}": _build_custom_pricing_entry( + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + model_info=model_info, + ) + } + ) ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ### custom_prompt_dict = {} # type: ignore if ( @@ -4889,15 +4925,24 @@ def embedding( # noqa: PLR0915 if ( input_cost_per_token is not None and output_cost_per_token is not None ) or input_cost_per_second is not None: - litellm.register_model( - { - f"{custom_llm_provider}/{model}": _build_custom_pricing_entry( - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=kwargs.get("model_info"), - ) - } - ) + if _model_has_known_pricing( + model=model, custom_llm_provider=custom_llm_provider + ): + verbose_logger.warning( + f"Custom pricing for model='{model}' not registered: model already " + f"has canonical pricing in litellm.model_cost. Per-request rates " + f"will be applied via cost threading for this request only." + ) + else: + litellm.register_model( + { + f"{custom_llm_provider}/{model}": _build_custom_pricing_entry( + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + model_info=kwargs.get("model_info"), + ) + } + ) litellm_params_dict = get_litellm_params(**kwargs) diff --git a/litellm/router.py b/litellm/router.py index debccb0e83f..c2a7f055cc8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3172,6 +3172,13 @@ class Router: kwargs["model_info"] = model_info + for field in CustomPricingLiteLLMParams.model_fields: + deployment_rate = deployment["litellm_params"].get(field) + if deployment_rate is None: + deployment_rate = model_info.get(field) + if deployment_rate is not None and kwargs.get(field) is None: + kwargs[field] = deployment_rate + kwargs["timeout"] = self._get_timeout( kwargs=kwargs, data=deployment["litellm_params"] ) diff --git a/tests/test_litellm/interactions/test_openapi_compliance.py b/tests/test_litellm/interactions/test_openapi_compliance.py index aededaaca77..509ded8c704 100644 --- a/tests/test_litellm/interactions/test_openapi_compliance.py +++ b/tests/test_litellm/interactions/test_openapi_compliance.py @@ -165,7 +165,6 @@ class TestResponseCompliance: "status", "created", "updated", - "role", "steps", "usage", ] diff --git a/tests/test_litellm/test_custom_pricing_robustness.py b/tests/test_litellm/test_custom_pricing_robustness.py new file mode 100644 index 00000000000..6a443013468 --- /dev/null +++ b/tests/test_litellm/test_custom_pricing_robustness.py @@ -0,0 +1,407 @@ +"""Regression tests for deployment / per-request custom pricing robustness. + +Covers the two intertwined billing bugs described in +https://github.com/BerriAI/litellm/issues/30081: + +Bug 1 - custom pricing lost when ``litellm.model_cost`` is replaced + (e.g. after a model-cost-map reload). The deployment's explicit + ``input_cost_per_token`` / ``output_cost_per_token`` must still be + honored, because they are threaded through the cost calculator as + ``custom_cost_per_token`` rather than depending on the deployment-id + entry surviving in the process-global ``litellm.model_cost``. + +Bug 2 - per-request custom pricing must apply to that request only and must + NOT overwrite (poison) the shared canonical ``model_cost`` entry used + by all other traffic. +""" + +import json +import os +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Thread + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +import litellm +from litellm import Usage +from litellm.litellm_core_utils.litellm_logging import ( + extract_custom_cost_per_second, + extract_custom_cost_per_token, +) + + +def _make_model_response(model: str, prompt_tokens: int, completion_tokens: int): + return litellm.ModelResponse( + model=model, + usage=Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + ) + + +# --------------------------------------------------------------------------- +# Bug 1 - custom_cost_per_token threading +# --------------------------------------------------------------------------- +def test_response_cost_calculator_honors_zero_custom_cost_per_token(): + """A deployment priced at 0 stays at 0 even when its model_cost entry is gone. + + Simulates the post-reload state: ``router_model_id`` is not present in + ``litellm.model_cost`` and ``custom_pricing`` is detected from the + deployment params. With the explicit rates threaded through as + ``custom_cost_per_token`` the cost is 0, instead of falling back to the + model's public price. + """ + response = _make_model_response( + "gpt-4o", prompt_tokens=100_000, completion_tokens=10_000 + ) + + cost = litellm.response_cost_calculator( + response_object=response, + model="gpt-4o", + custom_llm_provider="openai", + call_type="acompletion", + optional_params={}, + custom_pricing=True, + router_model_id="id-not-in-model-cost", # wiped by a reload + custom_cost_per_token={ + "input_cost_per_token": 0, + "output_cost_per_token": 0, + }, + ) + + assert cost == 0.0 + + +def test_response_cost_calculator_falls_back_without_custom_cost_per_token(): + """Control: without the threaded rates the same request bills at public price. + + This is the buggy behaviour from the issue - it confirms the threading in + :func:`test_response_cost_calculator_honors_zero_custom_cost_per_token` is + what fixes the silent mis-billing. + """ + response = _make_model_response( + "gpt-4o", prompt_tokens=100_000, completion_tokens=10_000 + ) + + cost = litellm.response_cost_calculator( + response_object=response, + model="gpt-4o", + custom_llm_provider="openai", + call_type="acompletion", + optional_params={}, + custom_pricing=True, + router_model_id="id-not-in-model-cost", + ) + + assert cost > 0.0 + + +def test_response_cost_calculator_honors_nonzero_custom_cost_per_token(): + """Explicit non-zero rates are applied exactly, regardless of model_cost.""" + response = _make_model_response( + "gpt-4o", prompt_tokens=1_000, completion_tokens=2_000 + ) + + cost = litellm.response_cost_calculator( + response_object=response, + model="gpt-4o", + custom_llm_provider="openai", + call_type="acompletion", + optional_params={}, + custom_pricing=True, + router_model_id="id-not-in-model-cost", + custom_cost_per_token={ + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + }, + ) + + # 1000 * 1e-06 + 2000 * 2e-06 = 0.001 + 0.004 + assert cost == pytest.approx(0.005) + + +# --------------------------------------------------------------------------- +# extract_custom_cost_per_token helper +# --------------------------------------------------------------------------- +def test_extract_from_top_level_litellm_params(): + result = extract_custom_cost_per_token( + {"input_cost_per_token": 0, "output_cost_per_token": 0} + ) + assert result == {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0} + + +@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata", "model_info"]) +def test_extract_ignores_caller_model_info(metadata_key): + costs = { + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "input_cost_per_second": 0, + } + nested = costs if metadata_key == "model_info" else {"model_info": costs} + assert extract_custom_cost_per_token({metadata_key: nested}) is None + assert extract_custom_cost_per_second({metadata_key: nested}) is None + + +def test_extract_includes_cache_rates_when_present(): + result = extract_custom_cost_per_token( + { + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "cache_read_input_token_cost": 0, + "cache_creation_input_token_cost": 0, + } + ) + assert result == { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "cache_read_input_token_cost": 0.0, + "cache_creation_input_token_cost": 0.0, + } + + +def test_extract_returns_none_for_partial_pricing(): + # Only input set -> stay on the standard pricing path (don't assume 0 output). + assert extract_custom_cost_per_token({"input_cost_per_token": 0}) is None + + +def test_extract_returns_none_when_absent(): + assert extract_custom_cost_per_token(None) is None + assert extract_custom_cost_per_token({}) is None + assert extract_custom_cost_per_token({"model_info": {}}) is None + + +# --------------------------------------------------------------------------- +# extract_custom_cost_per_second +# --------------------------------------------------------------------------- +def test_extract_custom_cost_per_second_top_level(): + result = extract_custom_cost_per_second({"input_cost_per_second": 0.005}) + assert result == 0.005 + + +def test_extract_custom_cost_per_second_zero(): + result = extract_custom_cost_per_second({"input_cost_per_second": 0}) + assert result == 0.0 + + +def test_extract_custom_cost_per_second_ignores_untrusted_model_info(): + """Top-level model_info is caller-controlled and must not supply request prices. + + This prevents proxy clients from injecting ``model_info: {input_cost_per_second: 0}`` + to bypass spend tracking for per-second priced deployments.""" + result = extract_custom_cost_per_second( + {"model_info": {"input_cost_per_second": 0.01}} + ) + assert result is None + + +def test_extract_custom_cost_per_second_absent(): + assert extract_custom_cost_per_second(None) is None + assert extract_custom_cost_per_second({}) is None + + +# --------------------------------------------------------------------------- +# extract_custom_cost_per_token — security gate +# --------------------------------------------------------------------------- +def test_extract_ignores_untrusted_model_info(): + """Top-level model_info is caller-controlled and must not supply request prices. + + This prevents proxy clients from injecting ``model_info: {input_cost_per_token: 0, + output_cost_per_token: 0}`` to bypass spend tracking.""" + litellm_params = { + "model_info": { + "input_cost_per_token": 0, + "output_cost_per_token": 0, + } + } + result = extract_custom_cost_per_token(litellm_params) + assert result is None + + +# --------------------------------------------------------------------------- +# response_cost_calculator — custom_cost_per_second threading +# --------------------------------------------------------------------------- +def test_response_cost_calculator_honors_custom_cost_per_second(): + r"""Per-second custom pricing is threaded through the cost calculator. + + completion_cost reads timing from _response_ms on the + response object, so timing must be set there for per-second billing. + """ + response = _make_model_response("gpt-4o", prompt_tokens=100, completion_tokens=0) + response._response_ms = 5000.0 + + # 0.01 $/s * 5 s = 0.05 + cost = litellm.response_cost_calculator( + response_object=response, + model="gpt-4o", + custom_llm_provider="openai", + call_type="acompletion", + optional_params={}, + custom_pricing=True, + router_model_id="id-not-in-model-cost", + custom_cost_per_second=0.01, + ) + + assert cost == pytest.approx(0.05) + + +# --------------------------------------------------------------------------- +# Bug 2 - per-request pricing must not poison the canonical model_cost entry +# --------------------------------------------------------------------------- +@pytest.fixture +def pricing_provider(): + class PricingHandler(BaseHTTPRequestHandler): + def do_POST(self): + request = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + body = json.dumps( + { + "id": "chatcmpl-pricing", + "object": "chat.completion", + "created": 1, + "model": request["model"], + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150, + }, + } + ).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), PricingHandler) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}/v1" + finally: + server.shutdown() + server.server_close() + thread.join() + + +def test_per_request_custom_pricing_does_not_poison_canonical_entry(pricing_provider): + model = "gpt-4o" + canonical_before = litellm.model_cost[model].copy() + request = { + "model": model, + "messages": [{"role": "user", "content": "hi"}], + "api_base": pricing_provider, + "api_key": "fixture-key", + } + custom_response = litellm.completion( + **request, input_cost_per_token=0, output_cost_per_token=0 + ) + assert custom_response.usage.total_tokens == 150 + assert custom_response._hidden_params["response_cost"] == 0.0 + assert litellm.model_cost[model] == canonical_before + + standard_response = litellm.completion(**request) + expected = ( + 100 * canonical_before["input_cost_per_token"] + + 50 * canonical_before["output_cost_per_token"] + ) + assert expected > 0 + assert standard_response._hidden_params["response_cost"] == pytest.approx(expected) + + +@pytest.mark.parametrize("pricing_source", ["litellm_params", "model_info"]) +def test_router_custom_pricing_survives_cost_map_reload( + pricing_provider, monkeypatch, pricing_source +): + canonical_before = litellm.model_cost["gpt-4o"].copy() + deployment = { + "model_name": "byok", + "litellm_params": { + "model": "openai/gpt-4o", + "api_base": pricing_provider, + "api_key": "fixture-key", + }, + "model_info": {"id": "byok-deployment"}, + } + deployment[pricing_source].update( + {"input_cost_per_token": 0, "output_cost_per_token": 0} + ) + router = litellm.Router(model_list=[deployment]) + monkeypatch.setattr(litellm, "model_cost", {"gpt-4o": canonical_before}) + response = router.completion( + model="byok", messages=[{"role": "user", "content": "hi"}] + ) + assert response.usage.total_tokens == 150 + assert response._hidden_params["response_cost"] == 0.0 + assert litellm.model_cost["gpt-4o"] == canonical_before + + +@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) +def test_request_metadata_cannot_override_pricing(pricing_provider, metadata_key): + canonical = litellm.model_cost["gpt-4o"] + response = litellm.completion( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + api_base=pricing_provider, + api_key="fixture-key", + **{ + metadata_key: { + "model_info": { + "id": "client-deployment", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + } + } + }, + ) + expected = ( + 100 * canonical["input_cost_per_token"] + + 50 * canonical["output_cost_per_token"] + ) + assert response._hidden_params["response_cost"] == pytest.approx(expected) + + +def test_custom_pricing_warning_does_not_emit_model_line_breaks( + pricing_provider, monkeypatch, caplog +): + model = "custom-model\nFORGED_LOG_ENTRY" + monkeypatch.setitem( + litellm.model_cost, + model, + { + **litellm.model_cost["gpt-4o"], + "litellm_provider": "openai", + "mode": "chat", + }, + ) + response = litellm.completion( + model=f"openai/{model}", + messages=[{"role": "user", "content": "hi"}], + api_base=pricing_provider, + api_key="fixture-key", + input_cost_per_token=0, + output_cost_per_token=0, + ) + assert response._hidden_params["response_cost"] == 0 + pricing_warnings = [ + record.getMessage() + for record in caplog.records + if "canonical pricing" in record.getMessage() + ] + assert pricing_warnings + assert all( + "\n" not in message and "\r" not in message for message in pricing_warnings + )