From 06a8444bdd8c9b42846e8b8888a4bc155b66b33f Mon Sep 17 00:00:00 2001 From: liming Date: Thu, 27 Aug 2026 15:13:59 +0800 Subject: [PATCH] fix(cost): apply deployment custom token rates before provider dispatch /v1/messages callers only passed custom_pricing=True, so unmapped anthropic models still looked up the public price map and logged $0 spend. Extract input/output rates from litellm_params and feed custom_cost_per_token into the existing early return. Fixes #25204 Co-authored-by: Cursor --- litellm/cost_calculator.py | 67 ++++++++ .../anthropic_passthrough_logging_handler.py | 1 + .../test_litellm_logging.py | 50 +++++- tests/test_litellm/test_cost_calculator.py | 158 ++++++++++++++++++ 4 files changed, 275 insertions(+), 1 deletion(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 37a79e2f6d4..38c0177479a 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -236,6 +236,70 @@ def _cost_per_token_custom_pricing_helper( return None +def extract_custom_cost_per_token( + litellm_params: object | None, +) -> CostPerToken | None: + """Return deployment token rates from litellm_params when both input and output are set. + + Rates may sit on litellm_params itself (UI / model_list) or under + metadata.model_info / litellm_metadata.model_info (/v1/messages, /v1/responses). + Optional cache rates are copied when present so the custom-pricing helper can + apply them instead of falling back to the input rate. + """ + if litellm_params is None: + return None + if not isinstance(litellm_params, dict): + dump = getattr(litellm_params, "model_dump", None) + if not callable(dump): + return None + dumped = dump() + if not isinstance(dumped, dict): + return None + litellm_params = dumped + + def _from_mapping(source: object) -> CostPerToken | None: + if not isinstance(source, dict): + return None + input_cost = source.get("input_cost_per_token") + output_cost = source.get("output_cost_per_token") + if input_cost is None or output_cost is None: + return None + result: CostPerToken = { + "input_cost_per_token": float(input_cost), + "output_cost_per_token": float(output_cost), + } + cache_read = source.get("cache_read_input_token_cost") + if cache_read is not None: + result["cache_read_input_token_cost"] = float(cache_read) + cache_creation = source.get("cache_creation_input_token_cost") + if cache_creation is not None: + result["cache_creation_input_token_cost"] = float(cache_creation) + return result + + from_top = _from_mapping(litellm_params) + if from_top is not None: + return from_top + for metadata_key in ("metadata", "litellm_metadata"): + metadata = litellm_params.get(metadata_key) or {} + from_info = _from_mapping(metadata.get("model_info") if isinstance(metadata, dict) else None) + if from_info is not None: + return from_info + return None + + +def _custom_cost_per_token_from_logging_obj( + litellm_logging_obj: LitellmLoggingObject | None, +) -> CostPerToken | None: + if litellm_logging_obj is None: + return None + extracted = extract_custom_cost_per_token(getattr(litellm_logging_obj, "litellm_params", None)) + if extracted is not None: + return extracted + details = getattr(litellm_logging_obj, "model_call_details", None) or {} + nested = details.get("litellm_params") if isinstance(details, dict) else None + return extract_custom_cost_per_token(nested) + + def _get_additional_costs( model: str, custom_llm_provider: str | None, @@ -1209,6 +1273,9 @@ def completion_cost( - For un-mapped Replicate models, the cost is calculated based on the total time used for the request. """ try: + if custom_cost_per_token is None: + custom_cost_per_token = _custom_cost_per_token_from_logging_obj(litellm_logging_obj) + call_type = _infer_call_type(call_type, completion_response) or "completion" if ( diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index a36a365f39a..c32afd37e2d 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -287,6 +287,7 @@ class AnthropicPassthroughLoggingHandler: custom_llm_provider=custom_llm_provider, custom_pricing=custom_pricing, router_model_id=router_model_id, + litellm_logging_obj=logging_obj, ) ) diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 0222e756ba1..8ad57556d92 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -17,7 +17,7 @@ from litellm.constants import SENTRY_DENYLIST, SENTRY_PII_DENYLIST from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging from litellm.litellm_core_utils.litellm_logging import set_callbacks -from litellm.types.utils import ModelResponse, TextCompletionResponse +from litellm.types.utils import ModelResponse, TextCompletionResponse, Usage @pytest.fixture @@ -280,6 +280,54 @@ def test_response_cost_calculator_uses_router_model_id_from_litellm_metadata(): litellm.model_cost.pop(custom_model_id, None) +def test_response_cost_calculator_unknown_anthropic_model_uses_litellm_params_rates(): + """Native /v1/messages cost calc should apply deployment rates for an + unmapped anthropic model. Do not register_model — that is the + completions-only workaround and is not the /messages path. + Regression for #25204. + """ + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + + unknown_model = "litellm-unmapped-custom-priced-qwen" + input_cost = 1.2e-05 + output_cost = 3.6e-05 + + logging_obj = LiteLLMLoggingObj( + model=unknown_model, + messages=[{"role": "user", "content": "Hi"}], + stream=False, + call_type="anthropic_messages", + start_time=time.time(), + litellm_call_id="test-messages-custom-pricing", + function_id="test-fn", + ) + logging_obj.update_environment_variables( + model=unknown_model, + user="", + optional_params={}, + litellm_params={ + "custom_llm_provider": "anthropic", + "input_cost_per_token": input_cost, + "output_cost_per_token": output_cost, + }, + ) + logging_obj.model_call_details["custom_llm_provider"] = "anthropic" + + response_obj = ModelResponse( + id="msg_test", + model=unknown_model, + choices=[], + usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120), + ) + + cost = logging_obj._response_cost_calculator(result=response_obj) + + assert cost is not None + expected_cost = (100 * input_cost) + (20 * output_cost) + assert cost == pytest.approx(expected_cost) + assert cost > 0 + + class TestGetRouterModelId: """Tests for the get_router_model_id helper method.""" diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 8fce9ba080c..09974c18c74 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -13,6 +13,7 @@ from litellm.cost_calculator import ( RealtimeAPITokenUsageProcessor, completion_cost, cost_per_token, + extract_custom_cost_per_token, handle_realtime_stream_cost_calculation, response_cost_calculator, ) @@ -953,6 +954,163 @@ def test_custom_pricing_cost_calc_uses_router_model_id_from_litellm_metadata(): assert custom_model_id not in (selected_model_no_custom or "") +def test_extract_custom_cost_per_token_from_litellm_params_and_model_info(): + assert extract_custom_cost_per_token(None) is None + assert extract_custom_cost_per_token({"input_cost_per_token": 1.2e-05}) is None + assert extract_custom_cost_per_token( + { + "input_cost_per_token": 1.2e-05, + "output_cost_per_token": 3.6e-05, + "cache_read_input_token_cost": 1.2e-06, + } + ) == { + "input_cost_per_token": 1.2e-05, + "output_cost_per_token": 3.6e-05, + "cache_read_input_token_cost": 1.2e-06, + } + assert extract_custom_cost_per_token( + { + "litellm_metadata": { + "model_info": { + "id": "deploy-1", + "input_cost_per_token": 0.0003, + "output_cost_per_token": 0.0015, + }, + }, + } + ) == { + "input_cost_per_token": 0.0003, + "output_cost_per_token": 0.0015, + } + + +def test_completion_cost_unknown_anthropic_model_uses_litellm_params_rates(): + """Unknown anthropic models logged $0 on /v1/messages even when the + deployment set input/output rates in litellm_params. + + The public price map has no entry, so provider dispatch must not run + before custom_cost_per_token is applied. Regression for #25204. + """ + import time + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + + unknown_model = "litellm-unmapped-custom-priced-qwen" + input_cost = 1.2e-05 + output_cost = 3.6e-05 + prompt_tokens = 100 + completion_tokens = 20 + + assert unknown_model not in litellm.model_cost + assert f"anthropic/{unknown_model}" not in litellm.model_cost + + logging_obj = LiteLLMLoggingObj( + model=unknown_model, + messages=[{"role": "user", "content": "Hi"}], + stream=False, + call_type="anthropic_messages", + start_time=time.time(), + litellm_call_id="test-unmapped-custom-pricing", + function_id="test-fn", + ) + logging_obj.update_environment_variables( + model=unknown_model, + user="", + optional_params={}, + litellm_params={ + "custom_llm_provider": "anthropic", + "input_cost_per_token": input_cost, + "output_cost_per_token": output_cost, + }, + ) + logging_obj.model_call_details["custom_llm_provider"] = "anthropic" + + response = ModelResponse( + id="test-id", + model=unknown_model, + choices=[], + usage=Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + ) + + cost = completion_cost( + completion_response=response, + model=unknown_model, + custom_llm_provider="anthropic", + call_type="anthropic_messages", + custom_pricing=True, + litellm_logging_obj=logging_obj, + ) + expected = prompt_tokens * input_cost + completion_tokens * output_cost + assert cost == pytest.approx(expected) + assert cost > 0 + + +def test_anthropic_passthrough_unknown_model_spend_uses_litellm_params_rates(): + """Passthrough /v1/messages must pass the logging object into + completion_cost so unmapped models pick up deployment rates. + Regression for #25204. + """ + import time + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( + AnthropicPassthroughLoggingHandler, + ) + + unknown_model = "litellm-unmapped-custom-priced-qwen" + input_cost = 1.2e-05 + output_cost = 3.6e-05 + + logging_obj = LiteLLMLoggingObj( + model=unknown_model, + messages=[{"role": "user", "content": "Hi"}], + stream=False, + call_type="anthropic_messages", + start_time=time.time(), + litellm_call_id="test-passthrough-custom-pricing", + function_id="test-fn", + ) + logging_obj.update_environment_variables( + model=unknown_model, + user="", + optional_params={}, + litellm_params={ + "custom_llm_provider": "anthropic", + "input_cost_per_token": input_cost, + "output_cost_per_token": output_cost, + }, + ) + logging_obj.model_call_details["custom_llm_provider"] = "anthropic" + + response = ModelResponse( + id="msg_test", + model=unknown_model, + choices=[], + usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120), + ) + + kwargs = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( + litellm_model_response=response, + model=unknown_model, + kwargs={}, + start_time=datetime.now(), + end_time=datetime.now(), + logging_obj=logging_obj, + ) + + expected = 100 * input_cost + 20 * output_cost + assert kwargs["response_cost"] == pytest.approx(expected) + assert logging_obj.model_call_details["response_cost"] == pytest.approx( + expected + ) + assert kwargs["response_cost"] > 0 + + def test_per_request_custom_pricing_with_router(): """When custom pricing is passed as per-request kwargs (not in model_list), _select_model_name_for_cost_calc should fall back to the model name