diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index ce63d34d48d..ea873a61f0d 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1483,6 +1483,12 @@ def _prices_only_a_billing_minimum(litellm_logging_obj: LitellmLoggingObject | N ) +def overrides_provider_stated_cost( + custom_pricing: bool | None, litellm_logging_obj: LitellmLoggingObject | None +) -> bool: + return custom_pricing is True and not _prices_only_a_billing_minimum(litellm_logging_obj) + + def _without_provider_stated_cost(usage: Usage | None) -> Usage | None: if usage is None or getattr(usage, "cost", None) is None: return usage @@ -1654,7 +1660,7 @@ def completion_cost( cache_read_input_tokens: int | None = None audio_transcription_file_duration: float = 0.0 provider_usage_object: Final = get_usage_object(completion_response=completion_response) - sets_prices: Final = custom_pricing is True and not _prices_only_a_billing_minimum(litellm_logging_obj) + sets_prices: Final = overrides_provider_stated_cost(custom_pricing, litellm_logging_obj) cost_per_token_usage_object: Final[Usage | None] = ( _without_provider_stated_cost(provider_usage_object) if sets_prices else provider_usage_object ) @@ -2357,7 +2363,9 @@ def response_cost_calculator( getattr(response_object, HIDDEN_PARAMS_ATTR), ) provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params) - if provider_response_cost is not None and not custom_pricing: + if provider_response_cost is not None and not overrides_provider_stated_cost( + custom_pricing, litellm_logging_obj + ): return provider_response_cost response_cost = completion_cost( diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index c1fc6479421..66f39fcd3c9 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -10,6 +10,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import ANTHROPIC_BATCHES_ROUTE +from litellm.cost_calculator import overrides_provider_stated_cost from litellm.litellm_core_utils.core_helpers import ( native_messages_reported_cost, override_stamped_response_cost, @@ -346,7 +347,7 @@ class AnthropicPassthroughLoggingHandler: if isinstance(litellm_model_response, ModelResponse) else None ) - if reported_cost is not None and not custom_pricing: + if reported_cost is not None and not overrides_provider_stated_cost(custom_pricing, logging_obj): return reported_cost return litellm.completion_cost( completion_response=litellm_model_response, diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 04882a5da12..d09784bf564 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -4,7 +4,7 @@ import asyncio import json import math import time -from collections.abc import AsyncIterator, Mapping, Sequence +from collections.abc import AsyncIterator, Callable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timedelta, timezone from itertools import chain @@ -1352,7 +1352,7 @@ def _estimate_request_input_cost_for_model( model_info=model_info, input_tokens=input_tokens, ) - for model_info in _get_model_cost_infos(model=model, llm_router=llm_router) + for model_info in _get_model_cost_infos(model=model, llm_router=llm_router, group_billing_minimum=min) ] valid_estimates: Final = [estimate for estimate in estimates if estimate is not None] return max(valid_estimates) if valid_estimates else None @@ -1521,6 +1521,7 @@ def _get_model_cost_info( def _get_model_cost_infos( model: str, llm_router: Router | None, + group_billing_minimum: Callable[[Sequence[int]], int] = max, ) -> Sequence[Mapping[str, object]]: """Cost-info candidates to estimate a request against for one model group. @@ -1528,7 +1529,9 @@ def _get_model_cost_infos( is unknown. Rather than guess, we estimate the cost against every eligible pricing shape in the group (the group's flat rates plus each deployment's tiered table) and let the caller reserve the maximum, so a cheaper sibling - deployment can never leave the request under-reserved. + deployment can never leave the request under-reserved. The deployments' + billed input minimums fold into one through ``group_billing_minimum``, the + largest for admission and the smallest for the input cost a disconnect is charged. """ try: group_info: Final = _get_model_cost_info(model=model, llm_router=llm_router) @@ -1541,12 +1544,14 @@ def _get_model_cost_infos( exc_info=True, ) return [] - billing_minimums: Final = [ - minimum - for deployment_info in deployment_infos - if (minimum := _to_int(deployment_info.get("min_billed_input_tokens"))) is not None - ] - base: Final = {**group_info, "min_billed_input_tokens": max(billing_minimums)} if billing_minimums else group_info + billing_minimums: Final = tuple( + _to_int(deployment_info.get("min_billed_input_tokens")) or 0 for deployment_info in deployment_infos + ) + base: Final = ( + {**group_info, "min_billed_input_tokens": group_billing_minimum(billing_minimums)} + if any(billing_minimums) + else group_info + ) tiered_tables: Final = [ table for deployment_info in deployment_infos if (table := _tiered_pricing_table(deployment_info)) is not None ] diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index 47b6af9aeb3..59743bfc522 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -2951,6 +2951,24 @@ def test_streamed_messages_bill_the_list_cost_reported_in_the_final_message_delt assert result["result"]._hidden_params["response_cost"] == 0.00412 +def test_streamed_messages_from_a_deployment_that_only_sets_a_billing_minimum_bill_the_reported_cost(): + logging_obj = _go_logging_obj() + logging_obj.litellm_params = {"metadata": {"model_info": {"min_billed_input_tokens": 100000}}} + + result = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route="/v1/messages", + request_body={"model": "opencode_go/claude-haiku-5-5", "stream": True}, + endpoint_type="messages", + start_time=datetime.now(), + all_chunks=_go_stream_chunks(), + end_time=datetime.now(), + ) + + assert result["kwargs"]["response_cost"] == 0.00412 + + def test_streamed_messages_from_other_providers_ignore_a_zero_usage_cost_field(): logging_obj = _go_logging_obj() logging_obj.model_call_details["custom_llm_provider"] = "anthropic" diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 78c229dd1f5..5918e17be79 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -253,6 +253,42 @@ def test_short_prompt_reserves_the_deployments_billed_input_minimum(deployment_m assert math.isclose(reserved, billed_tokens * input_cost_per_token) +@pytest.mark.parametrize("sibling_minimum", [0, None]) +def test_disconnect_charge_is_only_the_minimum_every_deployment_in_the_group_bills(sibling_minimum: int | None) -> None: + backend_model: Final = "vispark/vision-small" + backend_info: Final = litellm.get_model_info(model=backend_model) + built_in_minimum: Final = backend_info.get("min_billed_input_tokens") + input_cost_per_token: Final = backend_info.get("input_cost_per_token") + assert built_in_minimum is not None and built_in_minimum > 0 + assert input_cost_per_token is not None + largest_minimum: Final = built_in_minimum * 100 + router: Final = Router( + model_list=[ + Deployment( + model_name="mixed-minimum-group", + litellm_params=LiteLLM_Params(model=backend_model, api_key="sk-fake"), + model_info=ModelInfo(id=deployment_id, min_billed_input_tokens=minimum), + ).model_dump() + for deployment_id, minimum in (("largest-minimum", largest_minimum), ("sibling", sibling_minimum)) + ] + ) + prompt_tokens: Final = 12 + request_body: Final = {"model": "mixed-minimum-group", "messages": [{"role": "user", "content": "hi"}]} + token_counts: Final = {"mixed-minimum-group": prompt_tokens} + + disconnect_charge: Final = estimate_request_input_cost( + request_body=request_body, route="/chat/completions", llm_router=router, input_token_counts=token_counts + ) + admission_estimate: Final = estimate_request_max_cost( + request_body=request_body, route="/chat/completions", llm_router=router, input_token_counts=token_counts + ) + + shared_minimum: Final = built_in_minimum if sibling_minimum is None else sibling_minimum + assert disconnect_charge is not None and admission_estimate is not None + assert math.isclose(disconnect_charge, max(prompt_tokens, shared_minimum) * input_cost_per_token) + assert admission_estimate >= largest_minimum * input_cost_per_token + + @pytest.mark.asyncio @pytest.mark.parametrize( "expiry_offset, expected_max_budget", diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index c874e9964bc..c6bedb758bd 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10601,8 +10601,9 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates( 0.00266, ), ("kyma/sonar", "https://kymaapi.com/v1/chat/completions", 0.00721, 0.00721), + ("openrouter/openai/gpt-5.4-nano", "https://openrouter.ai/api/v1/chat/completions", 0.00721, 0.00721), ], - ids=["perplexity", "kyma"], + ids=["perplexity", "kyma", "openrouter"], ) def test_a_deployment_that_only_sets_a_billing_minimum_bills_the_provider_stated_cost( _local_model_cost_map, model: str, chat_url: str, usage_cost: object, stated_cost: float