fix: bill provider-stated cost for minimum-only deployments and charge a disconnect only the group's shared minimum

This commit is contained in:
mateo-berri 2026-10-29 17:13:14 -07:00
parent aa6ecf0878
commit dda069ea01
6 changed files with 82 additions and 13 deletions

View file

@ -1483,6 +1483,12 @@ def _prices_only_a_billing_minimum(litellm_logging_obj: LitellmLoggingObject | N
)
def overrides_provider_stated_cost(
custom_pricing: bool | None, litellm_logging_obj: LitellmLoggingObject | None
) -> bool:
return custom_pricing is True and not _prices_only_a_billing_minimum(litellm_logging_obj)
def _without_provider_stated_cost(usage: Usage | None) -> Usage | None:
if usage is None or getattr(usage, "cost", None) is None:
return usage
@ -1654,7 +1660,7 @@ def completion_cost(
cache_read_input_tokens: int | None = None
audio_transcription_file_duration: float = 0.0
provider_usage_object: Final = get_usage_object(completion_response=completion_response)
sets_prices: Final = custom_pricing is True and not _prices_only_a_billing_minimum(litellm_logging_obj)
sets_prices: Final = overrides_provider_stated_cost(custom_pricing, litellm_logging_obj)
cost_per_token_usage_object: Final[Usage | None] = (
_without_provider_stated_cost(provider_usage_object) if sets_prices else provider_usage_object
)
@ -2357,7 +2363,9 @@ def response_cost_calculator(
getattr(response_object, HIDDEN_PARAMS_ATTR),
)
provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
if provider_response_cost is not None and not custom_pricing:
if provider_response_cost is not None and not overrides_provider_stated_cost(
custom_pricing, litellm_logging_obj
):
return provider_response_cost
response_cost = completion_cost(

View file

@ -10,6 +10,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import ANTHROPIC_BATCHES_ROUTE
from litellm.cost_calculator import overrides_provider_stated_cost
from litellm.litellm_core_utils.core_helpers import (
native_messages_reported_cost,
override_stamped_response_cost,
@ -346,7 +347,7 @@ class AnthropicPassthroughLoggingHandler:
if isinstance(litellm_model_response, ModelResponse)
else None
)
if reported_cost is not None and not custom_pricing:
if reported_cost is not None and not overrides_provider_stated_cost(custom_pricing, logging_obj):
return reported_cost
return litellm.completion_cost(
completion_response=litellm_model_response,

View file

@ -4,7 +4,7 @@ import asyncio
import json
import math
import time
from collections.abc import AsyncIterator, Mapping, Sequence
from collections.abc import AsyncIterator, Callable, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from itertools import chain
@ -1352,7 +1352,7 @@ def _estimate_request_input_cost_for_model(
model_info=model_info,
input_tokens=input_tokens,
)
for model_info in _get_model_cost_infos(model=model, llm_router=llm_router)
for model_info in _get_model_cost_infos(model=model, llm_router=llm_router, group_billing_minimum=min)
]
valid_estimates: Final = [estimate for estimate in estimates if estimate is not None]
return max(valid_estimates) if valid_estimates else None
@ -1521,6 +1521,7 @@ def _get_model_cost_info(
def _get_model_cost_infos(
model: str,
llm_router: Router | None,
group_billing_minimum: Callable[[Sequence[int]], int] = max,
) -> Sequence[Mapping[str, object]]:
"""Cost-info candidates to estimate a request against for one model group.
@ -1528,7 +1529,9 @@ def _get_model_cost_infos(
is unknown. Rather than guess, we estimate the cost against every eligible
pricing shape in the group (the group's flat rates plus each deployment's
tiered table) and let the caller reserve the maximum, so a cheaper sibling
deployment can never leave the request under-reserved.
deployment can never leave the request under-reserved. The deployments'
billed input minimums fold into one through ``group_billing_minimum``, the
largest for admission and the smallest for the input cost a disconnect is charged.
"""
try:
group_info: Final = _get_model_cost_info(model=model, llm_router=llm_router)
@ -1541,12 +1544,14 @@ def _get_model_cost_infos(
exc_info=True,
)
return []
billing_minimums: Final = [
minimum
for deployment_info in deployment_infos
if (minimum := _to_int(deployment_info.get("min_billed_input_tokens"))) is not None
]
base: Final = {**group_info, "min_billed_input_tokens": max(billing_minimums)} if billing_minimums else group_info
billing_minimums: Final = tuple(
_to_int(deployment_info.get("min_billed_input_tokens")) or 0 for deployment_info in deployment_infos
)
base: Final = (
{**group_info, "min_billed_input_tokens": group_billing_minimum(billing_minimums)}
if any(billing_minimums)
else group_info
)
tiered_tables: Final = [
table for deployment_info in deployment_infos if (table := _tiered_pricing_table(deployment_info)) is not None
]

View file

@ -2951,6 +2951,24 @@ def test_streamed_messages_bill_the_list_cost_reported_in_the_final_message_delt
assert result["result"]._hidden_params["response_cost"] == 0.00412
def test_streamed_messages_from_a_deployment_that_only_sets_a_billing_minimum_bill_the_reported_cost():
logging_obj = _go_logging_obj()
logging_obj.litellm_params = {"metadata": {"model_info": {"min_billed_input_tokens": 100000}}}
result = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks(
litellm_logging_obj=logging_obj,
passthrough_success_handler_obj=MagicMock(),
url_route="/v1/messages",
request_body={"model": "opencode_go/claude-haiku-5-5", "stream": True},
endpoint_type="messages",
start_time=datetime.now(),
all_chunks=_go_stream_chunks(),
end_time=datetime.now(),
)
assert result["kwargs"]["response_cost"] == 0.00412
def test_streamed_messages_from_other_providers_ignore_a_zero_usage_cost_field():
logging_obj = _go_logging_obj()
logging_obj.model_call_details["custom_llm_provider"] = "anthropic"

View file

@ -253,6 +253,42 @@ def test_short_prompt_reserves_the_deployments_billed_input_minimum(deployment_m
assert math.isclose(reserved, billed_tokens * input_cost_per_token)
@pytest.mark.parametrize("sibling_minimum", [0, None])
def test_disconnect_charge_is_only_the_minimum_every_deployment_in_the_group_bills(sibling_minimum: int | None) -> None:
backend_model: Final = "vispark/vision-small"
backend_info: Final = litellm.get_model_info(model=backend_model)
built_in_minimum: Final = backend_info.get("min_billed_input_tokens")
input_cost_per_token: Final = backend_info.get("input_cost_per_token")
assert built_in_minimum is not None and built_in_minimum > 0
assert input_cost_per_token is not None
largest_minimum: Final = built_in_minimum * 100
router: Final = Router(
model_list=[
Deployment(
model_name="mixed-minimum-group",
litellm_params=LiteLLM_Params(model=backend_model, api_key="sk-fake"),
model_info=ModelInfo(id=deployment_id, min_billed_input_tokens=minimum),
).model_dump()
for deployment_id, minimum in (("largest-minimum", largest_minimum), ("sibling", sibling_minimum))
]
)
prompt_tokens: Final = 12
request_body: Final = {"model": "mixed-minimum-group", "messages": [{"role": "user", "content": "hi"}]}
token_counts: Final = {"mixed-minimum-group": prompt_tokens}
disconnect_charge: Final = estimate_request_input_cost(
request_body=request_body, route="/chat/completions", llm_router=router, input_token_counts=token_counts
)
admission_estimate: Final = estimate_request_max_cost(
request_body=request_body, route="/chat/completions", llm_router=router, input_token_counts=token_counts
)
shared_minimum: Final = built_in_minimum if sibling_minimum is None else sibling_minimum
assert disconnect_charge is not None and admission_estimate is not None
assert math.isclose(disconnect_charge, max(prompt_tokens, shared_minimum) * input_cost_per_token)
assert admission_estimate >= largest_minimum * input_cost_per_token
@pytest.mark.asyncio
@pytest.mark.parametrize(
"expiry_offset, expected_max_budget",

View file

@ -10601,8 +10601,9 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates(
0.00266,
),
("kyma/sonar", "https://kymaapi.com/v1/chat/completions", 0.00721, 0.00721),
("openrouter/openai/gpt-5.4-nano", "https://openrouter.ai/api/v1/chat/completions", 0.00721, 0.00721),
],
ids=["perplexity", "kyma"],
ids=["perplexity", "kyma", "openrouter"],
)
def test_a_deployment_that_only_sets_a_billing_minimum_bills_the_provider_stated_cost(
_local_model_cost_map, model: str, chat_url: str, usage_cost: object, stated_cost: float