mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix: bill provider-stated cost for minimum-only deployments and charge a disconnect only the group's shared minimum
This commit is contained in:
parent
aa6ecf0878
commit
dda069ea01
6 changed files with 82 additions and 13 deletions
|
|
@ -1483,6 +1483,12 @@ def _prices_only_a_billing_minimum(litellm_logging_obj: LitellmLoggingObject | N
|
|||
)
|
||||
|
||||
|
||||
def overrides_provider_stated_cost(
|
||||
custom_pricing: bool | None, litellm_logging_obj: LitellmLoggingObject | None
|
||||
) -> bool:
|
||||
return custom_pricing is True and not _prices_only_a_billing_minimum(litellm_logging_obj)
|
||||
|
||||
|
||||
def _without_provider_stated_cost(usage: Usage | None) -> Usage | None:
|
||||
if usage is None or getattr(usage, "cost", None) is None:
|
||||
return usage
|
||||
|
|
@ -1654,7 +1660,7 @@ def completion_cost(
|
|||
cache_read_input_tokens: int | None = None
|
||||
audio_transcription_file_duration: float = 0.0
|
||||
provider_usage_object: Final = get_usage_object(completion_response=completion_response)
|
||||
sets_prices: Final = custom_pricing is True and not _prices_only_a_billing_minimum(litellm_logging_obj)
|
||||
sets_prices: Final = overrides_provider_stated_cost(custom_pricing, litellm_logging_obj)
|
||||
cost_per_token_usage_object: Final[Usage | None] = (
|
||||
_without_provider_stated_cost(provider_usage_object) if sets_prices else provider_usage_object
|
||||
)
|
||||
|
|
@ -2357,7 +2363,9 @@ def response_cost_calculator(
|
|||
getattr(response_object, HIDDEN_PARAMS_ATTR),
|
||||
)
|
||||
provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
|
||||
if provider_response_cost is not None and not custom_pricing:
|
||||
if provider_response_cost is not None and not overrides_provider_stated_cost(
|
||||
custom_pricing, litellm_logging_obj
|
||||
):
|
||||
return provider_response_cost
|
||||
|
||||
response_cost = completion_cost(
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import ANTHROPIC_BATCHES_ROUTE
|
||||
from litellm.cost_calculator import overrides_provider_stated_cost
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
native_messages_reported_cost,
|
||||
override_stamped_response_cost,
|
||||
|
|
@ -346,7 +347,7 @@ class AnthropicPassthroughLoggingHandler:
|
|||
if isinstance(litellm_model_response, ModelResponse)
|
||||
else None
|
||||
)
|
||||
if reported_cost is not None and not custom_pricing:
|
||||
if reported_cost is not None and not overrides_provider_stated_cost(custom_pricing, logging_obj):
|
||||
return reported_cost
|
||||
return litellm.completion_cost(
|
||||
completion_response=litellm_model_response,
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import asyncio
|
|||
import json
|
||||
import math
|
||||
import time
|
||||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
from collections.abc import AsyncIterator, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from itertools import chain
|
||||
|
|
@ -1352,7 +1352,7 @@ def _estimate_request_input_cost_for_model(
|
|||
model_info=model_info,
|
||||
input_tokens=input_tokens,
|
||||
)
|
||||
for model_info in _get_model_cost_infos(model=model, llm_router=llm_router)
|
||||
for model_info in _get_model_cost_infos(model=model, llm_router=llm_router, group_billing_minimum=min)
|
||||
]
|
||||
valid_estimates: Final = [estimate for estimate in estimates if estimate is not None]
|
||||
return max(valid_estimates) if valid_estimates else None
|
||||
|
|
@ -1521,6 +1521,7 @@ def _get_model_cost_info(
|
|||
def _get_model_cost_infos(
|
||||
model: str,
|
||||
llm_router: Router | None,
|
||||
group_billing_minimum: Callable[[Sequence[int]], int] = max,
|
||||
) -> Sequence[Mapping[str, object]]:
|
||||
"""Cost-info candidates to estimate a request against for one model group.
|
||||
|
||||
|
|
@ -1528,7 +1529,9 @@ def _get_model_cost_infos(
|
|||
is unknown. Rather than guess, we estimate the cost against every eligible
|
||||
pricing shape in the group (the group's flat rates plus each deployment's
|
||||
tiered table) and let the caller reserve the maximum, so a cheaper sibling
|
||||
deployment can never leave the request under-reserved.
|
||||
deployment can never leave the request under-reserved. The deployments'
|
||||
billed input minimums fold into one through ``group_billing_minimum``, the
|
||||
largest for admission and the smallest for the input cost a disconnect is charged.
|
||||
"""
|
||||
try:
|
||||
group_info: Final = _get_model_cost_info(model=model, llm_router=llm_router)
|
||||
|
|
@ -1541,12 +1544,14 @@ def _get_model_cost_infos(
|
|||
exc_info=True,
|
||||
)
|
||||
return []
|
||||
billing_minimums: Final = [
|
||||
minimum
|
||||
for deployment_info in deployment_infos
|
||||
if (minimum := _to_int(deployment_info.get("min_billed_input_tokens"))) is not None
|
||||
]
|
||||
base: Final = {**group_info, "min_billed_input_tokens": max(billing_minimums)} if billing_minimums else group_info
|
||||
billing_minimums: Final = tuple(
|
||||
_to_int(deployment_info.get("min_billed_input_tokens")) or 0 for deployment_info in deployment_infos
|
||||
)
|
||||
base: Final = (
|
||||
{**group_info, "min_billed_input_tokens": group_billing_minimum(billing_minimums)}
|
||||
if any(billing_minimums)
|
||||
else group_info
|
||||
)
|
||||
tiered_tables: Final = [
|
||||
table for deployment_info in deployment_infos if (table := _tiered_pricing_table(deployment_info)) is not None
|
||||
]
|
||||
|
|
|
|||
|
|
@ -2951,6 +2951,24 @@ def test_streamed_messages_bill_the_list_cost_reported_in_the_final_message_delt
|
|||
assert result["result"]._hidden_params["response_cost"] == 0.00412
|
||||
|
||||
|
||||
def test_streamed_messages_from_a_deployment_that_only_sets_a_billing_minimum_bill_the_reported_cost():
|
||||
logging_obj = _go_logging_obj()
|
||||
logging_obj.litellm_params = {"metadata": {"model_info": {"min_billed_input_tokens": 100000}}}
|
||||
|
||||
result = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks(
|
||||
litellm_logging_obj=logging_obj,
|
||||
passthrough_success_handler_obj=MagicMock(),
|
||||
url_route="/v1/messages",
|
||||
request_body={"model": "opencode_go/claude-haiku-5-5", "stream": True},
|
||||
endpoint_type="messages",
|
||||
start_time=datetime.now(),
|
||||
all_chunks=_go_stream_chunks(),
|
||||
end_time=datetime.now(),
|
||||
)
|
||||
|
||||
assert result["kwargs"]["response_cost"] == 0.00412
|
||||
|
||||
|
||||
def test_streamed_messages_from_other_providers_ignore_a_zero_usage_cost_field():
|
||||
logging_obj = _go_logging_obj()
|
||||
logging_obj.model_call_details["custom_llm_provider"] = "anthropic"
|
||||
|
|
|
|||
|
|
@ -253,6 +253,42 @@ def test_short_prompt_reserves_the_deployments_billed_input_minimum(deployment_m
|
|||
assert math.isclose(reserved, billed_tokens * input_cost_per_token)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sibling_minimum", [0, None])
|
||||
def test_disconnect_charge_is_only_the_minimum_every_deployment_in_the_group_bills(sibling_minimum: int | None) -> None:
|
||||
backend_model: Final = "vispark/vision-small"
|
||||
backend_info: Final = litellm.get_model_info(model=backend_model)
|
||||
built_in_minimum: Final = backend_info.get("min_billed_input_tokens")
|
||||
input_cost_per_token: Final = backend_info.get("input_cost_per_token")
|
||||
assert built_in_minimum is not None and built_in_minimum > 0
|
||||
assert input_cost_per_token is not None
|
||||
largest_minimum: Final = built_in_minimum * 100
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
Deployment(
|
||||
model_name="mixed-minimum-group",
|
||||
litellm_params=LiteLLM_Params(model=backend_model, api_key="sk-fake"),
|
||||
model_info=ModelInfo(id=deployment_id, min_billed_input_tokens=minimum),
|
||||
).model_dump()
|
||||
for deployment_id, minimum in (("largest-minimum", largest_minimum), ("sibling", sibling_minimum))
|
||||
]
|
||||
)
|
||||
prompt_tokens: Final = 12
|
||||
request_body: Final = {"model": "mixed-minimum-group", "messages": [{"role": "user", "content": "hi"}]}
|
||||
token_counts: Final = {"mixed-minimum-group": prompt_tokens}
|
||||
|
||||
disconnect_charge: Final = estimate_request_input_cost(
|
||||
request_body=request_body, route="/chat/completions", llm_router=router, input_token_counts=token_counts
|
||||
)
|
||||
admission_estimate: Final = estimate_request_max_cost(
|
||||
request_body=request_body, route="/chat/completions", llm_router=router, input_token_counts=token_counts
|
||||
)
|
||||
|
||||
shared_minimum: Final = built_in_minimum if sibling_minimum is None else sibling_minimum
|
||||
assert disconnect_charge is not None and admission_estimate is not None
|
||||
assert math.isclose(disconnect_charge, max(prompt_tokens, shared_minimum) * input_cost_per_token)
|
||||
assert admission_estimate >= largest_minimum * input_cost_per_token
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"expiry_offset, expected_max_budget",
|
||||
|
|
|
|||
|
|
@ -10601,8 +10601,9 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates(
|
|||
0.00266,
|
||||
),
|
||||
("kyma/sonar", "https://kymaapi.com/v1/chat/completions", 0.00721, 0.00721),
|
||||
("openrouter/openai/gpt-5.4-nano", "https://openrouter.ai/api/v1/chat/completions", 0.00721, 0.00721),
|
||||
],
|
||||
ids=["perplexity", "kyma"],
|
||||
ids=["perplexity", "kyma", "openrouter"],
|
||||
)
|
||||
def test_a_deployment_that_only_sets_a_billing_minimum_bills_the_provider_stated_cost(
|
||||
_local_model_cost_map, model: str, chat_url: str, usage_cost: object, stated_cost: float
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue