mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-12 08:42:18 +00:00
fix(cost): bill a provider reported usage.cost on every path unless the deployment is priced
cost_per_token bills a finite non-negative usage.cost from any provider, split across input and output by the ratio of the token math, unless the caller priced the call itself. completion_cost drops the reported cost when the deployment has custom pricing or a base_model, so operator rates win on streams too, where stream_chunk_builder used to stamp usage.cost as the final response_cost ahead of custom pricing and margins. The xAI and Perplexity calculators lose their private copies of the same rule. Co-authored-by: AoHanBei <dongaohan21@gmail.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
parent
de5dc5b238
commit
c200c2f9f8
10 changed files with 381 additions and 61 deletions
|
|
@ -5,7 +5,7 @@ import math
|
|||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime
|
||||
from functools import lru_cache
|
||||
from functools import lru_cache, partial
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
|
||||
|
||||
|
|
@ -22,7 +22,7 @@ from litellm.constants import (
|
|||
OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES,
|
||||
REALTIME_METERED_USAGE_KEY,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, provider_reported_usage_cost
|
||||
from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
|
||||
from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
|
||||
|
|
@ -408,6 +408,19 @@ def _per_second_pricing_cost(
|
|||
return resolved_cost_per_second * seconds, 0.0
|
||||
|
||||
|
||||
def _billable_reported_cost(usage_object: Usage | None) -> float | None:
|
||||
return _billable_header_cost(provider_reported_usage_cost(usage_object, None))
|
||||
|
||||
|
||||
def _split_reported_cost(reported_cost: float, priced: tuple[float, float] | None) -> tuple[float, float]:
|
||||
"""Providers report one total, so attribute it to input and output in the ratio the token math gives."""
|
||||
priced_total: Final = (priced[0] + priced[1]) if priced is not None else 0.0
|
||||
if priced is None or priced_total <= 0:
|
||||
return 0.0, reported_cost
|
||||
input_cost: Final = reported_cost * priced[0] / priced_total
|
||||
return input_cost, reported_cost - input_cost
|
||||
|
||||
|
||||
def cost_per_token(
|
||||
model: str = "",
|
||||
prompt_tokens: int = 0,
|
||||
|
|
@ -444,6 +457,87 @@ def cost_per_token(
|
|||
request_model: str | None = None, # original request model for router detection
|
||||
custom_model_info: OCRPricing | None = None,
|
||||
bedrock_model_info: ModelInfo | None = None,
|
||||
) -> tuple[float, float]:
|
||||
"""Cost in USD for prompt and completion tokens.
|
||||
|
||||
A numeric ``usage.cost`` the provider reported is the bill, split by the ratio of our own token math,
|
||||
unless the caller priced the call itself with ``custom_cost_per_token`` or ``custom_cost_per_second``.
|
||||
Without a reported cost the token math is the bill.
|
||||
"""
|
||||
priced_by_caller: Final = custom_cost_per_token is not None or custom_cost_per_second is not None
|
||||
reported_cost: Final = None if priced_by_caller else _billable_reported_cost(usage_object)
|
||||
rated: Final = partial(
|
||||
_cost_per_token_from_rates,
|
||||
model=model,
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
response_time_ms=response_time_ms,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
region_name=region_name,
|
||||
prompt_characters=prompt_characters,
|
||||
completion_characters=completion_characters,
|
||||
cache_creation_input_tokens=cache_creation_input_tokens,
|
||||
cache_read_input_tokens=cache_read_input_tokens,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
number_of_queries=number_of_queries,
|
||||
usage_object=usage_object,
|
||||
rerank_billed_units=rerank_billed_units,
|
||||
call_type=call_type,
|
||||
audio_transcription_file_duration=audio_transcription_file_duration,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
response=response,
|
||||
request_model=request_model,
|
||||
custom_model_info=custom_model_info,
|
||||
bedrock_model_info=bedrock_model_info,
|
||||
)
|
||||
if reported_cost is None:
|
||||
return rated()
|
||||
try:
|
||||
return _split_reported_cost(reported_cost, rated())
|
||||
except Exception: # noqa: BLE001 # an unmapped model has no rates to split by, the reported total still bills
|
||||
verbose_logger.debug("No rates for model=%s, billing the provider reported cost %s", model, reported_cost)
|
||||
return _split_reported_cost(reported_cost, None)
|
||||
|
||||
|
||||
def _cost_per_token_from_rates(
|
||||
model: str = "",
|
||||
prompt_tokens: int = 0,
|
||||
completion_tokens: int = 0,
|
||||
response_time_ms: float | None = 0.0,
|
||||
custom_llm_provider: str | None = None,
|
||||
region_name=None,
|
||||
### CHARACTER PRICING ###
|
||||
prompt_characters: int | None = None,
|
||||
completion_characters: int | None = None,
|
||||
### PROMPT CACHING PRICING ### - used for anthropic
|
||||
cache_creation_input_tokens: int | None = 0,
|
||||
cache_read_input_tokens: int | None = 0,
|
||||
### CUSTOM PRICING ###
|
||||
custom_cost_per_token: CostPerToken | None = None,
|
||||
custom_cost_per_second: float | None = None,
|
||||
### NUMBER OF QUERIES ###
|
||||
number_of_queries: int | None = None,
|
||||
### USAGE OBJECT ###
|
||||
usage_object: Usage | None = None, # just read the usage object if provided
|
||||
### BILLED UNITS ###
|
||||
rerank_billed_units: RerankBilledUnits | None = None,
|
||||
### CALL TYPE ###
|
||||
call_type: CallTypesLiteral = "completion",
|
||||
audio_transcription_file_duration: float = 0.0, # for audio transcription calls - the file time in seconds
|
||||
### SERVICE TIER ###
|
||||
service_tier: str | None = None, # for OpenAI service tier pricing
|
||||
### DATA RESIDENCY ###
|
||||
data_residency: str | None = None, # for OpenAI regional-processing uplift (e.g. "eu", "us")
|
||||
### VERTEX LOCATION ###
|
||||
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
|
||||
response: object | None = None,
|
||||
### REQUEST MODEL ###
|
||||
request_model: str | None = None, # original request model for router detection
|
||||
custom_model_info: OCRPricing | None = None,
|
||||
bedrock_model_info: ModelInfo | None = None,
|
||||
) -> tuple[float, float]:
|
||||
"""
|
||||
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
|
||||
|
|
@ -1610,7 +1704,9 @@ def completion_cost(
|
|||
audio_transcription_file_duration: float = 0.0
|
||||
provider_usage_object: Final = get_usage_object(completion_response=completion_response)
|
||||
cost_per_token_usage_object: Final[Usage | None] = (
|
||||
_without_provider_stated_cost(provider_usage_object) if custom_pricing else provider_usage_object
|
||||
_without_provider_stated_cost(provider_usage_object)
|
||||
if custom_pricing or base_model is not None
|
||||
else provider_usage_object
|
||||
)
|
||||
rerank_billed_units: RerankBilledUnits | None = None
|
||||
|
||||
|
|
|
|||
|
|
@ -1350,9 +1350,6 @@ def stream_chunk_builder_text_completion(
|
|||
return TextCompletionResponse(**response)
|
||||
|
||||
|
||||
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value})
|
||||
|
||||
|
||||
def _logged_provider(logging_obj: Optional["Logging"]) -> object:
|
||||
if logging_obj is None:
|
||||
return None
|
||||
|
|
@ -1360,10 +1357,6 @@ def _logged_provider(logging_obj: Optional["Logging"]) -> object:
|
|||
return provider
|
||||
|
||||
|
||||
def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
|
||||
return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS
|
||||
|
||||
|
||||
def _stream_builder_response_cost(
|
||||
response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object]
|
||||
) -> float | None:
|
||||
|
|
@ -1374,9 +1367,6 @@ def _stream_builder_response_cost(
|
|||
for chunk in reversed(chunks)
|
||||
)
|
||||
return next((cost for cost in reported if cost is not None), None)
|
||||
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
|
||||
if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj):
|
||||
return float(usage_cost)
|
||||
if logging_obj is not None:
|
||||
return None
|
||||
provider_hint: Final = response._hidden_params.get( # pyright: ignore[reportPrivateUsage] # no public accessor
|
||||
|
|
|
|||
|
|
@ -24,22 +24,6 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
|
|||
Returns:
|
||||
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
|
||||
"""
|
||||
## USE PRE-CALCULATED COST FROM PERPLEXITY IF AVAILABLE
|
||||
## Perplexity returns accurate cost in usage.cost.total_cost including request fees.
|
||||
## By the time it reaches here, ResponseAPIUsage.parse_cost has already flattened
|
||||
## that dict down to a float, so both shapes must be accepted.
|
||||
cost_info: Final = getattr(usage, "cost", None)
|
||||
total_cost: float | None = None
|
||||
if isinstance(cost_info, dict):
|
||||
total_cost = cost_info.get("total_cost")
|
||||
elif isinstance(cost_info, (int, float)) and not isinstance(cost_info, bool):
|
||||
total_cost = float(cost_info)
|
||||
if total_cost is not None:
|
||||
# Return total cost as completion_cost (prompt_cost=0) since Perplexity
|
||||
# doesn't break down by input/output in their cost object
|
||||
return (0.0, float(total_cost))
|
||||
|
||||
## FALLBACK: Calculate cost manually if Perplexity doesn't provide it
|
||||
## GET MODEL INFO
|
||||
model_info: Final = get_model_info(model=model, custom_llm_provider="perplexity")
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
"""
|
||||
Helper util for handling XAI-specific cost calculation
|
||||
- Prefers the cost xAI reports on the response over recomputing it locally
|
||||
- Uses the generic cost calculator which already handles tiered pricing correctly
|
||||
- Handles XAI-specific reasoning token billing (billed as part of completion tokens)
|
||||
- Skips the web search surcharge when xAI reported a total that already covers it
|
||||
"""
|
||||
|
||||
import math
|
||||
|
|
@ -49,19 +49,6 @@ def _cost_reported_by_xai(usage: "Usage") -> float | None:
|
|||
return float(reported_cost)
|
||||
|
||||
|
||||
def _split_reported_cost(model: str, usage: Usage, reported_cost: float) -> tuple[float, float]:
|
||||
"""xAI reports one total, so attribute it to input and output in the ratio our own token math gives."""
|
||||
try:
|
||||
priced_input, priced_output = _token_math_cost_per_token(model, usage)
|
||||
except Exception: # noqa: BLE001 # generic_cost_per_token raises bare Exception for unmapped models
|
||||
return 0.0, reported_cost
|
||||
priced_total: Final = priced_input + priced_output
|
||||
if priced_total <= 0:
|
||||
return 0.0, reported_cost
|
||||
input_cost: Final = reported_cost * priced_input / priced_total
|
||||
return input_cost, reported_cost - input_cost
|
||||
|
||||
|
||||
def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
|
||||
"""
|
||||
Calculates the cost per token for a given XAI model, prompt tokens, and completion tokens.
|
||||
|
|
@ -74,9 +61,6 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
|
|||
Returns:
|
||||
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
|
||||
"""
|
||||
reported_cost: Final = _cost_reported_by_xai(usage)
|
||||
if reported_cost is not None:
|
||||
return _split_reported_cost(model, usage, reported_cost)
|
||||
return _token_math_cost_per_token(model, usage)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11878,7 +11878,89 @@ GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM: Final = GPT_5_6_TERRA_MESSAGES_BRIDGED_JS
|
|||
}
|
||||
)
|
||||
|
||||
|
||||
# An OpenAI-compatible gateway that reports what it charged under usage.cost (#47533). The reported total is the bill,
|
||||
# split across input and output by the ratio of the cost map rates (100 * 1e-6 : 20 * 3e-6), on plain and streamed
|
||||
# replies alike. A deployment the operator priced keeps the operator's rates on both.
|
||||
_GATEWAY_REPORTED_USAGE: Final = {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120, "cost": 0.037}
|
||||
_GATEWAY_REPORTED_EXPECTED: Final = ExactExpected(
|
||||
spend=0.037, input_cost=0.023125, output_cost=0.013875, prompt_tokens=100, completion_tokens=20
|
||||
)
|
||||
_GATEWAY_OPERATOR_PRICED: Final = Deployment(input_cost_per_token=2e-06, output_cost_per_token=6e-06)
|
||||
_GATEWAY_OPERATOR_PRICED_EXPECTED: Final = ExactExpected(
|
||||
spend=0.00032, input_cost=0.0002, output_cost=0.00012, prompt_tokens=100, completion_tokens=20
|
||||
)
|
||||
_GATEWAY_CHUNK: Final = {
|
||||
"id": "chatcmpl-$REQUEST_ID",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "gateway-reported-cost-fixture",
|
||||
}
|
||||
|
||||
GATEWAY_REPORTED_COST: Final = CostTrackingTestCase(
|
||||
name="gateway-reported-cost",
|
||||
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
model="gateway-reported-cost-fixture",
|
||||
endpoint="/v1/chat/completions",
|
||||
request={"model": "$MODEL", "messages": [{"role": "user", "content": "reported cost"}], "stream": False},
|
||||
response=JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
"id": "chatcmpl-$REQUEST_ID",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gateway-reported-cost-fixture",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": _GATEWAY_REPORTED_USAGE,
|
||||
},
|
||||
),
|
||||
expected=_GATEWAY_REPORTED_EXPECTED,
|
||||
)
|
||||
|
||||
GATEWAY_REPORTED_COST_STREAM: Final = CostTrackingTestCase(
|
||||
name="gateway-reported-cost-stream",
|
||||
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
model="gateway-reported-cost-fixture",
|
||||
endpoint="/v1/chat/completions",
|
||||
request={
|
||||
"model": "$MODEL",
|
||||
"messages": [{"role": "user", "content": "reported cost"}],
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
},
|
||||
response=SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=sse_frames(
|
||||
{**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]},
|
||||
{**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
|
||||
{**_GATEWAY_CHUNK, "choices": [], "usage": _GATEWAY_REPORTED_USAGE},
|
||||
done=True,
|
||||
),
|
||||
),
|
||||
expected=_GATEWAY_REPORTED_EXPECTED,
|
||||
)
|
||||
|
||||
GATEWAY_REPORTED_COST_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST.model_copy(
|
||||
update={
|
||||
"name": "gateway-reported-cost-operator_priced",
|
||||
"deployment": _GATEWAY_OPERATOR_PRICED,
|
||||
"expected": _GATEWAY_OPERATOR_PRICED_EXPECTED,
|
||||
}
|
||||
)
|
||||
|
||||
GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_STREAM.model_copy(
|
||||
update={
|
||||
"name": "gateway-reported-cost-operator_priced-stream",
|
||||
"deployment": _GATEWAY_OPERATOR_PRICED,
|
||||
"expected": _GATEWAY_OPERATOR_PRICED_EXPECTED,
|
||||
}
|
||||
)
|
||||
|
||||
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
||||
GATEWAY_REPORTED_COST,
|
||||
GATEWAY_REPORTED_COST_STREAM,
|
||||
GATEWAY_REPORTED_COST_OPERATOR_PRICED,
|
||||
GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM,
|
||||
GPT_5_6_SLASH_ALIAS,
|
||||
GPT_5_6_SLASH_ALIAS_STREAM,
|
||||
GPT_6_SOL_RESPONSES_DEPLOYMENT_JSON,
|
||||
|
|
@ -12120,6 +12202,10 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
)
|
||||
|
||||
PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
||||
StreamParityTestCase(plain=GATEWAY_REPORTED_COST, streamed=GATEWAY_REPORTED_COST_STREAM),
|
||||
StreamParityTestCase(
|
||||
plain=GATEWAY_REPORTED_COST_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM
|
||||
),
|
||||
StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM),
|
||||
StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_INCOMPLETE),
|
||||
StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_UNVALIDATED),
|
||||
|
|
|
|||
|
|
@ -26,6 +26,12 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
|
|||
"input_cost_per_token_above_128k_tokens": 2.2e-07,
|
||||
"output_cost_per_token_above_128k_tokens": 8.8e-07,
|
||||
},
|
||||
"gateway-reported-cost-fixture": {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 1e-06,
|
||||
"output_cost_per_token": 3e-06,
|
||||
},
|
||||
"openrouter/anthropic/claude-sonnet-5": {
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
|
|
|
|||
|
|
@ -2401,14 +2401,14 @@ def test_perplexity_streaming_dict_cost_bills_through_its_own_calculator():
|
|||
) == pytest.approx(0.00503)
|
||||
|
||||
|
||||
def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map():
|
||||
def test_openai_compatible_streaming_cost_is_the_reported_cost():
|
||||
import litellm
|
||||
from litellm.cost_calculator import (
|
||||
get_response_cost_from_hidden_params,
|
||||
response_cost_calculator,
|
||||
)
|
||||
|
||||
model = "openai/streams-cost-in-nanodollars"
|
||||
model = "openai/streams-with-a-reported-cost"
|
||||
litellm.register_model(
|
||||
{
|
||||
model: {
|
||||
|
|
@ -2423,7 +2423,7 @@ def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map():
|
|||
id="chatcmpl-openai-compatible",
|
||||
model=model,
|
||||
choices=[],
|
||||
usage=Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=3_144_000),
|
||||
usage=Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.037),
|
||||
)
|
||||
|
||||
CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response, "openai")
|
||||
|
|
@ -2435,7 +2435,7 @@ def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map():
|
|||
custom_llm_provider="openai",
|
||||
call_type="completion",
|
||||
optional_params={},
|
||||
) == pytest.approx(2e-5)
|
||||
) == pytest.approx(0.037)
|
||||
|
||||
|
||||
def test_xai_streaming_reported_cost_still_takes_the_margin(monkeypatch):
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import pytest
|
|||
|
||||
# Add the project root to Python path
|
||||
import litellm
|
||||
from litellm.cost_calculator import cost_per_token as billed_cost_per_token
|
||||
from litellm.llms.perplexity.cost_calculator import (
|
||||
cost_per_token as perplexity_cost_per_token,
|
||||
)
|
||||
|
|
@ -109,13 +110,13 @@ class TestPerplexityCostCalculator:
|
|||
"total_cost": 0.008,
|
||||
}
|
||||
|
||||
prompt_cost, completion_cost = perplexity_cost_per_token(model="sonar-pro", usage=usage)
|
||||
prompt_cost, completion_cost = billed_cost_per_token(
|
||||
model="perplexity/sonar-pro", custom_llm_provider="perplexity", usage_object=usage
|
||||
)
|
||||
|
||||
# When Perplexity provides total_cost, we use it directly
|
||||
# prompt_cost should be 0, completion_cost should be total_cost
|
||||
assert prompt_cost == 0.0
|
||||
assert completion_cost == 0.008
|
||||
assert prompt_cost + completion_cost == 0.008
|
||||
# When Perplexity provides total_cost, that total is the bill
|
||||
assert math.isclose(prompt_cost + completion_cost, 0.008, rel_tol=1e-10)
|
||||
assert prompt_cost > 0.0 and completion_cost > 0.0
|
||||
|
||||
def test_uses_perplexity_provided_cost_when_normalized_to_float(self):
|
||||
"""
|
||||
|
|
@ -135,10 +136,11 @@ class TestPerplexityCostCalculator:
|
|||
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
|
||||
usage.cost = 0.008
|
||||
|
||||
prompt_cost, completion_cost = perplexity_cost_per_token(model="sonar-pro", usage=usage)
|
||||
prompt_cost, completion_cost = billed_cost_per_token(
|
||||
model="perplexity/sonar-pro", custom_llm_provider="perplexity", usage_object=usage
|
||||
)
|
||||
|
||||
assert prompt_cost == 0.0
|
||||
assert completion_cost == 0.008
|
||||
assert math.isclose(prompt_cost + completion_cost, 0.008, rel_tol=1e-10)
|
||||
|
||||
OFF_PEAK_MODEL = "sonar-off-peak-test"
|
||||
OFF_PEAK_WINDOW = "14:00-00:00"
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import math
|
|||
import os
|
||||
|
||||
import litellm
|
||||
from litellm.cost_calculator import cost_per_token as billed_cost_per_token
|
||||
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
|
||||
StandardBuiltInToolCostTracking,
|
||||
)
|
||||
|
|
@ -115,7 +116,9 @@ class TestXAICostCalculator:
|
|||
model="grok-4.7", usage=Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300)
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(model="grok-4.7", usage=usage)
|
||||
prompt_cost, completion_cost = billed_cost_per_token(
|
||||
model="xai/grok-4.7", custom_llm_provider="xai", usage_object=usage
|
||||
)
|
||||
|
||||
assert math.isclose(prompt_cost + completion_cost, 0.0037756, rel_tol=1e-10)
|
||||
assert math.isclose(prompt_cost / completion_cost, priced_prompt / priced_completion, rel_tol=1e-10)
|
||||
|
|
@ -165,7 +168,10 @@ class TestXAICostCalculator:
|
|||
"""A reported zero is a real answer, not a missing value."""
|
||||
usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300, cost=0.0)
|
||||
|
||||
assert cost_per_token(model="grok-4-latest", usage=usage) == (0.0, 0.0)
|
||||
assert billed_cost_per_token(model="xai/grok-4-latest", custom_llm_provider="xai", usage_object=usage) == (
|
||||
0.0,
|
||||
0.0,
|
||||
)
|
||||
|
||||
def test_custom_pricing_beats_the_reported_cost(self):
|
||||
response = ModelResponse(
|
||||
|
|
|
|||
|
|
@ -10522,3 +10522,169 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates(
|
|||
assert entry is not None
|
||||
assert entry[1]["input_cost_per_token_above_300k_tokens"] == 0.03
|
||||
assert entry[1][tier_field] == (tier_override if tier_override is not None else 0.007)
|
||||
|
||||
|
||||
_REPORTED_USAGE: Final = {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120, "cost": 0.037}
|
||||
_REPORTED_COST_FIXTURE_MODEL: Final = "openai/reported-cost-fixture"
|
||||
_REPORTED_COST_FIXTURE_RATES: Final = {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 3e-6,
|
||||
}
|
||||
|
||||
|
||||
def _reported_cost_logging(stream: bool, litellm_params: Mapping[str, object]) -> Logging:
|
||||
logging_obj: Final = Logging(
|
||||
model=_REPORTED_COST_FIXTURE_MODEL,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=stream,
|
||||
call_type="completion",
|
||||
start_time=datetime.datetime(2026, 1, 1),
|
||||
litellm_call_id="reported-cost",
|
||||
function_id="reported-cost",
|
||||
)
|
||||
logging_obj.update_from_kwargs(
|
||||
kwargs={}, optional_params={}, litellm_params={"custom_llm_provider": "openai", **litellm_params}
|
||||
)
|
||||
logging_obj.model_call_details["custom_llm_provider"] = "openai"
|
||||
return logging_obj
|
||||
|
||||
|
||||
def _reported_cost_stream_chunk(content: str | None, finish_reason: str | None, usage: Usage | None) -> object:
|
||||
return litellm.ModelResponseStream(
|
||||
id="chatcmpl-reported",
|
||||
model="reported-cost-fixture",
|
||||
created=1,
|
||||
object="chat.completion.chunk",
|
||||
choices=[litellm.types.utils.StreamingChoices(index=0, delta=litellm.types.utils.Delta(content=content), finish_reason=finish_reason)],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
|
||||
def _bill_reported_cost(stream: bool, litellm_params: Mapping[str, object], hidden_params: Mapping[str, object]) -> float | None:
|
||||
"""The spend the proxy logs for one upstream reply carrying usage.cost, streamed or not."""
|
||||
logging_obj: Final = _reported_cost_logging(stream, litellm_params)
|
||||
if stream:
|
||||
response: Final[object] = litellm.stream_chunk_builder(
|
||||
[
|
||||
_reported_cost_stream_chunk("hi", None, None),
|
||||
_reported_cost_stream_chunk(None, "stop", None),
|
||||
_reported_cost_stream_chunk(None, None, Usage(**_REPORTED_USAGE)),
|
||||
],
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
else:
|
||||
response = ModelResponse(
|
||||
id="chatcmpl-reported",
|
||||
model="reported-cost-fixture",
|
||||
choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")],
|
||||
usage=Usage(**_REPORTED_USAGE),
|
||||
)
|
||||
assert isinstance(response, ModelResponse)
|
||||
response._hidden_params.update({"custom_llm_provider": "openai", **hidden_params})
|
||||
return logging_obj.response_cost_calculator(result=response)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
@pytest.mark.parametrize("mapped", [False, True])
|
||||
def test_provider_reported_usage_cost_is_billed_on_stream_and_non_stream(
|
||||
monkeypatch: pytest.MonkeyPatch, stream: bool, mapped: bool
|
||||
) -> None:
|
||||
"""Regression for #47533: a plain openai-compatible deployment billed usage.cost on streams and $0 on non-stream."""
|
||||
if mapped:
|
||||
monkeypatch.setitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, _REPORTED_COST_FIXTURE_RATES)
|
||||
else:
|
||||
monkeypatch.delitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, raising=False)
|
||||
|
||||
assert _bill_reported_cost(stream, {}, {}) == pytest.approx(0.037)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
def test_deployment_pricing_beats_the_provider_reported_usage_cost(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None:
|
||||
"""Streams used to bill usage.cost even when the operator priced the deployment; both paths bill the operator's rates."""
|
||||
deployment_id: Final = "reported-cost-deployment-priced-by-the-operator"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
deployment_id,
|
||||
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 2e-6, "output_cost_per_token": 6e-6},
|
||||
)
|
||||
litellm_params: Final = {
|
||||
"input_cost_per_token": 2e-6,
|
||||
"output_cost_per_token": 6e-6,
|
||||
"model_info": {"id": deployment_id},
|
||||
}
|
||||
|
||||
assert _bill_reported_cost(stream, litellm_params, {"model_id": deployment_id}) == pytest.approx(
|
||||
100 * 2e-6 + 20 * 6e-6
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
def test_base_model_pricing_beats_the_provider_reported_usage_cost(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None:
|
||||
base_model: Final = "openai/reported-cost-base-model-fixture"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
base_model,
|
||||
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 5e-6, "output_cost_per_token": 7e-6},
|
||||
)
|
||||
|
||||
assert _bill_reported_cost(stream, {"base_model": base_model}, {}) == pytest.approx(100 * 5e-6 + 20 * 7e-6)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
def test_provider_reported_usage_cost_takes_discount_then_margin(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None:
|
||||
monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.2})
|
||||
monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"percentage": 0.5, "fixed_amount": 0.001}})
|
||||
|
||||
assert _bill_reported_cost(stream, {}, {}) == pytest.approx(0.037 * 0.8 * 1.5 + 0.001)
|
||||
|
||||
|
||||
def test_provider_reported_usage_cost_splits_input_and_output_by_the_token_math_ratio(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, _REPORTED_COST_FIXTURE_RATES)
|
||||
usage: Final = Usage(**_REPORTED_USAGE)
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(
|
||||
model=_REPORTED_COST_FIXTURE_MODEL, custom_llm_provider="openai", usage_object=usage
|
||||
)
|
||||
|
||||
priced_prompt: Final = 100 * 1e-6
|
||||
priced_completion: Final = 20 * 3e-6
|
||||
assert prompt_cost + completion_cost == pytest.approx(0.037)
|
||||
assert prompt_cost == pytest.approx(0.037 * priced_prompt / (priced_prompt + priced_completion))
|
||||
|
||||
|
||||
def test_provider_reported_usage_cost_without_rates_is_all_output(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, raising=False)
|
||||
|
||||
assert cost_per_token(
|
||||
model=_REPORTED_COST_FIXTURE_MODEL, custom_llm_provider="openai", usage_object=Usage(**_REPORTED_USAGE)
|
||||
) == (0.0, 0.037)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reported", [-0.037, float("nan"), float("inf"), True])
|
||||
def test_unusable_provider_reported_usage_cost_falls_back_to_the_token_math(
|
||||
monkeypatch: pytest.MonkeyPatch, reported: object
|
||||
) -> None:
|
||||
monkeypatch.setitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, _REPORTED_COST_FIXTURE_RATES)
|
||||
usage: Final = Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120)
|
||||
setattr(usage, "cost", reported)
|
||||
|
||||
assert cost_per_token(model=_REPORTED_COST_FIXTURE_MODEL, custom_llm_provider="openai", usage_object=usage) == (
|
||||
pytest.approx(100 * 1e-6),
|
||||
pytest.approx(20 * 3e-6),
|
||||
)
|
||||
|
||||
|
||||
def test_caller_supplied_custom_cost_per_token_beats_the_provider_reported_usage_cost() -> None:
|
||||
assert cost_per_token(
|
||||
model=_REPORTED_COST_FIXTURE_MODEL,
|
||||
custom_llm_provider="openai",
|
||||
prompt_tokens=100,
|
||||
completion_tokens=20,
|
||||
usage_object=Usage(**_REPORTED_USAGE),
|
||||
custom_cost_per_token={"input_cost_per_token": 0.001, "output_cost_per_token": 0.002},
|
||||
) == (pytest.approx(0.1), pytest.approx(0.04))
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue