scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-07 07:58:09 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 229

fix(cost): bill a provider reported usage.cost on every path unless the deployment is priced

Browse source
This commit is contained in:
devin-ai-integration[bot] 2026-10-29 19:32:37 +00:00
parent cd42694b48
commit ea1e2ae08c
5 changed files with 249 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
litellm/cost_calculator.py
View file

@ -1522,9 +1522,9 @@ def _adjusted_provider_billed_cost(
)
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=None,
completion_tokens_cost_usd_dollar=None,
cost_for_built_in_tools_cost_usd_dollar=None,
prompt_tokens_cost_usd_dollar=0.0,
completion_tokens_cost_usd_dollar=adjusted_cost,
cost_for_built_in_tools_cost_usd_dollar=0.0,
total_cost_usd_dollar=adjusted_cost,
original_cost=reported_cost,
discount_percent=discount_percent,

13
litellm/litellm_core_utils/core_helpers.py
View file

@ -1017,11 +1017,20 @@ def chunk_usage(chunk: object) -> object:
return getattr(chunk, "usage", None)
USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"})
OWN_REPORTED_COST_READER_PROVIDERS: Final = frozenset({"xai", "perplexity", "opencode_go"})
def _carries_litellm_bill(response: object) -> bool:
"""A response the stream builder already priced carries litellm's own bill in ``usage.cost`` too."""
hidden: Final[object] = getattr(response, "_hidden_params", None)
stamped: Final = hidden.get("response_cost") if isinstance(hidden, Mapping) else None
return isinstance(stamped, (int, float)) and not isinstance(stamped, bool)
def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None:
if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS:
"""The charge an OpenAI-compatible upstream reports in ``usage.cost``, billed unless the provider has its own
reader or the number is litellm's own bill written back onto the usage."""
if custom_llm_provider in OWN_REPORTED_COST_READER_PROVIDERS or _carries_litellm_bill(response):
return None
cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider)
return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None

16
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -13,7 +13,7 @@ from typing_extensions import ReadOnly, Required
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import (
USAGE_COST_BILLED_PROVIDERS,
OWN_REPORTED_COST_READER_PROVIDERS,
chunk_usage,
provider_reported_usage_cost,
)
@ -1356,9 +1356,6 @@ def stream_chunk_builder_text_completion(
return TextCompletionResponse(**response)
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS
def _logged_provider(logging_obj: Optional["Logging"]) -> object:
if logging_obj is None:
return None
@ -1366,10 +1363,6 @@ def _logged_provider(logging_obj: Optional["Logging"]) -> object:
return provider
def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS
def _stream_builder_response_cost(
response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object]
) -> float | None:
@ -1380,9 +1373,6 @@ def _stream_builder_response_cost(
for chunk in reversed(chunks)
)
return next((cost for cost in reported if cost is not None), None)
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj):
return float(usage_cost)
if logging_obj is not None:
return None
provider_hint: Final = response._hidden_params.get( # pyright: ignore[reportPrivateUsage] # no public accessor
@ -1429,7 +1419,9 @@ def _set_stream_builder_response_cost(
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
if logging_obj is None:
return
if isinstance(getattr(usage, "cost", None), (int, float)):
if _logged_provider(logging_obj) in OWN_REPORTED_COST_READER_PROVIDERS and isinstance(
getattr(usage, "cost", None), (int, float)
):
return
computed_cost: Final = logging_obj.response_cost_calculator(result=response)
if not isinstance(computed_cost, (int, float)) or computed_cost <= 0:

225
tests/integration/cost_calculation/cases/openai.py
View file

@ -2,6 +2,7 @@
PARITY pairs the streamed case with its plain twin where both bill the same row."""
from collections.abc import Mapping
from typing import Final, cast
from integration.cost_calculation.cost_tracking_case import (
@ -11878,7 +11879,215 @@ GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM: Final = GPT_5_6_TERRA_MESSAGES_BRIDGED_JS
}
)
# An OpenAI-compatible gateway that reports what it charged under usage.cost (#47533). The reported total is the bill,
# split across input and output by the ratio of the cost map rates (100 * 1e-6 : 20 * 3e-6), on plain and streamed
# replies alike. A deployment the operator priced keeps the operator's rates on both.
_GATEWAY_REPORTED_USAGE: Final = {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120, "cost": 0.037}
_GATEWAY_REPORTED_EXPECTED: Final = ExactExpected(
spend=0.037, input_cost=0.023125, output_cost=0.013875, prompt_tokens=100, completion_tokens=20
)
_GATEWAY_OPERATOR_PRICED: Final = Deployment(input_cost_per_token=2e-06, output_cost_per_token=6e-06)
_GATEWAY_OPERATOR_PRICED_EXPECTED: Final = ExactExpected(
spend=0.00032, input_cost=0.0002, output_cost=0.00012, prompt_tokens=100, completion_tokens=20
)
_GATEWAY_CHUNK: Final = {
"id": "chatcmpl-$REQUEST_ID",
"object": "chat.completion.chunk",
"created": 1,
"model": "gateway-reported-cost-fixture",
}
GATEWAY_REPORTED_COST: Final = CostTrackingTestCase(
name="gateway-reported-cost",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gateway-reported-cost-fixture",
endpoint="/v1/chat/completions",
request={"model": "$MODEL", "messages": [{"role": "user", "content": "reported cost"}], "stream": False},
response=JsonResponse(
content_type="application/json",
body={
"id": "chatcmpl-$REQUEST_ID",
"object": "chat.completion",
"created": 1,
"model": "gateway-reported-cost-fixture",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
"usage": _GATEWAY_REPORTED_USAGE,
},
),
expected=_GATEWAY_REPORTED_EXPECTED,
)
GATEWAY_REPORTED_COST_STREAM: Final = CostTrackingTestCase(
name="gateway-reported-cost-stream",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gateway-reported-cost-fixture",
endpoint="/v1/chat/completions",
request={
"model": "$MODEL",
"messages": [{"role": "user", "content": "reported cost"}],
"stream": True,
"stream_options": {"include_usage": True},
},
response=SseResponse(
content_type="text/event-stream",
frames=sse_frames(
{**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]},
{**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
{**_GATEWAY_CHUNK, "choices": [], "usage": _GATEWAY_REPORTED_USAGE},
done=True,
),
),
expected=_GATEWAY_REPORTED_EXPECTED,
)
GATEWAY_REPORTED_COST_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST.model_copy(
update={
"name": "gateway-reported-cost-operator_priced",
"deployment": _GATEWAY_OPERATOR_PRICED,
"expected": _GATEWAY_OPERATOR_PRICED_EXPECTED,
}
)
GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_STREAM.model_copy(
update={
"name": "gateway-reported-cost-operator_priced-stream",
"deployment": _GATEWAY_OPERATOR_PRICED,
"expected": _GATEWAY_OPERATOR_PRICED_EXPECTED,
}
)
_GATEWAY_RESPONSES_USAGE: Final[Mapping[str, JsonValue]] = {"input_tokens": 100, "output_tokens": 20, "total_tokens": 120, "cost": 0.037}
_GATEWAY_RESPONSES_ITEM: Final[Mapping[str, JsonValue]] = {
"id": "msg_gateway",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "ok", "annotations": []}],
}
_GATEWAY_RESPONSES_BODY: Final[Mapping[str, JsonValue]] = {
"id": "resp_$REQUEST_ID",
"object": "response",
"created_at": 1,
"status": "completed",
"model": "gateway-reported-cost-fixture",
"output": [_GATEWAY_RESPONSES_ITEM],
"usage": _GATEWAY_RESPONSES_USAGE,
}
GATEWAY_REPORTED_COST_RESPONSES: Final = CostTrackingTestCase(
name="gateway-reported-cost-responses",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gateway-reported-cost-fixture",
endpoint="/v1/responses",
request={"model": "$MODEL", "input": "reported cost", "cache": {"no-cache": True}},
response=JsonResponse(content_type="application/json", body=_GATEWAY_RESPONSES_BODY),
expected=_GATEWAY_REPORTED_EXPECTED,
)
GATEWAY_REPORTED_COST_RESPONSES_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy(
update={
"name": "gateway-reported-cost-responses-stream",
"request": {**GATEWAY_REPORTED_COST_RESPONSES.request, "stream": True},
"response": SseResponse(
content_type="text/event-stream",
frames=sse_frames(
{
"type": "response.created",
"sequence_number": 0,
"response": {**_GATEWAY_RESPONSES_BODY, "status": "in_progress", "output": [], "usage": None},
},
{
"type": "response.output_text.delta",
"sequence_number": 1,
"item_id": "msg_gateway",
"output_index": 0,
"content_index": 0,
"delta": "ok",
},
{"type": "response.completed", "sequence_number": 2, "response": _GATEWAY_RESPONSES_BODY},
),
),
}
)
GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy(
update={
"name": "gateway-reported-cost-responses-operator_priced",
"deployment": _GATEWAY_OPERATOR_PRICED,
"expected": _GATEWAY_OPERATOR_PRICED_EXPECTED,
}
)
GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES_STREAM.model_copy(
update={
"name": "gateway-reported-cost-responses-operator_priced-stream",
"deployment": _GATEWAY_OPERATOR_PRICED,
"expected": _GATEWAY_OPERATOR_PRICED_EXPECTED,
}
)
_GATEWAY_ZERO_USAGE: Final = {**_GATEWAY_REPORTED_USAGE, "cost": 0}
GATEWAY_REPORTED_ZERO_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED.model_copy(
update={
"name": "gateway-reported-zero-operator_priced",
"response": JsonResponse(
content_type="application/json",
body={**GATEWAY_REPORTED_COST.response.body, "usage": _GATEWAY_ZERO_USAGE},
),
}
)
GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM.model_copy(
update={
"name": "gateway-reported-zero-operator_priced-stream",
"response": SseResponse(
content_type="text/event-stream",
frames=sse_frames(
{**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]},
{**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
{**_GATEWAY_CHUNK, "choices": [], "usage": _GATEWAY_ZERO_USAGE},
done=True,
),
),
}
)
_GATEWAY_OPENROUTER_OPERATOR_PRICED: Final = _GATEWAY_OPERATOR_PRICED.model_copy(
update={"model": "openrouter/vendor/gateway-reported-cost-fixture"}
)
GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED.model_copy(
update={
"name": "gateway-reported-cost-openrouter-operator_priced",
"deployment": _GATEWAY_OPENROUTER_OPERATOR_PRICED,
}
)
GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM: Final = (
GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM.model_copy(
update={
"name": "gateway-reported-cost-openrouter-operator_priced-stream",
"deployment": _GATEWAY_OPENROUTER_OPERATOR_PRICED,
}
)
)
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
GATEWAY_REPORTED_COST,
GATEWAY_REPORTED_COST_STREAM,
GATEWAY_REPORTED_COST_OPERATOR_PRICED,
GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM,
GATEWAY_REPORTED_COST_RESPONSES,
GATEWAY_REPORTED_COST_RESPONSES_STREAM,
GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED,
GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM,
GATEWAY_REPORTED_ZERO_OPERATOR_PRICED,
GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM,
GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED,
GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM,
GPT_5_6_SLASH_ALIAS,
GPT_5_6_SLASH_ALIAS_STREAM,
GPT_6_SOL_RESPONSES_DEPLOYMENT_JSON,
@ -12120,6 +12329,22 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
)
PARITY: Final[tuple[StreamParityTestCase, ...]] = (
StreamParityTestCase(plain=GATEWAY_REPORTED_COST, streamed=GATEWAY_REPORTED_COST_STREAM),
StreamParityTestCase(
plain=GATEWAY_REPORTED_COST_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM
),
StreamParityTestCase(
plain=GATEWAY_REPORTED_ZERO_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM
),
StreamParityTestCase(
plain=GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED,
streamed=GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM,
),
StreamParityTestCase(plain=GATEWAY_REPORTED_COST_RESPONSES, streamed=GATEWAY_REPORTED_COST_RESPONSES_STREAM),
StreamParityTestCase(
plain=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED,
streamed=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM,
),
StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM),
StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_INCOMPLETE),
StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_UNVALIDATED),

6
tests/integration/cost_calculation/cost_map.py
View file

@ -26,6 +26,12 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
"input_cost_per_token_above_128k_tokens": 2.2e-07,
"output_cost_per_token_above_128k_tokens": 8.8e-07,
},
"gateway-reported-cost-fixture": {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 3e-06,
},
"openrouter/anthropic/claude-sonnet-5": {
"litellm_provider": "openrouter",
"mode": "chat",