scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-10 08:28:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 108

fix(cost): show litellm's bill in usage.cost on chat non-stream and both Responses paths

Browse source 
A priced deployment showed its bill only on chat streams. The metadata pass now stamps the bill onto
ModelResponse and ResponsesAPIResponse usage, the Responses stream stamp overwrites the upstream's
number too, all through one stamp_billable_usage_cost guard, and the integration suite asserts a JSON
reply's usage.cost equals the spend row and gains /v1/responses twins of the reported-cost cases
This commit is contained in:
devin-ai-integration[bot] 2026-10-27 01:37:04 +00:00
parent 3490128d7d
commit a20609a6f2
8 changed files with 257 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
litellm/litellm_core_utils/hidden_params.py
View file

@ -147,6 +147,16 @@ def stamp_billed_usage_cost(obj: object, usage: object, cost: float) -> None:
setattr(usage, "cost", cost)
def stamp_billable_usage_cost(obj: object, usage: object, cost: object) -> float | None:
"""Stamp a priced bill, a $0 one included once the provider put a number there; an unpriceable reply keeps its usage."""
if isinstance(cost, bool) or not isinstance(cost, (int, float)) or cost < 0:
return None
if cost == 0 and getattr(usage, "cost", None) is None:
return None
stamp_billed_usage_cost(obj, usage, float(cost))
return float(cost)
def provider_stated_usage_cost(obj: object, usage_cost: object) -> object:
"""What the provider itself put in usage.cost, looking past a bill litellm stamped there."""
hidden_params: Final = get_hidden_params(obj)

26
litellm/litellm_core_utils/llm_response_utils/response_metadata.py
View file

@ -8,13 +8,16 @@ from pydantic import TypeAdapter, ValidationError
from litellm.constants import LITELLM_DETAILED_TIMING
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers
from litellm.litellm_core_utils.hidden_params import stamp_billable_usage_cost
from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
from litellm.litellm_core_utils.logging_utils import LiteLLMLoggingObject
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
from litellm.types.utils import (
EmbeddingResponse,
HiddenParams,
ModelResponse,
TranscriptionResponse,
Usage,
)
@ -159,16 +162,18 @@ class ResponseMetadata:
## ADD OTHER HIDDEN PARAMS
model_info: Final = kwargs.get("model_info", {}) or {}
model_id: Final = model_info.get("id", None)
response_cost: Final = logging_obj.response_cost_calculator(
result=self.result,
litellm_model_name=model,
router_model_id=model_id,
cache_hit=cache_hit,
)
self._stamp_billed_usage_cost(response_cost)
new_params: Final = {
"litellm_call_id": getattr(logging_obj, "litellm_call_id", None),
"api_base": get_api_base(model=model or "", optional_params=kwargs),
"model_id": model_id,
"response_cost": logging_obj.response_cost_calculator(
result=self.result,
litellm_model_name=model,
router_model_id=model_id,
cache_hit=cache_hit,
),
"response_cost": response_cost,
"additional_headers": process_response_headers(
self._get_additional_headers_from_hidden_params() or {},
preserve_litellm_internal_headers=True,
@ -179,6 +184,15 @@ class ResponseMetadata:
}
self._update_hidden_params(new_params)
def _stamp_billed_usage_cost(self, response_cost: object) -> None:
"""A client reads what litellm billed from usage.cost, streamed or not, priced deployment or not."""
if not isinstance(self.result, (ModelResponse, ResponsesAPIResponse)):
return
usage: Final = getattr(self.result, "usage", None)
if not isinstance(usage, (Usage, ResponseAPIUsage)):
return
stamp_billable_usage_cost(self.result, usage, response_cost)
def _update_hidden_params(self, new_params: Mapping[str, object]) -> None:
"""
Update hidden params - handles when self._hidden_params is a dict or HiddenParams object

11
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -13,7 +13,7 @@ from typing_extensions import ReadOnly, Required
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params, stamp_billed_usage_cost
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params, stamp_billable_usage_cost
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict
)
@ -1413,13 +1413,10 @@ def _set_stream_builder_response_cost(
def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None:
if logging_obj is None:
return
computed_cost: Final = logging_obj.response_cost_calculator(result=response)
if not isinstance(computed_cost, (int, float)) or computed_cost < 0:
billed: Final = stamp_billable_usage_cost(response, usage, logging_obj.response_cost_calculator(result=response))
if billed is None:
return
if computed_cost == 0 and getattr(usage, "cost", None) is None:
return
stamp_billed_usage_cost(response, usage, computed_cost)
_set_hidden_response_cost(response, float(computed_cost))
_set_hidden_response_cost(response, billed)
def _field_of(value: object, name: str) -> object:

7
litellm/responses/streaming_iterator.py
View file

@ -41,7 +41,7 @@ from litellm.exceptions import MidStreamFallbackError, RateLimitError
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base
from litellm.litellm_core_utils.hidden_params import USAGE_COST_BEFORE_LITELLM_STAMP, stamp_billed_usage_cost
from litellm.litellm_core_utils.hidden_params import USAGE_COST_BEFORE_LITELLM_STAMP, stamp_billable_usage_cost
from litellm.litellm_core_utils.internal_call_metadata import without_budget_reservation
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -1758,14 +1758,11 @@ def stamp_responses_usage_cost(
if usage_obj is None:
return
response_obj.usage = usage_obj # rebind-ok: the stamped cost has to ride on the response the client receives
if isinstance(getattr(usage_obj, "cost", None), (int, float)):
return
try:
cost: Final[float | None] = logging_obj.response_cost_calculator(result=response_obj)
except Exception:
return
if isinstance(cost, (int, float)) and cost > 0:
stamp_billed_usage_cost(response_obj, usage_obj, cost)
stamp_billable_usage_cost(response_obj, usage_obj, cost)
def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]:

16
tests/integration/cost_calculation/assertions.py
View file

@ -55,6 +55,18 @@ def assert_streamed_usage_cost(case_name: str, endpoint: str, response_text: str
assert approx_equal(seen, spend), f"{case_name}: streamed usage.cost {seen} != spend {spend}"
def assert_json_usage_cost(case_name: str, endpoint: str, response: httpx.Response, spend: float) -> None:
"""A JSON reply's usage.cost is what the row billed, the upstream's own figure included, so a client and the
spend log never disagree on a priced deployment."""
if endpoint not in STREAMED_COST_ENDPOINTS:
return
usage: Final = JSON_OBJECT.validate_json(response.text).get("usage")
cost: Final = usage.get("cost") if isinstance(usage, dict) else None
if not isinstance(cost, (int, float)) or isinstance(cost, bool):
return
assert approx_equal(float(cost), spend), f"{case_name}: usage.cost {cost} != spend {spend}"
def assert_breakdown(
case_name: str,
response_content_type: str,
@ -160,6 +172,8 @@ def assert_exact(
assert_total_is_sum_of_components(row, breakdown, case_name)
if response is not None and response_content_type == "text/event-stream":
assert_streamed_usage_cost(case_name, response.request.url.path, response.text, expected.spend)
if response is not None and response_content_type == "application/json":
assert_json_usage_cost(case_name, response.request.url.path, response, expected.spend)
def assert_recount(
@ -201,3 +215,5 @@ def assert_recount(
assert_total_is_sum_of_components(row, row.breakdown, case_name)
if response is not None and response_content_type == "text/event-stream":
assert_streamed_usage_cost(case_name, response.request.url.path, response.text, row.spend)
if response is not None and response_content_type == "application/json":
assert_json_usage_cost(case_name, response.request.url.path, response, row.spend)

80
tests/integration/cost_calculation/cases/openai.py
View file

@ -2,6 +2,7 @@
PARITY pairs the streamed case with its plain twin where both bill the same row."""
from collections.abc import Mapping
from typing import Final, cast
from integration.cost_calculation.cost_tracking_case import (
@ -11956,11 +11957,85 @@ GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_STRE
}
)
_GATEWAY_RESPONSES_USAGE: Final[Mapping[str, JsonValue]] = {"input_tokens": 100, "output_tokens": 20, "total_tokens": 120, "cost": 0.037}
_GATEWAY_RESPONSES_ITEM: Final[Mapping[str, JsonValue]] = {
"id": "msg_gateway",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "ok", "annotations": []}],
}
_GATEWAY_RESPONSES_BODY: Final[Mapping[str, JsonValue]] = {
"id": "resp_$REQUEST_ID",
"object": "response",
"created_at": 1,
"status": "completed",
"model": "gateway-reported-cost-fixture",
"output": [_GATEWAY_RESPONSES_ITEM],
"usage": _GATEWAY_RESPONSES_USAGE,
}
GATEWAY_REPORTED_COST_RESPONSES: Final = CostTrackingTestCase(
name="gateway-reported-cost-responses",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gateway-reported-cost-fixture",
endpoint="/v1/responses",
request={"model": "$MODEL", "input": "reported cost", "cache": {"no-cache": True}},
response=JsonResponse(content_type="application/json", body=_GATEWAY_RESPONSES_BODY),
expected=_GATEWAY_REPORTED_EXPECTED,
)
GATEWAY_REPORTED_COST_RESPONSES_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy(
update={
"name": "gateway-reported-cost-responses-stream",
"request": {**GATEWAY_REPORTED_COST_RESPONSES.request, "stream": True},
"response": SseResponse(
content_type="text/event-stream",
frames=sse_frames(
{
"type": "response.created",
"sequence_number": 0,
"response": {**_GATEWAY_RESPONSES_BODY, "status": "in_progress", "output": [], "usage": None},
},
{
"type": "response.output_text.delta",
"sequence_number": 1,
"item_id": "msg_gateway",
"output_index": 0,
"content_index": 0,
"delta": "ok",
},
{"type": "response.completed", "sequence_number": 2, "response": _GATEWAY_RESPONSES_BODY},
),
),
}
)
GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy(
update={
"name": "gateway-reported-cost-responses-operator_priced",
"deployment": _GATEWAY_OPERATOR_PRICED,
"expected": _GATEWAY_OPERATOR_PRICED_EXPECTED,
}
)
GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES_STREAM.model_copy(
update={
"name": "gateway-reported-cost-responses-operator_priced-stream",
"deployment": _GATEWAY_OPERATOR_PRICED,
"expected": _GATEWAY_OPERATOR_PRICED_EXPECTED,
}
)
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
GATEWAY_REPORTED_COST,
GATEWAY_REPORTED_COST_STREAM,
GATEWAY_REPORTED_COST_OPERATOR_PRICED,
GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM,
GATEWAY_REPORTED_COST_RESPONSES,
GATEWAY_REPORTED_COST_RESPONSES_STREAM,
GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED,
GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM,
GPT_5_6_SLASH_ALIAS,
GPT_5_6_SLASH_ALIAS_STREAM,
GPT_6_SOL_RESPONSES_DEPLOYMENT_JSON,
@ -12206,6 +12281,11 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
StreamParityTestCase(
plain=GATEWAY_REPORTED_COST_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM
),
StreamParityTestCase(plain=GATEWAY_REPORTED_COST_RESPONSES, streamed=GATEWAY_REPORTED_COST_RESPONSES_STREAM),
StreamParityTestCase(
plain=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED,
streamed=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM,
),
StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM),
StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_INCOMPLETE),
StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_UNVALIDATED),

27
tests/unit/responses/test_streaming_iterator.py
View file

@ -851,17 +851,36 @@ def test_stamp_responses_usage_cost_stamps_computed_cost():
logging_obj.response_cost_calculator.assert_called_once_with(result=response)
def test_stamp_responses_usage_cost_keeps_provider_reported_cost():
def test_stamp_responses_usage_cost_puts_the_bill_over_the_provider_reported_cost():
from litellm.litellm_core_utils.hidden_params import USAGE_COST_BEFORE_LITELLM_STAMP, provider_stated_usage_cost
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
response = _responses_api_response_with_usage()
setattr(response.usage, "cost", 0.5)
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj.response_cost_calculator.return_value = 0.00032
stamp_responses_usage_cost(response, logging_obj)
assert getattr(response.usage, "cost", None) == pytest.approx(0.5)
logging_obj.response_cost_calculator.assert_not_called()
assert getattr(response.usage, "cost", None) == pytest.approx(0.00032)
assert provider_stated_usage_cost(response, response.usage.cost) == pytest.approx(0.5)
assert response._hidden_params[USAGE_COST_BEFORE_LITELLM_STAMP] == pytest.approx(0.5)
def test_stamp_responses_usage_cost_stamps_a_zero_bill_only_over_a_provider_reported_cost():
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
unpriced = _responses_api_response_with_usage()
reported = _responses_api_response_with_usage()
setattr(reported.usage, "cost", 0.5)
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj.response_cost_calculator.return_value = 0.0
stamp_responses_usage_cost(unpriced, logging_obj)
stamp_responses_usage_cost(reported, logging_obj)
assert getattr(unpriced.usage, "cost", None) is None
assert getattr(reported.usage, "cost", None) == 0.0
def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse:
@ -889,13 +908,13 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage():
}
)
logging_obj = Mock(spec=LiteLLMLoggingObj)
logging_obj.response_cost_calculator.return_value = 3e-05
stamp_responses_usage_cost(response, logging_obj)
assert isinstance(response.usage, ResponseAPIUsage)
assert response.usage.cost == pytest.approx(3e-05)
assert response.usage.output_tokens_details.reasoning_tokens == 117
logging_obj.response_cost_calculator.assert_not_called()
def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost():

102
tests/unit/test_cost_calculator.py
View file

@ -24,6 +24,8 @@ from litellm.litellm_core_utils.fallback_generalizations import (
)
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.llm_response_utils.response_metadata import update_response_metadata
from litellm.responses.streaming_iterator import stamp_responses_usage_cost
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
convert_to_model_response_object,
)
@ -10685,6 +10687,106 @@ def test_a_zero_bill_is_stamped_over_the_provider_reported_usage_cost_on_a_strea
assert logging_obj.response_cost_calculator(result=response) == 0.0
_PRICED_DEPLOYMENT_ID: Final = "reported-cost-deployment-priced-by-the-operator"
_PRICED_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType(
{
"input_cost_per_token": 2e-6,
"output_cost_per_token": 6e-6,
"metadata": {"model_info": {"id": _PRICED_DEPLOYMENT_ID}},
}
)
def _reported_cost_usage() -> Usage:
return Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120, cost=0.037)
def _priced_logging(stream: bool, call_type: str) -> Logging:
"""The logging object the proxy hands a priced deployment's call, the deployment under metadata.model_info."""
logging_obj: Final = _reported_cost_logging(stream, _PRICED_LITELLM_PARAMS)
logging_obj.call_type = call_type
return logging_obj
def _reported_cost_responses_response() -> ResponsesAPIResponse:
return ResponsesAPIResponse(
id="resp_reported",
created_at=1,
status="completed",
model="reported-cost-fixture",
object="response",
output=[],
usage=ResponseAPIUsage(input_tokens=100, output_tokens=20, total_tokens=120, cost=0.037),
)
def _client_visible_reply(route: str) -> tuple[object, Logging]:
"""The object a client of the given route reads usage.cost from, after litellm priced it."""
match route:
case "chat":
logging_obj: Logging = _priced_logging(False, "completion")
response: object = ModelResponse(
id="chatcmpl-reported",
model="reported-cost-fixture",
choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")],
usage=_reported_cost_usage(),
)
update_response_metadata(
result=response,
logging_obj=logging_obj,
model=_REPORTED_COST_FIXTURE_MODEL,
kwargs={},
start_time=datetime.datetime(2026, 1, 1),
end_time=datetime.datetime(2026, 1, 1, 0, 0, 1),
)
case "chat-stream":
logging_obj = _priced_logging(True, "completion")
response = litellm.stream_chunk_builder(
[
_reported_cost_stream_chunk("hi", None, None),
_reported_cost_stream_chunk(None, "stop", None),
_reported_cost_stream_chunk(None, None, _reported_cost_usage()),
],
messages=[{"role": "user", "content": "hi"}],
logging_obj=logging_obj,
)
case "responses":
logging_obj = _priced_logging(False, "aresponses")
response = _reported_cost_responses_response()
update_response_metadata(
result=response,
logging_obj=logging_obj,
model=_REPORTED_COST_FIXTURE_MODEL,
kwargs={},
start_time=datetime.datetime(2026, 1, 1),
end_time=datetime.datetime(2026, 1, 1, 0, 0, 1),
)
case _:
logging_obj = _priced_logging(True, "aresponses")
response = _reported_cost_responses_response()
stamp_responses_usage_cost(response, logging_obj)
return response, logging_obj
@pytest.mark.parametrize("route", ["chat", "chat-stream", "responses", "responses-stream"])
def test_the_client_reads_what_litellm_billed_from_usage_cost_on_every_route(
monkeypatch: pytest.MonkeyPatch, route: str
) -> None:
"""A priced deployment showed its bill on chat streams only; the other three routes still showed the upstream's number."""
monkeypatch.setitem(
litellm.model_cost,
_PRICED_DEPLOYMENT_ID,
{"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 2e-6, "output_cost_per_token": 6e-6},
)
bill: Final = 100 * 2e-6 + 20 * 6e-6
response, logging_obj = _client_visible_reply(route)
usage: Final = getattr(response, "usage", None)
assert usage is not None and usage.cost == pytest.approx(bill)
assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx(bill)
def test_provider_reported_usage_cost_splits_input_and_output_by_the_token_math_ratio(
monkeypatch: pytest.MonkeyPatch,
) -> None: