fix(streaming): propagate provider cost through streaming chunk builder

OpenRouter (and any provider that sets usage.cost in streaming chunks)
had its cost silently dropped during stream reassembly. chunk_parser
correctly preserved usage.cost on each ModelResponseStream, but
_calculate_usage_per_chunk never extracted or accumulated it.
Changes:
- Add cost: Optional[float] to UsagePerChunk TypedDict
- Extract cost in _calculate_usage_per_chunk (handles both Usage
  objects via getattr and plain dicts via isinstance fallback)
- Set returned_usage.cost in calculate_usage when cost is present
- Bridge usage.cost -> _hidden_params[additional_headers][
  llm_provider-x-litellm-response-cost] in stream_chunk_builder so
  response_cost_calculator picks it up (matches non-streaming path)
Fixes #16021

Made-with: Cursor
This commit is contained in:
hyde-jyotiraditya 2026-04-30 14:46:19 +05:30
parent d3891e6eae
commit a03ba7ba72
4 changed files with 77 additions and 0 deletions

View file

@ -544,6 +544,7 @@ class ChunkProcessor:
web_search_requests: Optional[int] = None
completion_tokens_details: Optional[CompletionTokensDetails] = None
prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
cost: Optional[float] = None
for chunk in chunks:
usage_chunk: Optional[Usage] = None
if "usage" in chunk:
@ -589,6 +590,11 @@ class ChunkProcessor:
and usage_chunk.server_tool_use is not None
):
server_tool_use = usage_chunk.server_tool_use
_cost = getattr(usage_chunk, "cost", None)
if _cost is None and isinstance(usage_chunk, dict):
_cost = usage_chunk.get("cost")
if _cost is not None:
cost = _cost
if (
usage_chunk_dict["prompt_tokens_details"] is not None
and getattr(
@ -614,6 +620,7 @@ class ChunkProcessor:
web_search_requests=web_search_requests,
completion_tokens_details=completion_tokens_details,
prompt_tokens_details=prompt_tokens_details,
cost=cost,
)
def calculate_usage(
@ -711,6 +718,8 @@ class ChunkProcessor:
if server_tool_use is not None:
returned_usage.server_tool_use = server_tool_use
if calculated_usage_per_chunk["cost"] is not None:
returned_usage.cost = calculated_usage_per_chunk["cost"]
if web_search_requests is not None:
if returned_usage.prompt_tokens_details is None:
returned_usage.prompt_tokens_details = PromptTokensDetailsWrapper(

View file

@ -7462,6 +7462,12 @@ def stream_chunk_builder( # noqa: PLR0915
)
setattr(response, "usage", usage)
_provider_response_cost = getattr(usage, "cost", None)
if _provider_response_cost is not None:
response._hidden_params.setdefault("additional_headers", {})[
"llm_provider-x-litellm-response-cost"
] = _provider_response_cost
# Propagate provider_specific_fields from chunk hidden params when present.
for chunk in reversed(chunks):
if isinstance(chunk, dict):
@ -7640,6 +7646,12 @@ def stream_chunk_builder( # noqa: PLR0915
setattr(response, "usage", usage)
_provider_response_cost = getattr(usage, "cost", None)
if _provider_response_cost is not None:
response._hidden_params.setdefault("additional_headers", {})[
"llm_provider-x-litellm-response-cost"
] = _provider_response_cost
# Propagate provider_specific_fields from the last chunk (contains provider
# metadata like traffic_type set during streaming)
for chunk in reversed(chunks):

View file

@ -14,3 +14,4 @@ class UsagePerChunk(TypedDict):
web_search_requests: Optional[int]
completion_tokens_details: Optional[CompletionTokensDetails]
prompt_tokens_details: Optional[PromptTokensDetailsWrapper]
cost: Optional[float]

View file

@ -528,6 +528,61 @@ def test_openrouter_cost_tracking_streaming():
assert result2.usage.cost == 0.0001
def test_openrouter_streaming_cost_propagated_to_final_response():
"""
OpenRouter streams `usage.cost` in the final chunk. After
stream_chunk_builder rebuilds the response, the cost must land on
`_hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"]`
so response_cost_calculator picks it up (mirrors the non-streaming path).
"""
from litellm.main import stream_chunk_builder
handler = OpenRouterChatCompletionStreamingHandler(
streaming_response=None, sync_stream=True
)
chunk1 = {
"id": "gen-stream-789",
"created": 1234567890,
"model": "openrouter/anthropic/claude-sonnet-4.5",
"choices": [{"delta": {"content": "Hi", "reasoning": None}, "index": 0}],
}
chunk2 = {
"id": "gen-stream-789",
"created": 1234567890,
"model": "openrouter/anthropic/claude-sonnet-4.5",
"usage": {
"prompt_tokens": 7,
"completion_tokens": 3,
"total_tokens": 10,
"cost": 0.00042,
},
"choices": [
{
"delta": {"content": "", "reasoning": None},
"finish_reason": "stop",
"index": 0,
}
],
}
parsed_chunks = [handler.chunk_parser(chunk1), handler.chunk_parser(chunk2)]
final_response = stream_chunk_builder(
chunks=parsed_chunks,
messages=[{"role": "user", "content": "Hi"}],
)
assert final_response is not None
assert final_response.usage.cost == 0.00042
assert (
final_response._hidden_params["additional_headers"][
"llm_provider-x-litellm-response-cost"
]
== 0.00042
)
def test_openrouter_reasoning_models_allow_reasoning_effort_param():
"""
OpenRouter reasoning-capable models should accept the reasoning_effort param.