From a03ba7ba72d77a2705266422932dbc3ea7cb2142 Mon Sep 17 00:00:00 2001 From: hyde-jyotiraditya Date: Thu, 30 Apr 2026 14:46:19 +0530 Subject: [PATCH] fix(streaming): propagate provider cost through streaming chunk builder OpenRouter (and any provider that sets usage.cost in streaming chunks) had its cost silently dropped during stream reassembly. chunk_parser correctly preserved usage.cost on each ModelResponseStream, but _calculate_usage_per_chunk never extracted or accumulated it. Changes: - Add cost: Optional[float] to UsagePerChunk TypedDict - Extract cost in _calculate_usage_per_chunk (handles both Usage objects via getattr and plain dicts via isinstance fallback) - Set returned_usage.cost in calculate_usage when cost is present - Bridge usage.cost -> _hidden_params[additional_headers][ llm_provider-x-litellm-response-cost] in stream_chunk_builder so response_cost_calculator picks it up (matches non-streaming path) Fixes #16021 Made-with: Cursor --- .../streaming_chunk_builder_utils.py | 9 +++ litellm/main.py | 12 ++++ .../streaming_chunk_builder_utils.py | 1 + .../test_openrouter_chat_transformation.py | 55 +++++++++++++++++++ 4 files changed, 77 insertions(+) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index fe7c62c3842..4cbe88c1017 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -544,6 +544,7 @@ class ChunkProcessor: web_search_requests: Optional[int] = None completion_tokens_details: Optional[CompletionTokensDetails] = None prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None + cost: Optional[float] = None for chunk in chunks: usage_chunk: Optional[Usage] = None if "usage" in chunk: @@ -589,6 +590,11 @@ class ChunkProcessor: and usage_chunk.server_tool_use is not None ): server_tool_use = usage_chunk.server_tool_use + _cost = getattr(usage_chunk, "cost", None) + if _cost is None and isinstance(usage_chunk, dict): + _cost = usage_chunk.get("cost") + if _cost is not None: + cost = _cost if ( usage_chunk_dict["prompt_tokens_details"] is not None and getattr( @@ -614,6 +620,7 @@ class ChunkProcessor: web_search_requests=web_search_requests, completion_tokens_details=completion_tokens_details, prompt_tokens_details=prompt_tokens_details, + cost=cost, ) def calculate_usage( @@ -711,6 +718,8 @@ class ChunkProcessor: if server_tool_use is not None: returned_usage.server_tool_use = server_tool_use + if calculated_usage_per_chunk["cost"] is not None: + returned_usage.cost = calculated_usage_per_chunk["cost"] if web_search_requests is not None: if returned_usage.prompt_tokens_details is None: returned_usage.prompt_tokens_details = PromptTokensDetailsWrapper( diff --git a/litellm/main.py b/litellm/main.py index daa0fb063aa..136a7bf6fc1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7462,6 +7462,12 @@ def stream_chunk_builder( # noqa: PLR0915 ) setattr(response, "usage", usage) + _provider_response_cost = getattr(usage, "cost", None) + if _provider_response_cost is not None: + response._hidden_params.setdefault("additional_headers", {})[ + "llm_provider-x-litellm-response-cost" + ] = _provider_response_cost + # Propagate provider_specific_fields from chunk hidden params when present. for chunk in reversed(chunks): if isinstance(chunk, dict): @@ -7640,6 +7646,12 @@ def stream_chunk_builder( # noqa: PLR0915 setattr(response, "usage", usage) + _provider_response_cost = getattr(usage, "cost", None) + if _provider_response_cost is not None: + response._hidden_params.setdefault("additional_headers", {})[ + "llm_provider-x-litellm-response-cost" + ] = _provider_response_cost + # Propagate provider_specific_fields from the last chunk (contains provider # metadata like traffic_type set during streaming) for chunk in reversed(chunks): diff --git a/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py index a1f89dac5cf..c9f9d4e6baa 100644 --- a/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py @@ -14,3 +14,4 @@ class UsagePerChunk(TypedDict): web_search_requests: Optional[int] completion_tokens_details: Optional[CompletionTokensDetails] prompt_tokens_details: Optional[PromptTokensDetailsWrapper] + cost: Optional[float] diff --git a/tests/test_litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/test_litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py index f102319d6bf..92f9f93ee02 100644 --- a/tests/test_litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/test_litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -528,6 +528,61 @@ def test_openrouter_cost_tracking_streaming(): assert result2.usage.cost == 0.0001 +def test_openrouter_streaming_cost_propagated_to_final_response(): + """ + OpenRouter streams `usage.cost` in the final chunk. After + stream_chunk_builder rebuilds the response, the cost must land on + `_hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"]` + so response_cost_calculator picks it up (mirrors the non-streaming path). + """ + from litellm.main import stream_chunk_builder + + handler = OpenRouterChatCompletionStreamingHandler( + streaming_response=None, sync_stream=True + ) + + chunk1 = { + "id": "gen-stream-789", + "created": 1234567890, + "model": "openrouter/anthropic/claude-sonnet-4.5", + "choices": [{"delta": {"content": "Hi", "reasoning": None}, "index": 0}], + } + chunk2 = { + "id": "gen-stream-789", + "created": 1234567890, + "model": "openrouter/anthropic/claude-sonnet-4.5", + "usage": { + "prompt_tokens": 7, + "completion_tokens": 3, + "total_tokens": 10, + "cost": 0.00042, + }, + "choices": [ + { + "delta": {"content": "", "reasoning": None}, + "finish_reason": "stop", + "index": 0, + } + ], + } + + parsed_chunks = [handler.chunk_parser(chunk1), handler.chunk_parser(chunk2)] + + final_response = stream_chunk_builder( + chunks=parsed_chunks, + messages=[{"role": "user", "content": "Hi"}], + ) + + assert final_response is not None + assert final_response.usage.cost == 0.00042 + assert ( + final_response._hidden_params["additional_headers"][ + "llm_provider-x-litellm-response-cost" + ] + == 0.00042 + ) + + def test_openrouter_reasoning_models_allow_reasoning_effort_param(): """ OpenRouter reasoning-capable models should accept the reasoning_effort param.