mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
fix(streaming): propagate provider cost through streaming chunk builder
OpenRouter (and any provider that sets usage.cost in streaming chunks) had its cost silently dropped during stream reassembly. chunk_parser correctly preserved usage.cost on each ModelResponseStream, but _calculate_usage_per_chunk never extracted or accumulated it. Changes: - Add cost: Optional[float] to UsagePerChunk TypedDict - Extract cost in _calculate_usage_per_chunk (handles both Usage objects via getattr and plain dicts via isinstance fallback) - Set returned_usage.cost in calculate_usage when cost is present - Bridge usage.cost -> _hidden_params[additional_headers][ llm_provider-x-litellm-response-cost] in stream_chunk_builder so response_cost_calculator picks it up (matches non-streaming path) Fixes #16021 Made-with: Cursor
This commit is contained in:
parent
d3891e6eae
commit
a03ba7ba72
4 changed files with 77 additions and 0 deletions
|
|
@ -544,6 +544,7 @@ class ChunkProcessor:
|
|||
web_search_requests: Optional[int] = None
|
||||
completion_tokens_details: Optional[CompletionTokensDetails] = None
|
||||
prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
|
||||
cost: Optional[float] = None
|
||||
for chunk in chunks:
|
||||
usage_chunk: Optional[Usage] = None
|
||||
if "usage" in chunk:
|
||||
|
|
@ -589,6 +590,11 @@ class ChunkProcessor:
|
|||
and usage_chunk.server_tool_use is not None
|
||||
):
|
||||
server_tool_use = usage_chunk.server_tool_use
|
||||
_cost = getattr(usage_chunk, "cost", None)
|
||||
if _cost is None and isinstance(usage_chunk, dict):
|
||||
_cost = usage_chunk.get("cost")
|
||||
if _cost is not None:
|
||||
cost = _cost
|
||||
if (
|
||||
usage_chunk_dict["prompt_tokens_details"] is not None
|
||||
and getattr(
|
||||
|
|
@ -614,6 +620,7 @@ class ChunkProcessor:
|
|||
web_search_requests=web_search_requests,
|
||||
completion_tokens_details=completion_tokens_details,
|
||||
prompt_tokens_details=prompt_tokens_details,
|
||||
cost=cost,
|
||||
)
|
||||
|
||||
def calculate_usage(
|
||||
|
|
@ -711,6 +718,8 @@ class ChunkProcessor:
|
|||
|
||||
if server_tool_use is not None:
|
||||
returned_usage.server_tool_use = server_tool_use
|
||||
if calculated_usage_per_chunk["cost"] is not None:
|
||||
returned_usage.cost = calculated_usage_per_chunk["cost"]
|
||||
if web_search_requests is not None:
|
||||
if returned_usage.prompt_tokens_details is None:
|
||||
returned_usage.prompt_tokens_details = PromptTokensDetailsWrapper(
|
||||
|
|
|
|||
|
|
@ -7462,6 +7462,12 @@ def stream_chunk_builder( # noqa: PLR0915
|
|||
)
|
||||
setattr(response, "usage", usage)
|
||||
|
||||
_provider_response_cost = getattr(usage, "cost", None)
|
||||
if _provider_response_cost is not None:
|
||||
response._hidden_params.setdefault("additional_headers", {})[
|
||||
"llm_provider-x-litellm-response-cost"
|
||||
] = _provider_response_cost
|
||||
|
||||
# Propagate provider_specific_fields from chunk hidden params when present.
|
||||
for chunk in reversed(chunks):
|
||||
if isinstance(chunk, dict):
|
||||
|
|
@ -7640,6 +7646,12 @@ def stream_chunk_builder( # noqa: PLR0915
|
|||
|
||||
setattr(response, "usage", usage)
|
||||
|
||||
_provider_response_cost = getattr(usage, "cost", None)
|
||||
if _provider_response_cost is not None:
|
||||
response._hidden_params.setdefault("additional_headers", {})[
|
||||
"llm_provider-x-litellm-response-cost"
|
||||
] = _provider_response_cost
|
||||
|
||||
# Propagate provider_specific_fields from the last chunk (contains provider
|
||||
# metadata like traffic_type set during streaming)
|
||||
for chunk in reversed(chunks):
|
||||
|
|
|
|||
|
|
@ -14,3 +14,4 @@ class UsagePerChunk(TypedDict):
|
|||
web_search_requests: Optional[int]
|
||||
completion_tokens_details: Optional[CompletionTokensDetails]
|
||||
prompt_tokens_details: Optional[PromptTokensDetailsWrapper]
|
||||
cost: Optional[float]
|
||||
|
|
|
|||
|
|
@ -528,6 +528,61 @@ def test_openrouter_cost_tracking_streaming():
|
|||
assert result2.usage.cost == 0.0001
|
||||
|
||||
|
||||
def test_openrouter_streaming_cost_propagated_to_final_response():
|
||||
"""
|
||||
OpenRouter streams `usage.cost` in the final chunk. After
|
||||
stream_chunk_builder rebuilds the response, the cost must land on
|
||||
`_hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"]`
|
||||
so response_cost_calculator picks it up (mirrors the non-streaming path).
|
||||
"""
|
||||
from litellm.main import stream_chunk_builder
|
||||
|
||||
handler = OpenRouterChatCompletionStreamingHandler(
|
||||
streaming_response=None, sync_stream=True
|
||||
)
|
||||
|
||||
chunk1 = {
|
||||
"id": "gen-stream-789",
|
||||
"created": 1234567890,
|
||||
"model": "openrouter/anthropic/claude-sonnet-4.5",
|
||||
"choices": [{"delta": {"content": "Hi", "reasoning": None}, "index": 0}],
|
||||
}
|
||||
chunk2 = {
|
||||
"id": "gen-stream-789",
|
||||
"created": 1234567890,
|
||||
"model": "openrouter/anthropic/claude-sonnet-4.5",
|
||||
"usage": {
|
||||
"prompt_tokens": 7,
|
||||
"completion_tokens": 3,
|
||||
"total_tokens": 10,
|
||||
"cost": 0.00042,
|
||||
},
|
||||
"choices": [
|
||||
{
|
||||
"delta": {"content": "", "reasoning": None},
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
parsed_chunks = [handler.chunk_parser(chunk1), handler.chunk_parser(chunk2)]
|
||||
|
||||
final_response = stream_chunk_builder(
|
||||
chunks=parsed_chunks,
|
||||
messages=[{"role": "user", "content": "Hi"}],
|
||||
)
|
||||
|
||||
assert final_response is not None
|
||||
assert final_response.usage.cost == 0.00042
|
||||
assert (
|
||||
final_response._hidden_params["additional_headers"][
|
||||
"llm_provider-x-litellm-response-cost"
|
||||
]
|
||||
== 0.00042
|
||||
)
|
||||
|
||||
|
||||
def test_openrouter_reasoning_models_allow_reasoning_effort_param():
|
||||
"""
|
||||
OpenRouter reasoning-capable models should accept the reasoning_effort param.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue