fix(streaming): preserve prompt_tokens_details on OpenAI-compatible stream usage

OpenAI-compatible stream parsers copied only prompt/completion/total token
counts, so Anthropic /v1/messages streaming reported cache_read_input_tokens 0.
This commit is contained in:
li872 2026-09-14 11:51:08 +00:00
parent 30f33a949b
commit 223c5cf304
5 changed files with 171 additions and 23 deletions

View file

@ -1529,11 +1529,7 @@ class CustomStreamWrapper:
setattr(
model_response,
"usage",
litellm.Usage(
prompt_tokens=response_obj["usage"].get("prompt_tokens", None) or None,
completion_tokens=response_obj["usage"].get("completion_tokens", None) or None,
total_tokens=response_obj["usage"].get("total_tokens", None) or None,
),
litellm.Usage(**response_obj["usage"]),
)
elif isinstance(response_obj["usage"], Usage):
setattr(

View file

@ -11,6 +11,21 @@ from litellm.types.llms.openai import (
from litellm.types.utils import GenericStreamingChunk, Usage
def _usage_to_chat_completion_block(usage: Usage | None) -> ChatCompletionUsageBlock | None:
if usage is None:
return None
dumped: Final = usage.model_dump()
prompt_tokens_details: Final = dumped.get("prompt_tokens_details")
completion_tokens_details: Final = dumped.get("completion_tokens_details")
return ChatCompletionUsageBlock(
prompt_tokens=usage.prompt_tokens or 0,
completion_tokens=usage.completion_tokens or 0,
total_tokens=usage.total_tokens or 0,
prompt_tokens_details=prompt_tokens_details if isinstance(prompt_tokens_details, dict) else None,
completion_tokens_details=completion_tokens_details if isinstance(completion_tokens_details, dict) else None,
)
class ModelResponseIterator:
def __init__(self, streaming_response, sync_stream: bool):
self.streaming_response = streaming_response
@ -23,27 +38,17 @@ class ModelResponseIterator:
tool_use: ChatCompletionToolCallChunk | None = None
is_finished = False
finish_reason = ""
usage: ChatCompletionUsageBlock | None = None
# Usage-only final chunk (OpenAI ``stream_options.include_usage``)
# arrives with an empty ``choices`` list — return usage without
# indexing ``choices[0]``.
if len(processed_chunk.choices) == 0:
final_usage: Final = getattr(processed_chunk, "usage", None)
return GenericStreamingChunk(
text="",
tool_use=None,
is_finished=False,
finish_reason="",
usage=(
ChatCompletionUsageBlock(
prompt_tokens=final_usage.prompt_tokens or 0,
completion_tokens=final_usage.completion_tokens or 0,
total_tokens=final_usage.total_tokens or 0,
)
if final_usage is not None
else None
),
usage=_usage_to_chat_completion_block(getattr(processed_chunk, "usage", None)),
index=0,
)
@ -70,13 +75,7 @@ class ModelResponseIterator:
is_finished = True
finish_reason = processed_chunk.choices[0].finish_reason
usage_chunk: Final[Usage | None] = getattr(processed_chunk, "usage", None)
if usage_chunk is not None:
usage = ChatCompletionUsageBlock(
prompt_tokens=usage_chunk.prompt_tokens,
completion_tokens=usage_chunk.completion_tokens,
total_tokens=usage_chunk.total_tokens,
)
usage: Final = _usage_to_chat_completion_block(getattr(processed_chunk, "usage", None))
return GenericStreamingChunk(
text=text,

View file

@ -2511,6 +2511,70 @@ def test_usage_only_chunk_not_dropped_when_finish_reason_already_set(
assert result.usage is not None
def test_chunk_creator_preserves_dict_usage_prompt_cache_details(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
from types import SimpleNamespace
initialized_custom_stream_wrapper.custom_llm_provider = "openai"
initialized_custom_stream_wrapper.send_stream_usage = True
chunk = SimpleNamespace(
id="chatcmpl-cache",
choices=[],
system_fingerprint=None,
usage={
"prompt_tokens": 9888,
"completion_tokens": 1,
"total_tokens": 9889,
"prompt_tokens_details": {"cached_tokens": 6752, "audio_tokens": 0},
"completion_tokens_details": {"reasoning_tokens": 0},
},
)
result = initialized_custom_stream_wrapper.chunk_creator(chunk=chunk)
assert result is not None
assert result.usage is not None
assert result.usage.prompt_tokens == 9888
assert result.usage.prompt_tokens_details is not None
assert result.usage.prompt_tokens_details.cached_tokens == 6752
assert result.usage.completion_tokens_details is not None
assert result.usage.completion_tokens_details.reasoning_tokens == 0
def test_chunk_creator_preserves_openai_like_generic_usage_cache_details(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
from litellm.llms.databricks.streaming_utils import ModelResponseIterator
initialized_custom_stream_wrapper.custom_llm_provider = "openai_like"
initialized_custom_stream_wrapper.received_finish_reason = "stop"
parsed = ModelResponseIterator(streaming_response=None, sync_stream=True).chunk_parser(
chunk={
"id": "chatcmpl-cache",
"object": "chat.completion.chunk",
"created": 1,
"model": "m",
"choices": [],
"usage": {
"prompt_tokens": 9888,
"completion_tokens": 1,
"total_tokens": 9889,
"prompt_tokens_details": {"cached_tokens": 6752},
"completion_tokens_details": {"reasoning_tokens": 0},
},
}
)
result = initialized_custom_stream_wrapper.chunk_creator(chunk=parsed)
assert result is not None
assert result.usage is not None
assert result.usage.prompt_tokens_details is not None
assert result.usage.prompt_tokens_details.cached_tokens == 6752
def _run_dispatch(wrapper: CustomStreamWrapper, chunk):
model_response = wrapper.model_response_creator()
completion_obj = {"content": ""}

View file

@ -3253,6 +3253,46 @@ def test_translate_streaming_openai_response_to_anthropic_cache_tokens_from_prom
assert message_delta["usage"]["cache_creation_input_tokens"] == 20
def test_openai_like_stream_usage_chunk_keeps_cache_read_in_anthropic_delta():
from unittest.mock import MagicMock
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.llms.databricks.streaming_utils import ModelResponseIterator
parsed = ModelResponseIterator(streaming_response=None, sync_stream=True).chunk_parser(
chunk={
"id": "chatcmpl-cache",
"object": "chat.completion.chunk",
"created": 1,
"model": "m",
"choices": [],
"usage": {
"prompt_tokens": 120,
"completion_tokens": 50,
"total_tokens": 170,
"prompt_tokens_details": {"cached_tokens": 30, "audio_tokens": 0},
},
}
)
wrapper = CustomStreamWrapper(
completion_stream=None,
model="m",
logging_obj=MagicMock(),
custom_llm_provider="openai_like",
)
wrapper.received_finish_reason = "stop"
usage_chunk = wrapper.chunk_creator(chunk=parsed)
assert usage_chunk is not None
assert usage_chunk.usage is not None
adapter = LiteLLMAnthropicMessagesAdapter()
usage_delta = adapter._translate_openai_usage_to_anthropic_usage_delta(usage_chunk.usage)
assert usage_delta["input_tokens"] == 90
assert usage_delta["output_tokens"] == 50
assert usage_delta["cache_read_input_tokens"] == 30
def test_translate_streaming_openai_response_to_anthropic_cache_tokens_from_hidden_params_usage():
from litellm.types.utils import PromptTokensDetailsWrapper

View file

@ -48,6 +48,55 @@ def test_chunk_parser_empty_choices_without_usage():
assert result["usage"] is None
def test_chunk_parser_preserves_prompt_tokens_details_on_usage_only_chunk():
iterator = ModelResponseIterator(streaming_response=None, sync_stream=True)
usage_only_chunk = {
"id": "chatcmpl-x",
"object": "chat.completion.chunk",
"created": 1,
"model": "m",
"choices": [],
"usage": {
"prompt_tokens": 9888,
"completion_tokens": 1,
"total_tokens": 9889,
"prompt_tokens_details": {"cached_tokens": 6752, "audio_tokens": 0},
"completion_tokens_details": {"reasoning_tokens": 0},
},
}
result = iterator.chunk_parser(chunk=usage_only_chunk)
assert result["usage"] is not None
assert result["usage"]["prompt_tokens"] == 9888
assert result["usage"]["completion_tokens"] == 1
assert result["usage"]["prompt_tokens_details"]["cached_tokens"] == 6752
assert result["usage"]["completion_tokens_details"]["reasoning_tokens"] == 0
def test_chunk_parser_preserves_prompt_tokens_details_on_finish_chunk():
iterator = ModelResponseIterator(streaming_response=None, sync_stream=True)
chunk = {
"id": "chatcmpl-x",
"object": "chat.completion.chunk",
"created": 1,
"model": "m",
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
"usage": {
"prompt_tokens": 9888,
"completion_tokens": 1,
"total_tokens": 9889,
"prompt_tokens_details": {"cached_tokens": 6752},
},
}
result = iterator.chunk_parser(chunk=chunk)
assert result["is_finished"] is True
assert result["usage"] is not None
assert result["usage"]["prompt_tokens_details"]["cached_tokens"] == 6752
def test_chunk_parser_normal_content_chunk_still_works():
"""A regular content chunk is unaffected by the empty-choices guard."""
iterator = ModelResponseIterator(streaming_response=None, sync_stream=True)