From 223c5cf3044cf8a182174935ce8b1c0609ca1be4 Mon Sep 17 00:00:00 2001 From: li872 Date: Mon, 14 Sep 2026 11:51:08 +0000 Subject: [PATCH] fix(streaming): preserve prompt_tokens_details on OpenAI-compatible stream usage OpenAI-compatible stream parsers copied only prompt/completion/total token counts, so Anthropic /v1/messages streaming reported cache_read_input_tokens 0. --- .../litellm_core_utils/streaming_handler.py | 6 +- litellm/llms/databricks/streaming_utils.py | 35 +++++----- .../test_streaming_handler.py | 64 +++++++++++++++++++ ...al_pass_through_adapters_transformation.py | 40 ++++++++++++ .../test_databricks_streaming_utils.py | 49 ++++++++++++++ 5 files changed, 171 insertions(+), 23 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 6a5a8832cc6..249c27c72bc 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1529,11 +1529,7 @@ class CustomStreamWrapper: setattr( model_response, "usage", - litellm.Usage( - prompt_tokens=response_obj["usage"].get("prompt_tokens", None) or None, - completion_tokens=response_obj["usage"].get("completion_tokens", None) or None, - total_tokens=response_obj["usage"].get("total_tokens", None) or None, - ), + litellm.Usage(**response_obj["usage"]), ) elif isinstance(response_obj["usage"], Usage): setattr( diff --git a/litellm/llms/databricks/streaming_utils.py b/litellm/llms/databricks/streaming_utils.py index 92f82a3f8d7..d8ce9fdf95e 100644 --- a/litellm/llms/databricks/streaming_utils.py +++ b/litellm/llms/databricks/streaming_utils.py @@ -11,6 +11,21 @@ from litellm.types.llms.openai import ( from litellm.types.utils import GenericStreamingChunk, Usage +def _usage_to_chat_completion_block(usage: Usage | None) -> ChatCompletionUsageBlock | None: + if usage is None: + return None + dumped: Final = usage.model_dump() + prompt_tokens_details: Final = dumped.get("prompt_tokens_details") + completion_tokens_details: Final = dumped.get("completion_tokens_details") + return ChatCompletionUsageBlock( + prompt_tokens=usage.prompt_tokens or 0, + completion_tokens=usage.completion_tokens or 0, + total_tokens=usage.total_tokens or 0, + prompt_tokens_details=prompt_tokens_details if isinstance(prompt_tokens_details, dict) else None, + completion_tokens_details=completion_tokens_details if isinstance(completion_tokens_details, dict) else None, + ) + + class ModelResponseIterator: def __init__(self, streaming_response, sync_stream: bool): self.streaming_response = streaming_response @@ -23,27 +38,17 @@ class ModelResponseIterator: tool_use: ChatCompletionToolCallChunk | None = None is_finished = False finish_reason = "" - usage: ChatCompletionUsageBlock | None = None # Usage-only final chunk (OpenAI ``stream_options.include_usage``) # arrives with an empty ``choices`` list — return usage without # indexing ``choices[0]``. if len(processed_chunk.choices) == 0: - final_usage: Final = getattr(processed_chunk, "usage", None) return GenericStreamingChunk( text="", tool_use=None, is_finished=False, finish_reason="", - usage=( - ChatCompletionUsageBlock( - prompt_tokens=final_usage.prompt_tokens or 0, - completion_tokens=final_usage.completion_tokens or 0, - total_tokens=final_usage.total_tokens or 0, - ) - if final_usage is not None - else None - ), + usage=_usage_to_chat_completion_block(getattr(processed_chunk, "usage", None)), index=0, ) @@ -70,13 +75,7 @@ class ModelResponseIterator: is_finished = True finish_reason = processed_chunk.choices[0].finish_reason - usage_chunk: Final[Usage | None] = getattr(processed_chunk, "usage", None) - if usage_chunk is not None: - usage = ChatCompletionUsageBlock( - prompt_tokens=usage_chunk.prompt_tokens, - completion_tokens=usage_chunk.completion_tokens, - total_tokens=usage_chunk.total_tokens, - ) + usage: Final = _usage_to_chat_completion_block(getattr(processed_chunk, "usage", None)) return GenericStreamingChunk( text=text, diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 37e2031fdf4..9e9661e4111 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -2511,6 +2511,70 @@ def test_usage_only_chunk_not_dropped_when_finish_reason_already_set( assert result.usage is not None +def test_chunk_creator_preserves_dict_usage_prompt_cache_details( + initialized_custom_stream_wrapper: CustomStreamWrapper, +): + from types import SimpleNamespace + + initialized_custom_stream_wrapper.custom_llm_provider = "openai" + initialized_custom_stream_wrapper.send_stream_usage = True + + chunk = SimpleNamespace( + id="chatcmpl-cache", + choices=[], + system_fingerprint=None, + usage={ + "prompt_tokens": 9888, + "completion_tokens": 1, + "total_tokens": 9889, + "prompt_tokens_details": {"cached_tokens": 6752, "audio_tokens": 0}, + "completion_tokens_details": {"reasoning_tokens": 0}, + }, + ) + + result = initialized_custom_stream_wrapper.chunk_creator(chunk=chunk) + + assert result is not None + assert result.usage is not None + assert result.usage.prompt_tokens == 9888 + assert result.usage.prompt_tokens_details is not None + assert result.usage.prompt_tokens_details.cached_tokens == 6752 + assert result.usage.completion_tokens_details is not None + assert result.usage.completion_tokens_details.reasoning_tokens == 0 + + +def test_chunk_creator_preserves_openai_like_generic_usage_cache_details( + initialized_custom_stream_wrapper: CustomStreamWrapper, +): + from litellm.llms.databricks.streaming_utils import ModelResponseIterator + + initialized_custom_stream_wrapper.custom_llm_provider = "openai_like" + initialized_custom_stream_wrapper.received_finish_reason = "stop" + parsed = ModelResponseIterator(streaming_response=None, sync_stream=True).chunk_parser( + chunk={ + "id": "chatcmpl-cache", + "object": "chat.completion.chunk", + "created": 1, + "model": "m", + "choices": [], + "usage": { + "prompt_tokens": 9888, + "completion_tokens": 1, + "total_tokens": 9889, + "prompt_tokens_details": {"cached_tokens": 6752}, + "completion_tokens_details": {"reasoning_tokens": 0}, + }, + } + ) + + result = initialized_custom_stream_wrapper.chunk_creator(chunk=parsed) + + assert result is not None + assert result.usage is not None + assert result.usage.prompt_tokens_details is not None + assert result.usage.prompt_tokens_details.cached_tokens == 6752 + + def _run_dispatch(wrapper: CustomStreamWrapper, chunk): model_response = wrapper.model_response_creator() completion_obj = {"content": ""} diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 03b9840b1c3..ae045dc3a0d 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -3253,6 +3253,46 @@ def test_translate_streaming_openai_response_to_anthropic_cache_tokens_from_prom assert message_delta["usage"]["cache_creation_input_tokens"] == 20 +def test_openai_like_stream_usage_chunk_keeps_cache_read_in_anthropic_delta(): + from unittest.mock import MagicMock + + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.llms.databricks.streaming_utils import ModelResponseIterator + + parsed = ModelResponseIterator(streaming_response=None, sync_stream=True).chunk_parser( + chunk={ + "id": "chatcmpl-cache", + "object": "chat.completion.chunk", + "created": 1, + "model": "m", + "choices": [], + "usage": { + "prompt_tokens": 120, + "completion_tokens": 50, + "total_tokens": 170, + "prompt_tokens_details": {"cached_tokens": 30, "audio_tokens": 0}, + }, + } + ) + wrapper = CustomStreamWrapper( + completion_stream=None, + model="m", + logging_obj=MagicMock(), + custom_llm_provider="openai_like", + ) + wrapper.received_finish_reason = "stop" + usage_chunk = wrapper.chunk_creator(chunk=parsed) + assert usage_chunk is not None + assert usage_chunk.usage is not None + + adapter = LiteLLMAnthropicMessagesAdapter() + usage_delta = adapter._translate_openai_usage_to_anthropic_usage_delta(usage_chunk.usage) + + assert usage_delta["input_tokens"] == 90 + assert usage_delta["output_tokens"] == 50 + assert usage_delta["cache_read_input_tokens"] == 30 + + def test_translate_streaming_openai_response_to_anthropic_cache_tokens_from_hidden_params_usage(): from litellm.types.utils import PromptTokensDetailsWrapper diff --git a/tests/test_litellm/llms/databricks/test_databricks_streaming_utils.py b/tests/test_litellm/llms/databricks/test_databricks_streaming_utils.py index 5612864a841..448f91078bf 100644 --- a/tests/test_litellm/llms/databricks/test_databricks_streaming_utils.py +++ b/tests/test_litellm/llms/databricks/test_databricks_streaming_utils.py @@ -48,6 +48,55 @@ def test_chunk_parser_empty_choices_without_usage(): assert result["usage"] is None +def test_chunk_parser_preserves_prompt_tokens_details_on_usage_only_chunk(): + iterator = ModelResponseIterator(streaming_response=None, sync_stream=True) + usage_only_chunk = { + "id": "chatcmpl-x", + "object": "chat.completion.chunk", + "created": 1, + "model": "m", + "choices": [], + "usage": { + "prompt_tokens": 9888, + "completion_tokens": 1, + "total_tokens": 9889, + "prompt_tokens_details": {"cached_tokens": 6752, "audio_tokens": 0}, + "completion_tokens_details": {"reasoning_tokens": 0}, + }, + } + + result = iterator.chunk_parser(chunk=usage_only_chunk) + + assert result["usage"] is not None + assert result["usage"]["prompt_tokens"] == 9888 + assert result["usage"]["completion_tokens"] == 1 + assert result["usage"]["prompt_tokens_details"]["cached_tokens"] == 6752 + assert result["usage"]["completion_tokens_details"]["reasoning_tokens"] == 0 + + +def test_chunk_parser_preserves_prompt_tokens_details_on_finish_chunk(): + iterator = ModelResponseIterator(streaming_response=None, sync_stream=True) + chunk = { + "id": "chatcmpl-x", + "object": "chat.completion.chunk", + "created": 1, + "model": "m", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 9888, + "completion_tokens": 1, + "total_tokens": 9889, + "prompt_tokens_details": {"cached_tokens": 6752}, + }, + } + + result = iterator.chunk_parser(chunk=chunk) + + assert result["is_finished"] is True + assert result["usage"] is not None + assert result["usage"]["prompt_tokens_details"]["cached_tokens"] == 6752 + + def test_chunk_parser_normal_content_chunk_still_works(): """A regular content chunk is unaffected by the empty-choices guard.""" iterator = ModelResponseIterator(streaming_response=None, sync_stream=True)