From a8320749bb28b44cf4306435a2a6451e1cb69314 Mon Sep 17 00:00:00 2001 From: sami Date: Tue, 1 Sep 2026 12:01:43 +0300 Subject: [PATCH] fix(streaming): coerce a chunk's provider usage model to litellm Usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `usage` is not a declared field on ModelResponseStream, so assigning a provider SDK's model does not coerce it. A chunk carrying both choices and usage therefore travels on with openai's CompletionUsage: callers reading the streamed chunk, and the logging path, get PromptTokensDetails where every other route hands them PromptTokensDetailsWrapper. Coerce at the assignment rather than skipping it. Guarding on `isinstance(..., Usage)` looks equivalent and is not — for two providers this line is the only thing carrying usage forward: - azure_text: handle_azure_text_completion_chunk returns no usage key at all, and the dispatch branch sets none, so skipping drops usage entirely. stream_chunk_builder then re-derives prompt_tokens with token_counter — 9521 reported becomes 8. - text-completion-openai: the branch builds a deliberately lossy three-field Usage with no prompt_tokens_details, so skipping drops cached_tokens and bills them at the full input rate. Both are covered by tests that fail under that variant. Scope, stated precisely: this does NOT change billing on the chat path. ChunkProcessor._extract_usage_chunk already normalizes a raw provider model into Usage (9e25dd708f), so prompt_tokens and cached_tokens survive there either way — verified identical before and after on an openai chunk with cached_tokens=9504. What changes is the type the caller and the logging path observe, plus the two provider paths above. Tests are mutation-checked: reverting to the raw assignment fails the two coercion tests; the isinstance variant fails the azure_text and text-completion-openai tests. --- .../litellm_core_utils/streaming_handler.py | 26 +++- .../test_streaming_handler.py | 123 ++++++++++++++++++ 2 files changed, 148 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 6a5a8832cc6..0fb8c144d8f 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -92,6 +92,30 @@ def print_verbose(print_statement: object): pass +def _as_litellm_usage(usage: object) -> Usage: + """Coerce a chunk's usage to litellm's own `Usage`. + + `usage` is not a declared field on ModelResponseStream, so assigning a provider SDK's + model (openai's CompletionUsage, say) does not coerce it. The object then travels on + as the provider's type: callers reading the streamed chunk, and the logging path, get + PromptTokensDetails where every other route gives them PromptTokensDetailsWrapper. + + Coercing rather than skipping the assignment, because for two providers this line is + the only thing carrying usage forward: azure_text's chunk handler returns no usage at + all, and text-completion-openai builds a deliberately lossy three-field Usage that + drops prompt_tokens_details. Guarding the assignment on `isinstance(..., Usage)` + silently discards provider-reported tokens on the first and cached tokens on the + second. + """ + if isinstance(usage, Usage): + return usage + if isinstance(usage, BaseModel): + return Usage(**usage.model_dump()) + if isinstance(usage, dict): + return Usage(**usage) + return Usage() + + @dataclass(frozen=True, slots=True) class _ProviderChunkParsed: response_obj: dict[str, object] @@ -1658,7 +1682,7 @@ class CustomStreamWrapper: self.tool_call = True if hasattr(chunk, "usage") and chunk.usage is not None: - model_response.usage = chunk.usage + model_response.usage = _as_litellm_usage(chunk.usage) ## RETURN ARG result: Final = self.return_processed_chunk_logic( diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 37e2031fdf4..60bc0e6e8c3 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -4917,3 +4917,126 @@ class TestStableStreamingResponseId: ) wrapper.response_id = "chatcmpl-from-provider" assert wrapper.model_response_creator().id == "chatcmpl-from-provider" + + +def _openai_chunk_with_usage(usage): + """An openai chunk carrying BOTH choices and usage — the shape that reaches the + assignment under test. A usage-only final chunk returns earlier, via the + include_usage path, and never gets here.""" + from openai.types.chat import ChatCompletionChunk + + chunk = ChatCompletionChunk( + id="chatcmpl-1", + choices=[{"delta": {"content": "hi"}, "finish_reason": None, "index": 0}], + created=1, + model="gpt-4o", + object="chat.completion.chunk", + ) + setattr(chunk, "usage", usage) + return chunk + + +def _completion_usage(prompt_tokens=9521, cached_tokens=9504): + from openai.types.completion_usage import CompletionUsage, PromptTokensDetails + + return CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=7, + total_tokens=prompt_tokens + 7, + prompt_tokens_details=PromptTokensDetails(cached_tokens=cached_tokens), + ) + + +def test_provider_usage_model_is_coerced_to_litellm_usage(): + """A provider SDK usage model must not travel on as the provider's own type. + + `usage` is not a declared field on ModelResponseStream, so assigning openai's + CompletionUsage does not coerce it. The chunk handed to the caller and to logging then + carries PromptTokensDetails where every other route gives PromptTokensDetailsWrapper. + """ + from litellm.types.utils import Usage + + wrapper = litellm.CustomStreamWrapper( + completion_stream=None, + model="gpt-4o", + logging_obj=MagicMock(), + custom_llm_provider="openai", + ) + processed = wrapper.chunk_creator(chunk=_openai_chunk_with_usage(_completion_usage())) + + assert isinstance(processed.usage, Usage) + assert processed.usage.prompt_tokens == 9521 + assert processed.usage.prompt_tokens_details.cached_tokens == 9504 + + +def test_text_completion_openai_keeps_cached_tokens(): + """text-completion-openai builds a lossy three-field Usage that drops + prompt_tokens_details, so this assignment is what carries cached tokens forward. + Skipping it on a non-Usage instance bills cached tokens at the full input rate — the + exact harm this line exists to prevent.""" + from openai.types import Completion + + chunk = Completion( + id="cmpl-1", + choices=[{"text": "hi", "finish_reason": "stop", "index": 0, "logprobs": None}], + created=1, + model="gpt-3.5-turbo-instruct", + object="text_completion", + ) + setattr(chunk, "usage", _completion_usage()) + + wrapper = litellm.CustomStreamWrapper( + completion_stream=None, + model="gpt-3.5-turbo-instruct", + logging_obj=MagicMock(), + custom_llm_provider="text-completion-openai", + ) + processed = wrapper.chunk_creator(chunk=chunk) + + assert processed.usage.prompt_tokens_details.cached_tokens == 9504 + + +def test_azure_text_usage_survives(): + """azure_text's chunk handler returns no usage key at all, so this assignment is the + ONLY thing putting usage on the chunk. Without it, stream_chunk_builder falls back to + token_counter and reports a prompt count three orders of magnitude off.""" + from openai.types import Completion + + chunk = Completion( + id="cmpl-1", + choices=[{"text": "hi", "finish_reason": "stop", "index": 0, "logprobs": None}], + created=1, + model="gpt-3.5-turbo-instruct", + object="text_completion", + ) + setattr(chunk, "usage", _completion_usage()) + + wrapper = litellm.CustomStreamWrapper( + completion_stream=None, + model="gpt-3.5-turbo-instruct", + logging_obj=MagicMock(), + custom_llm_provider="azure_text", + ) + processed = wrapper.chunk_creator(chunk=chunk) + + assert processed.usage is not None + assert processed.usage.prompt_tokens == 9521 + + +def test_dict_usage_is_coerced(): + """Some providers attach a plain dict. Usage() over an unrecognised shape must not + raise inside the streaming path.""" + from litellm.types.utils import Usage + + wrapper = litellm.CustomStreamWrapper( + completion_stream=None, + model="gpt-4o", + logging_obj=MagicMock(), + custom_llm_provider="openai", + ) + chunk = _openai_chunk_with_usage({"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7}) + processed = wrapper.chunk_creator(chunk=chunk) + + assert isinstance(processed.usage, Usage) + assert processed.usage.prompt_tokens == 3 +