From fe6c698214a88bbd6a0a803d156d5c7fd182052c Mon Sep 17 00:00:00 2001 From: daleselaji-dev <265319989+daleselaji-dev@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:25:06 +0800 Subject: [PATCH 1/6] fix(streaming): preserve explicit zero usage --- .../streaming_chunk_builder_utils.py | 15 ++++++++-- .../streaming_chunk_builder_utils.py | 2 ++ .../test_streaming_chunk_builder_utils.py | 28 +++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index ee0518c4aec..673575a549f 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -761,6 +761,8 @@ class ChunkProcessor: # # Update usage information if needed prompt_tokens = 0 completion_tokens = 0 + prompt_tokens_provided = False + completion_tokens_provided = False # Anthropic's `message_start` SSE event carries usage.output_tokens=1 as a # cursor/placeholder; the real value only arrives in `message_delta`. # If a stream is cancelled before `message_delta` lands, the last-wins @@ -793,6 +795,8 @@ class ChunkProcessor: if usage_chunk is not None: usage_chunk_dict = self._usage_chunk_calculation_helper(usage_chunk) + prompt_tokens_provided = prompt_tokens_provided or "prompt_tokens" in usage_chunk + completion_tokens_provided = completion_tokens_provided or "completion_tokens" in usage_chunk if usage_chunk_dict["prompt_tokens"] is not None and usage_chunk_dict["prompt_tokens"] > 0: prompt_tokens = usage_chunk_dict["prompt_tokens"] if usage_chunk_dict["completion_tokens"] is not None and usage_chunk_dict["completion_tokens"] > 0: @@ -847,7 +851,9 @@ class ChunkProcessor: return UsagePerChunk( prompt_tokens=prompt_tokens, + prompt_tokens_provided=prompt_tokens_provided, completion_tokens=completion_tokens, + completion_tokens_provided=completion_tokens_provided, cache_creation_input_tokens=cache_creation_input_tokens, cache_read_input_tokens=cache_read_input_tokens, server_tool_use=server_tool_use, @@ -946,13 +952,18 @@ class ChunkProcessor: cost: Final[float | None] = calculated_usage_per_chunk["cost"] try: - returned_usage.prompt_tokens = prompt_tokens or token_counter(model=model, messages=messages) + returned_usage.prompt_tokens = ( + prompt_tokens + if calculated_usage_per_chunk["prompt_tokens_provided"] + else token_counter(model=model, messages=messages) + ) except Exception: # don't allow this failing to block a complete streaming response from being returned print_verbose("token_counter failed, assuming prompt tokens is 0") returned_usage.prompt_tokens = 0 returned_usage.completion_tokens = ( completion_tokens - or token_counter( + if calculated_usage_per_chunk["completion_tokens_provided"] + else token_counter( model=model, text=completion_output, count_response_tokens=True, # count_response_tokens is a Flag to tell token counter this is a response, No need to add extra tokens we do for input messages diff --git a/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py index 893b0bdbb9f..b76f06ccfae 100644 --- a/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py @@ -5,7 +5,9 @@ from ..utils import CompletionTokensDetails, PromptTokensDetailsWrapper, ServerT class UsagePerChunk(TypedDict): prompt_tokens: int + prompt_tokens_provided: bool completion_tokens: int + completion_tokens_provided: bool cache_creation_input_tokens: int | None cache_read_input_tokens: int | None server_tool_use: ServerToolUse | None diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index 44e77506b3f..9c959907ead 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -592,6 +592,34 @@ def test_stream_chunk_builder_litellm_usage_chunks(): assert usage.total_tokens == 77 +def test_stream_chunk_builder_preserves_explicit_zero_usage(): + chunk = ModelResponseStream( + id="chatcmpl-explicit-zero-usage", + created=1745513206, + model="gpt-5.5", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(content="partial output"), + ) + ], + usage=Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0), + ) + + usage = ChunkProcessor(chunks=[chunk]).calculate_usage( + chunks=[chunk], + model="gpt-5.5", + messages=[{"role": "user", "content": "a non-empty prompt"}], + completion_output="partial output", + ) + + assert usage.prompt_tokens == 0 + assert usage.completion_tokens == 0 + assert usage.total_tokens == 0 + + def test_get_model_from_chunks_azure_model_router(): """ Test that _get_model_from_chunks finds the actual model from Azure Model Router chunks. From 13fb6367c6ec59e6557c13783dfa55201a598eca Mon Sep 17 00:00:00 2001 From: daleselaji-dev <265319989+daleselaji-dev@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:53:52 +0800 Subject: [PATCH 2/6] fix(streaming): preserve Anthropic cursor fallback --- .../streaming_chunk_builder_utils.py | 32 +++++++++++++------ .../streaming_chunk_builder_utils.py | 2 -- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 673575a549f..f706cc18f0b 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -178,6 +178,7 @@ class ChunkProcessor: self.chunks = self._sort_chunks(chunks) self.messages = messages self.first_chunk = chunks[0] + self._usage_fields_provided: tuple[bool, bool] = (False, False) def _sort_chunks(self, chunks: list) -> list: if not chunks: @@ -795,13 +796,20 @@ class ChunkProcessor: if usage_chunk is not None: usage_chunk_dict = self._usage_chunk_calculation_helper(usage_chunk) - prompt_tokens_provided = prompt_tokens_provided or "prompt_tokens" in usage_chunk - completion_tokens_provided = completion_tokens_provided or "completion_tokens" in usage_chunk - if usage_chunk_dict["prompt_tokens"] is not None and usage_chunk_dict["prompt_tokens"] > 0: + if "prompt_tokens" in usage_chunk: + prompt_tokens_provided = True + if "prompt_tokens" in usage_chunk and ( + usage_chunk_dict["prompt_tokens"] > 0 or prompt_tokens == 0 + ): prompt_tokens = usage_chunk_dict["prompt_tokens"] - if usage_chunk_dict["completion_tokens"] is not None and usage_chunk_dict["completion_tokens"] > 0: + if "completion_tokens" in usage_chunk: + completion_tokens_provided = True + if "completion_tokens" in usage_chunk and ( + usage_chunk_dict["completion_tokens"] > 0 or completion_tokens == 0 + ): completion_tokens = usage_chunk_dict["completion_tokens"] - completion_usage_updates += 1 + if completion_tokens > 0: + completion_usage_updates += 1 if usage_chunk_dict["cache_creation_input_tokens"] is not None and ( usage_chunk_dict["cache_creation_input_tokens"] > 0 or cache_creation_input_tokens is None ): @@ -843,17 +851,20 @@ class ChunkProcessor: prompt_tokens_details = attach_cache_creation_token_details(prompt_tokens_details, cache_creation_token_details) + was_anthropic_cursor = completion_tokens == 1 completion_tokens = self._reset_anthropic_cursor_completion_tokens( chunks=chunks, - completion_tokens=completion_tokens, + completion_tokens=completion_tokens or 0, completion_usage_updates=completion_usage_updates, ) + if was_anthropic_cursor and completion_tokens == 0: + completion_tokens_provided = False + + self._usage_fields_provided = (prompt_tokens_provided, completion_tokens_provided) return UsagePerChunk( prompt_tokens=prompt_tokens, - prompt_tokens_provided=prompt_tokens_provided, completion_tokens=completion_tokens, - completion_tokens_provided=completion_tokens_provided, cache_creation_input_tokens=cache_creation_input_tokens, cache_read_input_tokens=cache_read_input_tokens, server_tool_use=server_tool_use, @@ -950,11 +961,12 @@ class ChunkProcessor: ] prompt_tokens_details: PromptTokensDetailsWrapper | None = calculated_usage_per_chunk["prompt_tokens_details"] cost: Final[float | None] = calculated_usage_per_chunk["cost"] + prompt_tokens_provided, completion_tokens_provided = self._usage_fields_provided try: returned_usage.prompt_tokens = ( prompt_tokens - if calculated_usage_per_chunk["prompt_tokens_provided"] + if prompt_tokens_provided else token_counter(model=model, messages=messages) ) except Exception: # don't allow this failing to block a complete streaming response from being returned @@ -962,7 +974,7 @@ class ChunkProcessor: returned_usage.prompt_tokens = 0 returned_usage.completion_tokens = ( completion_tokens - if calculated_usage_per_chunk["completion_tokens_provided"] + if completion_tokens_provided else token_counter( model=model, text=completion_output, diff --git a/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py index b76f06ccfae..893b0bdbb9f 100644 --- a/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py @@ -5,9 +5,7 @@ from ..utils import CompletionTokensDetails, PromptTokensDetailsWrapper, ServerT class UsagePerChunk(TypedDict): prompt_tokens: int - prompt_tokens_provided: bool completion_tokens: int - completion_tokens_provided: bool cache_creation_input_tokens: int | None cache_read_input_tokens: int | None server_tool_use: ServerToolUse | None From 9ba1183e93430ec2c24384043ca27c8a3655daeb Mon Sep 17 00:00:00 2001 From: daleselaji-dev <265319989+daleselaji-dev@users.noreply.github.com> Date: Sun, 23 Aug 2026 10:58:10 +0800 Subject: [PATCH 3/6] style: format streaming usage changes --- .../litellm_core_utils/streaming_chunk_builder_utils.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index f706cc18f0b..51e2f51401c 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -798,9 +798,7 @@ class ChunkProcessor: usage_chunk_dict = self._usage_chunk_calculation_helper(usage_chunk) if "prompt_tokens" in usage_chunk: prompt_tokens_provided = True - if "prompt_tokens" in usage_chunk and ( - usage_chunk_dict["prompt_tokens"] > 0 or prompt_tokens == 0 - ): + if "prompt_tokens" in usage_chunk and (usage_chunk_dict["prompt_tokens"] > 0 or prompt_tokens == 0): prompt_tokens = usage_chunk_dict["prompt_tokens"] if "completion_tokens" in usage_chunk: completion_tokens_provided = True @@ -965,9 +963,7 @@ class ChunkProcessor: try: returned_usage.prompt_tokens = ( - prompt_tokens - if prompt_tokens_provided - else token_counter(model=model, messages=messages) + prompt_tokens if prompt_tokens_provided else token_counter(model=model, messages=messages) ) except Exception: # don't allow this failing to block a complete streaming response from being returned print_verbose("token_counter failed, assuming prompt tokens is 0") From b16dfa2cd93cac81dd6b7450dbb0e51bfca17ed0 Mon Sep 17 00:00:00 2001 From: daleselaji-dev <265319989+daleselaji-dev@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:21:08 +0800 Subject: [PATCH 4/6] fix(streaming): keep usage aggregation within lint budget --- .../streaming_chunk_builder_utils.py | 51 ++++++++++++++----- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 51e2f51401c..88317817a1e 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -751,6 +751,32 @@ class ChunkProcessor: return Usage(**usage_chunk) return usage_chunk + @staticmethod + def _update_usage_token_counts( + usage_chunk: Usage, + usage_chunk_dict: "_UsageSummary", + prompt_tokens: int, + completion_tokens: int, + completion_usage_updates: int, + ) -> tuple[int, int, bool, bool, int]: + prompt_tokens_provided = "prompt_tokens" in usage_chunk + completion_tokens_provided = "completion_tokens" in usage_chunk + + if prompt_tokens_provided and (usage_chunk_dict["prompt_tokens"] > 0 or prompt_tokens == 0): + prompt_tokens = usage_chunk_dict["prompt_tokens"] + if completion_tokens_provided and (usage_chunk_dict["completion_tokens"] > 0 or completion_tokens == 0): + completion_tokens = usage_chunk_dict["completion_tokens"] + if completion_tokens > 0: + completion_usage_updates += 1 + + return ( + prompt_tokens, + completion_tokens, + prompt_tokens_provided, + completion_tokens_provided, + completion_usage_updates, + ) + def _calculate_usage_per_chunk( self, chunks: Sequence["_UsageBearingChunk | ModelResponse"], @@ -796,18 +822,19 @@ class ChunkProcessor: if usage_chunk is not None: usage_chunk_dict = self._usage_chunk_calculation_helper(usage_chunk) - if "prompt_tokens" in usage_chunk: - prompt_tokens_provided = True - if "prompt_tokens" in usage_chunk and (usage_chunk_dict["prompt_tokens"] > 0 or prompt_tokens == 0): - prompt_tokens = usage_chunk_dict["prompt_tokens"] - if "completion_tokens" in usage_chunk: - completion_tokens_provided = True - if "completion_tokens" in usage_chunk and ( - usage_chunk_dict["completion_tokens"] > 0 or completion_tokens == 0 - ): - completion_tokens = usage_chunk_dict["completion_tokens"] - if completion_tokens > 0: - completion_usage_updates += 1 + ( + prompt_tokens, + completion_tokens, + prompt_tokens_provided, + completion_tokens_provided, + completion_usage_updates, + ) = self._update_usage_token_counts( + usage_chunk=usage_chunk, + usage_chunk_dict=usage_chunk_dict, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + completion_usage_updates=completion_usage_updates, + ) if usage_chunk_dict["cache_creation_input_tokens"] is not None and ( usage_chunk_dict["cache_creation_input_tokens"] > 0 or cache_creation_input_tokens is None ): From 34cd139f537434c392d4c491eabb34399d00b96e Mon Sep 17 00:00:00 2001 From: daleselaji-dev <265319989+daleselaji-dev@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:23:42 +0800 Subject: [PATCH 5/6] fix(streaming): avoid rebinding usage helper arguments --- .../streaming_chunk_builder_utils.py | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 88317817a1e..4c0b06322be 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -762,19 +762,30 @@ class ChunkProcessor: prompt_tokens_provided = "prompt_tokens" in usage_chunk completion_tokens_provided = "completion_tokens" in usage_chunk - if prompt_tokens_provided and (usage_chunk_dict["prompt_tokens"] > 0 or prompt_tokens == 0): - prompt_tokens = usage_chunk_dict["prompt_tokens"] - if completion_tokens_provided and (usage_chunk_dict["completion_tokens"] > 0 or completion_tokens == 0): - completion_tokens = usage_chunk_dict["completion_tokens"] - if completion_tokens > 0: - completion_usage_updates += 1 + updated_prompt_tokens = ( + usage_chunk_dict["prompt_tokens"] + if prompt_tokens_provided and (usage_chunk_dict["prompt_tokens"] > 0 or prompt_tokens == 0) + else prompt_tokens + ) + updated_completion_tokens = ( + usage_chunk_dict["completion_tokens"] + if completion_tokens_provided and (usage_chunk_dict["completion_tokens"] > 0 or completion_tokens == 0) + else completion_tokens + ) + updated_completion_usage_updates = ( + completion_usage_updates + 1 + if completion_tokens_provided + and (usage_chunk_dict["completion_tokens"] > 0 or completion_tokens == 0) + and updated_completion_tokens > 0 + else completion_usage_updates + ) return ( - prompt_tokens, - completion_tokens, + updated_prompt_tokens, + updated_completion_tokens, prompt_tokens_provided, completion_tokens_provided, - completion_usage_updates, + updated_completion_usage_updates, ) def _calculate_usage_per_chunk( From 0eacc03b55180ab010745efee6c401bb6a5bdca4 Mon Sep 17 00:00:00 2001 From: daleselaji-dev <265319989+daleselaji-dev@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:22:59 +0800 Subject: [PATCH 6/6] fix(streaming): annotate aggregated token counts --- litellm/litellm_core_utils/streaming_chunk_builder_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 4c0b06322be..176b007cb6e 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -136,8 +136,8 @@ class _UsageBearingChunk(TypedDict, total=False): class _UsageSummary(TypedDict): - prompt_tokens: int | None - completion_tokens: int | None + prompt_tokens: int + completion_tokens: int cache_creation_input_tokens: int | None cache_read_input_tokens: int | None completion_tokens_details: CompletionTokensDetails | None