From 4c97a0471d22fcf9190cc3a43ba8747fa791dc1f Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 26 Oct 2026 17:12:21 +0000 Subject: [PATCH] fix(L6): Emit finish chunk before terminal usage chunk in continuous usage streams Signed-off-by: Alberto Perdomo --- .../litellm_core_utils/streaming_handler.py | 4 +-- .../test_streaming_handler.py | 28 +++++++++++++++++-- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 11e6d255e9f..9deac7923b6 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1554,9 +1554,7 @@ class CustomStreamWrapper: except Exception: model_response.choices[0].delta = Delta() else: - if self.send_stream_usage is True: - if self._continuous_usage_stats: - self.sent_stream_usage = True + if self.send_stream_usage is True and not self._continuous_usage_stats: model_response.choices = [] return model_response self._record_usage_only_chunk(model_response=model_response) diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index a0eafbd33a2..db0f879a08d 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -6749,12 +6749,34 @@ async def test_continuous_usage_stats_keeps_usage_on_every_chunk(sync_mode: bool emitted: Final = [chunk async for chunk in wrapper] if not sync_mode else list(wrapper) - usage_chunks: Final = [chunk for chunk in emitted if getattr(chunk, "usage", None) is not None] - assert [chunk.usage.total_tokens for chunk in usage_chunks] == [11, 12, 13] - assert emitted[-1].choices == [] + assert all(getattr(chunk, "usage", None) is not None for chunk in emitted) + assert [chunk.usage.total_tokens for chunk in emitted] == [11, 12, 13] assert litellm.stream_chunk_builder(chunks=emitted).usage.total_tokens == 13 +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_continuous_usage_stats_finishes_before_the_final_usage_chunk(sync_mode: bool): + content_usage: Final = {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11} + final_usage: Final = {"prompt_tokens": 10, "completion_tokens": 3, "total_tokens": 13} + chunks: Final = [ + _vllm_chunk({"index": 0, "delta": {"content": "Hi"}, "finish_reason": None}, usage=content_usage), + _vllm_chunk({"index": 0, "delta": {"content": "!"}, "finish_reason": "length"}, usage=final_usage), + ModelResponseStream(id="c", created=1, model="hosted_vllm/m", choices=[], usage=Usage(**final_usage)), + ] + wrapper: Final = _vllm_wrapper(chunks, stream_options={"include_usage": True, "continuous_usage_stats": True}) + + emitted: Final = [chunk async for chunk in wrapper] if not sync_mode else list(wrapper) + + finish_indexes: Final = [i for i, chunk in enumerate(emitted) if chunk.choices and chunk.choices[0].finish_reason] + assert len(finish_indexes) == 1 + assert emitted[finish_indexes[0]].choices[0].finish_reason == "length" + assert finish_indexes[0] < len(emitted) - 1 + terminal_chunk: Final = emitted[-1] + assert not (terminal_chunk.choices and terminal_chunk.choices[0].finish_reason) + assert terminal_chunk.usage.total_tokens == final_usage["total_tokens"] + + @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_include_usage_without_continuous_usage_stats_still_emits_only_a_final_usage_chunk(sync_mode: bool):