From 1d35bad5e2054a0aecc8afa39099d722b16462fd Mon Sep 17 00:00:00 2001 From: Arafel187 Date: Fri, 11 Sep 2026 19:01:10 +0300 Subject: [PATCH 1/2] fix(streaming): update cache tokens on explicit zero update (#40736) When an earlier usage chunk reports positive cache_creation_input_tokens and a later chunk explicitly reports cache_creation_input_tokens as 0 alongside updated prompt or cache-read tokens, the stream merger was retaining the earlier positive value due to the > 0 or ... is None guard. This caused downstream consumers deriving uncached input as prompt_tokens - cache_read_input_tokens - cache_creation_input_tokens to calculate negative token usage. This fix updates the merge conditions for cache_creation_input_tokens and cache_read_input_tokens so that an explicit zero update is respected when prompt tokens or the companion cache metric are provided, while still preserving accumulated cache metrics when later chunks omit prompt/cache details. Closes #40736 --- .../streaming_chunk_builder_utils.py | 16 +++- .../test_streaming_chunk_builder_utils.py | 75 +++++++++++++++++++ 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 90698296142..85bb40b0b24 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -874,11 +874,23 @@ class ChunkProcessor: completion_tokens = usage_chunk_dict["completion_tokens"] completion_usage_updates += 1 if usage_chunk_dict["cache_creation_input_tokens"] is not None and ( - usage_chunk_dict["cache_creation_input_tokens"] > 0 or cache_creation_input_tokens is None + usage_chunk_dict["cache_creation_input_tokens"] > 0 + or cache_creation_input_tokens is None + or (usage_chunk_dict["prompt_tokens"] is not None and usage_chunk_dict["prompt_tokens"] > 0) + or ( + usage_chunk_dict["cache_read_input_tokens"] is not None + and usage_chunk_dict["cache_read_input_tokens"] > 0 + ) ): cache_creation_input_tokens = usage_chunk_dict["cache_creation_input_tokens"] if usage_chunk_dict["cache_read_input_tokens"] is not None and ( - usage_chunk_dict["cache_read_input_tokens"] > 0 or cache_read_input_tokens is None + usage_chunk_dict["cache_read_input_tokens"] > 0 + or cache_read_input_tokens is None + or (usage_chunk_dict["prompt_tokens"] is not None and usage_chunk_dict["prompt_tokens"] > 0) + or ( + usage_chunk_dict["cache_creation_input_tokens"] is not None + and usage_chunk_dict["cache_creation_input_tokens"] > 0 + ) ): cache_read_input_tokens = usage_chunk_dict["cache_read_input_tokens"] if usage_chunk_dict["completion_tokens_details"] is not None: diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index efe4209c1c9..b62ad4150a9 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1603,3 +1603,78 @@ def test_calculate_usage_falls_back_to_prompt_counter_when_mock_stream_has_no_ad ) assert usage.prompt_tokens == 77 + + +def test_streaming_usage_merger_updates_cache_creation_to_zero_on_later_update() -> None: + """Regression test for issue #40736: + When an earlier usage chunk reports positive cache_creation_input_tokens and a later chunk + explicitly reports cache_creation_input_tokens=0 alongside positive cache_read_input_tokens + or prompt_tokens, the explicit zero update must replace the earlier positive cache-write + count so downstream derived uncached tokens does not become negative. + """ + c1 = ModelResponseStream( + model="claude-3-opus", + usage=Usage( + prompt_tokens=58354, + completion_tokens=1, + cache_read_input_tokens=0, + cache_creation_input_tokens=58352, + ), + ) + c2 = ModelResponseStream( + model="claude-3-opus", + usage=Usage( + prompt_tokens=58354, + completion_tokens=408, + cache_read_input_tokens=58352, + cache_creation_input_tokens=0, + ), + ) + + chunks = [c1, c2] + usage = ChunkProcessor(chunks=chunks).calculate_usage( + chunks=chunks, + model="claude-3-opus", + completion_output="done", + ) + + assert usage.prompt_tokens == 58354 + assert usage.completion_tokens == 408 + assert usage.cache_creation_input_tokens == 0 + assert usage.cache_read_input_tokens == 58352 + uncached_input = ( + usage.prompt_tokens + - (usage.cache_read_input_tokens or 0) + - (usage.cache_creation_input_tokens or 0) + ) + assert uncached_input == 2 + + +def test_streaming_usage_merger_preserves_cache_counts_when_subsequent_chunk_omits_them() -> None: + """Ensure that if a subsequent usage chunk omits cache token fields entirely, + the earlier accumulated cache counts are preserved. + """ + c1 = ModelResponseStream( + model="claude-3-opus", + usage=Usage( + prompt_tokens=58354, + completion_tokens=1, + cache_creation_input_tokens=58352, + ), + ) + c2 = ModelResponseStream( + model="claude-3-opus", + usage=Usage( + prompt_tokens=58354, + completion_tokens=408, + ), + ) + + chunks = [c1, c2] + usage = ChunkProcessor(chunks=chunks).calculate_usage( + chunks=chunks, + model="claude-3-opus", + completion_output="done", + ) + + assert usage.cache_creation_input_tokens == 58352 From 2f47e3fee676e2b23e1f166ac739364e3f8625ea Mon Sep 17 00:00:00 2001 From: Arafel187 Date: Fri, 11 Sep 2026 19:31:06 +0300 Subject: [PATCH 2/2] chore: trigger CI on litellm_internal_staging base