From f99b6dbd8b68b0a57601a8360499175c7a173369 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Mon, 26 Oct 2026 11:33:42 +0000 Subject: [PATCH] chore: Apply implementation step Signed-off-by: Alberto Perdomo --- .../llm_response_utils/convert_dict_to_response.py | 1 + litellm/litellm_core_utils/streaming_handler.py | 7 +++++-- litellm/proxy/common_request_processing.py | 11 ++++++++++- litellm/proxy/proxy_server.py | 12 ++++++++++++ 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index f3c60679c81..d2b0b9124f9 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -790,6 +790,7 @@ def convert_to_model_response_object( logprobs=logprobs, enhancements=enhancements, provider_specific_fields=provider_specific_fields, + **{f: v for f, v in provider_specific_fields.items() if f not in _CHOICES_FIELDS}, ) if provider_sent_logprobs and logprobs is None: choice.logprobs = None diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 78fb82c98e7..973aeb13b8a 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -297,6 +297,9 @@ class CustomStreamWrapper: self.messages = getattr(logging_obj, "messages", None) self.sent_stream_usage = False self.send_stream_usage = True if self.check_send_stream_usage(self.stream_options) else False + self._continuous_usage_stats = ( + isinstance(self.stream_options, dict) and self.stream_options.get("continuous_usage_stats") is True + ) self.tool_call = False self.chunks: list = [] # keep track of the returned chunks - used for calculating the input/output tokens for stream options self._repeated_messages_count = 1 @@ -1873,7 +1876,7 @@ class CustomStreamWrapper: # ModelResponseStream declares `usage` as a field, so # hasattr(response, "usage") is always True — must check # `is not None` to avoid running this path on every chunk. - if getattr(response, "usage", None) is not None: + if getattr(response, "usage", None) is not None and not self._continuous_usage_stats: usage_to_preserve = response.usage if usage_to_preserve: response_hidden_params_for_usage = cast( # cast-ok: preserve dynamic mapping behavior @@ -2088,7 +2091,7 @@ class CustomStreamWrapper: hasattr(processed_chunk, "usage") and getattr(processed_chunk, "usage", None) is not None ) - if _has_usage: + if _has_usage and not self._continuous_usage_stats: # Store a copy ONLY when usage stripping below will mutate # the chunk. For non-usage chunks (vast majority), store # directly to avoid expensive model_copy() per chunk. diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 5f798fce6cf..b4f20df9e25 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -552,7 +552,16 @@ def stream_usage_tracking_updates( return scrub if not supports_stream_options(): return scrub - merged_stream_options: Final = {**stream_options} if isinstance(stream_options, dict) else {} + # The proxy injects include_usage for its own cost tracking, so + # caller-supplied continuous_usage_stats must not ride along and leak + # per-chunk usage to a caller who never asked for usage. A caller that + # wants cumulative usage sends include_usage too and takes the early return + # above, keeping both flags intact for the backend. + merged_stream_options: Final = ( + {k: v for k, v in stream_options.items() if k != "continuous_usage_stats"} + if isinstance(stream_options, dict) + else {} + ) return { "stream_options": {**merged_stream_options, "include_usage": True}, "_litellm_strip_stream_usage": True, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3de3dccc532..d6eabf23451 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9856,6 +9856,16 @@ def _restamp_streaming_chunk_model( return chunk, model_mismatch_logged +_OPENAI_CHUNK_CHOICE_FIELDS: Final = frozenset({"index", "delta", "finish_reason", "logprobs", "enhancements"}) + + +def _streaming_choice_carries_provider_extras(choice: object) -> bool: + extra: Final = getattr(choice, "__pydantic_extra__", None) + if not isinstance(extra, dict): + return False + return any(key not in _OPENAI_CHUNK_CHOICE_FIELDS for key in extra) + + def _fast_serialize_simple_model_response_stream( chunk: ModelResponseStream, ) -> bytes | None: @@ -9878,6 +9888,8 @@ def _fast_serialize_simple_model_response_stream( choice: Final = choices[0] if getattr(choice, "logprobs", None) is not None or getattr(choice, "enhancements", None) is not None: return None + if _streaming_choice_carries_provider_extras(choice): + return None delta: Final[object] = getattr(choice, "delta", None) if delta is None: