From 45d1c82eb4c7a5c0fcbfa43072810eec3fdf72d8 Mon Sep 17 00:00:00 2001 From: Ed Summers Date: Thu, 15 Oct 2026 01:17:34 -0400 Subject: [PATCH] fix(streaming): keep provider fields on the final usage chunk Provider fields sent on the usage-only chunk at the end of a stream (for example GreenPT's impact object) never reached the client. The stream wrapper rebuilt the final usage chunk without them, and the proxy dropped them in its fast serializer and when stripping the usage chunk it injected for spend tracking. Record the usage-only chunk's non-OpenAI fields and stamp them onto the final usage chunk, reusing the selection rule from preserve_upstream_non_openai_attributes. In the proxy, fall back to the canonical serializer when a chunk carries such fields, and keep an injected usage chunk that carries them, minus the usage. Fixes #46613 Co-Authored-By: Claude Opus 5.5 --- litellm/litellm_core_utils/core_helpers.py | 33 ++++++--- .../litellm_core_utils/streaming_handler.py | 14 +++- litellm/proxy/proxy_server.py | 19 ++++- .../test_streaming_handler.py | 46 +++++++++++++ .../proxy_server/test_streaming_helpers.py | 69 +++++++++++++++++++ 5 files changed, 171 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 00c6ce97e78..c1c3d29c6fa 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -634,19 +634,36 @@ def process_response_headers( return {**additional_headers, **({"request-id": request_id} if request_id is not None else {})} +def _upstream_non_openai_keys( + response_type: "type[ModelResponseStream]", original_chunk: "ModelResponseStream" +) -> tuple[str, ...]: + # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings + expected_keys: Final = frozenset(response_type.model_fields) | frozenset({"usage"}) + return tuple( + key + for key in (*type(original_chunk).model_fields, *(original_chunk.__pydantic_extra__ or {})) + if key not in expected_keys + ) + + +def _dump_keys(original_chunk: "ModelResponseStream", keys: tuple[str, ...]) -> Mapping[str, object]: + return original_chunk.model_dump(include=frozenset(keys)) + + +def upstream_non_openai_attributes( + response_type: "type[ModelResponseStream]", original_chunk: "ModelResponseStream" +) -> Mapping[str, object]: + keys: Final = _upstream_non_openai_keys(response_type, original_chunk) + return MappingProxyType(_dump_keys(original_chunk, keys) if keys else {}) + + def preserve_upstream_non_openai_attributes( model_response: "ModelResponseStream", original_chunk: "ModelResponseStream" ): """ Preserve non-OpenAI attributes from the original chunk. """ - # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings - expected_keys: Final = frozenset(type(model_response).model_fields) | frozenset({"usage"}) - candidate_keys: Final = tuple( - key - for key in (*type(original_chunk).model_fields, *(original_chunk.__pydantic_extra__ or {})) - if key not in expected_keys - ) + candidate_keys: Final = _upstream_non_openai_keys(type(model_response), original_chunk) if not candidate_keys: return candidate_values: Final[tuple[tuple[str, object], ...]] = tuple( @@ -662,7 +679,7 @@ def preserve_upstream_non_openai_attributes( if all(value is None or isinstance(value, (str, int, float, bool)) for _, value in candidate_values): _assign_attributes(model_response, candidate_values) return - _assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items()) + _assign_attributes(model_response, _dump_keys(original_chunk, candidate_keys).items()) def _assign_attributes(target: object, values: Iterable[tuple[str, object]]) -> None: diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 2df470edc59..8d67f914c7d 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -55,7 +55,7 @@ from litellm.types.utils import ( from litellm.types.utils import GenericStreamingChunk as GChunk from ..exceptions import OpenAIError, Timeout -from .core_helpers import map_finish_reason, process_response_headers +from .core_helpers import map_finish_reason, process_response_headers, upstream_non_openai_attributes from .exception_mapping_utils import exception_type, with_deployment_api_base from .llm_response_utils.get_api_base import get_api_base from .rules import Rules @@ -138,6 +138,12 @@ def _skips_stream_fallback(status_code: int | None) -> bool: return status_code is not None and 400 <= status_code < 500 and status_code != 429 +def _stamp_upstream_attributes(response: ModelResponseStream, attributes: Mapping[str, object]) -> None: + for key, value in attributes.items(): + if value is not None: + setattr(response, key, value) + + def is_async_iterable(obj: object) -> bool: """ Check if an object is an async iterable (can be used with 'async for'). @@ -376,6 +382,7 @@ class CustomStreamWrapper: self.stream_options = stream_options or getattr(logging_obj, "stream_options", None) self.messages = getattr(logging_obj, "messages", None) self.sent_stream_usage = False + self._usage_chunk_upstream_attributes: Mapping[str, object] = MappingProxyType({}) self.send_stream_usage = True if self.check_send_stream_usage(self.stream_options) else False self.caller_requested_stream_usage: Final = ( self.check_send_stream_usage(getattr(logging_obj, "stream_options", None)) @@ -1696,6 +1703,9 @@ class CustomStreamWrapper: model_response.choices[0].delta = plain_delta if plain_delta is not None else Delta() plain_delta_created = plain_delta is not None # rebind-ok: the first chunk builds its delta else: + self._usage_chunk_upstream_attributes = upstream_non_openai_attributes( + ModelResponseStream, original_chunk + ) if self.send_stream_usage is True: model_response.choices = [] return model_response @@ -2161,6 +2171,7 @@ class CustomStreamWrapper: if self.sent_stream_usage is False and self.send_stream_usage is True: self.sent_stream_usage = True + _stamp_upstream_attributes(response, self._usage_chunk_upstream_attributes) return response self.close() raise # Re-raise StopIteration @@ -2362,6 +2373,7 @@ class CustomStreamWrapper: if self.sent_stream_usage is False and self.send_stream_usage is True: self.sent_stream_usage = True + _stamp_upstream_attributes(response, self._usage_chunk_upstream_attributes) return response _deferred_cb: Final = getattr( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ca0f8892b9d..f7c7352b2f8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10068,6 +10068,7 @@ def _fast_serialize_simple_model_response_stream( getattr(chunk, "provider_specific_fields", None) is not None or getattr(chunk, "system_fingerprint", None) is not None or getattr(chunk, "usage", None) is not None + or _has_provider_extras(chunk) ): return None @@ -10143,6 +10144,20 @@ def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes: return chunk.model_dump_json(exclude_none=True, exclude_unset=True) +_FAST_SERIALIZED_EXTRAS: Final = frozenset({"service_tier", "usage"}) + + +def _has_provider_extras(chunk: ModelResponseStream) -> bool: + extras: Final = chunk.model_extra or {} + return any(value is not None for key, value in extras.items() if key not in _FAST_SERIALIZED_EXTRAS) + + +def _without_injected_usage(chunk: object) -> ModelResponseStream | None: + if not isinstance(chunk, ModelResponseStream) or not _has_provider_extras(chunk): + return None + return chunk.model_copy(update={"usage": None}) + + def _is_injected_stream_usage_artifact(chunk: object) -> bool: if not isinstance(chunk, ModelResponseStream): return False @@ -10552,7 +10567,9 @@ async def async_data_generator( fallback_errors=fallback_errors, ) fallback_metadata_event_sent = True - continue + if (kept_chunk := _without_injected_usage(chunk)) is None: + continue + chunk = kept_chunk if error_state is not None: error_state.observe_chunk(chunk) diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index cc1fa1b8803..4c043e0a6d6 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -875,6 +875,52 @@ def test_finish_reason_with_holding_chunk_preserves_non_openai_attributes( assert getattr(returned_chunk, "custom_field", None) == {"key": "value"} +def _stream_ending_with_provider_usage_chunk() -> list[ModelResponseStream]: + def chunk(choices: list[StreamingChoices], usage: Usage | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-usage-fields", + created=1742056047, + model=None, + object="chat.completion.chunk", + choices=choices, + usage=usage, + ) + + usage_chunk: Final = chunk([], Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30)) + setattr(usage_chunk, "provider_metrics", {"energy_wh": 0.5}) + return [ + chunk([StreamingChoices(index=0, delta=Delta(content="Hello!", role="assistant"))]), + chunk([StreamingChoices(finish_reason="stop", index=0, delta=Delta())]), + usage_chunk, + ] + + +@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"]) +@pytest.mark.asyncio +async def test_final_usage_chunk_keeps_provider_fields_from_usage_only_chunk(sync_mode: bool): + response: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator(model_responses=_stream_ending_with_provider_usage_chunk()), + model="gpt-5.5", + custom_llm_provider="openai", + logging_obj=Logging( + model="gpt-5.5", + messages=[{"role": "user", "content": "Hey"}], + stream=True, + call_type="completion", + start_time=time.time(), + litellm_call_id="usage-fields", + function_id="usage-fields", + ), + stream_options={"include_usage": True}, + ) + + chunks: Final = list(response) if sync_mode else [chunk async for chunk in response] + + assert chunks[-1].usage is not None + assert chunks[-1].usage.total_tokens == 30 + assert getattr(chunks[-1], "provider_metrics", None) == {"energy_wh": 0.5} + + def test_set_response_id_propagation_empty_to_valid( initialized_custom_stream_wrapper: CustomStreamWrapper, ): diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 9a3516d20d6..31d924a17ad 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -553,6 +553,23 @@ def test_fast_serialize_simple_model_response_stream_with_usage_returns_none_inv assert _fast_serialize_simple_model_response_stream(chunk) is None +def test_serialize_streaming_chunk_keeps_provider_fields(): + chunk = _simple_chunk() + setattr(chunk, "provider_metrics", {"energy_wh": 0.5}) + + payload = json.loads(_serialize_streaming_chunk(chunk)) + + assert payload["provider_metrics"] == {"energy_wh": 0.5} + assert payload["choices"][0]["delta"] == {"role": "assistant", "content": "hi"} + + +def test_serialize_streaming_chunk_ignores_unset_provider_fields_for_fast_path(): + chunk = _simple_chunk() + setattr(chunk, "citations", None) + + assert isinstance(_serialize_streaming_chunk(chunk), bytes) + + # --------------------------------------------------------------------------- # _serialize_streaming_chunk # --------------------------------------------------------------------------- @@ -686,6 +703,58 @@ def _patch_logging_flags(monkeypatch, needs_wrap=False, needs_per_chunk=False): ) +def _injected_usage_chunk(**provider_fields: object) -> ModelResponseStream: + chunk: Final = ModelResponseStream( + id="chatcmpl-test", + created=0, + model="gpt-4", + object="chat.completion.chunk", + usage=Usage(prompt_tokens=3, completion_tokens=2, total_tokens=5), + ) + for key, value in provider_fields.items(): + setattr(chunk, key, value) + return chunk + + +def _sse_payload(line: str | bytes) -> dict[str, object]: + text: Final = line.decode() if isinstance(line, bytes) else line + return json.loads(text.removeprefix("data: ").removesuffix("\n\n")) + + +async def _stream_with_injected_usage(usage_chunk: ModelResponseStream) -> list[dict[str, object]]: + out: Final = [ + line + async for line in async_data_generator( + response=_async_iter([_simple_chunk(content="hello"), usage_chunk]), + user_api_key_dict=_user_auth(), + request_data={"model": "gpt-4", "_litellm_strip_stream_usage": True}, + ) + ] + assert out[-1] == "data: [DONE]\n\n" + return [_sse_payload(line) for line in out[:-1]] + + +@pytest.mark.asyncio +async def test_async_data_generator_keeps_provider_fields_from_stripped_usage_chunk(monkeypatch): + _patch_logging_flags(monkeypatch) + + payloads: Final = await _stream_with_injected_usage(_injected_usage_chunk(provider_metrics={"energy_wh": 0.5})) + + assert len(payloads) == 2 + assert payloads[-1]["provider_metrics"] == {"energy_wh": 0.5} + assert "usage" not in payloads[-1] + + +@pytest.mark.asyncio +async def test_async_data_generator_drops_stripped_usage_chunk_without_provider_fields(monkeypatch): + _patch_logging_flags(monkeypatch) + + payloads: Final = await _stream_with_injected_usage(_injected_usage_chunk(citations=None)) + + assert len(payloads) == 1 + assert "usage" not in payloads[0] + + @pytest.mark.asyncio async def test_async_data_generator_yields_sse_chunks_and_done(monkeypatch): _patch_logging_flags(monkeypatch)