From 6b6956a3d1dcec3a39c0678220bac8c96ff397a1 Mon Sep 17 00:00:00 2001 From: Ed Summers Date: Thu, 15 Oct 2026 01:17:34 -0400 Subject: [PATCH 1/2] fix(streaming): keep provider fields on the final usage chunk Provider fields sent on the usage-only chunk at the end of a stream (for example GreenPT's impact object) never reached the client. The stream wrapper rebuilt the final usage chunk without them, and the proxy dropped them in its fast serializer and when stripping the usage chunk it injected for spend tracking. Record the usage-only chunk's non-OpenAI fields and stamp them onto the final usage chunk, reusing the selection rule from preserve_upstream_non_openai_attributes. In the proxy, fall back to the canonical serializer when a chunk carries such fields, and keep an injected usage chunk that carries them, minus the usage. Fixes #46613 Co-Authored-By: Claude Opus 5.5 --- litellm/litellm_core_utils/core_helpers.py | 33 ++++++--- .../litellm_core_utils/streaming_handler.py | 14 +++- litellm/proxy/proxy_server.py | 19 ++++- .../test_streaming_handler.py | 46 +++++++++++++ .../proxy_server/test_streaming_helpers.py | 69 +++++++++++++++++++ 5 files changed, 171 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 2f81154ab83..7be02147cc9 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -679,19 +679,36 @@ def process_response_headers( return {**additional_headers, **({"request-id": request_id} if request_id is not None else {})} +def _upstream_non_openai_keys( + response_type: "type[ModelResponseStream]", original_chunk: "ModelResponseStream" +) -> tuple[str, ...]: + # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings + expected_keys: Final = frozenset(response_type.model_fields) | frozenset({"usage"}) + return tuple( + key + for key in (*type(original_chunk).model_fields, *(original_chunk.__pydantic_extra__ or {})) + if key not in expected_keys + ) + + +def _dump_keys(original_chunk: "ModelResponseStream", keys: tuple[str, ...]) -> Mapping[str, object]: + return original_chunk.model_dump(include=frozenset(keys)) + + +def upstream_non_openai_attributes( + response_type: "type[ModelResponseStream]", original_chunk: "ModelResponseStream" +) -> Mapping[str, object]: + keys: Final = _upstream_non_openai_keys(response_type, original_chunk) + return MappingProxyType(_dump_keys(original_chunk, keys) if keys else {}) + + def preserve_upstream_non_openai_attributes( model_response: "ModelResponseStream", original_chunk: "ModelResponseStream" ): """ Preserve non-OpenAI attributes from the original chunk. """ - # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings - expected_keys: Final = frozenset(type(model_response).model_fields) | frozenset({"usage"}) - candidate_keys: Final = tuple( - key - for key in (*type(original_chunk).model_fields, *(original_chunk.__pydantic_extra__ or {})) - if key not in expected_keys - ) + candidate_keys: Final = _upstream_non_openai_keys(type(model_response), original_chunk) if not candidate_keys: return candidate_values: Final[tuple[tuple[str, object], ...]] = tuple( @@ -707,7 +724,7 @@ def preserve_upstream_non_openai_attributes( if all(value is None or isinstance(value, (str, int, float, bool)) for _, value in candidate_values): _assign_attributes(model_response, candidate_values) return - _assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items()) + _assign_attributes(model_response, _dump_keys(original_chunk, candidate_keys).items()) def _assign_attributes(target: object, values: Iterable[tuple[str, object]]) -> None: diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 098a404ea14..09b9be35525 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -56,7 +56,7 @@ from litellm.types.utils import ( from litellm.types.utils import GenericStreamingChunk as GChunk from ..exceptions import OpenAIError, Timeout -from .core_helpers import map_finish_reason, process_response_headers +from .core_helpers import map_finish_reason, process_response_headers, upstream_non_openai_attributes from .exception_mapping_utils import exception_type, with_deployment_api_base from .llm_response_utils.get_api_base import get_api_base from .rules import Rules @@ -151,6 +151,12 @@ def _skips_stream_fallback(status_code: int | None) -> bool: return status_code is not None and 400 <= status_code < 500 and status_code != 429 +def _stamp_upstream_attributes(response: ModelResponseStream, attributes: Mapping[str, object]) -> None: + for key, value in attributes.items(): + if value is not None: + setattr(response, key, value) + + def is_async_iterable(obj: object) -> bool: """ Check if an object is an async iterable (can be used with 'async for'). @@ -389,6 +395,7 @@ class CustomStreamWrapper: self.stream_options = stream_options or getattr(logging_obj, "stream_options", None) self.messages = getattr(logging_obj, "messages", None) self.sent_stream_usage = False + self._usage_chunk_upstream_attributes: Mapping[str, object] = MappingProxyType({}) self.send_stream_usage = True if self.check_send_stream_usage(self.stream_options) else False self.caller_requested_stream_usage: Final = ( self.check_send_stream_usage(getattr(logging_obj, "stream_options", None)) @@ -1721,6 +1728,9 @@ class CustomStreamWrapper: model_response.choices[0].delta = plain_delta if plain_delta is not None else Delta() plain_delta_created = plain_delta is not None # rebind-ok: the first chunk builds its delta else: + self._usage_chunk_upstream_attributes = upstream_non_openai_attributes( + ModelResponseStream, original_chunk + ) if self.send_stream_usage is True: model_response.choices = [] return model_response @@ -2211,6 +2221,7 @@ class CustomStreamWrapper: if self.sent_stream_usage is False and self.send_stream_usage is True: self.sent_stream_usage = True + _stamp_upstream_attributes(response, self._usage_chunk_upstream_attributes) return response self.close() raise # Re-raise StopIteration @@ -2412,6 +2423,7 @@ class CustomStreamWrapper: if self.sent_stream_usage is False and self.send_stream_usage is True: self.sent_stream_usage = True + _stamp_upstream_attributes(response, self._usage_chunk_upstream_attributes) return response _deferred_cb: Final = getattr( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f078267c6b2..b3c59424597 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10171,6 +10171,7 @@ def _fast_serialize_simple_model_response_stream( getattr(chunk, "provider_specific_fields", None) is not None or getattr(chunk, "system_fingerprint", None) is not None or getattr(chunk, "usage", None) is not None + or _has_provider_extras(chunk) ): return None @@ -10246,6 +10247,20 @@ def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes: return chunk.model_dump_json(exclude_none=True, exclude_unset=True) +_FAST_SERIALIZED_EXTRAS: Final = frozenset({"service_tier", "usage"}) + + +def _has_provider_extras(chunk: ModelResponseStream) -> bool: + extras: Final = chunk.model_extra or {} + return any(value is not None for key, value in extras.items() if key not in _FAST_SERIALIZED_EXTRAS) + + +def _without_injected_usage(chunk: object) -> ModelResponseStream | None: + if not isinstance(chunk, ModelResponseStream) or not _has_provider_extras(chunk): + return None + return chunk.model_copy(update={"usage": None}) + + async def _apply_streaming_chunk_hooks( *, chunk: Any, @@ -10632,7 +10647,9 @@ async def async_data_generator( fallback_errors=fallback_errors, ) fallback_metadata_event_sent = True - continue + if (kept_chunk := _without_injected_usage(chunk)) is None: + continue + chunk = kept_chunk if error_state is not None: error_state.observe_chunk(chunk) diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index b9c8feb223d..c94573b3169 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -994,6 +994,52 @@ def test_finish_reason_with_holding_chunk_preserves_non_openai_attributes( assert getattr(returned_chunk, "custom_field", None) == {"key": "value"} +def _stream_ending_with_provider_usage_chunk() -> list[ModelResponseStream]: + def chunk(choices: list[StreamingChoices], usage: Usage | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-usage-fields", + created=1742056047, + model=None, + object="chat.completion.chunk", + choices=choices, + usage=usage, + ) + + usage_chunk: Final = chunk([], Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30)) + setattr(usage_chunk, "provider_metrics", {"energy_wh": 0.5}) + return [ + chunk([StreamingChoices(index=0, delta=Delta(content="Hello!", role="assistant"))]), + chunk([StreamingChoices(finish_reason="stop", index=0, delta=Delta())]), + usage_chunk, + ] + + +@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"]) +@pytest.mark.asyncio +async def test_final_usage_chunk_keeps_provider_fields_from_usage_only_chunk(sync_mode: bool): + response: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator(model_responses=_stream_ending_with_provider_usage_chunk()), + model="gpt-5.5", + custom_llm_provider="openai", + logging_obj=Logging( + model="gpt-5.5", + messages=[{"role": "user", "content": "Hey"}], + stream=True, + call_type="completion", + start_time=time.time(), + litellm_call_id="usage-fields", + function_id="usage-fields", + ), + stream_options={"include_usage": True}, + ) + + chunks: Final = list(response) if sync_mode else [chunk async for chunk in response] + + assert chunks[-1].usage is not None + assert chunks[-1].usage.total_tokens == 30 + assert getattr(chunks[-1], "provider_metrics", None) == {"energy_wh": 0.5} + + def test_set_response_id_propagation_empty_to_valid( initialized_custom_stream_wrapper: CustomStreamWrapper, ): diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index c37475bd539..5f6a268b6c0 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -553,6 +553,23 @@ def test_fast_serialize_simple_model_response_stream_with_usage_returns_none_inv assert _fast_serialize_simple_model_response_stream(chunk) is None +def test_serialize_streaming_chunk_keeps_provider_fields(): + chunk = _simple_chunk() + setattr(chunk, "provider_metrics", {"energy_wh": 0.5}) + + payload = json.loads(_serialize_streaming_chunk(chunk)) + + assert payload["provider_metrics"] == {"energy_wh": 0.5} + assert payload["choices"][0]["delta"] == {"role": "assistant", "content": "hi"} + + +def test_serialize_streaming_chunk_ignores_unset_provider_fields_for_fast_path(): + chunk = _simple_chunk() + setattr(chunk, "citations", None) + + assert isinstance(_serialize_streaming_chunk(chunk), bytes) + + # --------------------------------------------------------------------------- # _serialize_streaming_chunk # --------------------------------------------------------------------------- @@ -686,6 +703,58 @@ def _patch_logging_flags(monkeypatch, needs_wrap=False, needs_per_chunk=False): ) +def _injected_usage_chunk(**provider_fields: object) -> ModelResponseStream: + chunk: Final = ModelResponseStream( + id="chatcmpl-test", + created=0, + model="gpt-4", + object="chat.completion.chunk", + usage=Usage(prompt_tokens=3, completion_tokens=2, total_tokens=5), + ) + for key, value in provider_fields.items(): + setattr(chunk, key, value) + return chunk + + +def _sse_payload(line: str | bytes) -> dict[str, object]: + text: Final = line.decode() if isinstance(line, bytes) else line + return json.loads(text.removeprefix("data: ").removesuffix("\n\n")) + + +async def _stream_with_injected_usage(usage_chunk: ModelResponseStream) -> list[dict[str, object]]: + out: Final = [ + line + async for line in async_data_generator( + response=_async_iter([_simple_chunk(content="hello"), usage_chunk]), + user_api_key_dict=_user_auth(), + request_data={"model": "gpt-4", "_litellm_strip_stream_usage": True}, + ) + ] + assert out[-1] == "data: [DONE]\n\n" + return [_sse_payload(line) for line in out[:-1]] + + +@pytest.mark.asyncio +async def test_async_data_generator_keeps_provider_fields_from_stripped_usage_chunk(monkeypatch): + _patch_logging_flags(monkeypatch) + + payloads: Final = await _stream_with_injected_usage(_injected_usage_chunk(provider_metrics={"energy_wh": 0.5})) + + assert len(payloads) == 2 + assert payloads[-1]["provider_metrics"] == {"energy_wh": 0.5} + assert "usage" not in payloads[-1] + + +@pytest.mark.asyncio +async def test_async_data_generator_drops_stripped_usage_chunk_without_provider_fields(monkeypatch): + _patch_logging_flags(monkeypatch) + + payloads: Final = await _stream_with_injected_usage(_injected_usage_chunk(citations=None)) + + assert len(payloads) == 1 + assert "usage" not in payloads[0] + + @pytest.mark.asyncio async def test_async_data_generator_yields_sse_chunks_and_done(monkeypatch): _patch_logging_flags(monkeypatch) From e2edb28a79d49b823c6b803d1d3ef10f1a3de02b Mon Sep 17 00:00:00 2001 From: Ed Summers Date: Tue, 27 Oct 2026 09:05:33 -0400 Subject: [PATCH 2/2] fix(streaming): type upstream chunk helpers against BaseModel The upstream chunk is an OpenAI ChatCompletionChunk, not a ModelResponseStream, and main now types it as a protocol. Accept any pydantic model in the helpers and narrow at the call site so the rebase does not add a reportArgumentType error Co-Authored-By: Claude Opus 5.5 --- litellm/litellm_core_utils/core_helpers.py | 14 +++++--------- litellm/litellm_core_utils/streaming_handler.py | 7 ++++--- .../litellm_core_utils/test_streaming_handler.py | 2 +- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 7be02147cc9..cc07e9f0114 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -9,7 +9,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable import httpx -from pydantic import TypeAdapter, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import TypeIs # noqa: TID251 # narrows a duck-typed value to the Protocol its caller reads import litellm @@ -679,9 +679,7 @@ def process_response_headers( return {**additional_headers, **({"request-id": request_id} if request_id is not None else {})} -def _upstream_non_openai_keys( - response_type: "type[ModelResponseStream]", original_chunk: "ModelResponseStream" -) -> tuple[str, ...]: +def _upstream_non_openai_keys(response_type: type[BaseModel], original_chunk: BaseModel) -> tuple[str, ...]: # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings expected_keys: Final = frozenset(response_type.model_fields) | frozenset({"usage"}) return tuple( @@ -691,13 +689,11 @@ def _upstream_non_openai_keys( ) -def _dump_keys(original_chunk: "ModelResponseStream", keys: tuple[str, ...]) -> Mapping[str, object]: - return original_chunk.model_dump(include=frozenset(keys)) +def _dump_keys(original_chunk: BaseModel, keys: tuple[str, ...]) -> Mapping[str, object]: + return original_chunk.model_dump(include=MappingProxyType(dict.fromkeys(keys, True))) -def upstream_non_openai_attributes( - response_type: "type[ModelResponseStream]", original_chunk: "ModelResponseStream" -) -> Mapping[str, object]: +def upstream_non_openai_attributes(response_type: type[BaseModel], original_chunk: BaseModel) -> Mapping[str, object]: keys: Final = _upstream_non_openai_keys(response_type, original_chunk) return MappingProxyType(_dump_keys(original_chunk, keys) if keys else {}) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 09b9be35525..dabf33af034 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1728,9 +1728,10 @@ class CustomStreamWrapper: model_response.choices[0].delta = plain_delta if plain_delta is not None else Delta() plain_delta_created = plain_delta is not None # rebind-ok: the first chunk builds its delta else: - self._usage_chunk_upstream_attributes = upstream_non_openai_attributes( - ModelResponseStream, original_chunk - ) + if isinstance(original_chunk, BaseModel): + self._usage_chunk_upstream_attributes = upstream_non_openai_attributes( + ModelResponseStream, original_chunk + ) if self.send_stream_usage is True: model_response.choices = [] return model_response diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index c94573b3169..86517e52844 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -1026,7 +1026,7 @@ async def test_final_usage_chunk_keeps_provider_fields_from_usage_only_chunk(syn messages=[{"role": "user", "content": "Hey"}], stream=True, call_type="completion", - start_time=time.time(), + start_time=datetime.datetime.now(), litellm_call_id="usage-fields", function_id="usage-fields", ),