fix(streaming): keep provider fields on the final usage chunk

Provider fields sent on the usage-only chunk at the end of a stream
(for example GreenPT's impact object) never reached the client. The
stream wrapper rebuilt the final usage chunk without them, and the proxy
dropped them in its fast serializer and when stripping the usage chunk
it injected for spend tracking.

Record the usage-only chunk's non-OpenAI fields and stamp them onto the
final usage chunk, reusing the selection rule from
preserve_upstream_non_openai_attributes. In the proxy, fall back to the
canonical serializer when a chunk carries such fields, and keep an
injected usage chunk that carries them, minus the usage.

Fixes #46613

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ed Summers 2026-10-15 01:17:34 -04:00
parent cfe0100e1f
commit 45d1c82eb4
No known key found for this signature in database
GPG key ID: D012FF557AFFF80A
5 changed files with 171 additions and 10 deletions

View file

@ -634,19 +634,36 @@ def process_response_headers(
return {**additional_headers, **({"request-id": request_id} if request_id is not None else {})}
def _upstream_non_openai_keys(
response_type: "type[ModelResponseStream]", original_chunk: "ModelResponseStream"
) -> tuple[str, ...]:
# Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
expected_keys: Final = frozenset(response_type.model_fields) | frozenset({"usage"})
return tuple(
key
for key in (*type(original_chunk).model_fields, *(original_chunk.__pydantic_extra__ or {}))
if key not in expected_keys
)
def _dump_keys(original_chunk: "ModelResponseStream", keys: tuple[str, ...]) -> Mapping[str, object]:
return original_chunk.model_dump(include=frozenset(keys))
def upstream_non_openai_attributes(
response_type: "type[ModelResponseStream]", original_chunk: "ModelResponseStream"
) -> Mapping[str, object]:
keys: Final = _upstream_non_openai_keys(response_type, original_chunk)
return MappingProxyType(_dump_keys(original_chunk, keys) if keys else {})
def preserve_upstream_non_openai_attributes(
model_response: "ModelResponseStream", original_chunk: "ModelResponseStream"
):
"""
Preserve non-OpenAI attributes from the original chunk.
"""
# Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
expected_keys: Final = frozenset(type(model_response).model_fields) | frozenset({"usage"})
candidate_keys: Final = tuple(
key
for key in (*type(original_chunk).model_fields, *(original_chunk.__pydantic_extra__ or {}))
if key not in expected_keys
)
candidate_keys: Final = _upstream_non_openai_keys(type(model_response), original_chunk)
if not candidate_keys:
return
candidate_values: Final[tuple[tuple[str, object], ...]] = tuple(
@ -662,7 +679,7 @@ def preserve_upstream_non_openai_attributes(
if all(value is None or isinstance(value, (str, int, float, bool)) for _, value in candidate_values):
_assign_attributes(model_response, candidate_values)
return
_assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items())
_assign_attributes(model_response, _dump_keys(original_chunk, candidate_keys).items())
def _assign_attributes(target: object, values: Iterable[tuple[str, object]]) -> None:

View file

@ -55,7 +55,7 @@ from litellm.types.utils import (
from litellm.types.utils import GenericStreamingChunk as GChunk
from ..exceptions import OpenAIError, Timeout
from .core_helpers import map_finish_reason, process_response_headers
from .core_helpers import map_finish_reason, process_response_headers, upstream_non_openai_attributes
from .exception_mapping_utils import exception_type, with_deployment_api_base
from .llm_response_utils.get_api_base import get_api_base
from .rules import Rules
@ -138,6 +138,12 @@ def _skips_stream_fallback(status_code: int | None) -> bool:
return status_code is not None and 400 <= status_code < 500 and status_code != 429
def _stamp_upstream_attributes(response: ModelResponseStream, attributes: Mapping[str, object]) -> None:
for key, value in attributes.items():
if value is not None:
setattr(response, key, value)
def is_async_iterable(obj: object) -> bool:
"""
Check if an object is an async iterable (can be used with 'async for').
@ -376,6 +382,7 @@ class CustomStreamWrapper:
self.stream_options = stream_options or getattr(logging_obj, "stream_options", None)
self.messages = getattr(logging_obj, "messages", None)
self.sent_stream_usage = False
self._usage_chunk_upstream_attributes: Mapping[str, object] = MappingProxyType({})
self.send_stream_usage = True if self.check_send_stream_usage(self.stream_options) else False
self.caller_requested_stream_usage: Final = (
self.check_send_stream_usage(getattr(logging_obj, "stream_options", None))
@ -1696,6 +1703,9 @@ class CustomStreamWrapper:
model_response.choices[0].delta = plain_delta if plain_delta is not None else Delta()
plain_delta_created = plain_delta is not None # rebind-ok: the first chunk builds its delta
else:
self._usage_chunk_upstream_attributes = upstream_non_openai_attributes(
ModelResponseStream, original_chunk
)
if self.send_stream_usage is True:
model_response.choices = []
return model_response
@ -2161,6 +2171,7 @@ class CustomStreamWrapper:
if self.sent_stream_usage is False and self.send_stream_usage is True:
self.sent_stream_usage = True
_stamp_upstream_attributes(response, self._usage_chunk_upstream_attributes)
return response
self.close()
raise # Re-raise StopIteration
@ -2362,6 +2373,7 @@ class CustomStreamWrapper:
if self.sent_stream_usage is False and self.send_stream_usage is True:
self.sent_stream_usage = True
_stamp_upstream_attributes(response, self._usage_chunk_upstream_attributes)
return response
_deferred_cb: Final = getattr(

View file

@ -10068,6 +10068,7 @@ def _fast_serialize_simple_model_response_stream(
getattr(chunk, "provider_specific_fields", None) is not None
or getattr(chunk, "system_fingerprint", None) is not None
or getattr(chunk, "usage", None) is not None
or _has_provider_extras(chunk)
):
return None
@ -10143,6 +10144,20 @@ def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes:
return chunk.model_dump_json(exclude_none=True, exclude_unset=True)
_FAST_SERIALIZED_EXTRAS: Final = frozenset({"service_tier", "usage"})
def _has_provider_extras(chunk: ModelResponseStream) -> bool:
extras: Final = chunk.model_extra or {}
return any(value is not None for key, value in extras.items() if key not in _FAST_SERIALIZED_EXTRAS)
def _without_injected_usage(chunk: object) -> ModelResponseStream | None:
if not isinstance(chunk, ModelResponseStream) or not _has_provider_extras(chunk):
return None
return chunk.model_copy(update={"usage": None})
def _is_injected_stream_usage_artifact(chunk: object) -> bool:
if not isinstance(chunk, ModelResponseStream):
return False
@ -10552,7 +10567,9 @@ async def async_data_generator(
fallback_errors=fallback_errors,
)
fallback_metadata_event_sent = True
continue
if (kept_chunk := _without_injected_usage(chunk)) is None:
continue
chunk = kept_chunk
if error_state is not None:
error_state.observe_chunk(chunk)

View file

@ -875,6 +875,52 @@ def test_finish_reason_with_holding_chunk_preserves_non_openai_attributes(
assert getattr(returned_chunk, "custom_field", None) == {"key": "value"}
def _stream_ending_with_provider_usage_chunk() -> list[ModelResponseStream]:
def chunk(choices: list[StreamingChoices], usage: Usage | None = None) -> ModelResponseStream:
return ModelResponseStream(
id="chatcmpl-usage-fields",
created=1742056047,
model=None,
object="chat.completion.chunk",
choices=choices,
usage=usage,
)
usage_chunk: Final = chunk([], Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30))
setattr(usage_chunk, "provider_metrics", {"energy_wh": 0.5})
return [
chunk([StreamingChoices(index=0, delta=Delta(content="Hello!", role="assistant"))]),
chunk([StreamingChoices(finish_reason="stop", index=0, delta=Delta())]),
usage_chunk,
]
@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"])
@pytest.mark.asyncio
async def test_final_usage_chunk_keeps_provider_fields_from_usage_only_chunk(sync_mode: bool):
response: Final = CustomStreamWrapper(
completion_stream=ModelResponseListIterator(model_responses=_stream_ending_with_provider_usage_chunk()),
model="gpt-5.5",
custom_llm_provider="openai",
logging_obj=Logging(
model="gpt-5.5",
messages=[{"role": "user", "content": "Hey"}],
stream=True,
call_type="completion",
start_time=time.time(),
litellm_call_id="usage-fields",
function_id="usage-fields",
),
stream_options={"include_usage": True},
)
chunks: Final = list(response) if sync_mode else [chunk async for chunk in response]
assert chunks[-1].usage is not None
assert chunks[-1].usage.total_tokens == 30
assert getattr(chunks[-1], "provider_metrics", None) == {"energy_wh": 0.5}
def test_set_response_id_propagation_empty_to_valid(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):

View file

@ -553,6 +553,23 @@ def test_fast_serialize_simple_model_response_stream_with_usage_returns_none_inv
assert _fast_serialize_simple_model_response_stream(chunk) is None
def test_serialize_streaming_chunk_keeps_provider_fields():
chunk = _simple_chunk()
setattr(chunk, "provider_metrics", {"energy_wh": 0.5})
payload = json.loads(_serialize_streaming_chunk(chunk))
assert payload["provider_metrics"] == {"energy_wh": 0.5}
assert payload["choices"][0]["delta"] == {"role": "assistant", "content": "hi"}
def test_serialize_streaming_chunk_ignores_unset_provider_fields_for_fast_path():
chunk = _simple_chunk()
setattr(chunk, "citations", None)
assert isinstance(_serialize_streaming_chunk(chunk), bytes)
# ---------------------------------------------------------------------------
# _serialize_streaming_chunk
# ---------------------------------------------------------------------------
@ -686,6 +703,58 @@ def _patch_logging_flags(monkeypatch, needs_wrap=False, needs_per_chunk=False):
)
def _injected_usage_chunk(**provider_fields: object) -> ModelResponseStream:
chunk: Final = ModelResponseStream(
id="chatcmpl-test",
created=0,
model="gpt-4",
object="chat.completion.chunk",
usage=Usage(prompt_tokens=3, completion_tokens=2, total_tokens=5),
)
for key, value in provider_fields.items():
setattr(chunk, key, value)
return chunk
def _sse_payload(line: str | bytes) -> dict[str, object]:
text: Final = line.decode() if isinstance(line, bytes) else line
return json.loads(text.removeprefix("data: ").removesuffix("\n\n"))
async def _stream_with_injected_usage(usage_chunk: ModelResponseStream) -> list[dict[str, object]]:
out: Final = [
line
async for line in async_data_generator(
response=_async_iter([_simple_chunk(content="hello"), usage_chunk]),
user_api_key_dict=_user_auth(),
request_data={"model": "gpt-4", "_litellm_strip_stream_usage": True},
)
]
assert out[-1] == "data: [DONE]\n\n"
return [_sse_payload(line) for line in out[:-1]]
@pytest.mark.asyncio
async def test_async_data_generator_keeps_provider_fields_from_stripped_usage_chunk(monkeypatch):
_patch_logging_flags(monkeypatch)
payloads: Final = await _stream_with_injected_usage(_injected_usage_chunk(provider_metrics={"energy_wh": 0.5}))
assert len(payloads) == 2
assert payloads[-1]["provider_metrics"] == {"energy_wh": 0.5}
assert "usage" not in payloads[-1]
@pytest.mark.asyncio
async def test_async_data_generator_drops_stripped_usage_chunk_without_provider_fields(monkeypatch):
_patch_logging_flags(monkeypatch)
payloads: Final = await _stream_with_injected_usage(_injected_usage_chunk(citations=None))
assert len(payloads) == 1
assert "usage" not in payloads[0]
@pytest.mark.asyncio
async def test_async_data_generator_yields_sse_chunks_and_done(monkeypatch):
_patch_logging_flags(monkeypatch)