diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 20e10f63f83..55311da2751 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -680,19 +680,32 @@ def process_response_headers( return {**additional_headers, **({"request-id": request_id} if request_id is not None else {})} +def _upstream_non_openai_keys(response_type: type[BaseModel], original_chunk: BaseModel) -> tuple[str, ...]: + # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings + expected_keys: Final = frozenset(response_type.model_fields) | frozenset({"usage"}) + return tuple( + key + for key in (*type(original_chunk).model_fields, *(original_chunk.__pydantic_extra__ or {})) + if key not in expected_keys + ) + + +def _dump_keys(original_chunk: BaseModel, keys: tuple[str, ...]) -> Mapping[str, object]: + return original_chunk.model_dump(include=MappingProxyType(dict.fromkeys(keys, True))) + + +def upstream_non_openai_attributes(response_type: type[BaseModel], original_chunk: BaseModel) -> Mapping[str, object]: + keys: Final = _upstream_non_openai_keys(response_type, original_chunk) + return MappingProxyType(_dump_keys(original_chunk, keys) if keys else {}) + + def preserve_upstream_non_openai_attributes( model_response: "ModelResponseStream", original_chunk: "ModelResponseStream" ): """ Preserve non-OpenAI attributes from the original chunk. """ - # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings - expected_keys: Final = frozenset(type(model_response).model_fields) | frozenset({"usage"}) - candidate_keys: Final = tuple( - key - for key in (*type(original_chunk).model_fields, *(original_chunk.__pydantic_extra__ or {})) - if key not in expected_keys - ) + candidate_keys: Final = _upstream_non_openai_keys(type(model_response), original_chunk) if not candidate_keys: return candidate_values: Final[tuple[tuple[str, object], ...]] = tuple( @@ -708,7 +721,7 @@ def preserve_upstream_non_openai_attributes( if all(value is None or isinstance(value, (str, int, float, bool)) for _, value in candidate_values): _assign_attributes(model_response, candidate_values) return - _assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items()) + _assign_attributes(model_response, _dump_keys(original_chunk, candidate_keys).items()) def _assign_attributes(target: BaseModel, values: Iterable[tuple[str, object]]) -> None: diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index b1b3e20f218..40446cd7ac8 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -57,7 +57,7 @@ from litellm.types.utils import ( from litellm.types.utils import GenericStreamingChunk as GChunk from ..exceptions import OpenAIError, Timeout -from .core_helpers import map_finish_reason, process_response_headers +from .core_helpers import map_finish_reason, process_response_headers, upstream_non_openai_attributes from .exception_mapping_utils import exception_type, with_deployment_api_base from .llm_response_utils.get_api_base import get_api_base from .rules import Rules @@ -152,6 +152,12 @@ def _skips_stream_fallback(status_code: int | None) -> bool: return status_code is not None and 400 <= status_code < 500 and status_code != 429 +def _stamp_upstream_attributes(response: ModelResponseStream, attributes: Mapping[str, object]) -> None: + for key, value in attributes.items(): + if value is not None: + setattr(response, key, value) + + def is_async_iterable(obj: object) -> bool: """ Check if an object is an async iterable (can be used with 'async for'). @@ -390,6 +396,7 @@ class CustomStreamWrapper: self.stream_options = stream_options or getattr(logging_obj, "stream_options", None) self.messages = getattr(logging_obj, "messages", None) self.sent_stream_usage = False + self._usage_chunk_upstream_attributes: Mapping[str, object] = MappingProxyType({}) self.send_stream_usage = True if self.check_send_stream_usage(self.stream_options) else False self.caller_requested_stream_usage: Final = ( self.check_send_stream_usage(getattr(logging_obj, "stream_options", None)) @@ -1718,6 +1725,10 @@ class CustomStreamWrapper: model_response.choices[0].delta = plain_delta if plain_delta is not None else Delta() plain_delta_created = plain_delta is not None # rebind-ok: the first chunk builds its delta else: + if isinstance(original_chunk, BaseModel): + self._usage_chunk_upstream_attributes = upstream_non_openai_attributes( + ModelResponseStream, original_chunk + ) if self.send_stream_usage is True: model_response.choices = [] return model_response @@ -2208,6 +2219,7 @@ class CustomStreamWrapper: if self.sent_stream_usage is False and self.send_stream_usage is True: self.sent_stream_usage = True + _stamp_upstream_attributes(response, self._usage_chunk_upstream_attributes) return response self.close() raise # Re-raise StopIteration @@ -2409,6 +2421,7 @@ class CustomStreamWrapper: if self.sent_stream_usage is False and self.send_stream_usage is True: self.sent_stream_usage = True + _stamp_upstream_attributes(response, self._usage_chunk_upstream_attributes) return response _deferred_cb: Final = getattr( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f078267c6b2..b3c59424597 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10171,6 +10171,7 @@ def _fast_serialize_simple_model_response_stream( getattr(chunk, "provider_specific_fields", None) is not None or getattr(chunk, "system_fingerprint", None) is not None or getattr(chunk, "usage", None) is not None + or _has_provider_extras(chunk) ): return None @@ -10246,6 +10247,20 @@ def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes: return chunk.model_dump_json(exclude_none=True, exclude_unset=True) +_FAST_SERIALIZED_EXTRAS: Final = frozenset({"service_tier", "usage"}) + + +def _has_provider_extras(chunk: ModelResponseStream) -> bool: + extras: Final = chunk.model_extra or {} + return any(value is not None for key, value in extras.items() if key not in _FAST_SERIALIZED_EXTRAS) + + +def _without_injected_usage(chunk: object) -> ModelResponseStream | None: + if not isinstance(chunk, ModelResponseStream) or not _has_provider_extras(chunk): + return None + return chunk.model_copy(update={"usage": None}) + + async def _apply_streaming_chunk_hooks( *, chunk: Any, @@ -10632,7 +10647,9 @@ async def async_data_generator( fallback_errors=fallback_errors, ) fallback_metadata_event_sent = True - continue + if (kept_chunk := _without_injected_usage(chunk)) is None: + continue + chunk = kept_chunk if error_state is not None: error_state.observe_chunk(chunk) diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 148b7193ef6..f63efdcbd54 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -994,6 +994,52 @@ def test_finish_reason_with_holding_chunk_preserves_non_openai_attributes( assert getattr(returned_chunk, "custom_field", None) == {"key": "value"} +def _stream_ending_with_provider_usage_chunk() -> list[ModelResponseStream]: + def chunk(choices: list[StreamingChoices], usage: Usage | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-usage-fields", + created=1742056047, + model=None, + object="chat.completion.chunk", + choices=choices, + usage=usage, + ) + + usage_chunk: Final = chunk([], Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30)) + setattr(usage_chunk, "provider_metrics", {"energy_wh": 0.5}) + return [ + chunk([StreamingChoices(index=0, delta=Delta(content="Hello!", role="assistant"))]), + chunk([StreamingChoices(finish_reason="stop", index=0, delta=Delta())]), + usage_chunk, + ] + + +@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"]) +@pytest.mark.asyncio +async def test_final_usage_chunk_keeps_provider_fields_from_usage_only_chunk(sync_mode: bool): + response: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator(model_responses=_stream_ending_with_provider_usage_chunk()), + model="gpt-5.5", + custom_llm_provider="openai", + logging_obj=Logging( + model="gpt-5.5", + messages=[{"role": "user", "content": "Hey"}], + stream=True, + call_type="completion", + start_time=datetime.datetime.now(), + litellm_call_id="usage-fields", + function_id="usage-fields", + ), + stream_options={"include_usage": True}, + ) + + chunks: Final = list(response) if sync_mode else [chunk async for chunk in response] + + assert chunks[-1].usage is not None + assert chunks[-1].usage.total_tokens == 30 + assert getattr(chunks[-1], "provider_metrics", None) == {"energy_wh": 0.5} + + def test_set_response_id_propagation_empty_to_valid( initialized_custom_stream_wrapper: CustomStreamWrapper, ): diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index c37475bd539..5f6a268b6c0 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -553,6 +553,23 @@ def test_fast_serialize_simple_model_response_stream_with_usage_returns_none_inv assert _fast_serialize_simple_model_response_stream(chunk) is None +def test_serialize_streaming_chunk_keeps_provider_fields(): + chunk = _simple_chunk() + setattr(chunk, "provider_metrics", {"energy_wh": 0.5}) + + payload = json.loads(_serialize_streaming_chunk(chunk)) + + assert payload["provider_metrics"] == {"energy_wh": 0.5} + assert payload["choices"][0]["delta"] == {"role": "assistant", "content": "hi"} + + +def test_serialize_streaming_chunk_ignores_unset_provider_fields_for_fast_path(): + chunk = _simple_chunk() + setattr(chunk, "citations", None) + + assert isinstance(_serialize_streaming_chunk(chunk), bytes) + + # --------------------------------------------------------------------------- # _serialize_streaming_chunk # --------------------------------------------------------------------------- @@ -686,6 +703,58 @@ def _patch_logging_flags(monkeypatch, needs_wrap=False, needs_per_chunk=False): ) +def _injected_usage_chunk(**provider_fields: object) -> ModelResponseStream: + chunk: Final = ModelResponseStream( + id="chatcmpl-test", + created=0, + model="gpt-4", + object="chat.completion.chunk", + usage=Usage(prompt_tokens=3, completion_tokens=2, total_tokens=5), + ) + for key, value in provider_fields.items(): + setattr(chunk, key, value) + return chunk + + +def _sse_payload(line: str | bytes) -> dict[str, object]: + text: Final = line.decode() if isinstance(line, bytes) else line + return json.loads(text.removeprefix("data: ").removesuffix("\n\n")) + + +async def _stream_with_injected_usage(usage_chunk: ModelResponseStream) -> list[dict[str, object]]: + out: Final = [ + line + async for line in async_data_generator( + response=_async_iter([_simple_chunk(content="hello"), usage_chunk]), + user_api_key_dict=_user_auth(), + request_data={"model": "gpt-4", "_litellm_strip_stream_usage": True}, + ) + ] + assert out[-1] == "data: [DONE]\n\n" + return [_sse_payload(line) for line in out[:-1]] + + +@pytest.mark.asyncio +async def test_async_data_generator_keeps_provider_fields_from_stripped_usage_chunk(monkeypatch): + _patch_logging_flags(monkeypatch) + + payloads: Final = await _stream_with_injected_usage(_injected_usage_chunk(provider_metrics={"energy_wh": 0.5})) + + assert len(payloads) == 2 + assert payloads[-1]["provider_metrics"] == {"energy_wh": 0.5} + assert "usage" not in payloads[-1] + + +@pytest.mark.asyncio +async def test_async_data_generator_drops_stripped_usage_chunk_without_provider_fields(monkeypatch): + _patch_logging_flags(monkeypatch) + + payloads: Final = await _stream_with_injected_usage(_injected_usage_chunk(citations=None)) + + assert len(payloads) == 1 + assert "usage" not in payloads[0] + + @pytest.mark.asyncio async def test_async_data_generator_yields_sse_chunks_and_done(monkeypatch): _patch_logging_flags(monkeypatch)