From ca348cf742603956792666ba2acb0210584c7a9c Mon Sep 17 00:00:00 2001 From: yelnyboy Date: Thu, 22 Oct 2026 11:38:48 +0800 Subject: [PATCH 1/3] fix(streaming): preserve native finish_reason on streaming chunks Streaming chunks currently drop the provider-native finish_reason that Choices already preserves under provider_specific_fields. Mirror that behavior for streaming by stashing the raw value at the constructor and the terminal chunk emission, and carry it through to the assembled response used for logging. --- .../streaming_chunk_builder_utils.py | 20 ++++++ .../litellm_core_utils/streaming_handler.py | 12 +++- litellm/types/utils.py | 13 ++-- .../litellm_core_utils/test_core_helpers.py | 67 ++++++++++++++++++- .../test_streaming_handler.py | 34 ++++++++++ 5 files changed, 138 insertions(+), 8 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 39f675099d7..4a23245c388 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -406,6 +406,19 @@ class ChunkProcessor: None, ) + @staticmethod + def _get_native_finish_reason_from_chunks(chunks: Sequence["_BaseChunk"]) -> str | None: + for chunk in reversed(chunks): + choices = chunk.get("choices") + if not choices: + continue + provider_specific_fields = getattr(choices[0], "provider_specific_fields", None) + if provider_specific_fields and "native_finish_reason" in provider_specific_fields: + native_finish_reason = provider_specific_fields["native_finish_reason"] + if isinstance(native_finish_reason, str): + return native_finish_reason + return None + def build_base_response(self, chunks: Sequence["_BaseChunk"]) -> ModelResponse: chunk = self.first_chunk id: Final = ChunkProcessor._get_chunk_id(chunks) @@ -429,6 +442,8 @@ class ChunkProcessor: if chunk_finish_reason is not None: finish_reason = chunk_finish_reason + native_finish_reason: Final = ChunkProcessor._get_native_finish_reason_from_chunks(chunks) + # Initialize the response dictionary response = ModelResponse( **{ @@ -452,6 +467,11 @@ class ChunkProcessor: "index": 0, "message": {"role": role, "content": ""}, "finish_reason": finish_reason, + **( + {"provider_specific_fields": {"native_finish_reason": native_finish_reason}} + if native_finish_reason is not None + else {} + ), } ], "usage": { diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index f994baa52b1..2250882fecf 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1234,10 +1234,16 @@ class CustomStreamWrapper: if _is_delta_empty: model_response.choices[0].delta = Delta(content=None) # ensure empty delta chunk returned # get any function call arguments - model_response.choices[0].finish_reason = map_finish_reason( + native_finish_reason: Final = self.received_finish_reason + mapped_finish_reason: Final = map_finish_reason( finish_reason=self.received_finish_reason - ) # ensure consistent output to openai - + ) + model_response.choices[0].finish_reason = mapped_finish_reason # ensure consistent output to openai + if native_finish_reason is not None and native_finish_reason != mapped_finish_reason: + model_response.choices[0].provider_specific_fields = { + **(model_response.choices[0].provider_specific_fields or {}), + "native_finish_reason": native_finish_reason, + } self.sent_last_chunk = True return model_response diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5939a350970..acaa4af8d5b 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2082,6 +2082,8 @@ class UnreportedUsage(Usage): class StreamingChoices(OpenAIObject): + provider_specific_fields: dict[str, Any] | None = Field(default=None) + def __init__( self, finish_reason=None, @@ -2094,11 +2096,14 @@ class StreamingChoices(OpenAIObject): # Fix Perplexity return both delta and message cause OpenWebUI repect text # https://github.com/BerriAI/litellm/issues/8455 params.pop("message", None) + mapped: Final = map_finish_reason(finish_reason) if finish_reason else None + if mapped is not None and finish_reason != mapped: + params["provider_specific_fields"] = { + **(params.get("provider_specific_fields") or {}), + "native_finish_reason": finish_reason, + } super().__init__(**params) - if finish_reason: - self.finish_reason = map_finish_reason(finish_reason) - else: - self.finish_reason = None + self.finish_reason = mapped self.index = index if delta is not None: if isinstance(delta, Delta): diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index d66c37ba8c2..2f48d6c5a15 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -32,7 +32,15 @@ from litellm.litellm_core_utils.core_helpers import ( unbind_budget_reservation_from_callbacks, ) from litellm.proxy._types import UserAPIKeyAuth -from litellm.types.utils import Delta, ImageResponse, ModelResponseStream, StreamingChoices, TranscriptionResponse +from litellm.types.utils import ( + Choices, + Delta, + ImageResponse, + Message, + ModelResponseStream, + StreamingChoices, + TranscriptionResponse, +) def test_parent_otel_span_reads_dict_like_metadata_by_index(): @@ -371,6 +379,63 @@ class TestFinishReasonMapOutputsAreValid: ) +class TestStreamingChoicesNativeFinishReason: + """StreamingChoices must preserve the provider-native finish reason the same + way Choices does, so streaming consumers can read the original provider value + when map_finish_reason translates or defaults it.""" + + def test_unknown_provider_reason_is_preserved(self): + choice: Final = StreamingChoices(finish_reason="network_error") + + assert choice.finish_reason == "stop" + assert choice.provider_specific_fields == {"native_finish_reason": "network_error"} + + def test_translated_dialect_reason_is_preserved(self): + choice: Final = StreamingChoices(finish_reason="end_turn") + + assert choice.finish_reason == "stop" + assert choice.provider_specific_fields == {"native_finish_reason": "end_turn"} + + def test_openai_reason_is_not_stashed(self): + choice: Final = StreamingChoices(finish_reason="stop") + + assert choice.finish_reason == "stop" + assert choice.provider_specific_fields is None + + def test_none_finish_reason_keeps_provider_specific_fields_absent(self): + choice: Final = StreamingChoices(finish_reason=None) + + assert choice.finish_reason is None + assert choice.provider_specific_fields is None + + def test_matches_choices_behavior(self): + streaming: Final = StreamingChoices(finish_reason="network_error") + non_streaming: Final = Choices( + finish_reason="network_error", index=0, message=Message(role="assistant", content="") + ) + + assert streaming.finish_reason == non_streaming.finish_reason == "stop" + assert streaming.provider_specific_fields == non_streaming.provider_specific_fields + + def test_survives_proxy_chunk_serialization(self): + """The proxy serialises chunks with exclude_none and exclude_unset, the + preserved value has to survive that round trip to reach the client.""" + chunk: Final = ModelResponseStream( + id="chatcmpl-native-finish-reason", + model="gpt-4o", + choices=[ + StreamingChoices(index=0, delta=Delta(content=""), finish_reason="network_error") + ], + ) + + payload: Final = json.loads(chunk.model_dump_json(exclude_none=True, exclude_unset=True)) + + assert payload["choices"][0]["finish_reason"] == "stop" + assert payload["choices"][0]["provider_specific_fields"] == { + "native_finish_reason": "network_error" + } + + class TestRedactNestedMatchAndRegexKeys: def test_redacts_match_and_regex_recursively(self): payload = { diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index c13cea51183..a1cc251b4df 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -953,6 +953,40 @@ def test_logged_response_keeps_received_model_id_when_the_finish_chunk_is_synthe assert logged._hidden_params.get("received_model_id") == upstream_id +def test_terminal_chunk_preserves_native_finish_reason(logging_obj: Logging): + """A finish-only chunk whose raw finish_reason is not an OpenAI reason must surface + the provider value under provider_specific_fields.native_finish_reason while the + OpenAI-facing finish_reason stays normalized.""" + finish_choice: Final = StreamingChoices(index=0, delta=Delta()) + finish_choice.finish_reason = "network_error" + wrapper: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator( + model_responses=[ + ModelResponseStream( + id="chatcmpl-native-finish", + model="glm-5", + choices=[finish_choice], + ) + ] + ), + model="glm-5", + logging_obj=logging_obj, + custom_llm_provider="openai", + ) + + emitted: Final = list(wrapper) + + terminal: Final = [chunk for chunk in emitted if chunk.choices and chunk.choices[0].finish_reason is not None] + assert len(terminal) == 1 + assert terminal[0].choices[0].finish_reason == "stop" + assert terminal[0].choices[0].provider_specific_fields == {"native_finish_reason": "network_error"} + + logged: Final = litellm.stream_chunk_builder(chunks=wrapper.chunks) + assert logged is not None + assert logged.choices[0].finish_reason == "stop" + assert logged.choices[0].provider_specific_fields == {"native_finish_reason": "network_error"} + + @pytest.mark.asyncio async def test_streaming_completion_start_time(logging_obj: Logging): """Test that the start time is set correctly""" From 5d443e0818740fdfd6bae4d3bd78d5106b452155 Mon Sep 17 00:00:00 2001 From: yelnyboy Date: Thu, 22 Oct 2026 14:16:48 +0800 Subject: [PATCH 2/3] fix(streaming): keep native finish_reason when chunks are passed as dicts stream_chunk_builder is a public API and callers may pass model_dump()-ed chunks, the helper now reads provider_specific_fields from dict-shaped choices the same way the finish_reason loop does --- .../streaming_chunk_builder_utils.py | 5 ++- .../test_streaming_chunk_builder_utils.py | 34 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 4a23245c388..67c3c0c4f19 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -412,7 +412,10 @@ class ChunkProcessor: choices = chunk.get("choices") if not choices: continue - provider_specific_fields = getattr(choices[0], "provider_specific_fields", None) + choice = choices[0] + provider_specific_fields = getattr(choice, "provider_specific_fields", None) + if provider_specific_fields is None and isinstance(choice, Mapping): + provider_specific_fields = choice.get("provider_specific_fields") if provider_specific_fields and "native_finish_reason" in provider_specific_fields: native_finish_reason = provider_specific_fields["native_finish_reason"] if isinstance(native_finish_reason, str): diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index 5c56245cf33..1630afbff60 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -2517,3 +2517,37 @@ def test_stream_chunk_builder_keeps_reasoning_items_on_text_only_replies(): assert response is not None assert response.choices[0].message.content == "Hello world" assert response.choices[0].message.reasoning_items == reasoning_items + + +def test_stream_chunk_builder_preserves_native_finish_reason_from_dict_chunks(): + """SDK callers may pass model_dump()-ed chunks back to stream_chunk_builder, the + preserved native finish reason has to survive that round trip too.""" + chunks: Final[list[dict[str, object]]] = [ + { + "id": "chatcmpl-native-dict", + "object": "chat.completion.chunk", + "created": 1, + "model": "glm-5", + "choices": [{"index": 0, "delta": {"content": "hi"}, "finish_reason": None}], + }, + { + "id": "chatcmpl-native-dict", + "object": "chat.completion.chunk", + "created": 1, + "model": "glm-5", + "choices": [ + { + "index": 0, + "delta": {}, + "finish_reason": "stop", + "provider_specific_fields": {"native_finish_reason": "network_error"}, + } + ], + }, + ] + + response: Final = stream_chunk_builder(chunks=chunks) + + assert response is not None + assert response.choices[0].finish_reason == "stop" + assert response.choices[0].provider_specific_fields == {"native_finish_reason": "network_error"} From 662d5ba25f6a01c050a24c0c0384b3b24858dff5 Mon Sep 17 00:00:00 2001 From: yelnyboy Date: Thu, 22 Oct 2026 14:31:53 +0800 Subject: [PATCH 3/3] style(streaming): single-line map_finish_reason call to satisfy ruff format --- litellm/litellm_core_utils/streaming_handler.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 2250882fecf..84be34aeebc 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1235,9 +1235,7 @@ class CustomStreamWrapper: model_response.choices[0].delta = Delta(content=None) # ensure empty delta chunk returned # get any function call arguments native_finish_reason: Final = self.received_finish_reason - mapped_finish_reason: Final = map_finish_reason( - finish_reason=self.received_finish_reason - ) + mapped_finish_reason: Final = map_finish_reason(finish_reason=self.received_finish_reason) model_response.choices[0].finish_reason = mapped_finish_reason # ensure consistent output to openai if native_finish_reason is not None and native_finish_reason != mapped_finish_reason: model_response.choices[0].provider_specific_fields = {