From b19363b73a5a4b6d0076ccd24461834bf42ca68a Mon Sep 17 00:00:00 2001 From: inchang-ing <197932532+inchang-ing@users.noreply.github.com> Date: Sat, 17 Oct 2026 01:30:00 +0000 Subject: [PATCH] fix(streaming_handler): coerce list delta content to text Rebased onto current main, where the running-text sites funnel through _accumulate_streamed_delta. That helper only appends string content, so a provider streaming delta.content as a list of content parts now loses the answer text silently instead of raising on it. A single shared helper, delta_content_as_str, joins the text parts of a content-part list into one string. The openai/azure chunk handler, the wrapper running text, and the end-of-stream chunk builder all read delta content through it, so the accumulated text and the aggregated response both carry the joined parts. --- .../streaming_chunk_builder_utils.py | 20 +++++++++-- .../litellm_core_utils/streaming_handler.py | 9 +++-- .../test_streaming_chunk_builder_utils.py | 36 +++++++++++++++++++ .../test_streaming_handler.py | 36 +++++++++++++++++++ 4 files changed, 97 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index d505192184e..b6ef7eab932 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -700,7 +700,7 @@ class ChunkProcessor: content = delta.get(delta_key, "") if content is None: continue # openai v1.0.0 sets content = None for chunks - content_list.append(content) + content_list.append(delta_content_as_str(content)) # Combine the "content" strings into a single string || combine the 'function' strings into a single string combined_content: Final = "".join(content_list) @@ -1381,12 +1381,28 @@ def _delta_carries_more_than_text(delta: Mapping[str, object]) -> bool: return any(delta.get(field) is not None for field in _NON_TEXT_DELTA_FIELDS) +def delta_content_as_str(content: object) -> str: + """Coerce a streamed delta content value to plain text. + + OpenAI-compatible providers are supposed to stream delta.content as a + string, but some stream a list of content parts instead. Join the text + parts into one string and map everything else to an empty string, so the + running answer text neither loses the content nor raises on a list. + """ + if isinstance(content, str): + return content + if not isinstance(content, list): + return "" + parts: Final = cast("Sequence[Mapping[str, str]]", content) # cast-ok: json content parts are dicts + return "".join(part.get("text", "") for part in parts) + + def _simple_text_part(choices: Sequence[object]) -> str | None: deltas: Final = tuple(_stream_choice_delta(choice) for choice in choices) if any(_delta_carries_more_than_text(delta) for delta in deltas): return None content: Final = deltas[0].get("content") - return content if isinstance(content, str) else "" + return delta_content_as_str(content) def stream_chunk_builder( diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 78fb82c98e7..c90b7ccdce5 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -27,6 +27,7 @@ from litellm.litellm_core_utils.model_response_utils import ( is_model_response_stream_empty, ) from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject +from litellm.litellm_core_utils.streaming_chunk_builder_utils import delta_content_as_str from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer from litellm.types.llms.base import LiteLLMBaseModel @@ -668,7 +669,11 @@ class CustomStreamWrapper: usage = None if str_line and str_line.choices and len(str_line.choices) > 0: if str_line.choices[0].delta is not None and str_line.choices[0].delta.content is not None: - text = str_line.choices[0].delta.content + delta_content: Final = cast( # cast-ok: provider delta content is str | list | None + "str | Sequence[Mapping[str, str]] | None", + str_line.choices[0].delta.content, + ) + text = delta_content_as_str(delta_content) else: # function/tool calling chunk - when content is None. in this case we just return the original chunk from openai pass if str_line.choices[0].finish_reason: @@ -2366,7 +2371,7 @@ class CustomStreamWrapper: # text and the disqualifying-content latch stay in step across all three. get: Final = getattr(delta, "get", None) content: Final = get("content") if callable(get) else None - self.response_uptil_now += content if isinstance(content, str) else "" + self.response_uptil_now += delta_content_as_str(content) if not self._emitted_disqualifying_content and self._delta_disqualifies_continuation(delta): self._emitted_disqualifying_content = True diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index f8cd0416b1c..1eafd47f706 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -2180,3 +2180,39 @@ def test_stream_chunk_builder_keeps_reasoning_items_on_text_only_replies(): assert response is not None assert response.choices[0].message.content == "Hello world" assert response.choices[0].message.reasoning_items == reasoning_items + + +def test_stream_chunk_builder_joins_content_part_list_delta(): + """A provider streaming delta.content as a content-part list must aggregate as text.""" + chunks = [ + ModelResponseStream( + id="chatcmpl-123", + created=1, + model="gpt-4.1-mini", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content=[{"type": "text", "text": "Hello "}], role="assistant"), + ) + ], + ), + ModelResponseStream( + id="chatcmpl-123", + created=2, + model="gpt-4.1-mini", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(content=[{"type": "text", "text": "world"}], role=None), + ) + ], + ), + ] + + response = stream_chunk_builder(chunks=chunks) + assert response is not None + assert response.choices[0].message.content == "Hello world" diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 93f700b1c2a..b1eb5dd8b5a 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -6689,3 +6689,39 @@ async def test_leaving_an_async_with_block_closes_the_provider_stream(): async with _wrapper_over(stream): pass assert stream.close_calls == 1 + + +def _chunk_with_list_content(text: str, chunk_id: str, created: int) -> ModelResponseStream: + return ModelResponseStream( + id=chunk_id, + object="chat.completion.chunk", + created=created, + model="gpt-5.6", + choices=[ + StreamingChoices( + index=0, + delta=Delta(role="assistant", content=[{"type": "text", "text": text}]), + finish_reason=None, + ) + ], + ) + + +def test_streaming_delta_content_list_accumulates_as_str(): + """A provider streaming delta.content as a content-part list must still accumulate as text.""" + chunks: Final = [ + _chunk_with_list_content("hello ", "chunk1", 1741037890), + _chunk_with_list_content("world", "chunk2", 1741037891), + ] + + response: Final = CustomStreamWrapper( + completion_stream=iter(chunks), + model="gpt-5.6", + logging_obj=MagicMock(), + custom_llm_provider="openai", + ) + + for _ in response: + pass + + assert response.response_uptil_now == "hello world"