From edf75d67830b0b6578d22eb567f8d64d057356fe Mon Sep 17 00:00:00 2001 From: Tanmay Mandal Date: Sun, 10 May 2026 15:58:39 +0530 Subject: [PATCH] fix(custom_llm): allow streaming/astreaming to yield ModelResponseStream directly --- .../litellm_core_utils/streaming_handler.py | 15 +++ tests/local_testing/test_custom_llm.py | 88 +++++++++++++++++- .../test_streaming_handler.py | 92 +++++++++++++++++++ 3 files changed, 194 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index fa7faf3035d..f27c18e9b65 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1128,6 +1128,21 @@ class CustomStreamWrapper: completion_obj: Dict[str, Any] = {"content": ""} from litellm.types.utils import GenericStreamingChunk as GChunk + if isinstance(chunk, ModelResponseStream): + _has_content = bool( + chunk.choices + and chunk.choices[0].delta is not None + and chunk.choices[0].delta.content + ) + if self.received_finish_reason is not None: + if not _has_content: + raise StopIteration + if chunk.choices and chunk.choices[0].finish_reason: + self.received_finish_reason = chunk.choices[0].finish_reason + if not _has_content: + return None + return chunk + if ( isinstance(chunk, dict) and generic_chunk_has_all_required_fields( diff --git a/tests/local_testing/test_custom_llm.py b/tests/local_testing/test_custom_llm.py index 34ab6c043b9..ea15c3db9d0 100644 --- a/tests/local_testing/test_custom_llm.py +++ b/tests/local_testing/test_custom_llm.py @@ -44,7 +44,14 @@ from litellm import ( image_generation, ) from litellm.utils import ModelResponseIterator -from litellm.types.utils import ImageResponse, ImageObject, EmbeddingResponse +from litellm.types.utils import ( + ImageResponse, + ImageObject, + EmbeddingResponse, + ModelResponseStream, + StreamingChoices, + Delta, +) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @@ -644,3 +651,82 @@ async def test_simple_aembedding(): "embedding": [0.1, 0.2, 0.3], "index": 1, } + + +# ── Tests for ModelResponseStream passthrough in custom providers (issue #27389) ── + + +class ModelResponseStreamLLM(MyCustomLLM): + """Subclass that overrides streaming/astreaming to yield ModelResponseStream directly.""" + + def __init__(self, finish_reason: str = "stop"): + self._finish_reason = finish_reason + + def streaming(self, *args, **kwargs) -> Iterator[ModelResponseStream]: # type: ignore + yield ModelResponseStream( + id="test-stream-id", + choices=[ + StreamingChoices( + index=0, + delta=Delta(content="Hello world"), + finish_reason=self._finish_reason, + ) + ], + ) + + async def astreaming(self, *args, **kwargs) -> AsyncIterator[ModelResponseStream]: # type: ignore + yield ModelResponseStream( + id="test-stream-id", + choices=[ + StreamingChoices( + index=0, + delta=Delta(content="Hello world"), + finish_reason=self._finish_reason, + ) + ], + ) + + +@pytest.mark.parametrize( + "finish_reason", ["stop", "tool_calls", "length", "content_filter"] +) +def test_custom_llm_streaming_model_response_stream(finish_reason): + my_custom_llm = ModelResponseStreamLLM(finish_reason=finish_reason) + litellm.custom_provider_map = [ + {"provider": "custom_llm", "custom_handler": my_custom_llm} + ] + resp = completion( + model="custom_llm/my-fake-model", + messages=[{"role": "user", "content": "Hello world!"}], + stream=True, + ) + + for chunk in resp: + print(chunk) + if chunk.choices[0].finish_reason is None: + assert isinstance(chunk.choices[0].delta.content, str) + else: + assert chunk.choices[0].finish_reason == finish_reason + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "finish_reason", ["stop", "tool_calls", "length", "content_filter"] +) +async def test_custom_llm_astreaming_model_response_stream(finish_reason): + my_custom_llm = ModelResponseStreamLLM(finish_reason=finish_reason) + litellm.custom_provider_map = [ + {"provider": "custom_llm", "custom_handler": my_custom_llm} + ] + resp = await litellm.acompletion( + model="custom_llm/my-fake-model", + messages=[{"role": "user", "content": "Hello world!"}], + stream=True, + ) + + async for chunk in resp: + print(chunk) + if chunk.choices[0].finish_reason is None: + assert isinstance(chunk.choices[0].delta.content, str) + else: + assert chunk.choices[0].finish_reason == finish_reason diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 49d3c51e340..e7b82b42997 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -2124,3 +2124,95 @@ def test_gemini_legacy_vertex_tool_calls_finish_reason_with_stop_enum(): f"Expected 'tool_calls' but got {final.choices[0].finish_reason!r}. " "STOP enum was not normalised through map_finish_reason()." ) + + +@pytest.mark.parametrize( + "finish_reason", ["stop", "tool_calls", "length", "content_filter"] +) +def test_chunk_creator_passes_through_model_response_stream( + initialized_custom_stream_wrapper: CustomStreamWrapper, + finish_reason: str, +): + """ + chunk_creator must pass ModelResponseStream chunks from custom providers + straight through and preserve finish_reason exactly — not force-cast to GChunk. + Regression test for issue #27389. + """ + initialized_custom_stream_wrapper.custom_llm_provider = "my-custom-provider" + litellm._custom_providers.append("my-custom-provider") + + chunk = ModelResponseStream( + id="test-id", + choices=[ + StreamingChoices( + index=0, + delta=Delta(content="Hello", role="assistant"), + finish_reason=finish_reason, + ) + ], + ) + + result = initialized_custom_stream_wrapper.chunk_creator(chunk=chunk) + + litellm._custom_providers.remove("my-custom-provider") + + assert result is not None + assert initialized_custom_stream_wrapper.received_finish_reason == finish_reason + + +def test_chunk_creator_drops_empty_finish_chunk( + initialized_custom_stream_wrapper: CustomStreamWrapper, +): + """ + A ModelResponseStream chunk with finish_reason but no content should return + None so finish_reason_handler() synthesises the final chunk — mirrors GChunk + behaviour via is_chunk_non_empty. + """ + initialized_custom_stream_wrapper.custom_llm_provider = "my-custom-provider" + litellm._custom_providers.append("my-custom-provider") + + chunk = ModelResponseStream( + id="test-id", + choices=[ + StreamingChoices( + index=0, + delta=Delta(content=""), + finish_reason="stop", + ) + ], + ) + + result = initialized_custom_stream_wrapper.chunk_creator(chunk=chunk) + + litellm._custom_providers.remove("my-custom-provider") + + assert result is None + assert initialized_custom_stream_wrapper.received_finish_reason == "stop" + + +def test_chunk_creator_stops_iteration_on_trailing_chunk( + initialized_custom_stream_wrapper: CustomStreamWrapper, +): + """ + After received_finish_reason is set, any empty trailing chunk (e.g. provider + metadata flush) must raise StopIteration to end the stream cleanly. + """ + initialized_custom_stream_wrapper.custom_llm_provider = "my-custom-provider" + initialized_custom_stream_wrapper.received_finish_reason = "stop" + litellm._custom_providers.append("my-custom-provider") + + trailing_chunk = ModelResponseStream( + id="test-id", + choices=[ + StreamingChoices( + index=0, + delta=Delta(content=None), + finish_reason="stop", + ) + ], + ) + + with pytest.raises(StopIteration): + initialized_custom_stream_wrapper.chunk_creator(chunk=trailing_chunk) + + litellm._custom_providers.remove("my-custom-provider")