From 0dee0143e04d7d63a9a721afb74a7d1019142955 Mon Sep 17 00:00:00 2001 From: alex107ivanov <30668368+alex107ivanov@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:17:47 +0400 Subject: [PATCH] fix: preserve responses streaming flag (#30189) * fix: preserve responses streaming flag * test: cover async responses streaming flag --- .../handler.py | 4 ++ litellm/llms/custom_httpx/llm_http_handler.py | 2 + ...itellm_responses_transformation_handler.py | 55 ++++++++++++++ .../custom_httpx/test_llm_http_handler.py | 71 +++++++++++++++++++ 4 files changed, 132 insertions(+) diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 87c26b776e8..d27cfefda73 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -171,6 +171,8 @@ class ResponsesToCompletionBridgeHandler: model_response = validated_kwargs["model_response"] logging_obj = validated_kwargs["logging_obj"] custom_llm_provider = validated_kwargs["custom_llm_provider"] + if kwargs.get("stream") is True and "stream" not in optional_params: + optional_params = {**optional_params, "stream": True} request_data = self.transformation_handler.transform_request( model=model, @@ -263,6 +265,8 @@ class ResponsesToCompletionBridgeHandler: model_response = validated_kwargs["model_response"] logging_obj = validated_kwargs["logging_obj"] custom_llm_provider = validated_kwargs["custom_llm_provider"] + if kwargs.get("stream") is True and "stream" not in optional_params: + optional_params = {**optional_params, "stream": True} try: request_data = self.transformation_handler.transform_request( diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 7617a5abbbe..141019afcbb 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2305,6 +2305,7 @@ class BaseLLMHTTPHandler: if extra_body: data.update(extra_body) + stream = bool(stream or data.get("stream")) # Preserve the OpenAI-style request context (not sent to the provider) for streaming # hooks/metadata; the streaming iterator now consumes this to run deployment hooks @@ -2467,6 +2468,7 @@ class BaseLLMHTTPHandler: if extra_body: data.update(extra_body) + stream = bool(stream or data.get("stream")) # Preserve the OpenAI-style request context (not sent to the provider) for streaming # hooks/metadata; the streaming iterator now consumes this to run deployment hooks diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py index 734033ed6be..a5bc01c2b74 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_handler.py @@ -125,6 +125,33 @@ def test_completion_skips_rewrapping_preformatted_cached_chat_stream(): assert result is stream +def test_completion_preserves_top_level_stream_flag_in_responses_request(): + stream = MagicMock(spec=CustomStreamWrapper) + stream.custom_llm_provider = "cached_response" + bridge = ResponsesToCompletionBridgeHandler() + kwargs = _bridge_kwargs(stream=False) + kwargs["stream"] = True + kwargs["optional_params"].pop("stream") + + with ( + patch.object( + bridge.transformation_handler, + "transform_request", + return_value={"model": "gpt-5.4", "input": "hi"}, + ) as transform_request, + patch("litellm.responses", return_value=stream), + patch.object( + bridge, + "_apply_post_stream_processing", + side_effect=lambda s, *a, **kw: s, + ), + ): + result = bridge.completion(**kwargs) + + assert result is stream + assert transform_request.call_args.kwargs["optional_params"]["stream"] is True + + @pytest.mark.asyncio async def test_acompletion_skips_rewrapping_preformatted_cached_chat_stream(): stream = MagicMock(spec=CustomStreamWrapper) @@ -148,3 +175,31 @@ async def test_acompletion_skips_rewrapping_preformatted_cached_chat_stream(): post.assert_called_once() assert result is stream + + +@pytest.mark.asyncio +async def test_acompletion_preserves_top_level_stream_flag_in_responses_request(): + stream = MagicMock(spec=CustomStreamWrapper) + stream.custom_llm_provider = "cached_response" + bridge = ResponsesToCompletionBridgeHandler() + kwargs = _bridge_kwargs(stream=False) + kwargs["stream"] = True + kwargs["optional_params"].pop("stream") + + with ( + patch.object( + bridge.transformation_handler, + "transform_request", + return_value={"model": "gpt-5.4", "input": "hi"}, + ) as transform_request, + patch("litellm.aresponses", new=AsyncMock(return_value=stream)), + patch.object( + bridge, + "_apply_post_stream_processing", + side_effect=lambda s, *a, **kw: s, + ), + ): + result = await bridge.acompletion(**kwargs) + + assert result is stream + assert transform_request.call_args.kwargs["optional_params"]["stream"] is True diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index 7321abcee46..f7d445d0788 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -81,6 +81,77 @@ def test_prepare_fake_stream_request(): assert result_data["messages"] == [{"role": "user", "content": "Hello"}] +def test_response_api_handler_streams_when_provider_transform_adds_stream(): + handler = BaseLLMHTTPHandler() + config = Mock() + config.validate_environment.return_value = {} + config.get_complete_url.return_value = "https://chatgpt.example.com/responses" + config.transform_responses_api_request.return_value = { + "model": "gpt-5.3-codex", + "input": "hi", + "stream": True, + } + config.sign_request.return_value = ({}, None) + client = HTTPHandler(client=httpx.Client()) + client.post = Mock( + return_value=httpx.Response( + 200, + request=httpx.Request("POST", "https://chatgpt.example.com/responses"), + ) + ) + logging_obj = Mock() + + handler.response_api_handler( + model="gpt-5.3-codex", + input="hi", + responses_api_provider_config=config, + response_api_optional_request_params={}, + custom_llm_provider="chatgpt", + litellm_params=GenericLiteLLMParams(), + logging_obj=logging_obj, + client=client, + ) + + assert client.post.call_args.kwargs["stream"] is True + assert client.post.call_args.kwargs["json"]["stream"] is True + + +@pytest.mark.asyncio +async def test_async_response_api_handler_streams_when_provider_transform_adds_stream(): + handler = BaseLLMHTTPHandler() + config = Mock() + config.validate_environment.return_value = {} + config.get_complete_url.return_value = "https://chatgpt.example.com/responses" + config.transform_responses_api_request.return_value = { + "model": "gpt-5.3-codex", + "input": "hi", + "stream": True, + } + config.sign_request.return_value = ({}, None) + client = AsyncHTTPHandler() + client.post = AsyncMock( + return_value=httpx.Response( + 200, + request=httpx.Request("POST", "https://chatgpt.example.com/responses"), + ) + ) + logging_obj = Mock() + + await handler.async_response_api_handler( + model="gpt-5.3-codex", + input="hi", + responses_api_provider_config=config, + response_api_optional_request_params={}, + custom_llm_provider="chatgpt", + litellm_params=GenericLiteLLMParams(), + logging_obj=logging_obj, + client=client, + ) + + assert client.post.call_args.kwargs["stream"] is True + assert client.post.call_args.kwargs["json"]["stream"] is True + + def test_get_agentic_loop_settings_defaults_and_overrides(): handler = BaseLLMHTTPHandler()