From 22bda311339b7761307ae7e65a49025a7463c7e4 Mon Sep 17 00:00:00 2001 From: Remi Mabon Date: Wed, 19 Aug 2026 08:28:27 +0000 Subject: [PATCH] fix(responses): derive streaming capability from base_model for custom deployment names The Responses API decides native vs fake streaming by looking up the routed model name in the model cost map. A custom deployment name (for example an Azure PTU deployment) misses the map, supports_native_streaming() turns the miss into False, and litellm silently strips stream=true from the provider request: the client gets SSE events synthesized only after the complete provider response, losing all first-token latency Deployments with custom names already declare their underlying model via model_info.base_model, which litellm trusts for cost tracking. This change consults the same base_model as the capability fallback when the routed model name is unknown: - supports_native_streaming() accepts an optional base_model and falls back to it only when the routed model is not in the model cost map; a model found in the map still answers from its own entry - ResponsesAPIConfig.should_fake_stream() gains the base_model parameter (base, openai, volcengine, manus configs; azure inherits openai) - responses() passes the router-injected model_info.base_model through Behavior for deployments without base_model is unchanged --- .../llms/base_llm/responses/transformation.py | 3 +- .../llms/manus/responses/transformation.py | 1 + .../llms/openai/responses/transformation.py | 2 + .../volcengine/responses/transformation.py | 1 + litellm/responses/main.py | 8 ++- litellm/utils.py | 48 +++++++------ .../test_openai_responses_transformation.py | 35 +++++++++ .../test_responses_fake_stream_base_model.py | 28 ++++++++ tests/test_litellm/test_utils.py | 71 +++++++++++++++++++ 9 files changed, 173 insertions(+), 24 deletions(-) create mode 100644 tests/test_litellm/responses/test_responses_fake_stream_base_model.py diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index 1365941fe2a..48dbd48226e 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -237,8 +237,9 @@ class BaseResponsesAPIConfig(ABC): model: str | None, stream: bool | None, custom_llm_provider: str | None = None, + base_model: str | None = None, ) -> bool: - """Returns True if litellm should fake a stream for the given model and stream value""" + """Returns True if litellm should fake a stream for the given model, stream value and base_model fallback""" return False def supports_native_websocket(self) -> bool: diff --git a/litellm/llms/manus/responses/transformation.py b/litellm/llms/manus/responses/transformation.py index f5733a152ba..8dce51646b1 100644 --- a/litellm/llms/manus/responses/transformation.py +++ b/litellm/llms/manus/responses/transformation.py @@ -52,6 +52,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): model: str | None, stream: bool | None, custom_llm_provider: str | None = None, + base_model: str | None = None, ) -> bool: """ Manus API doesn't support real-time streaming. diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index b2a69564908..9fd8dde838d 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -427,6 +427,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): model: str | None, stream: bool | None, custom_llm_provider: str | None = None, + base_model: str | None = None, ) -> bool: if stream is not True: return False @@ -436,6 +437,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): litellm.utils.supports_native_streaming( model=model, custom_llm_provider=custom_llm_provider, + base_model=base_model, ) is False ): diff --git a/litellm/llms/volcengine/responses/transformation.py b/litellm/llms/volcengine/responses/transformation.py index 82c4f7f61a7..68793c778e9 100644 --- a/litellm/llms/volcengine/responses/transformation.py +++ b/litellm/llms/volcengine/responses/transformation.py @@ -407,6 +407,7 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): model: str | None, stream: bool | None, custom_llm_provider: str | None = None, + base_model: str | None = None, ) -> bool: """ Volcengine Responses API supports native streaming; never fall back to fake stream. diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 34058e8eca7..83dd29201df 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -1111,6 +1111,9 @@ def responses( if custom_llm_provider is None: raise ValueError("custom_llm_provider is required but passed as None") + deployment_model_info: Final = kwargs.get("model_info") + base_model: Final = deployment_model_info.get("base_model") if isinstance(deployment_model_info, dict) else None + response = base_llm_http_handler.response_api_handler( model=model, input=input, @@ -1125,7 +1128,10 @@ def responses( _is_async=_is_async, client=kwargs.get("client"), fake_stream=responses_api_provider_config.should_fake_stream( - model=model, stream=stream, custom_llm_provider=custom_llm_provider + model=model, + stream=stream, + custom_llm_provider=custom_llm_provider, + base_model=base_model, ), litellm_metadata=kwargs.get("litellm_metadata", {}), shared_session=kwargs.get("shared_session"), diff --git a/litellm/utils.py b/litellm/utils.py index b8a9ea37e05..7db8925a8b1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2301,38 +2301,42 @@ def supports_url_context(model: str, custom_llm_provider: str | None = None) -> ) -def supports_native_streaming(model: str, custom_llm_provider: str | None) -> bool: +def supports_native_streaming(model: str, custom_llm_provider: str | None, base_model: str | None = None) -> bool: """ Check if the given model supports native streaming and return a boolean value. Parameters: model (str): The model name to be checked. custom_llm_provider (str): The provider to be checked. + base_model (str): Optional fallback looked up when ``model`` is not in the + model cost map, e.g. a custom Azure deployment name whose underlying + model is identified by the deployment's ``model_info.base_model``. Returns: bool: True if the model supports native streaming, False otherwise. - - Raises: - Exception: If the given model is not found in model_prices_and_context_window.json. + A model found in the model cost map answers from its own entry; ``base_model`` + is only consulted when ``model`` itself is unknown. """ - try: - model, custom_llm_provider, _, _ = litellm.get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider - ) - - model_info: Final = _get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) - supports_native_streaming = model_info.get("supports_native_streaming", True) - if supports_native_streaming is None: - supports_native_streaming = True - return supports_native_streaming - except Exception as e: - verbose_logger.debug( - "Model not found or error in checking supports_native_streaming support. You passed model=%s, custom_llm_provider=%s. Error: %s", - model, - custom_llm_provider, - e, - ) - return False + for candidate in (model, base_model): + if candidate is None: + continue + try: + resolved_model, resolved_provider, _, _ = litellm.get_llm_provider( + model=candidate, custom_llm_provider=custom_llm_provider + ) + supports = _get_model_info_helper(model=resolved_model, custom_llm_provider=resolved_provider).get( + "supports_native_streaming", True + ) + except Exception as e: + verbose_logger.debug( + "Model not found or error in checking supports_native_streaming support. You passed model=%s, custom_llm_provider=%s. Error: %s", + candidate, + custom_llm_provider, + e, + ) + continue + return supports is not False + return False def supports_response_schema(model: str, custom_llm_provider: str | None = None) -> bool: diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index 13b96dc9943..efb135ab303 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -1598,3 +1598,38 @@ class TestPromptCacheOptionsOnResponsesPath: "text": "hi", "prompt_cache_breakpoint": {"mode": "explicit"}, } +class TestShouldFakeStreamBaseModelFallback: + def setup_method(self): + self.config = OpenAIResponsesAPIConfig() + + def test_unknown_model_with_streaming_base_model_streams_natively(self): + assert ( + self.config.should_fake_stream( + model="azure/my-custom-ptu-deployment", + stream=True, + custom_llm_provider="azure", + base_model="gpt-4o", + ) + is False + ) + + def test_unknown_model_without_base_model_fake_streams(self): + assert ( + self.config.should_fake_stream( + model="azure/my-custom-ptu-deployment", + stream=True, + custom_llm_provider="azure", + ) + is True + ) + + def test_stream_not_requested_never_fake_streams(self): + assert ( + self.config.should_fake_stream( + model="azure/my-custom-ptu-deployment", + stream=None, + custom_llm_provider="azure", + base_model="gpt-4o", + ) + is False + ) diff --git a/tests/test_litellm/responses/test_responses_fake_stream_base_model.py b/tests/test_litellm/responses/test_responses_fake_stream_base_model.py new file mode 100644 index 00000000000..105338a2bd7 --- /dev/null +++ b/tests/test_litellm/responses/test_responses_fake_stream_base_model.py @@ -0,0 +1,28 @@ +from unittest.mock import MagicMock, patch + +import litellm + + +def _fake_stream_passed_to_handler(**extra_kwargs): + with patch( + "litellm.responses.main.base_llm_http_handler.response_api_handler", + return_value=MagicMock(), + ) as mock_handler: + litellm.responses( + model="azure/my-custom-ptu-deployment", + input="hello", + stream=True, + api_key="fake-api-key", + api_base="https://fake.openai.azure.com", + api_version="2025-01-01-preview", + **extra_kwargs, + ) + return mock_handler.call_args.kwargs["fake_stream"] + + +def test_responses_streams_natively_when_model_info_base_model_streams(): + assert _fake_stream_passed_to_handler(model_info={"base_model": "azure/gpt-4o"}) is False + + +def test_responses_fake_streams_without_base_model(): + assert _fake_stream_passed_to_handler() is True diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 075b455e4b5..745b3880d73 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -4989,3 +4989,74 @@ def test_completion_does_not_leak_rust_flag_into_provider_request_body(): create_kwargs = mock_client.chat.completions.with_raw_response.create.call_args.kwargs assert "rust" not in create_kwargs assert "rust" not in (create_kwargs.get("extra_body") or {}) + + +class TestSupportsNativeStreamingBaseModelFallback: + def test_unknown_model_falls_back_to_streaming_base_model(self): + assert ( + litellm.utils.supports_native_streaming( + model="azure/my-custom-ptu-deployment", + custom_llm_provider="azure", + base_model="gpt-4o", + ) + is True + ) + + def test_unknown_model_falls_back_to_provider_qualified_base_model(self): + assert ( + litellm.utils.supports_native_streaming( + model="azure/my-custom-ptu-deployment", + custom_llm_provider="azure", + base_model="azure/gpt-4o", + ) + is True + ) + + def test_unknown_model_with_cross_provider_base_model_returns_false(self): + assert ( + litellm.utils.supports_native_streaming( + model="azure/my-custom-ptu-deployment", + custom_llm_provider="azure", + base_model="gemini/gemini-2.5-flash", + ) + is False + ) + + def test_unknown_model_falls_back_to_non_streaming_base_model(self): + assert ( + litellm.utils.supports_native_streaming( + model="my-custom-o1-pro-deployment", + custom_llm_provider="openai", + base_model="o1-pro", + ) + is False + ) + + def test_known_model_capability_wins_over_base_model(self): + assert ( + litellm.utils.supports_native_streaming( + model="o1-pro", + custom_llm_provider="openai", + base_model="gpt-4o", + ) + is False + ) + + def test_unknown_model_and_unknown_base_model_returns_false(self): + assert ( + litellm.utils.supports_native_streaming( + model="azure/my-custom-ptu-deployment", + custom_llm_provider="azure", + base_model="my-equally-unknown-base-model", + ) + is False + ) + + def test_unknown_model_without_base_model_returns_false(self): + assert ( + litellm.utils.supports_native_streaming( + model="azure/my-custom-ptu-deployment", + custom_llm_provider="azure", + ) + is False + )