From 9fb6a008299bf8c4b54542f1249c592a8f57fc0b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:20:56 -0700 Subject: [PATCH] fix(bigmodel): keep context-window and content-policy errors on streamed Responses replays --- litellm/llms/custom_httpx/llm_http_handler.py | 62 +++++++++++-------- .../llms/bigmodel/test_bigmodel_provider.py | 24 +++++++ 2 files changed, 60 insertions(+), 26 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 7ef54fd7ba6..4e1813e2aee 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -3037,6 +3037,17 @@ class BaseLLMHTTPHandler: }, ) + replay_as_stream: Final = partial( + MockResponsesAPIStreamingIterator, + model=model, + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + litellm_metadata=litellm_metadata, + custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, + ) + try: if is_stream_request: response = sync_httpx_client.post( @@ -3047,12 +3058,14 @@ class BaseLLMHTTPHandler: **body_kwargs, ) if caller_requested_stream: - if fake_stream is True or ( + if fake_stream is True: + return replay_as_stream(response=response) + if not ( responses_api_provider_config.may_answer_stream_with_json() and _is_sized_json_body(response) and _is_json_document(response.read()) ): - return MockResponsesAPIStreamingIterator( + return SyncResponsesAPIStreamingIterator( response=response, model=model, logging_obj=logging_obj, @@ -3062,17 +3075,6 @@ class BaseLLMHTTPHandler: request_data=request_context, call_type=CallTypes.responses.value, ) - - return SyncResponsesAPIStreamingIterator( - response=response, - model=model, - logging_obj=logging_obj, - responses_api_provider_config=responses_api_provider_config, - litellm_metadata=litellm_metadata, - custom_llm_provider=custom_llm_provider, - request_data=request_context, - call_type=CallTypes.responses.value, - ) response.read() else: response = sync_httpx_client.post( @@ -3087,6 +3089,9 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) + if is_stream_request and caller_requested_stream: + return replay_as_stream(response=response) + initial_response: Final = responses_api_provider_config.transform_response_api_response( model=model, raw_response=response, @@ -3242,6 +3247,17 @@ class BaseLLMHTTPHandler: }, ) + replay_as_stream: Final = partial( + MockResponsesAPIStreamingIterator, + model=model, + logging_obj=logging_obj, + responses_api_provider_config=responses_api_provider_config, + litellm_metadata=litellm_metadata, + custom_llm_provider=custom_llm_provider, + request_data=request_context, + call_type=CallTypes.responses.value, + ) + try: if is_stream_request: response = await async_httpx_client.post( @@ -3254,12 +3270,14 @@ class BaseLLMHTTPHandler: ) if caller_requested_stream: - if fake_stream is True or ( + if fake_stream is True: + return replay_as_stream(response=response) + if not ( responses_api_provider_config.may_answer_stream_with_json() and _is_sized_json_body(response) and _is_json_document(await response.aread()) ): - return MockResponsesAPIStreamingIterator( + return ResponsesAPIStreamingIterator( response=response, model=model, logging_obj=logging_obj, @@ -3269,17 +3287,6 @@ class BaseLLMHTTPHandler: request_data=request_context, call_type=CallTypes.responses.value, ) - - return ResponsesAPIStreamingIterator( - response=response, - model=model, - logging_obj=logging_obj, - responses_api_provider_config=responses_api_provider_config, - litellm_metadata=litellm_metadata, - custom_llm_provider=custom_llm_provider, - request_data=request_context, - call_type=CallTypes.responses.value, - ) await response.aread() else: response = await async_httpx_client.post( @@ -3296,6 +3303,9 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) + if is_stream_request and caller_requested_stream: + return replay_as_stream(response=response) + initial_response: Final = responses_api_provider_config.transform_response_api_response( model=model, raw_response=response, diff --git a/tests/unit/llms/bigmodel/test_bigmodel_provider.py b/tests/unit/llms/bigmodel/test_bigmodel_provider.py index 60754cebe9f..cdf98c8f9a4 100644 --- a/tests/unit/llms/bigmodel/test_bigmodel_provider.py +++ b/tests/unit/llms/bigmodel/test_bigmodel_provider.py @@ -573,6 +573,30 @@ def test_prompt_too_long_and_unsafe_content_raise_the_errors_fallbacks_catch( assert raised.value.status_code == 400 +@pytest.mark.parametrize( + ("code", "error_class"), + ((1261, litellm.ContextWindowExceededError), (1301, litellm.ContentPolicyViolationError)), +) +@pytest.mark.parametrize("is_async", (False, True)) +@pytest.mark.asyncio +@respx.mock +async def test_streamed_prompt_too_long_and_unsafe_content_raise_the_errors_fallbacks_catch( + code: int, error_class: type[Exception], is_async: bool +) -> None: + respx.post(f"{BASE}/api/v1/responses").respond(json={"code": code, "msg": "rejected by BigModel", "success": False}) + params: Final = {"model": f"bigmodel/{MODEL}", "input": "Hello", "stream": True} + + async def call() -> None: + if is_async: + await litellm.aresponses(**params) + return + litellm.responses(**params) + + with pytest.raises(error_class, match="rejected by BigModel") as raised: + await call() + assert raised.value.status_code == 400 + + @pytest.mark.parametrize("route", ("responses", "messages")) @pytest.mark.asyncio @respx.mock