From f46231ddf00dd8757a7ab9d333ae1a138e274933 Mon Sep 17 00:00:00 2001 From: FU-max-boop Date: Fri, 29 May 2026 11:44:32 +0800 Subject: [PATCH] Cover Responses stream error events in CI path --- ...t_base_responses_api_streaming_iterator.py | 179 ------------------ .../responses/test_streaming_error_events.py | 171 +++++++++++++++++ 2 files changed, 171 insertions(+), 179 deletions(-) create mode 100644 tests/test_litellm/responses/test_streaming_error_events.py diff --git a/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py b/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py index 95d251fa9c4..1bcc17ccbaf 100644 --- a/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py +++ b/tests/llm_responses_api_testing/test_base_responses_api_streaming_iterator.py @@ -19,20 +19,16 @@ from datetime import datetime from typing import Any, Dict, Optional from unittest.mock import Mock, patch -import httpx import pytest sys.path.insert(0, os.path.abspath("../..")) -import litellm from litellm.constants import STREAM_SSE_DONE_STRING from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( - ErrorEvent, - ErrorEventError, ResponseCompletedEvent, ResponseFailedEvent, ResponseIncompleteEvent, @@ -578,181 +574,6 @@ class TestBaseResponsesAPIStreamingIterator: submit_args = mock_executor.submit.call_args assert submit_args[0][0] == mock_logging_obj.failure_handler - @pytest.mark.asyncio - async def test_streaming_error_event_raises_litellm_exception(self): - """ - OpenAI can send a top-level `error` event before response.failed. The - iterator should raise that as a LiteLLM exception instead of yielding it - as a normal stream chunk. - """ - from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator - - error_chunk = { - "type": "error", - "sequence_number": 2, - "error": { - "type": "invalid_request_error", - "code": "context_length_exceeded", - "message": "Input exceeds the model context window.", - "param": "input", - }, - } - - async def mock_aiter_bytes(): - yield f"data: {json.dumps(error_chunk)}\n\n".encode("utf-8") - - mock_response = Mock() - mock_response.headers = {} - mock_response.aiter_bytes = mock_aiter_bytes - - mock_logging_obj = Mock(spec=LiteLLMLoggingObj) - mock_logging_obj.model_call_details = {"litellm_params": {}} - mock_logging_obj.async_failure_handler = Mock() - mock_logging_obj.failure_handler = Mock() - mock_config = Mock(spec=BaseResponsesAPIConfig) - mock_config.transform_streaming_response.return_value = ErrorEvent( - type=ResponsesAPIStreamEvents.ERROR, - sequence_number=2, - error=ErrorEventError( - type="invalid_request_error", - code="context_length_exceeded", - message="Input exceeds the model context window.", - param="input", - ), - ) - - iterator = ResponsesAPIStreamingIterator( - response=mock_response, - model="gpt-5.4-mini", - responses_api_provider_config=mock_config, - logging_obj=mock_logging_obj, - custom_llm_provider="openai", - ) - - with ( - pytest.raises(litellm.ContextWindowExceededError) as exc_info, - patch( - "litellm.responses.streaming_iterator.run_async_function" - ) as mock_run_async, - patch("litellm.responses.streaming_iterator.executor") as mock_executor, - ): - await iterator.__anext__() - - assert "context window" in str(exc_info.value) - assert iterator.finished is True - mock_run_async.assert_called_once() - assert ( - mock_run_async.call_args.kwargs["async_function"] - == mock_logging_obj.async_failure_handler - ) - mock_executor.submit.assert_called_once() - - def test_sync_streaming_error_event_raises_litellm_exception(self): - """ - Sync Responses API streams should raise top-level `error` events too. - """ - from litellm.responses.streaming_iterator import ( - SyncResponsesAPIStreamingIterator, - ) - - error_chunk = { - "type": "error", - "sequence_number": 2, - "error": { - "type": "rate_limit_error", - "code": "rate_limit_exceeded", - "message": "Too many requests.", - "param": None, - }, - } - - mock_response = Mock() - mock_response.headers = {} - mock_response.iter_bytes.return_value = [ - f"data: {json.dumps(error_chunk)}\n\n".encode("utf-8") - ] - - mock_logging_obj = Mock(spec=LiteLLMLoggingObj) - mock_logging_obj.model_call_details = {"litellm_params": {}} - mock_logging_obj.async_failure_handler = Mock() - mock_logging_obj.failure_handler = Mock() - mock_config = Mock(spec=BaseResponsesAPIConfig) - mock_config.transform_streaming_response.return_value = ErrorEvent( - type=ResponsesAPIStreamEvents.ERROR, - sequence_number=2, - error=ErrorEventError( - type="rate_limit_error", - code="rate_limit_exceeded", - message="Too many requests.", - param=None, - ), - ) - - iterator = SyncResponsesAPIStreamingIterator( - response=mock_response, - model="gpt-5.4-mini", - responses_api_provider_config=mock_config, - logging_obj=mock_logging_obj, - custom_llm_provider="openai", - ) - - with ( - pytest.raises(litellm.RateLimitError), - patch( - "litellm.responses.streaming_iterator.run_async_function" - ) as mock_run_async, - patch("litellm.responses.streaming_iterator.executor") as mock_executor, - ): - next(iterator) - - assert iterator.finished is True - mock_run_async.assert_called_once() - mock_executor.submit.assert_called_once() - - def test_error_event_exception_mapping(self): - """Provider error metadata should keep useful LiteLLM exception types.""" - mock_logging_obj = Mock(spec=LiteLLMLoggingObj) - mock_logging_obj.model_call_details = {"litellm_params": {}} - iterator = BaseResponsesAPIStreamingIterator( - response=httpx.Response( - 400, request=httpx.Request("POST", "https://api.example.test") - ), - model="gpt-5.4-mini", - responses_api_provider_config=Mock(spec=BaseResponsesAPIConfig), - logging_obj=mock_logging_obj, - custom_llm_provider="openai", - ) - - auth_event = ErrorEvent( - type=ResponsesAPIStreamEvents.ERROR, - sequence_number=1, - error=ErrorEventError( - type="authentication_error", - code="invalid_api_key", - message="Invalid API key.", - param=None, - ), - ) - default_event = ErrorEvent( - type=ResponsesAPIStreamEvents.ERROR, - sequence_number=2, - error=ErrorEventError( - type="invalid_request_error", - code="bad_request", - message="Bad request.", - param="input", - ), - ) - - assert isinstance( - iterator._exception_from_error_event(auth_event), - litellm.AuthenticationError, - ) - assert isinstance( - iterator._exception_from_error_event(default_event), - litellm.BadRequestError, - ) - def test_process_chunk_response_incomplete_calls_success_handler(self): """ Test that a RESPONSE_INCOMPLETE event routes to success handlers. diff --git a/tests/test_litellm/responses/test_streaming_error_events.py b/tests/test_litellm/responses/test_streaming_error_events.py new file mode 100644 index 00000000000..16e5c790967 --- /dev/null +++ b/tests/test_litellm/responses/test_streaming_error_events.py @@ -0,0 +1,171 @@ +import json +from unittest.mock import Mock, patch + +import httpx +import pytest + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.responses.streaming_iterator import ( + BaseResponsesAPIStreamingIterator, + ResponsesAPIStreamingIterator, + SyncResponsesAPIStreamingIterator, +) +from litellm.types.llms.openai import ( + ErrorEvent, + ErrorEventError, + ResponsesAPIStreamEvents, +) + + +def _mock_logging_obj() -> Mock: + logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.model_call_details = {"litellm_params": {}} + logging_obj.async_failure_handler = Mock() + logging_obj.failure_handler = Mock() + return logging_obj + + +@pytest.mark.asyncio +async def test_async_responses_stream_error_event_raises_litellm_exception(): + error_chunk = { + "type": "error", + "sequence_number": 2, + "error": { + "type": "invalid_request_error", + "code": "context_length_exceeded", + "message": "Input exceeds the model context window.", + "param": "input", + }, + } + + async def mock_aiter_bytes(): + yield f"data: {json.dumps(error_chunk)}\n\n".encode("utf-8") + + mock_response = Mock() + mock_response.headers = {} + mock_response.aiter_bytes = mock_aiter_bytes + mock_logging_obj = _mock_logging_obj() + mock_config = Mock(spec=BaseResponsesAPIConfig) + mock_config.transform_streaming_response.return_value = ErrorEvent( + type=ResponsesAPIStreamEvents.ERROR, + sequence_number=2, + error=ErrorEventError( + type="invalid_request_error", + code="context_length_exceeded", + message="Input exceeds the model context window.", + param="input", + ), + ) + iterator = ResponsesAPIStreamingIterator( + response=mock_response, + model="gpt-5.4-mini", + responses_api_provider_config=mock_config, + logging_obj=mock_logging_obj, + custom_llm_provider="openai", + ) + + with ( + pytest.raises(litellm.ContextWindowExceededError), + patch( + "litellm.responses.streaming_iterator.run_async_function" + ) as mock_run_async, + patch("litellm.responses.streaming_iterator.executor") as mock_executor, + ): + await iterator.__anext__() + + assert iterator.finished is True + mock_run_async.assert_called_once() + mock_executor.submit.assert_called_once() + + +def test_sync_responses_stream_error_event_raises_litellm_exception(): + error_chunk = { + "type": "error", + "sequence_number": 2, + "error": { + "type": "rate_limit_error", + "code": "rate_limit_exceeded", + "message": "Too many requests.", + "param": None, + }, + } + + mock_response = Mock() + mock_response.headers = {} + mock_response.iter_bytes.return_value = [ + f"data: {json.dumps(error_chunk)}\n\n".encode("utf-8") + ] + mock_logging_obj = _mock_logging_obj() + mock_config = Mock(spec=BaseResponsesAPIConfig) + mock_config.transform_streaming_response.return_value = ErrorEvent( + type=ResponsesAPIStreamEvents.ERROR, + sequence_number=2, + error=ErrorEventError( + type="rate_limit_error", + code="rate_limit_exceeded", + message="Too many requests.", + param=None, + ), + ) + iterator = SyncResponsesAPIStreamingIterator( + response=mock_response, + model="gpt-5.4-mini", + responses_api_provider_config=mock_config, + logging_obj=mock_logging_obj, + custom_llm_provider="openai", + ) + + with ( + pytest.raises(litellm.RateLimitError), + patch( + "litellm.responses.streaming_iterator.run_async_function" + ) as mock_run_async, + patch("litellm.responses.streaming_iterator.executor") as mock_executor, + ): + next(iterator) + + assert iterator.finished is True + mock_run_async.assert_called_once() + mock_executor.submit.assert_called_once() + + +def test_responses_stream_error_event_exception_mapping_fallbacks(): + iterator = BaseResponsesAPIStreamingIterator( + response=httpx.Response( + 400, request=httpx.Request("POST", "https://api.example.test") + ), + model="gpt-5.4-mini", + responses_api_provider_config=Mock(spec=BaseResponsesAPIConfig), + logging_obj=_mock_logging_obj(), + custom_llm_provider="openai", + ) + + auth_event = ErrorEvent( + type=ResponsesAPIStreamEvents.ERROR, + sequence_number=1, + error=ErrorEventError( + type="authentication_error", + code="invalid_api_key", + message="Invalid API key.", + param=None, + ), + ) + default_event = ErrorEvent( + type=ResponsesAPIStreamEvents.ERROR, + sequence_number=2, + error=ErrorEventError( + type="invalid_request_error", + code="bad_request", + message="Bad request.", + param="input", + ), + ) + + assert isinstance( + iterator._exception_from_error_event(auth_event), litellm.AuthenticationError + ) + assert isinstance( + iterator._exception_from_error_event(default_event), litellm.BadRequestError + )