From df1b3c849b76b4dff04441a5838427b5b6be9827 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 18 Sep 2026 13:58:53 -0700 Subject: [PATCH] fix(responses): keep upstream error details in response.failed RateLimitError and InternalServerError now carry the provider body, so the OpenAI exception mapper keeps upstream codes like cyber_policy and the upstream message instead of a generic mapped one The proxy's response.failed event prefers the upstream body's code, message, and type over the mapped exception's, and numeric error codes in an error event map to their own HTTP status --- litellm/exceptions.py | 6 ++-- .../exception_mapping_utils.py | 5 +++ litellm/proxy/_lazy_openapi_snapshot.json | 2 +- .../common_utils/responses_stream_errors.py | 21 +++++++++++- litellm/responses/streaming_iterator.py | 13 +++++--- .../test_exception_mapping_utils.py | 23 +++++++++++++ .../proxy_server/test_streaming_helpers.py | 16 +++++++++ .../response_api_endpoints/test_endpoints.py | 17 ++++++---- .../test_streaming_iterator_error_events.py | 33 +++++++++++++++++++ 9 files changed, 120 insertions(+), 16 deletions(-) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index c638e8a0f86..de9f5c692a1 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -464,6 +464,7 @@ class RateLimitError(openai.RateLimitError): rate_limit_type: str | RateLimitType | None = None, headers: dict[str, str] | None = None, detail: Any = None, + body: object | None = None, ): self.status_code = 429 self.message = f"litellm.RateLimitError: {message}" @@ -507,7 +508,7 @@ class RateLimitError(openai.RateLimitError): ), ) super().__init__( - self.message, response=self.response, body=None + self.message, response=self.response, body=body ) # Call the base class constructor with the parameters it needs self.code = "429" self.type = "throttling_error" @@ -765,6 +766,7 @@ class InternalServerError(openai.InternalServerError): litellm_debug_info: str | None = None, max_retries: int | None = None, num_retries: int | None = None, + body: object | None = None, ): self.status_code = 500 self.message = f"litellm.InternalServerError: {message}" @@ -783,7 +785,7 @@ class InternalServerError(openai.InternalServerError): ), ) super().__init__( - self.message, response=self.response, body=None + self.message, response=self.response, body=body ) # Call the base class constructor with the parameters it needs def __str__(self): diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 70675966dfc..61e2698dd6f 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -307,6 +307,7 @@ def _map_openai_exception( model=model, llm_provider=custom_llm_provider, response=response, + body=getattr(original_exception, "body", None), ) elif ExceptionCheckers.is_error_str_context_window_exceeded(error_str): raise ContextWindowExceededError( @@ -381,6 +382,7 @@ def _map_openai_exception( message=f"{exception_provider} - {message}", model=model, llm_provider=custom_llm_provider, + body=getattr(original_exception, "body", None), ) elif "Request too large" in error_str: raise RateLimitError( @@ -389,6 +391,7 @@ def _map_openai_exception( llm_provider=custom_llm_provider, response=response, litellm_debug_info=extra_information, + body=getattr(original_exception, "body", None), ) elif ( "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable" @@ -460,6 +463,7 @@ def _map_openai_exception( llm_provider=custom_llm_provider, response=response, litellm_debug_info=extra_information, + body=getattr(original_exception, "body", None), ) elif original_exception.status_code == 500: raise InternalServerError( @@ -468,6 +472,7 @@ def _map_openai_exception( llm_provider=custom_llm_provider, response=response, litellm_debug_info=extra_information, + body=getattr(original_exception, "body", None), ) elif original_exception.status_code == 502: raise BadGatewayError( diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 8a8d08c6887..2aa2cf15ac1 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -19394,7 +19394,7 @@ } } }, - "description": "\n Unified rate-limit error.\n\n Every rate-limit condition surfaced by litellm \u2014 whether it originated from\n an upstream LLM provider, a vendor batch endpoint, or one of litellm's own\n proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\n max-iterations, etc.) \u2014 is raised as an instance of this class.\n\n The :attr:`category` attribute lets callers distinguish the source. See\n :class:`RateLimitErrorCategory` for the available values.\n " + "description": "\nUnified rate-limit error.\n\nEvery rate-limit condition surfaced by litellm \u2014 whether it originated from\nan upstream LLM provider, a vendor batch endpoint, or one of litellm's own\nproxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\nmax-iterations, etc.) \u2014 is raised as an instance of this class.\n\nThe :attr:`category` attribute lets callers distinguish the source. See\n:class:`RateLimitErrorCategory` for the available values.\n" }, "500": { "content": { diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 706b4c298d7..63f881c126e 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -36,6 +36,11 @@ class _FailureDetails(BaseModel): type: str | None = None status_code: int | None = None + @field_validator("message", mode="before") + @classmethod + def normalize_message(cls, value: object) -> str | None: + return value if isinstance(value, str) else None + @field_validator("code", mode="before") @classmethod def normalize_code(cls, value: object) -> str | int | None: @@ -54,6 +59,20 @@ def _original_failure(exception: Exception) -> Exception: return current +def _failure_details(original: Exception) -> _FailureDetails: + mapped: Final = _FailureDetails.model_validate(original) + body: Final = getattr(original, "body", None) + if not isinstance(body, Mapping): + return mapped + upstream: Final = _FailureDetails.model_validate(body) + return _FailureDetails( + message=upstream.message or mapped.message, + code=upstream.code if upstream.code is not None else mapped.code, + type=upstream.type or mapped.type, + status_code=mapped.status_code, + ) + + def _response_error_code(details: _FailureDetails) -> str: for value in (details.code, details.type): if value == "insufficient_quota": @@ -110,7 +129,7 @@ class ResponsesStreamErrorState: if self.terminal_emitted: return None original: Final = _original_failure(exception) - details: Final = _FailureDetails.model_validate(original) + details: Final = _failure_details(original) response: Final = ResponsesAPIResponse.model_validate( MappingProxyType( { diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 8d766cf1cd0..9e7cfc0fbe7 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -211,18 +211,21 @@ def _error_event_fields(error_obj: object) -> tuple[str, str | None, str | None] raw_code = None message: Final = str(raw_message) if raw_message is not None else "Response API in-stream error" error_type: Final = raw_type if isinstance(raw_type, str) else None - code: Final = raw_code if isinstance(raw_code, str) else None + code: Final = str(raw_code) if isinstance(raw_code, (str, int)) and not isinstance(raw_code, bool) else None return message, error_type, code +def _status_code_for_error_field(field: str) -> int | None: + if field.isdecimal() and 400 <= int(field) <= 599: + return int(field) + return _ERROR_CODE_HTTP_STATUS.get(field) + + def _status_code_for_error_fields(error_type: str | None, error_code: str | None) -> int: fields: Final = tuple(field for field in (error_code, error_type) if field is not None) if any(field.startswith("rate_limit") or field == "insufficient_quota" for field in fields): return 429 - return next( - (_ERROR_CODE_HTTP_STATUS[field] for field in fields if field in _ERROR_CODE_HTTP_STATUS), - 500, - ) + return next((status for status in map(_status_code_for_error_field, fields) if status is not None), 500) def _mid_stream_fallback_eligible(mapped_exception: Exception) -> bool: diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index acc6248bf3e..0970526956e 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -1437,6 +1437,29 @@ def test_openai_compatible_vendor_400_keeps_body_but_not_headers(): assert not exc_info.value.response.headers +@pytest.mark.parametrize( + ("status_code", "mapped_class"), [(429, litellm.RateLimitError), (500, litellm.InternalServerError)] +) +def test_openai_429_and_500_keep_body(status_code: int, mapped_class: type[openai.APIError]): + with pytest.raises(mapped_class) as exc_info: + exception_type( + model="gpt-5.4-mini", + original_exception=_openai_handler_error( + "server_error", {}, status_code=status_code, message="upstream cannot complete this response" + ), + custom_llm_provider="openai", + completion_kwargs={}, + extra_kwargs={}, + ) + + assert exc_info.value.body == { + **_GUARDRAIL_BLOCK_ERROR, + "type": "server_error", + "code": str(status_code), + "message": "upstream cannot complete this response", + } + + def test_litellm_proxy_repeated_response_header_keeps_each_value(): repeated = [("x-litellm-call-id", "call-guardrail"), ("set-cookie", "a=1"), ("set-cookie", "b=2")] diff --git a/tests/test_litellm/proxy/proxy_server/test_streaming_helpers.py b/tests/test_litellm/proxy/proxy_server/test_streaming_helpers.py index 53e055882e8..9adc2b80741 100644 --- a/tests/test_litellm/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/test_litellm/proxy/proxy_server/test_streaming_helpers.py @@ -883,6 +883,13 @@ async def test_async_data_generator_mid_stream_exception_yields_error_payload( assert any(isinstance(item, str) and item.startswith('data: {"error":') for item in out) +_UPSTREAM_BODY: Final = { + "code": "cyber_policy", + "message": "Upstream rejected request: flagged for possible cybersecurity risk", + "type": None, +} + + @pytest.mark.asyncio @pytest.mark.parametrize( "terminal,upstream_error,expected_code", @@ -906,6 +913,13 @@ async def test_async_data_generator_mid_stream_exception_yields_error_payload( ), "server_error", id="structured_provider_error_fields", ), + pytest.param( + "upstream_failure", + litellm.InternalServerError( + message="Upstream rejected request", llm_provider="openai", model="gpt-6-astra", body=_UPSTREAM_BODY + ), + "cyber_policy", id="upstream_body_code_and_message", + ), *( pytest.param( "upstream_failure", HTTPException(status_code=status, detail="Upstream rejected request"), @@ -995,6 +1009,8 @@ async def test_responses_stream_keeps_tool_deltas_and_only_emits_a_valid_termina assert "serialize" in failure.response.error["message"].lower() else: assert "Upstream rejected request" in failure.response.error["message"] + if isinstance(upstream_error, litellm.InternalServerError): + assert failure.response.error["message"] == _UPSTREAM_BODY["message"] if isinstance(upstream_error, (HTTPException, litellm.AuthenticationError)): assert upstream_error.status_code == original_status assert payloads[-1]["sequence_number"] > payloads[1]["sequence_number"] diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 7b79e1b8613..1fb3fef8fb3 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -24,6 +24,7 @@ from litellm.proxy.proxy_server import app ("/v1/responses", "numeric_rate_limit"), ("/v1/responses", "server_error"), ("/v1/responses", "response_failed"), + ("/v1/responses", "cyber_policy"), ("/cursor/chat/completions", "server_error"), ("/v1/chat/completions", "server_error"), ], @@ -31,7 +32,7 @@ from litellm.proxy.proxy_server import app async def test_streaming_upstream_errors_keep_the_client_protocol( monkeypatch: pytest.MonkeyPatch, path: str, - error_kind: Literal["rate_limit", "numeric_rate_limit", "server_error", "response_failed"], + error_kind: Literal["rate_limit", "numeric_rate_limit", "server_error", "response_failed", "cyber_policy"], ) -> None: import litellm.proxy.proxy_server as ps from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -40,7 +41,7 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( message: Final = "Upstream cannot complete this response" code: Final = { "rate_limit": "rate_limit_exceeded", "numeric_rate_limit": "429", - "server_error": "server_error", "response_failed": "server_error", + "server_error": "server_error", "response_failed": "server_error", "cyber_policy": "cyber_policy", }[error_kind] error: Final = {"message": message, "code": code, "type": None, "param": "input"} response: Final = {"id": "resp_upstream", "object": "response", "created_at": 1, @@ -55,13 +56,13 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( failed: Final = ( {"type": "response.failed", "sequence_number": 9, "response": {**response, "status": "failed", "error": error}} - if error_kind == "response_failed" else {"type": "error", "error": error} + if error_kind in ("response_failed", "cyber_policy") else {"type": "error", "error": error} ) chat: Final = {"id": "chatcmpl_partial", "object": "chat.completion.chunk", "created": 1, "model": model, "choices": [{"index": 0, "delta": {"content": "partial"}, "finish_reason": None}]} is_chat: Final = path == "/v1/chat/completions" - partial: Final = path != "/v1/responses" or error_kind in ("numeric_rate_limit", "response_failed") + partial: Final = path != "/v1/responses" or error_kind in ("numeric_rate_limit", "response_failed", "cyber_policy") response_events: Final = (created, tool_added, tool_delta, failed) if partial else (failed,) upstream_events: Final = (chat, {"error": error}) if is_chat else response_events wire: Final = "".join("data: " + json.dumps(event) + "\n\n" for event in upstream_events) @@ -108,9 +109,11 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( assert events[0]["sequence_number"] == 0 assert events[0]["response"]["id"].startswith("resp_") assert events[-1]["response"]["status"] == "failed" - assert events[-1]["response"]["error"]["code"] == ( - "rate_limit_exceeded" if error_kind in ("rate_limit", "numeric_rate_limit") else "server_error" - ) + assert events[-1]["response"]["error"]["code"] == { + "rate_limit": "rate_limit_exceeded", "numeric_rate_limit": "rate_limit_exceeded", + "server_error": "server_error", "response_failed": "server_error", "cyber_policy": "cyber_policy", + }[error_kind] + assert events[-1]["response"]["error"]["message"] == message else: assert events[0]["object"] == "chat.completion.chunk", result.text assert "response.failed" not in result.text diff --git a/tests/test_litellm/responses/test_streaming_iterator_error_events.py b/tests/test_litellm/responses/test_streaming_iterator_error_events.py index 3d7c220804a..e7cf09909fe 100644 --- a/tests/test_litellm/responses/test_streaming_iterator_error_events.py +++ b/tests/test_litellm/responses/test_streaming_iterator_error_events.py @@ -340,6 +340,36 @@ def test_maybe_raise_for_response_failed_event_with_dict_error(): assert exc_info.value.status_code == 429 +@pytest.mark.parametrize("code", [429, "429"]) +def test_response_failed_numeric_code_maps_to_its_http_status(code: int | str): + iterator = _make_iterator() + mock_response_obj = Mock() + mock_response_obj.error = {"code": code, "message": "throttled"} + chunk = Mock() + chunk.type = "response.failed" + chunk.response = mock_response_obj + with pytest.raises(MidStreamFallbackError) as exc_info: + iterator._maybe_raise_for_error_event(chunk) + assert exc_info.value.status_code == 429 + assert isinstance(exc_info.value.original_exception, litellm.RateLimitError) + + +def test_response_failed_unknown_code_keeps_upstream_code_and_message_on_mapped_exception(): + iterator = _make_iterator() + upstream_message = "This content was flagged for possible cybersecurity risk." + mock_response_obj = Mock() + mock_response_obj.error = {"code": "cyber_policy", "message": upstream_message} + chunk = Mock() + chunk.type = "response.failed" + chunk.response = mock_response_obj + with pytest.raises(MidStreamFallbackError) as exc_info: + iterator._maybe_raise_for_error_event(chunk) + mapped = exc_info.value.original_exception + assert isinstance(mapped, litellm.InternalServerError) + assert mapped.code == "cyber_policy" + assert mapped.body == {"message": upstream_message, "type": None, "code": "cyber_policy"} + + def test_maybe_raise_for_error_event_null_error_obj(): """error chunk with no error field: message and code default; wrapped as 500.""" iterator = _make_iterator() @@ -523,6 +553,9 @@ def test_every_openai_sdk_response_error_code_has_explicit_status_mapping(): ("failed_to_download_image", 400), ("image_file_not_found", 400), ("totally_unknown_future_code", 500), + ("429", 429), + ("503", 503), + ("200", 500), ], ) def test_status_code_for_documented_response_error_codes(code: str, expected_status: int):