diff --git a/litellm/exceptions.py b/litellm/exceptions.py index c638e8a0f86..de9f5c692a1 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -464,6 +464,7 @@ class RateLimitError(openai.RateLimitError): rate_limit_type: str | RateLimitType | None = None, headers: dict[str, str] | None = None, detail: Any = None, + body: object | None = None, ): self.status_code = 429 self.message = f"litellm.RateLimitError: {message}" @@ -507,7 +508,7 @@ class RateLimitError(openai.RateLimitError): ), ) super().__init__( - self.message, response=self.response, body=None + self.message, response=self.response, body=body ) # Call the base class constructor with the parameters it needs self.code = "429" self.type = "throttling_error" @@ -765,6 +766,7 @@ class InternalServerError(openai.InternalServerError): litellm_debug_info: str | None = None, max_retries: int | None = None, num_retries: int | None = None, + body: object | None = None, ): self.status_code = 500 self.message = f"litellm.InternalServerError: {message}" @@ -783,7 +785,7 @@ class InternalServerError(openai.InternalServerError): ), ) super().__init__( - self.message, response=self.response, body=None + self.message, response=self.response, body=body ) # Call the base class constructor with the parameters it needs def __str__(self): diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 70675966dfc..61e2698dd6f 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -307,6 +307,7 @@ def _map_openai_exception( model=model, llm_provider=custom_llm_provider, response=response, + body=getattr(original_exception, "body", None), ) elif ExceptionCheckers.is_error_str_context_window_exceeded(error_str): raise ContextWindowExceededError( @@ -381,6 +382,7 @@ def _map_openai_exception( message=f"{exception_provider} - {message}", model=model, llm_provider=custom_llm_provider, + body=getattr(original_exception, "body", None), ) elif "Request too large" in error_str: raise RateLimitError( @@ -389,6 +391,7 @@ def _map_openai_exception( llm_provider=custom_llm_provider, response=response, litellm_debug_info=extra_information, + body=getattr(original_exception, "body", None), ) elif ( "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable" @@ -460,6 +463,7 @@ def _map_openai_exception( llm_provider=custom_llm_provider, response=response, litellm_debug_info=extra_information, + body=getattr(original_exception, "body", None), ) elif original_exception.status_code == 500: raise InternalServerError( @@ -468,6 +472,7 @@ def _map_openai_exception( llm_provider=custom_llm_provider, response=response, litellm_debug_info=extra_information, + body=getattr(original_exception, "body", None), ) elif original_exception.status_code == 502: raise BadGatewayError( diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 8a8d08c6887..2aa2cf15ac1 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -19394,7 +19394,7 @@ } } }, - "description": "\n Unified rate-limit error.\n\n Every rate-limit condition surfaced by litellm \u2014 whether it originated from\n an upstream LLM provider, a vendor batch endpoint, or one of litellm's own\n proxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\n max-iterations, etc.) \u2014 is raised as an instance of this class.\n\n The :attr:`category` attribute lets callers distinguish the source. See\n :class:`RateLimitErrorCategory` for the available values.\n " + "description": "\nUnified rate-limit error.\n\nEvery rate-limit condition surfaced by litellm \u2014 whether it originated from\nan upstream LLM provider, a vendor batch endpoint, or one of litellm's own\nproxy-side limiters (parallel-requests, dynamic-rate, batch-rate, budget,\nmax-iterations, etc.) \u2014 is raised as an instance of this class.\n\nThe :attr:`category` attribute lets callers distinguish the source. See\n:class:`RateLimitErrorCategory` for the available values.\n" }, "500": { "content": { diff --git a/litellm/proxy/common_utils/responses_stream_errors.py b/litellm/proxy/common_utils/responses_stream_errors.py index 706b4c298d7..63f881c126e 100644 --- a/litellm/proxy/common_utils/responses_stream_errors.py +++ b/litellm/proxy/common_utils/responses_stream_errors.py @@ -36,6 +36,11 @@ class _FailureDetails(BaseModel): type: str | None = None status_code: int | None = None + @field_validator("message", mode="before") + @classmethod + def normalize_message(cls, value: object) -> str | None: + return value if isinstance(value, str) else None + @field_validator("code", mode="before") @classmethod def normalize_code(cls, value: object) -> str | int | None: @@ -54,6 +59,20 @@ def _original_failure(exception: Exception) -> Exception: return current +def _failure_details(original: Exception) -> _FailureDetails: + mapped: Final = _FailureDetails.model_validate(original) + body: Final = getattr(original, "body", None) + if not isinstance(body, Mapping): + return mapped + upstream: Final = _FailureDetails.model_validate(body) + return _FailureDetails( + message=upstream.message or mapped.message, + code=upstream.code if upstream.code is not None else mapped.code, + type=upstream.type or mapped.type, + status_code=mapped.status_code, + ) + + def _response_error_code(details: _FailureDetails) -> str: for value in (details.code, details.type): if value == "insufficient_quota": @@ -110,7 +129,7 @@ class ResponsesStreamErrorState: if self.terminal_emitted: return None original: Final = _original_failure(exception) - details: Final = _FailureDetails.model_validate(original) + details: Final = _failure_details(original) response: Final = ResponsesAPIResponse.model_validate( MappingProxyType( { diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 8d766cf1cd0..9e7cfc0fbe7 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -211,18 +211,21 @@ def _error_event_fields(error_obj: object) -> tuple[str, str | None, str | None] raw_code = None message: Final = str(raw_message) if raw_message is not None else "Response API in-stream error" error_type: Final = raw_type if isinstance(raw_type, str) else None - code: Final = raw_code if isinstance(raw_code, str) else None + code: Final = str(raw_code) if isinstance(raw_code, (str, int)) and not isinstance(raw_code, bool) else None return message, error_type, code +def _status_code_for_error_field(field: str) -> int | None: + if field.isdecimal() and 400 <= int(field) <= 599: + return int(field) + return _ERROR_CODE_HTTP_STATUS.get(field) + + def _status_code_for_error_fields(error_type: str | None, error_code: str | None) -> int: fields: Final = tuple(field for field in (error_code, error_type) if field is not None) if any(field.startswith("rate_limit") or field == "insufficient_quota" for field in fields): return 429 - return next( - (_ERROR_CODE_HTTP_STATUS[field] for field in fields if field in _ERROR_CODE_HTTP_STATUS), - 500, - ) + return next((status for status in map(_status_code_for_error_field, fields) if status is not None), 500) def _mid_stream_fallback_eligible(mapped_exception: Exception) -> bool: diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index acc6248bf3e..0970526956e 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -1437,6 +1437,29 @@ def test_openai_compatible_vendor_400_keeps_body_but_not_headers(): assert not exc_info.value.response.headers +@pytest.mark.parametrize( + ("status_code", "mapped_class"), [(429, litellm.RateLimitError), (500, litellm.InternalServerError)] +) +def test_openai_429_and_500_keep_body(status_code: int, mapped_class: type[openai.APIError]): + with pytest.raises(mapped_class) as exc_info: + exception_type( + model="gpt-5.4-mini", + original_exception=_openai_handler_error( + "server_error", {}, status_code=status_code, message="upstream cannot complete this response" + ), + custom_llm_provider="openai", + completion_kwargs={}, + extra_kwargs={}, + ) + + assert exc_info.value.body == { + **_GUARDRAIL_BLOCK_ERROR, + "type": "server_error", + "code": str(status_code), + "message": "upstream cannot complete this response", + } + + def test_litellm_proxy_repeated_response_header_keeps_each_value(): repeated = [("x-litellm-call-id", "call-guardrail"), ("set-cookie", "a=1"), ("set-cookie", "b=2")] diff --git a/tests/test_litellm/proxy/proxy_server/test_streaming_helpers.py b/tests/test_litellm/proxy/proxy_server/test_streaming_helpers.py index 53e055882e8..9adc2b80741 100644 --- a/tests/test_litellm/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/test_litellm/proxy/proxy_server/test_streaming_helpers.py @@ -883,6 +883,13 @@ async def test_async_data_generator_mid_stream_exception_yields_error_payload( assert any(isinstance(item, str) and item.startswith('data: {"error":') for item in out) +_UPSTREAM_BODY: Final = { + "code": "cyber_policy", + "message": "Upstream rejected request: flagged for possible cybersecurity risk", + "type": None, +} + + @pytest.mark.asyncio @pytest.mark.parametrize( "terminal,upstream_error,expected_code", @@ -906,6 +913,13 @@ async def test_async_data_generator_mid_stream_exception_yields_error_payload( ), "server_error", id="structured_provider_error_fields", ), + pytest.param( + "upstream_failure", + litellm.InternalServerError( + message="Upstream rejected request", llm_provider="openai", model="gpt-6-astra", body=_UPSTREAM_BODY + ), + "cyber_policy", id="upstream_body_code_and_message", + ), *( pytest.param( "upstream_failure", HTTPException(status_code=status, detail="Upstream rejected request"), @@ -995,6 +1009,8 @@ async def test_responses_stream_keeps_tool_deltas_and_only_emits_a_valid_termina assert "serialize" in failure.response.error["message"].lower() else: assert "Upstream rejected request" in failure.response.error["message"] + if isinstance(upstream_error, litellm.InternalServerError): + assert failure.response.error["message"] == _UPSTREAM_BODY["message"] if isinstance(upstream_error, (HTTPException, litellm.AuthenticationError)): assert upstream_error.status_code == original_status assert payloads[-1]["sequence_number"] > payloads[1]["sequence_number"] diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 7b79e1b8613..1fb3fef8fb3 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -24,6 +24,7 @@ from litellm.proxy.proxy_server import app ("/v1/responses", "numeric_rate_limit"), ("/v1/responses", "server_error"), ("/v1/responses", "response_failed"), + ("/v1/responses", "cyber_policy"), ("/cursor/chat/completions", "server_error"), ("/v1/chat/completions", "server_error"), ], @@ -31,7 +32,7 @@ from litellm.proxy.proxy_server import app async def test_streaming_upstream_errors_keep_the_client_protocol( monkeypatch: pytest.MonkeyPatch, path: str, - error_kind: Literal["rate_limit", "numeric_rate_limit", "server_error", "response_failed"], + error_kind: Literal["rate_limit", "numeric_rate_limit", "server_error", "response_failed", "cyber_policy"], ) -> None: import litellm.proxy.proxy_server as ps from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -40,7 +41,7 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( message: Final = "Upstream cannot complete this response" code: Final = { "rate_limit": "rate_limit_exceeded", "numeric_rate_limit": "429", - "server_error": "server_error", "response_failed": "server_error", + "server_error": "server_error", "response_failed": "server_error", "cyber_policy": "cyber_policy", }[error_kind] error: Final = {"message": message, "code": code, "type": None, "param": "input"} response: Final = {"id": "resp_upstream", "object": "response", "created_at": 1, @@ -55,13 +56,13 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( failed: Final = ( {"type": "response.failed", "sequence_number": 9, "response": {**response, "status": "failed", "error": error}} - if error_kind == "response_failed" else {"type": "error", "error": error} + if error_kind in ("response_failed", "cyber_policy") else {"type": "error", "error": error} ) chat: Final = {"id": "chatcmpl_partial", "object": "chat.completion.chunk", "created": 1, "model": model, "choices": [{"index": 0, "delta": {"content": "partial"}, "finish_reason": None}]} is_chat: Final = path == "/v1/chat/completions" - partial: Final = path != "/v1/responses" or error_kind in ("numeric_rate_limit", "response_failed") + partial: Final = path != "/v1/responses" or error_kind in ("numeric_rate_limit", "response_failed", "cyber_policy") response_events: Final = (created, tool_added, tool_delta, failed) if partial else (failed,) upstream_events: Final = (chat, {"error": error}) if is_chat else response_events wire: Final = "".join("data: " + json.dumps(event) + "\n\n" for event in upstream_events) @@ -108,9 +109,11 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( assert events[0]["sequence_number"] == 0 assert events[0]["response"]["id"].startswith("resp_") assert events[-1]["response"]["status"] == "failed" - assert events[-1]["response"]["error"]["code"] == ( - "rate_limit_exceeded" if error_kind in ("rate_limit", "numeric_rate_limit") else "server_error" - ) + assert events[-1]["response"]["error"]["code"] == { + "rate_limit": "rate_limit_exceeded", "numeric_rate_limit": "rate_limit_exceeded", + "server_error": "server_error", "response_failed": "server_error", "cyber_policy": "cyber_policy", + }[error_kind] + assert events[-1]["response"]["error"]["message"] == message else: assert events[0]["object"] == "chat.completion.chunk", result.text assert "response.failed" not in result.text diff --git a/tests/test_litellm/responses/test_streaming_iterator_error_events.py b/tests/test_litellm/responses/test_streaming_iterator_error_events.py index 3d7c220804a..e7cf09909fe 100644 --- a/tests/test_litellm/responses/test_streaming_iterator_error_events.py +++ b/tests/test_litellm/responses/test_streaming_iterator_error_events.py @@ -340,6 +340,36 @@ def test_maybe_raise_for_response_failed_event_with_dict_error(): assert exc_info.value.status_code == 429 +@pytest.mark.parametrize("code", [429, "429"]) +def test_response_failed_numeric_code_maps_to_its_http_status(code: int | str): + iterator = _make_iterator() + mock_response_obj = Mock() + mock_response_obj.error = {"code": code, "message": "throttled"} + chunk = Mock() + chunk.type = "response.failed" + chunk.response = mock_response_obj + with pytest.raises(MidStreamFallbackError) as exc_info: + iterator._maybe_raise_for_error_event(chunk) + assert exc_info.value.status_code == 429 + assert isinstance(exc_info.value.original_exception, litellm.RateLimitError) + + +def test_response_failed_unknown_code_keeps_upstream_code_and_message_on_mapped_exception(): + iterator = _make_iterator() + upstream_message = "This content was flagged for possible cybersecurity risk." + mock_response_obj = Mock() + mock_response_obj.error = {"code": "cyber_policy", "message": upstream_message} + chunk = Mock() + chunk.type = "response.failed" + chunk.response = mock_response_obj + with pytest.raises(MidStreamFallbackError) as exc_info: + iterator._maybe_raise_for_error_event(chunk) + mapped = exc_info.value.original_exception + assert isinstance(mapped, litellm.InternalServerError) + assert mapped.code == "cyber_policy" + assert mapped.body == {"message": upstream_message, "type": None, "code": "cyber_policy"} + + def test_maybe_raise_for_error_event_null_error_obj(): """error chunk with no error field: message and code default; wrapped as 500.""" iterator = _make_iterator() @@ -523,6 +553,9 @@ def test_every_openai_sdk_response_error_code_has_explicit_status_mapping(): ("failed_to_download_image", 400), ("image_file_not_found", 400), ("totally_unknown_future_code", 500), + ("429", 429), + ("503", 503), + ("200", 500), ], ) def test_status_code_for_documented_response_error_codes(code: str, expected_status: int):