diff --git a/litellm/litellm_core_utils/realtime_errors.py b/litellm/litellm_core_utils/realtime_errors.py index 07f7148815c..e7c0a14e6ab 100644 --- a/litellm/litellm_core_utils/realtime_errors.py +++ b/litellm/litellm_core_utils/realtime_errors.py @@ -46,7 +46,7 @@ def client_close_code(upstream_code: int) -> int: return int(CloseCode.INTERNAL_ERROR) -def upstream_handshake_close_code(status_code: int) -> int: +def refusal_close_code(status_code: int) -> int: from websockets.frames import CloseCode refusal_codes: Final = MappingProxyType( @@ -66,6 +66,6 @@ async def close_after_upstream_handshake_refusal(websocket: _ClientWebSocket, st except Exception: # noqa: BLE001 # best-effort notice: a dead client socket must not skip the close below pass await websocket.close( - code=upstream_handshake_close_code(status_code), + code=refusal_close_code(status_code), reason=websocket_close_reason(message, fallback="Upstream handshake rejected"), ) diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index ef59deb2e5e..f694a34825c 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -40,6 +40,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup +from litellm.litellm_core_utils.realtime_errors import refusal_close_code, websocket_close_reason from litellm.litellm_core_utils.token_counter import usage_tokenizer_model from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, first_validation_error from litellm.llms.openai.responses.guardrail_translation.handler import ( @@ -2081,22 +2082,36 @@ def _routing_hints_from_first_ws_frame(first_message: str) -> Mapping[str, objec return MappingProxyType({key: value for key, value in hints.items() if value is not None}) -def _responses_ws_failure_frame(failure: Exception) -> str: - if isinstance(failure, EndpointNotSupportedByModelError): - return json.dumps( - { - "type": "error", - "status": 400, - "error": {"type": "invalid_request_error", "message": failure.message.split("\n\n", 1)[0]}, - } - ) - proxy_exception: Final = handle_exception_on_proxy(failure) - status: Final = int(proxy_exception.code) if str(proxy_exception.code).isdigit() else 500 +@dataclass(frozen=True, slots=True) +class _ResponsesWsFailureReply: + frame: str + close_code: int + close_reason: str + + +def _responses_ws_error_frame(status: int, message: str) -> str: error_type: Final = ( "rate_limit_exceeded" if status == 429 else "invalid_request_error" if 400 <= status < 500 else "server_error" ) - return json.dumps( - {"type": "error", "status": status, "error": {"type": error_type, "message": proxy_exception.message}} + return json.dumps({"type": "error", "status": status, "error": {"type": error_type, "message": message}}) + + +def _responses_ws_failure_reply(failure: Exception) -> _ResponsesWsFailureReply: + if isinstance(failure, EndpointNotSupportedByModelError): + return _ResponsesWsFailureReply( + frame=_responses_ws_error_frame(400, failure.message.split("\n\n", 1)[0]), + close_code=1008, + close_reason="Pre-call error", + ) + proxy_exception: Final = handle_exception_on_proxy(failure) + status: Final = int(proxy_exception.code) if str(proxy_exception.code).isdigit() else 500 + close_code: Final = refusal_close_code(status) + return _ResponsesWsFailureReply( + frame=_responses_ws_error_frame(status, proxy_exception.message), + close_code=close_code, + close_reason="Internal server error" + if close_code == 1011 + else websocket_close_reason(proxy_exception.message.split("\n\n", 1)[0], fallback="Request refused"), ) @@ -2445,15 +2460,11 @@ async def _responses_websocket_session( raise except Exception as e: crash_row: Final = bill_socket_failure(e) + reply: Final = _responses_ws_failure_reply(e) with contextlib.suppress(Exception): - await websocket.send_text(_responses_ws_failure_frame(e)) + await websocket.send_text(reply.frame) with contextlib.suppress(Exception): - await websocket.close( - code=1008 if isinstance(e, EndpointNotSupportedByModelError) else 1011, - reason="Pre-call error" - if isinstance(e, EndpointNotSupportedByModelError) - else "Internal server error", - ) + await websocket.close(code=reply.close_code, reason=reply.close_reason) return crash_row return bill_socket_failure(outcome) if isinstance(outcome, Exception) else None diff --git a/tests/unit/litellm_core_utils/test_realtime_errors.py b/tests/unit/litellm_core_utils/test_realtime_errors.py index 999c660c286..f1b797f1262 100644 --- a/tests/unit/litellm_core_utils/test_realtime_errors.py +++ b/tests/unit/litellm_core_utils/test_realtime_errors.py @@ -8,7 +8,7 @@ from litellm.litellm_core_utils.realtime_errors import ( client_close_code, close_after_upstream_handshake_refusal, realtime_error_event, - upstream_handshake_close_code, + refusal_close_code, websocket_close_reason, ) from litellm.types.realtime import RealtimeErrorEvent @@ -62,8 +62,8 @@ def test_client_close_code_only_forwards_codes_a_server_may_send(upstream_code, ("status_code", "expected"), [(401, 1008), (403, 1008), (429, 1013), (500, 1011)], ) -def test_upstream_handshake_close_code_maps_http_status_to_close_code(status_code: int, expected: int): - assert upstream_handshake_close_code(status_code) == expected +def test_refusal_close_code_maps_http_status_to_close_code(status_code: int, expected: int): + assert refusal_close_code(status_code) == expected class _RecordingWebSocket: diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 1e4dded801e..f617e47a38f 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -2033,44 +2033,58 @@ class TestResponsesWSStreamingFirstMessage: @pytest.mark.parametrize( - ("failure", "expected_status", "expected_type"), + ("failure", "expected_status", "expected_type", "expected_close_code"), [ - (ValueError("No deployments available - crossed budget: Exceeded budget for deployment"), 429, "rate_limit_exceeded"), - (litellm.BadRequestError(message="bad input", model="m", llm_provider="openai"), 400, "invalid_request_error"), - (RuntimeError("boom"), 500, "server_error"), - (ValueError(f"{RouterErrors.no_deployments_with_tag_routing.value}=['free']"), 401, "invalid_request_error"), - (ProxyException(message="Authentication Error", type="auth_error", param=None, code=401), 401, "invalid_request_error"), - (HTTPException(status_code=403, detail="key not allowed"), 403, "invalid_request_error"), + ( + ValueError("No deployments available - crossed budget: Exceeded budget for deployment"), + 429, + "rate_limit_exceeded", + 1013, + ), + (litellm.BadRequestError(message="bad input", model="m", llm_provider="openai"), 400, "invalid_request_error", 1011), + (RuntimeError("boom"), 500, "server_error", 1011), + (ValueError(f"{RouterErrors.no_deployments_with_tag_routing.value}=['free']"), 401, "invalid_request_error", 1008), + ( + ProxyException(message="Authentication Error", type="auth_error", param=None, code=401), + 401, + "invalid_request_error", + 1008, + ), + (HTTPException(status_code=403, detail="key not allowed"), 403, "invalid_request_error", 1008), ], ids=["crossed-budget", "bad-request", "unexpected", "tag-routing", "proxy-auth", "http-exception"], ) -def test_responses_ws_failure_frame_uses_the_http_status_for_the_same_failure( - failure: Exception, expected_status: int, expected_type: str +def test_responses_ws_failure_reply_uses_the_http_status_for_the_same_failure( + failure: Exception, expected_status: int, expected_type: str, expected_close_code: int ): - from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_failure_frame + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_failure_reply from litellm.proxy.utils import handle_exception_on_proxy - frame: Final = json.loads(_responses_ws_failure_frame(failure)) + reply: Final = _responses_ws_failure_reply(failure) + frame: Final = json.loads(reply.frame) assert frame["status"] == expected_status assert frame["error"]["type"] == expected_type assert frame["error"]["message"] == handle_exception_on_proxy(failure).message + assert reply.close_code == expected_close_code -def test_responses_ws_failure_frame_shows_only_the_refusal_for_a_model_that_refuses_the_endpoint(): +def test_responses_ws_failure_reply_shows_only_the_refusal_for_a_model_that_refuses_the_endpoint(): from litellm.exceptions import EndpointNotSupportedByModelError - from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_failure_frame + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_failure_reply refusal: Final = "/v1/responses is not supported by model=embed-only. Supported endpoints: /v1/embeddings" failure: Final = EndpointNotSupportedByModelError( message=f"{refusal}\n\nLiteLLM: model group 'embed-only' failed with the error above", model="embed-only" ) + reply: Final = _responses_ws_failure_reply(failure) - assert json.loads(_responses_ws_failure_frame(failure)) == { + assert json.loads(reply.frame) == { "type": "error", "status": 400, "error": {"type": "invalid_request_error", "message": refusal}, } + assert (reply.close_code, reply.close_reason) == (1008, "Pre-call error") class TestWSSessionCostTracking: @@ -3059,10 +3073,41 @@ class TestResponsesWSFirstFrameModelAuth: assert closes == [(1011, "Internal server error")] @pytest.mark.asyncio - async def test_endpoint_sends_an_error_frame_when_routing_rejects_the_connection(self): + @pytest.mark.parametrize( + ("rejection", "expected_status", "expected_close"), + [ + ( + litellm.RateLimitError( + message="origin deployment is cooling down", model="gpt-4o-mini", llm_provider="openai" + ), + 429, + (1013, "litellm.RateLimitError: origin deployment is cooling down"), + ), + ( + litellm.RateLimitError( + message="Model rate limit exceeded. ITPM limit=350, current usage=430" + "\n\nLiteLLM: model group 'gpt-4o-mini' failed with the error above. No fallback was attempted.", + model="gpt-4o-mini", + llm_provider="openai", + ), + 429, + (1013, "litellm.RateLimitError: Model rate limit exceeded. ITPM limit=350, current usage=430"), + ), + ( + litellm.InternalServerError(message="upstream exploded", model="gpt-4o-mini", llm_provider="openai"), + 500, + (1011, "Internal server error"), + ), + ], + ids=["cooling-down", "deployment-limit", "server-error"], + ) + async def test_endpoint_sends_an_error_frame_when_routing_rejects_the_connection( + self, rejection: Exception, expected_status: int, expected_close: tuple[int, str] + ): from litellm.proxy.response_api_endpoints.endpoints import ( responses_websocket_endpoint, ) + from litellm.proxy.utils import handle_exception_on_proxy ws = MagicMock() ws.headers = {} @@ -3080,9 +3125,6 @@ class TestResponsesWSFirstFrameModelAuth: processor.common_processing_pre_call_logic = AsyncMock( return_value=({"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock()) ) - rejection = litellm.RateLimitError( - message="origin deployment is cooling down", model="gpt-4o-mini", llm_provider="openai" - ) proxy_logging_obj = MagicMock() proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None) @@ -3115,10 +3157,9 @@ class TestResponsesWSFirstFrameModelAuth: frame = json.loads(ws.send_text.await_args.args[0]) assert frame["type"] == "error" - assert frame["status"] == 429 - assert frame["error"]["type"] == "rate_limit_exceeded" - assert "cooling down" in frame["error"]["message"] - ws.close.assert_awaited_once_with(code=1011, reason="Internal server error") + assert frame["status"] == expected_status + assert frame["error"]["message"] == handle_exception_on_proxy(rejection).message + ws.close.assert_awaited_once_with(code=expected_close[0], reason=expected_close[1]) booked = proxy_logging_obj.post_call_failure_hook.await_args.kwargs assert booked["original_exception"] is rejection assert booked["user_api_key_dict"] is user_api_key_dict