fix(responses): close a refused Responses WebSocket open with the refusal's close code

A Responses API WebSocket whose first response.create the router refused
with a 429 (a deployment itpm/otpm limit, a cooldown) got the right 429
error frame, then a 1011 "Internal server error" close, which OpenAI SDK
clients surface as a server crash.

The failure frame and the close now come from one derivation of the
failure's HTTP status. 429 closes with 1013 Try Again Later and 401/403
with 1008, each with the failure's first paragraph as the close reason.
Every other failure keeps 1011 "Internal server error", and an endpoint
the model does not support keeps 1008 "Pre-call error".

upstream_handshake_close_code is renamed to refusal_close_code since it
now serves the proxy's own refusals too.
This commit is contained in:
mateo-berri 2026-10-30 09:23:34 -07:00
parent 5d5e3685c1
commit fa2bec1998
4 changed files with 99 additions and 47 deletions

View file

@ -46,7 +46,7 @@ def client_close_code(upstream_code: int) -> int:
return int(CloseCode.INTERNAL_ERROR)
def upstream_handshake_close_code(status_code: int) -> int:
def refusal_close_code(status_code: int) -> int:
from websockets.frames import CloseCode
refusal_codes: Final = MappingProxyType(
@ -66,6 +66,6 @@ async def close_after_upstream_handshake_refusal(websocket: _ClientWebSocket, st
except Exception: # noqa: BLE001 # best-effort notice: a dead client socket must not skip the close below
pass
await websocket.close(
code=upstream_handshake_close_code(status_code),
code=refusal_close_code(status_code),
reason=websocket_close_reason(message, fallback="Upstream handshake rejected"),
)

View file

@ -40,6 +40,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata
from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.litellm_core_utils.realtime_errors import refusal_close_code, websocket_close_reason
from litellm.litellm_core_utils.token_counter import usage_tokenizer_model
from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, first_validation_error
from litellm.llms.openai.responses.guardrail_translation.handler import (
@ -2081,22 +2082,36 @@ def _routing_hints_from_first_ws_frame(first_message: str) -> Mapping[str, objec
return MappingProxyType({key: value for key, value in hints.items() if value is not None})
def _responses_ws_failure_frame(failure: Exception) -> str:
if isinstance(failure, EndpointNotSupportedByModelError):
return json.dumps(
{
"type": "error",
"status": 400,
"error": {"type": "invalid_request_error", "message": failure.message.split("\n\n", 1)[0]},
}
)
proxy_exception: Final = handle_exception_on_proxy(failure)
status: Final = int(proxy_exception.code) if str(proxy_exception.code).isdigit() else 500
@dataclass(frozen=True, slots=True)
class _ResponsesWsFailureReply:
frame: str
close_code: int
close_reason: str
def _responses_ws_error_frame(status: int, message: str) -> str:
error_type: Final = (
"rate_limit_exceeded" if status == 429 else "invalid_request_error" if 400 <= status < 500 else "server_error"
)
return json.dumps(
{"type": "error", "status": status, "error": {"type": error_type, "message": proxy_exception.message}}
return json.dumps({"type": "error", "status": status, "error": {"type": error_type, "message": message}})
def _responses_ws_failure_reply(failure: Exception) -> _ResponsesWsFailureReply:
if isinstance(failure, EndpointNotSupportedByModelError):
return _ResponsesWsFailureReply(
frame=_responses_ws_error_frame(400, failure.message.split("\n\n", 1)[0]),
close_code=1008,
close_reason="Pre-call error",
)
proxy_exception: Final = handle_exception_on_proxy(failure)
status: Final = int(proxy_exception.code) if str(proxy_exception.code).isdigit() else 500
close_code: Final = refusal_close_code(status)
return _ResponsesWsFailureReply(
frame=_responses_ws_error_frame(status, proxy_exception.message),
close_code=close_code,
close_reason="Internal server error"
if close_code == 1011
else websocket_close_reason(proxy_exception.message.split("\n\n", 1)[0], fallback="Request refused"),
)
@ -2445,15 +2460,11 @@ async def _responses_websocket_session(
raise
except Exception as e:
crash_row: Final = bill_socket_failure(e)
reply: Final = _responses_ws_failure_reply(e)
with contextlib.suppress(Exception):
await websocket.send_text(_responses_ws_failure_frame(e))
await websocket.send_text(reply.frame)
with contextlib.suppress(Exception):
await websocket.close(
code=1008 if isinstance(e, EndpointNotSupportedByModelError) else 1011,
reason="Pre-call error"
if isinstance(e, EndpointNotSupportedByModelError)
else "Internal server error",
)
await websocket.close(code=reply.close_code, reason=reply.close_reason)
return crash_row
return bill_socket_failure(outcome) if isinstance(outcome, Exception) else None

View file

@ -8,7 +8,7 @@ from litellm.litellm_core_utils.realtime_errors import (
client_close_code,
close_after_upstream_handshake_refusal,
realtime_error_event,
upstream_handshake_close_code,
refusal_close_code,
websocket_close_reason,
)
from litellm.types.realtime import RealtimeErrorEvent
@ -62,8 +62,8 @@ def test_client_close_code_only_forwards_codes_a_server_may_send(upstream_code,
("status_code", "expected"),
[(401, 1008), (403, 1008), (429, 1013), (500, 1011)],
)
def test_upstream_handshake_close_code_maps_http_status_to_close_code(status_code: int, expected: int):
assert upstream_handshake_close_code(status_code) == expected
def test_refusal_close_code_maps_http_status_to_close_code(status_code: int, expected: int):
assert refusal_close_code(status_code) == expected
class _RecordingWebSocket:

View file

@ -2033,44 +2033,58 @@ class TestResponsesWSStreamingFirstMessage:
@pytest.mark.parametrize(
("failure", "expected_status", "expected_type"),
("failure", "expected_status", "expected_type", "expected_close_code"),
[
(ValueError("No deployments available - crossed budget: Exceeded budget for deployment"), 429, "rate_limit_exceeded"),
(litellm.BadRequestError(message="bad input", model="m", llm_provider="openai"), 400, "invalid_request_error"),
(RuntimeError("boom"), 500, "server_error"),
(ValueError(f"{RouterErrors.no_deployments_with_tag_routing.value}=['free']"), 401, "invalid_request_error"),
(ProxyException(message="Authentication Error", type="auth_error", param=None, code=401), 401, "invalid_request_error"),
(HTTPException(status_code=403, detail="key not allowed"), 403, "invalid_request_error"),
(
ValueError("No deployments available - crossed budget: Exceeded budget for deployment"),
429,
"rate_limit_exceeded",
1013,
),
(litellm.BadRequestError(message="bad input", model="m", llm_provider="openai"), 400, "invalid_request_error", 1011),
(RuntimeError("boom"), 500, "server_error", 1011),
(ValueError(f"{RouterErrors.no_deployments_with_tag_routing.value}=['free']"), 401, "invalid_request_error", 1008),
(
ProxyException(message="Authentication Error", type="auth_error", param=None, code=401),
401,
"invalid_request_error",
1008,
),
(HTTPException(status_code=403, detail="key not allowed"), 403, "invalid_request_error", 1008),
],
ids=["crossed-budget", "bad-request", "unexpected", "tag-routing", "proxy-auth", "http-exception"],
)
def test_responses_ws_failure_frame_uses_the_http_status_for_the_same_failure(
failure: Exception, expected_status: int, expected_type: str
def test_responses_ws_failure_reply_uses_the_http_status_for_the_same_failure(
failure: Exception, expected_status: int, expected_type: str, expected_close_code: int
):
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_failure_frame
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_failure_reply
from litellm.proxy.utils import handle_exception_on_proxy
frame: Final = json.loads(_responses_ws_failure_frame(failure))
reply: Final = _responses_ws_failure_reply(failure)
frame: Final = json.loads(reply.frame)
assert frame["status"] == expected_status
assert frame["error"]["type"] == expected_type
assert frame["error"]["message"] == handle_exception_on_proxy(failure).message
assert reply.close_code == expected_close_code
def test_responses_ws_failure_frame_shows_only_the_refusal_for_a_model_that_refuses_the_endpoint():
def test_responses_ws_failure_reply_shows_only_the_refusal_for_a_model_that_refuses_the_endpoint():
from litellm.exceptions import EndpointNotSupportedByModelError
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_failure_frame
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_failure_reply
refusal: Final = "/v1/responses is not supported by model=embed-only. Supported endpoints: /v1/embeddings"
failure: Final = EndpointNotSupportedByModelError(
message=f"{refusal}\n\nLiteLLM: model group 'embed-only' failed with the error above", model="embed-only"
)
reply: Final = _responses_ws_failure_reply(failure)
assert json.loads(_responses_ws_failure_frame(failure)) == {
assert json.loads(reply.frame) == {
"type": "error",
"status": 400,
"error": {"type": "invalid_request_error", "message": refusal},
}
assert (reply.close_code, reply.close_reason) == (1008, "Pre-call error")
class TestWSSessionCostTracking:
@ -3059,10 +3073,41 @@ class TestResponsesWSFirstFrameModelAuth:
assert closes == [(1011, "Internal server error")]
@pytest.mark.asyncio
async def test_endpoint_sends_an_error_frame_when_routing_rejects_the_connection(self):
@pytest.mark.parametrize(
("rejection", "expected_status", "expected_close"),
[
(
litellm.RateLimitError(
message="origin deployment is cooling down", model="gpt-4o-mini", llm_provider="openai"
),
429,
(1013, "litellm.RateLimitError: origin deployment is cooling down"),
),
(
litellm.RateLimitError(
message="Model rate limit exceeded. ITPM limit=350, current usage=430"
"\n\nLiteLLM: model group 'gpt-4o-mini' failed with the error above. No fallback was attempted.",
model="gpt-4o-mini",
llm_provider="openai",
),
429,
(1013, "litellm.RateLimitError: Model rate limit exceeded. ITPM limit=350, current usage=430"),
),
(
litellm.InternalServerError(message="upstream exploded", model="gpt-4o-mini", llm_provider="openai"),
500,
(1011, "Internal server error"),
),
],
ids=["cooling-down", "deployment-limit", "server-error"],
)
async def test_endpoint_sends_an_error_frame_when_routing_rejects_the_connection(
self, rejection: Exception, expected_status: int, expected_close: tuple[int, str]
):
from litellm.proxy.response_api_endpoints.endpoints import (
responses_websocket_endpoint,
)
from litellm.proxy.utils import handle_exception_on_proxy
ws = MagicMock()
ws.headers = {}
@ -3080,9 +3125,6 @@ class TestResponsesWSFirstFrameModelAuth:
processor.common_processing_pre_call_logic = AsyncMock(
return_value=({"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock())
)
rejection = litellm.RateLimitError(
message="origin deployment is cooling down", model="gpt-4o-mini", llm_provider="openai"
)
proxy_logging_obj = MagicMock()
proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None)
@ -3115,10 +3157,9 @@ class TestResponsesWSFirstFrameModelAuth:
frame = json.loads(ws.send_text.await_args.args[0])
assert frame["type"] == "error"
assert frame["status"] == 429
assert frame["error"]["type"] == "rate_limit_exceeded"
assert "cooling down" in frame["error"]["message"]
ws.close.assert_awaited_once_with(code=1011, reason="Internal server error")
assert frame["status"] == expected_status
assert frame["error"]["message"] == handle_exception_on_proxy(rejection).message
ws.close.assert_awaited_once_with(code=expected_close[0], reason=expected_close[1])
booked = proxy_logging_obj.post_call_failure_hook.await_args.kwargs
assert booked["original_exception"] is rejection
assert booked["user_api_key_dict"] is user_api_key_dict