mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(responses): close a refused Responses WebSocket open with the refusal's close code
A Responses API WebSocket whose first response.create the router refused with a 429 (a deployment itpm/otpm limit, a cooldown) got the right 429 error frame, then a 1011 "Internal server error" close, which OpenAI SDK clients surface as a server crash. The failure frame and the close now come from one derivation of the failure's HTTP status. 429 closes with 1013 Try Again Later and 401/403 with 1008, each with the failure's first paragraph as the close reason. Every other failure keeps 1011 "Internal server error", and an endpoint the model does not support keeps 1008 "Pre-call error". upstream_handshake_close_code is renamed to refusal_close_code since it now serves the proxy's own refusals too.
This commit is contained in:
parent
5d5e3685c1
commit
fa2bec1998
4 changed files with 99 additions and 47 deletions
|
|
@ -46,7 +46,7 @@ def client_close_code(upstream_code: int) -> int:
|
|||
return int(CloseCode.INTERNAL_ERROR)
|
||||
|
||||
|
||||
def upstream_handshake_close_code(status_code: int) -> int:
|
||||
def refusal_close_code(status_code: int) -> int:
|
||||
from websockets.frames import CloseCode
|
||||
|
||||
refusal_codes: Final = MappingProxyType(
|
||||
|
|
@ -66,6 +66,6 @@ async def close_after_upstream_handshake_refusal(websocket: _ClientWebSocket, st
|
|||
except Exception: # noqa: BLE001 # best-effort notice: a dead client socket must not skip the close below
|
||||
pass
|
||||
await websocket.close(
|
||||
code=upstream_handshake_close_code(status_code),
|
||||
code=refusal_close_code(status_code),
|
||||
reason=websocket_close_reason(message, fallback="Upstream handshake rejected"),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata
|
|||
from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
|
||||
from litellm.litellm_core_utils.realtime_errors import refusal_close_code, websocket_close_reason
|
||||
from litellm.litellm_core_utils.token_counter import usage_tokenizer_model
|
||||
from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, first_validation_error
|
||||
from litellm.llms.openai.responses.guardrail_translation.handler import (
|
||||
|
|
@ -2081,22 +2082,36 @@ def _routing_hints_from_first_ws_frame(first_message: str) -> Mapping[str, objec
|
|||
return MappingProxyType({key: value for key, value in hints.items() if value is not None})
|
||||
|
||||
|
||||
def _responses_ws_failure_frame(failure: Exception) -> str:
|
||||
if isinstance(failure, EndpointNotSupportedByModelError):
|
||||
return json.dumps(
|
||||
{
|
||||
"type": "error",
|
||||
"status": 400,
|
||||
"error": {"type": "invalid_request_error", "message": failure.message.split("\n\n", 1)[0]},
|
||||
}
|
||||
)
|
||||
proxy_exception: Final = handle_exception_on_proxy(failure)
|
||||
status: Final = int(proxy_exception.code) if str(proxy_exception.code).isdigit() else 500
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ResponsesWsFailureReply:
|
||||
frame: str
|
||||
close_code: int
|
||||
close_reason: str
|
||||
|
||||
|
||||
def _responses_ws_error_frame(status: int, message: str) -> str:
|
||||
error_type: Final = (
|
||||
"rate_limit_exceeded" if status == 429 else "invalid_request_error" if 400 <= status < 500 else "server_error"
|
||||
)
|
||||
return json.dumps(
|
||||
{"type": "error", "status": status, "error": {"type": error_type, "message": proxy_exception.message}}
|
||||
return json.dumps({"type": "error", "status": status, "error": {"type": error_type, "message": message}})
|
||||
|
||||
|
||||
def _responses_ws_failure_reply(failure: Exception) -> _ResponsesWsFailureReply:
|
||||
if isinstance(failure, EndpointNotSupportedByModelError):
|
||||
return _ResponsesWsFailureReply(
|
||||
frame=_responses_ws_error_frame(400, failure.message.split("\n\n", 1)[0]),
|
||||
close_code=1008,
|
||||
close_reason="Pre-call error",
|
||||
)
|
||||
proxy_exception: Final = handle_exception_on_proxy(failure)
|
||||
status: Final = int(proxy_exception.code) if str(proxy_exception.code).isdigit() else 500
|
||||
close_code: Final = refusal_close_code(status)
|
||||
return _ResponsesWsFailureReply(
|
||||
frame=_responses_ws_error_frame(status, proxy_exception.message),
|
||||
close_code=close_code,
|
||||
close_reason="Internal server error"
|
||||
if close_code == 1011
|
||||
else websocket_close_reason(proxy_exception.message.split("\n\n", 1)[0], fallback="Request refused"),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -2445,15 +2460,11 @@ async def _responses_websocket_session(
|
|||
raise
|
||||
except Exception as e:
|
||||
crash_row: Final = bill_socket_failure(e)
|
||||
reply: Final = _responses_ws_failure_reply(e)
|
||||
with contextlib.suppress(Exception):
|
||||
await websocket.send_text(_responses_ws_failure_frame(e))
|
||||
await websocket.send_text(reply.frame)
|
||||
with contextlib.suppress(Exception):
|
||||
await websocket.close(
|
||||
code=1008 if isinstance(e, EndpointNotSupportedByModelError) else 1011,
|
||||
reason="Pre-call error"
|
||||
if isinstance(e, EndpointNotSupportedByModelError)
|
||||
else "Internal server error",
|
||||
)
|
||||
await websocket.close(code=reply.close_code, reason=reply.close_reason)
|
||||
return crash_row
|
||||
return bill_socket_failure(outcome) if isinstance(outcome, Exception) else None
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from litellm.litellm_core_utils.realtime_errors import (
|
|||
client_close_code,
|
||||
close_after_upstream_handshake_refusal,
|
||||
realtime_error_event,
|
||||
upstream_handshake_close_code,
|
||||
refusal_close_code,
|
||||
websocket_close_reason,
|
||||
)
|
||||
from litellm.types.realtime import RealtimeErrorEvent
|
||||
|
|
@ -62,8 +62,8 @@ def test_client_close_code_only_forwards_codes_a_server_may_send(upstream_code,
|
|||
("status_code", "expected"),
|
||||
[(401, 1008), (403, 1008), (429, 1013), (500, 1011)],
|
||||
)
|
||||
def test_upstream_handshake_close_code_maps_http_status_to_close_code(status_code: int, expected: int):
|
||||
assert upstream_handshake_close_code(status_code) == expected
|
||||
def test_refusal_close_code_maps_http_status_to_close_code(status_code: int, expected: int):
|
||||
assert refusal_close_code(status_code) == expected
|
||||
|
||||
|
||||
class _RecordingWebSocket:
|
||||
|
|
|
|||
|
|
@ -2033,44 +2033,58 @@ class TestResponsesWSStreamingFirstMessage:
|
|||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("failure", "expected_status", "expected_type"),
|
||||
("failure", "expected_status", "expected_type", "expected_close_code"),
|
||||
[
|
||||
(ValueError("No deployments available - crossed budget: Exceeded budget for deployment"), 429, "rate_limit_exceeded"),
|
||||
(litellm.BadRequestError(message="bad input", model="m", llm_provider="openai"), 400, "invalid_request_error"),
|
||||
(RuntimeError("boom"), 500, "server_error"),
|
||||
(ValueError(f"{RouterErrors.no_deployments_with_tag_routing.value}=['free']"), 401, "invalid_request_error"),
|
||||
(ProxyException(message="Authentication Error", type="auth_error", param=None, code=401), 401, "invalid_request_error"),
|
||||
(HTTPException(status_code=403, detail="key not allowed"), 403, "invalid_request_error"),
|
||||
(
|
||||
ValueError("No deployments available - crossed budget: Exceeded budget for deployment"),
|
||||
429,
|
||||
"rate_limit_exceeded",
|
||||
1013,
|
||||
),
|
||||
(litellm.BadRequestError(message="bad input", model="m", llm_provider="openai"), 400, "invalid_request_error", 1011),
|
||||
(RuntimeError("boom"), 500, "server_error", 1011),
|
||||
(ValueError(f"{RouterErrors.no_deployments_with_tag_routing.value}=['free']"), 401, "invalid_request_error", 1008),
|
||||
(
|
||||
ProxyException(message="Authentication Error", type="auth_error", param=None, code=401),
|
||||
401,
|
||||
"invalid_request_error",
|
||||
1008,
|
||||
),
|
||||
(HTTPException(status_code=403, detail="key not allowed"), 403, "invalid_request_error", 1008),
|
||||
],
|
||||
ids=["crossed-budget", "bad-request", "unexpected", "tag-routing", "proxy-auth", "http-exception"],
|
||||
)
|
||||
def test_responses_ws_failure_frame_uses_the_http_status_for_the_same_failure(
|
||||
failure: Exception, expected_status: int, expected_type: str
|
||||
def test_responses_ws_failure_reply_uses_the_http_status_for_the_same_failure(
|
||||
failure: Exception, expected_status: int, expected_type: str, expected_close_code: int
|
||||
):
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_failure_frame
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_failure_reply
|
||||
from litellm.proxy.utils import handle_exception_on_proxy
|
||||
|
||||
frame: Final = json.loads(_responses_ws_failure_frame(failure))
|
||||
reply: Final = _responses_ws_failure_reply(failure)
|
||||
frame: Final = json.loads(reply.frame)
|
||||
|
||||
assert frame["status"] == expected_status
|
||||
assert frame["error"]["type"] == expected_type
|
||||
assert frame["error"]["message"] == handle_exception_on_proxy(failure).message
|
||||
assert reply.close_code == expected_close_code
|
||||
|
||||
|
||||
def test_responses_ws_failure_frame_shows_only_the_refusal_for_a_model_that_refuses_the_endpoint():
|
||||
def test_responses_ws_failure_reply_shows_only_the_refusal_for_a_model_that_refuses_the_endpoint():
|
||||
from litellm.exceptions import EndpointNotSupportedByModelError
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_failure_frame
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_failure_reply
|
||||
|
||||
refusal: Final = "/v1/responses is not supported by model=embed-only. Supported endpoints: /v1/embeddings"
|
||||
failure: Final = EndpointNotSupportedByModelError(
|
||||
message=f"{refusal}\n\nLiteLLM: model group 'embed-only' failed with the error above", model="embed-only"
|
||||
)
|
||||
reply: Final = _responses_ws_failure_reply(failure)
|
||||
|
||||
assert json.loads(_responses_ws_failure_frame(failure)) == {
|
||||
assert json.loads(reply.frame) == {
|
||||
"type": "error",
|
||||
"status": 400,
|
||||
"error": {"type": "invalid_request_error", "message": refusal},
|
||||
}
|
||||
assert (reply.close_code, reply.close_reason) == (1008, "Pre-call error")
|
||||
|
||||
|
||||
class TestWSSessionCostTracking:
|
||||
|
|
@ -3059,10 +3073,41 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
assert closes == [(1011, "Internal server error")]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_endpoint_sends_an_error_frame_when_routing_rejects_the_connection(self):
|
||||
@pytest.mark.parametrize(
|
||||
("rejection", "expected_status", "expected_close"),
|
||||
[
|
||||
(
|
||||
litellm.RateLimitError(
|
||||
message="origin deployment is cooling down", model="gpt-4o-mini", llm_provider="openai"
|
||||
),
|
||||
429,
|
||||
(1013, "litellm.RateLimitError: origin deployment is cooling down"),
|
||||
),
|
||||
(
|
||||
litellm.RateLimitError(
|
||||
message="Model rate limit exceeded. ITPM limit=350, current usage=430"
|
||||
"\n\nLiteLLM: model group 'gpt-4o-mini' failed with the error above. No fallback was attempted.",
|
||||
model="gpt-4o-mini",
|
||||
llm_provider="openai",
|
||||
),
|
||||
429,
|
||||
(1013, "litellm.RateLimitError: Model rate limit exceeded. ITPM limit=350, current usage=430"),
|
||||
),
|
||||
(
|
||||
litellm.InternalServerError(message="upstream exploded", model="gpt-4o-mini", llm_provider="openai"),
|
||||
500,
|
||||
(1011, "Internal server error"),
|
||||
),
|
||||
],
|
||||
ids=["cooling-down", "deployment-limit", "server-error"],
|
||||
)
|
||||
async def test_endpoint_sends_an_error_frame_when_routing_rejects_the_connection(
|
||||
self, rejection: Exception, expected_status: int, expected_close: tuple[int, str]
|
||||
):
|
||||
from litellm.proxy.response_api_endpoints.endpoints import (
|
||||
responses_websocket_endpoint,
|
||||
)
|
||||
from litellm.proxy.utils import handle_exception_on_proxy
|
||||
|
||||
ws = MagicMock()
|
||||
ws.headers = {}
|
||||
|
|
@ -3080,9 +3125,6 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
processor.common_processing_pre_call_logic = AsyncMock(
|
||||
return_value=({"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock())
|
||||
)
|
||||
rejection = litellm.RateLimitError(
|
||||
message="origin deployment is cooling down", model="gpt-4o-mini", llm_provider="openai"
|
||||
)
|
||||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
|
||||
proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None)
|
||||
|
|
@ -3115,10 +3157,9 @@ class TestResponsesWSFirstFrameModelAuth:
|
|||
|
||||
frame = json.loads(ws.send_text.await_args.args[0])
|
||||
assert frame["type"] == "error"
|
||||
assert frame["status"] == 429
|
||||
assert frame["error"]["type"] == "rate_limit_exceeded"
|
||||
assert "cooling down" in frame["error"]["message"]
|
||||
ws.close.assert_awaited_once_with(code=1011, reason="Internal server error")
|
||||
assert frame["status"] == expected_status
|
||||
assert frame["error"]["message"] == handle_exception_on_proxy(rejection).message
|
||||
ws.close.assert_awaited_once_with(code=expected_close[0], reason=expected_close[1])
|
||||
booked = proxy_logging_obj.post_call_failure_hook.await_args.kwargs
|
||||
assert booked["original_exception"] is rejection
|
||||
assert booked["user_api_key_dict"] is user_api_key_dict
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue