fix(proxy): send a fail-closed per-turn socket refusal with its 503 status

This commit is contained in:
mateo-berri 2026-10-31 01:55:57 -07:00
parent cbb4303d25
commit 24dfb223d0
4 changed files with 60 additions and 15 deletions

View file

@ -49,6 +49,7 @@ from litellm.constants import (
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY,
)
from litellm.cost_calculator import get_usage_object
from litellm.exceptions import ServiceUnavailableError
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.add_retry_fallback_headers import (
ensure_response_additional_headers,
@ -3045,9 +3046,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
tpm_windows: Final = _socket_turn_tpm_windows(
(*descriptors, *(d for d in agent_windows if (d["key"], d["value"]) not in built_scopes))
)
response: Final = await self.should_rate_limit(
tpm_windows, parent_otel_span=user_api_key_dict.parent_otel_span, read_only=True, skip_parallel_check=True
)
try:
response: Final = await self.should_rate_limit(
tpm_windows,
parent_otel_span=user_api_key_dict.parent_otel_span,
read_only=True,
skip_parallel_check=True,
)
except RateLimitUnverifiableError as unverifiable:
raise ServiceUnavailableError(
message=RATE_LIMIT_UNVERIFIABLE_MESSAGE, llm_provider="litellm_proxy", model=requested_model or ""
) from unverifiable
if response["overall_code"] == "OVER_LIMIT":
self._handle_rate_limit_error(response, tpm_windows, requested_model)

View file

@ -38,7 +38,7 @@ from litellm.constants import (
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY,
STREAM_SSE_DONE_STRING,
)
from litellm.exceptions import MidStreamFallbackError, RateLimitError
from litellm.exceptions import MidStreamFallbackError, RateLimitError, ServiceUnavailableError
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import (
process_response_headers,
@ -2205,16 +2205,18 @@ def turn_refusal_frame(refusal: RateLimitError) -> str:
)
def _turn_admission_error_frame(failure: Exception) -> str:
return json.dumps({"type": "error", "status": 500, "error": {"type": "server_error", "message": str(failure)}})
def _turn_admission_error_frame(failure: Exception, status: int) -> str:
return json.dumps({"type": "error", "status": status, "error": {"type": "server_error", "message": str(failure)}})
def _turn_rejection_frame(rejection: Exception) -> str:
match rejection:
case RateLimitError():
return turn_refusal_frame(rejection)
case ServiceUnavailableError():
return _turn_admission_error_frame(rejection, rejection.status_code)
case _:
return _turn_admission_error_frame(rejection)
return _turn_admission_error_frame(rejection, 500)
RESPONSES_WS_LOGGED_EVENT_TYPES: Final = [

View file

@ -24,6 +24,7 @@ from litellm import Router
from litellm.caching.caching import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY
from litellm.exceptions import ServiceUnavailableError
from litellm.integrations.custom_secret_manager import CustomSecretManager
from litellm.litellm_core_utils import ptu_pricing
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -7926,6 +7927,22 @@ async def test_fail_closed_read_only_check_rejects_with_503_when_redis_counters_
assert response["overall_code"] == "OK"
@pytest.mark.asyncio
async def test_fail_closed_socket_turn_check_refuses_with_503_when_redis_counters_are_unreachable():
auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-fail-closed-socket-turn"), tpm_limit=1000)
with pytest.raises(ServiceUnavailableError) as exc:
await _handler_with_redis(_UnreachableRedis(), fail_closed=True).enforce_token_limits_for_frame(
user_api_key_dict=auth, requested_model="gpt-5.4"
)
assert exc.value.status_code == 503
assert "fail_closed_rate_limit_enforcement" in str(exc.value)
await _handler_with_redis(_UnreachableRedis(), fail_closed=False).enforce_token_limits_for_frame(
user_api_key_dict=auth, requested_model="gpt-5.4"
)
@pytest.mark.parametrize("stored_counter_value, expected_code", [(1, "OK"), (2, "OVER_LIMIT"), (3, "OVER_LIMIT")])
@pytest.mark.asyncio
async def test_read_only_check_reports_the_redis_counters_without_incrementing_them(

View file

@ -1627,6 +1627,19 @@ def _out_of_tokens() -> Exception:
return RateLimitError(message="key is out of tokens", llm_provider="", model="")
def _counters_unverifiable() -> Exception:
from litellm.exceptions import ServiceUnavailableError
return ServiceUnavailableError(message="counters could not be verified", llm_provider="litellm_proxy", model="")
_TURN_REFUSALS: Final = pytest.mark.parametrize(
"refusal, refusal_frame",
[(_out_of_tokens(), (429, "rate_limit_exceeded")), (_counters_unverifiable(), (503, "server_error"))],
ids=["out-of-tokens", "counters-unverifiable"],
)
class _ClientSocket:
"""A client socket that, like Starlette's, stops receiving once the server closes it."""
@ -2225,8 +2238,9 @@ class TestWebSocketTurnAdmission:
@pytest.mark.asyncio
@_TURN_REFUSALS
async def test_managed_socket_checks_token_limits_before_every_later_turn_and_marks_each_turn(
self, monkeypatch: pytest.MonkeyPatch
self, monkeypatch: pytest.MonkeyPatch, refusal: Exception, refusal_frame: tuple[int, str]
) -> None:
import litellm
from litellm.constants import RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY
@ -2253,7 +2267,7 @@ class TestWebSocketTurnAdmission:
return _empty()
monkeypatch.setattr(litellm, "aresponses", fake_aresponses)
token_limits: Final = _TokenLimits(_out_of_tokens(), None)
token_limits: Final = _TokenLimits(refusal, None)
monkeypatch.setattr(litellm, "callbacks", [token_limits])
deployment_admission: Final = AsyncMock(return_value={})
user_api_key_dict: Final = UserAPIKeyAuth(api_key="hashed-key", tpm_limit=1000)
@ -2286,12 +2300,13 @@ class TestWebSocketTurnAdmission:
assert [socket_call_id for _, socket_call_id in answered] == ["socket-open", "socket-open"]
assert token_limits.checked == [(user_api_key_dict, "claude-ws"), (user_api_key_dict, "claude-ws")]
assert deployment_admission.await_count == 1
assert [(status, error["type"]) for status, error in _errors(websocket)] == [(429, "rate_limit_exceeded")]
assert [(status, error["type"]) for status, error in _errors(websocket)] == [refusal_frame]
@pytest.mark.asyncio
@_TURN_REFUSALS
@pytest.mark.parametrize(
"keyed, forwarded, refusals",
[(True, ["one", "three"], [(429, "rate_limit_exceeded")]), (False, ["one", "two", "three"], [])],
"keyed, forwarded, refused",
[(True, ["one", "three"], True), (False, ["one", "two", "three"], False)],
ids=["keyed", "keyless"],
)
async def test_native_socket_checks_token_limits_before_every_later_turn_of_a_keyed_socket(
@ -2299,7 +2314,9 @@ class TestWebSocketTurnAdmission:
monkeypatch: pytest.MonkeyPatch,
keyed: bool,
forwarded: list[str],
refusals: list[tuple[int, str]],
refused: bool,
refusal: Exception,
refusal_frame: tuple[int, str],
) -> None:
from unittest.mock import patch
@ -2307,7 +2324,7 @@ class TestWebSocketTurnAdmission:
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.proxy._types import UserAPIKeyAuth
token_limits: Final = _TokenLimits(_out_of_tokens(), None)
token_limits: Final = _TokenLimits(refusal, None)
monkeypatch.setattr(litellm, "callbacks", [token_limits])
provider: Final = _ProviderResponsesSocket()
config: Final = MagicMock(spec=OpenAIResponsesAPIConfig)
@ -2335,7 +2352,7 @@ class TestWebSocketTurnAdmission:
assert [frame["input"] for frame in provider.sent] == forwarded
assert token_limits.checked == ([(user_api_key_dict, "gpt-5.4")] * 2 if keyed else [])
assert [(status, error["type"]) for status, error in _errors(websocket)] == refusals
assert [(status, error["type"]) for status, error in _errors(websocket)] == ([refusal_frame] if refused else [])
@pytest.mark.asyncio
async def test_native_relay_runs_the_guardrail_before_the_deployment_counts_a_turn(self):