diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 76d6c903bdd..c83551a815c 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -49,6 +49,7 @@ from litellm.constants import ( RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY, ) from litellm.cost_calculator import get_usage_object +from litellm.exceptions import ServiceUnavailableError from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.add_retry_fallback_headers import ( ensure_response_additional_headers, @@ -3045,9 +3046,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): tpm_windows: Final = _socket_turn_tpm_windows( (*descriptors, *(d for d in agent_windows if (d["key"], d["value"]) not in built_scopes)) ) - response: Final = await self.should_rate_limit( - tpm_windows, parent_otel_span=user_api_key_dict.parent_otel_span, read_only=True, skip_parallel_check=True - ) + try: + response: Final = await self.should_rate_limit( + tpm_windows, + parent_otel_span=user_api_key_dict.parent_otel_span, + read_only=True, + skip_parallel_check=True, + ) + except RateLimitUnverifiableError as unverifiable: + raise ServiceUnavailableError( + message=RATE_LIMIT_UNVERIFIABLE_MESSAGE, llm_provider="litellm_proxy", model=requested_model or "" + ) from unverifiable if response["overall_code"] == "OVER_LIMIT": self._handle_rate_limit_error(response, tpm_windows, requested_model) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index e252b6bd6b6..803b85ab3a5 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -38,7 +38,7 @@ from litellm.constants import ( RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY, STREAM_SSE_DONE_STRING, ) -from litellm.exceptions import MidStreamFallbackError, RateLimitError +from litellm.exceptions import MidStreamFallbackError, RateLimitError, ServiceUnavailableError from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import ( process_response_headers, @@ -2205,16 +2205,18 @@ def turn_refusal_frame(refusal: RateLimitError) -> str: ) -def _turn_admission_error_frame(failure: Exception) -> str: - return json.dumps({"type": "error", "status": 500, "error": {"type": "server_error", "message": str(failure)}}) +def _turn_admission_error_frame(failure: Exception, status: int) -> str: + return json.dumps({"type": "error", "status": status, "error": {"type": "server_error", "message": str(failure)}}) def _turn_rejection_frame(rejection: Exception) -> str: match rejection: case RateLimitError(): return turn_refusal_frame(rejection) + case ServiceUnavailableError(): + return _turn_admission_error_frame(rejection, rejection.status_code) case _: - return _turn_admission_error_frame(rejection) + return _turn_admission_error_frame(rejection, 500) RESPONSES_WS_LOGGED_EVENT_TYPES: Final = [ diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 8d2833e71c0..bed185ba8f5 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -24,6 +24,7 @@ from litellm import Router from litellm.caching.caching import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY +from litellm.exceptions import ServiceUnavailableError from litellm.integrations.custom_secret_manager import CustomSecretManager from litellm.litellm_core_utils import ptu_pricing from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -7926,6 +7927,22 @@ async def test_fail_closed_read_only_check_rejects_with_503_when_redis_counters_ assert response["overall_code"] == "OK" +@pytest.mark.asyncio +async def test_fail_closed_socket_turn_check_refuses_with_503_when_redis_counters_are_unreachable(): + auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-fail-closed-socket-turn"), tpm_limit=1000) + + with pytest.raises(ServiceUnavailableError) as exc: + await _handler_with_redis(_UnreachableRedis(), fail_closed=True).enforce_token_limits_for_frame( + user_api_key_dict=auth, requested_model="gpt-5.4" + ) + assert exc.value.status_code == 503 + assert "fail_closed_rate_limit_enforcement" in str(exc.value) + + await _handler_with_redis(_UnreachableRedis(), fail_closed=False).enforce_token_limits_for_frame( + user_api_key_dict=auth, requested_model="gpt-5.4" + ) + + @pytest.mark.parametrize("stored_counter_value, expected_code", [(1, "OK"), (2, "OVER_LIMIT"), (3, "OVER_LIMIT")]) @pytest.mark.asyncio async def test_read_only_check_reports_the_redis_counters_without_incrementing_them( diff --git a/tests/unit/responses/test_responses_websocket_all_providers.py b/tests/unit/responses/test_responses_websocket_all_providers.py index 47025a64b95..7e044b55f5a 100644 --- a/tests/unit/responses/test_responses_websocket_all_providers.py +++ b/tests/unit/responses/test_responses_websocket_all_providers.py @@ -1627,6 +1627,19 @@ def _out_of_tokens() -> Exception: return RateLimitError(message="key is out of tokens", llm_provider="", model="") +def _counters_unverifiable() -> Exception: + from litellm.exceptions import ServiceUnavailableError + + return ServiceUnavailableError(message="counters could not be verified", llm_provider="litellm_proxy", model="") + + +_TURN_REFUSALS: Final = pytest.mark.parametrize( + "refusal, refusal_frame", + [(_out_of_tokens(), (429, "rate_limit_exceeded")), (_counters_unverifiable(), (503, "server_error"))], + ids=["out-of-tokens", "counters-unverifiable"], +) + + class _ClientSocket: """A client socket that, like Starlette's, stops receiving once the server closes it.""" @@ -2225,8 +2238,9 @@ class TestWebSocketTurnAdmission: @pytest.mark.asyncio + @_TURN_REFUSALS async def test_managed_socket_checks_token_limits_before_every_later_turn_and_marks_each_turn( - self, monkeypatch: pytest.MonkeyPatch + self, monkeypatch: pytest.MonkeyPatch, refusal: Exception, refusal_frame: tuple[int, str] ) -> None: import litellm from litellm.constants import RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY @@ -2253,7 +2267,7 @@ class TestWebSocketTurnAdmission: return _empty() monkeypatch.setattr(litellm, "aresponses", fake_aresponses) - token_limits: Final = _TokenLimits(_out_of_tokens(), None) + token_limits: Final = _TokenLimits(refusal, None) monkeypatch.setattr(litellm, "callbacks", [token_limits]) deployment_admission: Final = AsyncMock(return_value={}) user_api_key_dict: Final = UserAPIKeyAuth(api_key="hashed-key", tpm_limit=1000) @@ -2286,12 +2300,13 @@ class TestWebSocketTurnAdmission: assert [socket_call_id for _, socket_call_id in answered] == ["socket-open", "socket-open"] assert token_limits.checked == [(user_api_key_dict, "claude-ws"), (user_api_key_dict, "claude-ws")] assert deployment_admission.await_count == 1 - assert [(status, error["type"]) for status, error in _errors(websocket)] == [(429, "rate_limit_exceeded")] + assert [(status, error["type"]) for status, error in _errors(websocket)] == [refusal_frame] @pytest.mark.asyncio + @_TURN_REFUSALS @pytest.mark.parametrize( - "keyed, forwarded, refusals", - [(True, ["one", "three"], [(429, "rate_limit_exceeded")]), (False, ["one", "two", "three"], [])], + "keyed, forwarded, refused", + [(True, ["one", "three"], True), (False, ["one", "two", "three"], False)], ids=["keyed", "keyless"], ) async def test_native_socket_checks_token_limits_before_every_later_turn_of_a_keyed_socket( @@ -2299,7 +2314,9 @@ class TestWebSocketTurnAdmission: monkeypatch: pytest.MonkeyPatch, keyed: bool, forwarded: list[str], - refusals: list[tuple[int, str]], + refused: bool, + refusal: Exception, + refusal_frame: tuple[int, str], ) -> None: from unittest.mock import patch @@ -2307,7 +2324,7 @@ class TestWebSocketTurnAdmission: from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.proxy._types import UserAPIKeyAuth - token_limits: Final = _TokenLimits(_out_of_tokens(), None) + token_limits: Final = _TokenLimits(refusal, None) monkeypatch.setattr(litellm, "callbacks", [token_limits]) provider: Final = _ProviderResponsesSocket() config: Final = MagicMock(spec=OpenAIResponsesAPIConfig) @@ -2335,7 +2352,7 @@ class TestWebSocketTurnAdmission: assert [frame["input"] for frame in provider.sent] == forwarded assert token_limits.checked == ([(user_api_key_dict, "gpt-5.4")] * 2 if keyed else []) - assert [(status, error["type"]) for status, error in _errors(websocket)] == refusals + assert [(status, error["type"]) for status, error in _errors(websocket)] == ([refusal_frame] if refused else []) @pytest.mark.asyncio async def test_native_relay_runs_the_guardrail_before_the_deployment_counts_a_turn(self):