mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): send a fail-closed per-turn socket refusal with its 503 status
This commit is contained in:
parent
cbb4303d25
commit
24dfb223d0
4 changed files with 60 additions and 15 deletions
|
|
@ -49,6 +49,7 @@ from litellm.constants import (
|
|||
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY,
|
||||
)
|
||||
from litellm.cost_calculator import get_usage_object
|
||||
from litellm.exceptions import ServiceUnavailableError
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
||||
ensure_response_additional_headers,
|
||||
|
|
@ -3045,9 +3046,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
tpm_windows: Final = _socket_turn_tpm_windows(
|
||||
(*descriptors, *(d for d in agent_windows if (d["key"], d["value"]) not in built_scopes))
|
||||
)
|
||||
response: Final = await self.should_rate_limit(
|
||||
tpm_windows, parent_otel_span=user_api_key_dict.parent_otel_span, read_only=True, skip_parallel_check=True
|
||||
)
|
||||
try:
|
||||
response: Final = await self.should_rate_limit(
|
||||
tpm_windows,
|
||||
parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
read_only=True,
|
||||
skip_parallel_check=True,
|
||||
)
|
||||
except RateLimitUnverifiableError as unverifiable:
|
||||
raise ServiceUnavailableError(
|
||||
message=RATE_LIMIT_UNVERIFIABLE_MESSAGE, llm_provider="litellm_proxy", model=requested_model or ""
|
||||
) from unverifiable
|
||||
if response["overall_code"] == "OVER_LIMIT":
|
||||
self._handle_rate_limit_error(response, tpm_windows, requested_model)
|
||||
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ from litellm.constants import (
|
|||
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY,
|
||||
STREAM_SSE_DONE_STRING,
|
||||
)
|
||||
from litellm.exceptions import MidStreamFallbackError, RateLimitError
|
||||
from litellm.exceptions import MidStreamFallbackError, RateLimitError, ServiceUnavailableError
|
||||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
process_response_headers,
|
||||
|
|
@ -2205,16 +2205,18 @@ def turn_refusal_frame(refusal: RateLimitError) -> str:
|
|||
)
|
||||
|
||||
|
||||
def _turn_admission_error_frame(failure: Exception) -> str:
|
||||
return json.dumps({"type": "error", "status": 500, "error": {"type": "server_error", "message": str(failure)}})
|
||||
def _turn_admission_error_frame(failure: Exception, status: int) -> str:
|
||||
return json.dumps({"type": "error", "status": status, "error": {"type": "server_error", "message": str(failure)}})
|
||||
|
||||
|
||||
def _turn_rejection_frame(rejection: Exception) -> str:
|
||||
match rejection:
|
||||
case RateLimitError():
|
||||
return turn_refusal_frame(rejection)
|
||||
case ServiceUnavailableError():
|
||||
return _turn_admission_error_frame(rejection, rejection.status_code)
|
||||
case _:
|
||||
return _turn_admission_error_frame(rejection)
|
||||
return _turn_admission_error_frame(rejection, 500)
|
||||
|
||||
|
||||
RESPONSES_WS_LOGGED_EVENT_TYPES: Final = [
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from litellm import Router
|
|||
from litellm.caching.caching import DualCache
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY
|
||||
from litellm.exceptions import ServiceUnavailableError
|
||||
from litellm.integrations.custom_secret_manager import CustomSecretManager
|
||||
from litellm.litellm_core_utils import ptu_pricing
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -7926,6 +7927,22 @@ async def test_fail_closed_read_only_check_rejects_with_503_when_redis_counters_
|
|||
assert response["overall_code"] == "OK"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fail_closed_socket_turn_check_refuses_with_503_when_redis_counters_are_unreachable():
|
||||
auth: Final = UserAPIKeyAuth(api_key=hash_token("sk-fail-closed-socket-turn"), tpm_limit=1000)
|
||||
|
||||
with pytest.raises(ServiceUnavailableError) as exc:
|
||||
await _handler_with_redis(_UnreachableRedis(), fail_closed=True).enforce_token_limits_for_frame(
|
||||
user_api_key_dict=auth, requested_model="gpt-5.4"
|
||||
)
|
||||
assert exc.value.status_code == 503
|
||||
assert "fail_closed_rate_limit_enforcement" in str(exc.value)
|
||||
|
||||
await _handler_with_redis(_UnreachableRedis(), fail_closed=False).enforce_token_limits_for_frame(
|
||||
user_api_key_dict=auth, requested_model="gpt-5.4"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stored_counter_value, expected_code", [(1, "OK"), (2, "OVER_LIMIT"), (3, "OVER_LIMIT")])
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_only_check_reports_the_redis_counters_without_incrementing_them(
|
||||
|
|
|
|||
|
|
@ -1627,6 +1627,19 @@ def _out_of_tokens() -> Exception:
|
|||
return RateLimitError(message="key is out of tokens", llm_provider="", model="")
|
||||
|
||||
|
||||
def _counters_unverifiable() -> Exception:
|
||||
from litellm.exceptions import ServiceUnavailableError
|
||||
|
||||
return ServiceUnavailableError(message="counters could not be verified", llm_provider="litellm_proxy", model="")
|
||||
|
||||
|
||||
_TURN_REFUSALS: Final = pytest.mark.parametrize(
|
||||
"refusal, refusal_frame",
|
||||
[(_out_of_tokens(), (429, "rate_limit_exceeded")), (_counters_unverifiable(), (503, "server_error"))],
|
||||
ids=["out-of-tokens", "counters-unverifiable"],
|
||||
)
|
||||
|
||||
|
||||
class _ClientSocket:
|
||||
"""A client socket that, like Starlette's, stops receiving once the server closes it."""
|
||||
|
||||
|
|
@ -2225,8 +2238,9 @@ class TestWebSocketTurnAdmission:
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_TURN_REFUSALS
|
||||
async def test_managed_socket_checks_token_limits_before_every_later_turn_and_marks_each_turn(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
self, monkeypatch: pytest.MonkeyPatch, refusal: Exception, refusal_frame: tuple[int, str]
|
||||
) -> None:
|
||||
import litellm
|
||||
from litellm.constants import RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY
|
||||
|
|
@ -2253,7 +2267,7 @@ class TestWebSocketTurnAdmission:
|
|||
return _empty()
|
||||
|
||||
monkeypatch.setattr(litellm, "aresponses", fake_aresponses)
|
||||
token_limits: Final = _TokenLimits(_out_of_tokens(), None)
|
||||
token_limits: Final = _TokenLimits(refusal, None)
|
||||
monkeypatch.setattr(litellm, "callbacks", [token_limits])
|
||||
deployment_admission: Final = AsyncMock(return_value={})
|
||||
user_api_key_dict: Final = UserAPIKeyAuth(api_key="hashed-key", tpm_limit=1000)
|
||||
|
|
@ -2286,12 +2300,13 @@ class TestWebSocketTurnAdmission:
|
|||
assert [socket_call_id for _, socket_call_id in answered] == ["socket-open", "socket-open"]
|
||||
assert token_limits.checked == [(user_api_key_dict, "claude-ws"), (user_api_key_dict, "claude-ws")]
|
||||
assert deployment_admission.await_count == 1
|
||||
assert [(status, error["type"]) for status, error in _errors(websocket)] == [(429, "rate_limit_exceeded")]
|
||||
assert [(status, error["type"]) for status, error in _errors(websocket)] == [refusal_frame]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_TURN_REFUSALS
|
||||
@pytest.mark.parametrize(
|
||||
"keyed, forwarded, refusals",
|
||||
[(True, ["one", "three"], [(429, "rate_limit_exceeded")]), (False, ["one", "two", "three"], [])],
|
||||
"keyed, forwarded, refused",
|
||||
[(True, ["one", "three"], True), (False, ["one", "two", "three"], False)],
|
||||
ids=["keyed", "keyless"],
|
||||
)
|
||||
async def test_native_socket_checks_token_limits_before_every_later_turn_of_a_keyed_socket(
|
||||
|
|
@ -2299,7 +2314,9 @@ class TestWebSocketTurnAdmission:
|
|||
monkeypatch: pytest.MonkeyPatch,
|
||||
keyed: bool,
|
||||
forwarded: list[str],
|
||||
refusals: list[tuple[int, str]],
|
||||
refused: bool,
|
||||
refusal: Exception,
|
||||
refusal_frame: tuple[int, str],
|
||||
) -> None:
|
||||
from unittest.mock import patch
|
||||
|
||||
|
|
@ -2307,7 +2324,7 @@ class TestWebSocketTurnAdmission:
|
|||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
token_limits: Final = _TokenLimits(_out_of_tokens(), None)
|
||||
token_limits: Final = _TokenLimits(refusal, None)
|
||||
monkeypatch.setattr(litellm, "callbacks", [token_limits])
|
||||
provider: Final = _ProviderResponsesSocket()
|
||||
config: Final = MagicMock(spec=OpenAIResponsesAPIConfig)
|
||||
|
|
@ -2335,7 +2352,7 @@ class TestWebSocketTurnAdmission:
|
|||
|
||||
assert [frame["input"] for frame in provider.sent] == forwarded
|
||||
assert token_limits.checked == ([(user_api_key_dict, "gpt-5.4")] * 2 if keyed else [])
|
||||
assert [(status, error["type"]) for status, error in _errors(websocket)] == refusals
|
||||
assert [(status, error["type"]) for status, error in _errors(websocket)] == ([refusal_frame] if refused else [])
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_relay_runs_the_guardrail_before_the_deployment_counts_a_turn(self):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue