diff --git a/litellm/__init__.py b/litellm/__init__.py index 078c4348206..462f89c29e4 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1301,6 +1301,7 @@ from .exceptions import ( ImageFetchError, NotFoundError, RateLimitError, + ParallelRequestLimitError, ServiceUnavailableError, OpenAIError, ContextWindowExceededError, diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 77fb9c1faef..b4230ecdf65 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -353,6 +353,49 @@ class RateLimitError(openai.RateLimitError): # type: ignore return _message +class ParallelRequestLimitError(RateLimitError): # type: ignore + def __init__( + self, + message: str, + llm_provider: Optional[str] = "litellm", + model: Optional[str] = "unknown", + headers: Optional[dict] = None, + response: Optional[httpx.Response] = None, + litellm_debug_info: Optional[str] = None, + max_retries: Optional[int] = None, + num_retries: Optional[int] = None, + ): + # Store headers for later access (similar to FastAPI HTTPException) + self.headers = headers or {} + + # Create a response with custom headers if provided + if response is None: + response_headers = headers + response = httpx.Response( + status_code=429, + headers=response_headers, + request=httpx.Request( + method="POST", + url="https://litellm.ai/parallel-request-limiter", + ), + ) + + # Initialize parent with appropriate defaults for parallel request limiting + super().__init__( + message=message, + llm_provider=llm_provider or "litellm", + model=model or "unknown", + response=response, + litellm_debug_info=litellm_debug_info, + max_retries=max_retries, + num_retries=num_retries, + ) + + # Update the message prefix to be more specific + self.message = "litellm.ParallelRequestLimitError: {}".format(message) + self.detail = message # Store original detail for FastAPI compatibility + + # sub class of rate limit error - meant to give more granularity for error handling context window exceeded errors class ContextWindowExceededError(BadRequestError): # type: ignore def __init__( @@ -748,6 +791,7 @@ LITELLM_EXCEPTION_TYPES = [ Timeout, PermissionDeniedError, RateLimitError, + ParallelRequestLimitError, ContextWindowExceededError, RejectedRequestError, ContentPolicyViolationError, diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index eda380b5165..a1416c34568 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -24,6 +24,7 @@ from fastapi import HTTPException from litellm import DualCache from litellm._logging import verbose_proxy_logger +from litellm.exceptions import ParallelRequestLimitError from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject @@ -702,9 +703,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): f"Limit resets at: {reset_time_formatted}" ) - raise HTTPException( - status_code=429, - detail=detail, + raise ParallelRequestLimitError( + message=detail, headers={ "retry-after": str(self.window_size), "rate_limit_type": str(status["rate_limit_type"]), diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index 511eb5bbb89..d25e2638537 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -14,6 +14,7 @@ from fastapi import HTTPException import litellm from litellm import Router from litellm.caching.caching import DualCache +from litellm.exceptions import ParallelRequestLimitError from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.hooks.parallel_request_limiter_v3 import ( _PROXY_MaxParallelRequestsHandler_v3 as _PROXY_MaxParallelRequestsHandler, @@ -92,7 +93,7 @@ async def test_sliding_window_rate_limit_v3(monkeypatch): ) # Fourth request should fail (counter would be 4, limit is 3, so 4 > 3) - with pytest.raises(HTTPException) as exc_info: + with pytest.raises(ParallelRequestLimitError) as exc_info: await parallel_request_handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=local_cache, @@ -374,7 +375,7 @@ async def test_normal_router_call_tpm_v3(monkeypatch, rate_limit_object): await local_cache.async_increment_cache(key=counter_key, value=15, ttl=2) # Use up most of our 10 token limit # Make another request to test rate limiting - this should fail as we've consumed tokens - with pytest.raises(HTTPException) as exc_info: + with pytest.raises(ParallelRequestLimitError) as exc_info: await parallel_request_handler.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=local_cache, @@ -777,7 +778,7 @@ async def test_tpm_api_key_rate_limits_v3(): async def mock_should_rate_limit(descriptors, **kwargs): nonlocal captured_descriptors captured_descriptors = descriptors - # Return Error response to ensure HTTPException + # Return Error response to ensure ParallelRequestLimitError return { "overall_code": "OVER_LIMIT", "statuses": [{'code': 'OK', 'current_limit': 2, 'limit_remaining': 1, 'rate_limit_type': 'requests', 'descriptor_key': 'model_per_key'}, @@ -795,7 +796,7 @@ async def test_tpm_api_key_rate_limits_v3(): data={"model": model}, call_type="", ) - except HTTPException as e: + except ParallelRequestLimitError as e: error=e assert e.status_code == 429 assert "rate_limit_type" in e.headers @@ -853,7 +854,7 @@ async def test_rpm_api_key_rate_limits_v3(): async def mock_should_rate_limit(descriptors, **kwargs): nonlocal captured_descriptors captured_descriptors = descriptors - # Return Error response to ensure HTTPException + # Return Error response to ensure ParallelRequestLimitError return { "overall_code": "OVER_LIMIT", "statuses": [{'code': 'OVER_LIMIT', 'current_limit': 2, 'limit_remaining': -2, 'rate_limit_type': 'requests', 'descriptor_key': 'model_per_key'}, @@ -871,7 +872,7 @@ async def test_rpm_api_key_rate_limits_v3(): data={"model": model}, call_type="", ) - except HTTPException as e: + except ParallelRequestLimitError as e: error=e assert e.status_code == 429 assert "rate_limit_type" in e.headers @@ -922,7 +923,7 @@ async def test_team_member_rate_limits_v3(): async def mock_should_rate_limit(descriptors, **kwargs): nonlocal captured_descriptors captured_descriptors = descriptors - # Return OK response to avoid HTTPException + # Return OK response to avoid ParallelRequestLimitError return { "overall_code": "OK", "statuses": []