diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 85c232ad3f4..9c625b3c377 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -41,6 +41,100 @@ from ..exceptions import ( error_response_without_credentials, ) +_VALIDATION_STRUCTURED_MARKERS: Final = frozenset( + { + "bad_request", + "invalid_argument", + "invalid_request", + "invalid_request_error", + "validation", + "validation_error", + "validation_exception", + } +) +_STRUCTURED_ERROR_SIGNAL_KEYS: Final = ( + "code", + "error_code", + "status", + "status_code", + "type", + "error_type", + "__type", +) +_ERROR_PAYLOAD_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + + +def _normalise_structured_marker(value: object) -> str: + """Normalize provider code/type values for marker comparison.""" + if not isinstance(value, (int, str)): + return "" + value_string: Final = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", str(value)) + return re.sub(r"[^a-z0-9]+", "_", value_string.lower()).strip("_") + + +def _parse_structured_error_payload(error_text: str) -> Mapping[str, object] | None: + """Parse a JSON error body, including SDK prefixes around the JSON.""" + start: Final = error_text.find("{") + end: Final = error_text.rfind("}") + candidates: Final = (error_text.strip(),) + ((error_text[start : end + 1],) if start >= 0 and end > start else ()) + + for candidate in candidates: + try: + return _ERROR_PAYLOAD_ADAPTER.validate_json(candidate) + except ValidationError: + continue + return None + + +def _response_json_payload(response: httpx.Response | None) -> Mapping[str, object] | None: + if response is None: + return None + try: + return _ERROR_PAYLOAD_ADAPTER.validate_json(response.content) + except (ValidationError, httpx.ResponseNotRead): + return None + + +def _error_payload_mapping(payload: object) -> Mapping[str, object] | None: + try: + return _ERROR_PAYLOAD_ADAPTER.validate_python(payload) + except ValidationError: + return None + + +def _error_object_has_validation_marker(error_object: Mapping[str, object]) -> bool: + return any( + _normalise_structured_marker(error_object.get(key)) in _VALIDATION_STRUCTURED_MARKERS + for key in _STRUCTURED_ERROR_SIGNAL_KEYS + ) + + +def _payload_has_validation_marker(payload: Mapping[str, object]) -> bool: + nested_error: Final = _error_payload_mapping(payload.get("error")) + return _error_object_has_validation_marker(payload) or ( + nested_error is not None and _error_object_has_validation_marker(nested_error) + ) + + +def _structured_validation_signal( + *, + error_str: str, + response: httpx.Response | None = None, + error_body: object | None = None, +) -> bool: + """Return whether the provider supplies an explicit validation signal.""" + payloads: Final = tuple( + payload + for payload in ( + _error_payload_mapping(error_body), + _response_json_payload(response), + _parse_structured_error_payload(error_str), + ) + if payload is not None + ) + + return any(_payload_has_validation_marker(payload) for payload in payloads) + class ExceptionCheckers: """ @@ -48,7 +142,13 @@ class ExceptionCheckers: """ @staticmethod - def is_error_str_rate_limit(error_str: str, status_code: int | None = None) -> bool: + def is_error_str_rate_limit( + error_str: str, + status_code: int | None = None, + *, + response: httpx.Response | None = None, + error_body: object | None = None, + ) -> bool: """ Check if an error string indicates a rate limit error. @@ -57,7 +157,10 @@ class ExceptionCheckers: status_code: The HTTP status the provider returned, when known. Gates only the bare-number branch: providers echo the request back in validation errors and 429 is an ordinary token id, so an echoed prompt can put a standalone 429 in - the body of a 400. The phrase branches stay ungated (#11455). + the body of a 400. The legacy rate-limit phrase stays ungated (#11455). + New quota phrases require no structured validation signal. + response: The provider response, when available, for structured error fields. + error_body: The parsed provider error body, when available. Returns: True if the error indicates a rate limit, False otherwise @@ -76,13 +179,34 @@ class ExceptionCheckers: if re.search(r"rate[\s_\-]*limit", _error_str_lower): return True - ####################################### - # Mistral API returns this error string - ######################################### - if "service tier capacity exceeded" in _error_str_lower: - return True + if _structured_validation_signal( + error_str=error_str, + response=response, + error_body=error_body, + ): + return False - return False + # Common provider rate-limit and quota exhaustion phrases across Gemini, Bedrock, Mistral, Azure, and Anthropic + rate_limit_patterns: Final = ( + "service tier capacity exceeded", + "too many requests", + "quota exceeded", + "quota_exceeded", + "insufficient_quota", + "exceeded your current quota", + "resource_exhausted", + "resource has been exhausted", + "request_rate_too_large", + "provisioned throughput exceeded", + "capacity temporarily exceeded", + "concurrency limit reached", + "concurrent requests limit exceeded", + "requests per minute exceeded", + "tokens per minute exceeded", + "requests per day exceeded", + "tokens per day exceeded", + ) + return any(pattern in _error_str_lower for pattern in rate_limit_patterns) @staticmethod def is_error_str_context_window_exceeded(error_str: str) -> bool: @@ -360,7 +484,10 @@ def _map_openai_exception( exception_provider = custom_llm_provider[0].upper() + custom_llm_provider[1:] + "Exception" if ExceptionCheckers.is_error_str_rate_limit( - error_str, status_code=getattr(original_exception, "status_code", None) + error_str, + status_code=getattr(original_exception, "status_code", None), + response=getattr(original_exception, "response", None), + error_body=getattr(original_exception, "body", None), ): raise RateLimitError( message=f"RateLimitError: {exception_provider} - {message}", @@ -614,6 +741,9 @@ def _map_anthropic_exception( exception_provider: str, extra_information: str, ) -> None: + error_payload: Final = _parse_structured_error_payload(error_str) + structured_error: Final = _error_payload_mapping(error_payload.get("error")) if error_payload is not None else None + structured_message: Final = structured_error.get("message") if structured_error is not None else None if ( "prompt is too long" in error_str or "prompt: length" in error_str @@ -630,6 +760,25 @@ def _map_anthropic_exception( model=model, llm_provider="anthropic", ) + elif getattr(original_exception, "status_code", None) != 429 and ( + ( + isinstance(structured_message, str) + and structured_message.startswith("You have reached your specified API usage limits.") + ) + or ExceptionCheckers.is_error_str_rate_limit( + error_str, + status_code=getattr(original_exception, "status_code", None), + response=getattr(original_exception, "response", None), + error_body=getattr(original_exception, "body", None), + ) + ): + raise RateLimitError( + message=f"AnthropicException - {error_str}", + model=model, + llm_provider="anthropic", + response=getattr(original_exception, "response", None), + litellm_debug_info=extra_information, + ) if "Invalid API Key" in error_str: raise AuthenticationError( message=f"AnthropicError - {error_str}", @@ -1007,6 +1156,19 @@ def _map_bedrock_exception( llm_provider="bedrock", response=getattr(original_exception, "response", None), ) + elif getattr(original_exception, "status_code", None) != 429 and ExceptionCheckers.is_error_str_rate_limit( + error_str, + status_code=getattr(original_exception, "status_code", None), + response=getattr(original_exception, "response", None), + error_body=getattr(original_exception, "body", None), + ): + raise RateLimitError( + message=f"BedrockException - {error_str}", + model=model, + llm_provider="bedrock", + response=getattr(original_exception, "response", None), + litellm_debug_info=extra_information, + ) elif "Conversation blocks and tool result blocks cannot be provided in the same turn." in error_str: raise BadRequestError( message=f"BedrockException - {error_str}\n. Enable 'litellm.modify_params=True' (for PROXY do: `litellm_settings::modify_params: True`) to insert a dummy assistant message and fix this error.", @@ -1354,11 +1516,25 @@ def _map_vertex_exception( ), ) elif ( - "429 Quota exceeded" in error_str - or "Quota exceeded for" in error_str - or "Resource exhausted" in error_str - or "IndexError: list index out of range" in error_str - or "429 Unable to submit request because the service is temporarily out of capacity." in error_str + getattr(original_exception, "status_code", None) != 429 + and not _structured_validation_signal( + error_str=error_str, + response=getattr(original_exception, "response", None), + error_body=getattr(original_exception, "body", None), + ) + and ( + "429 Quota exceeded" in error_str + or "Quota exceeded for" in error_str + or "Resource exhausted" in error_str + or "IndexError: list index out of range" in error_str + or "429 Unable to submit request because the service is temporarily out of capacity." in error_str + or ExceptionCheckers.is_error_str_rate_limit( + error_str, + status_code=getattr(original_exception, "status_code", None), + response=getattr(original_exception, "response", None), + error_body=getattr(original_exception, "body", None), + ) + ) ): raise RateLimitError( message=f"litellm.RateLimitError: {custom_llm_provider}Exception - {error_str}", @@ -1880,6 +2056,19 @@ def _map_together_ai_exception( model=model, llm_provider="together_ai", ) + elif getattr(original_exception, "status_code", None) != 429 and ExceptionCheckers.is_error_str_rate_limit( + error_str, + status_code=getattr(original_exception, "status_code", None), + response=getattr(original_exception, "response", None), + error_body=getattr(original_exception, "body", None), + ): + raise RateLimitError( + message=f"TogetherAIException - {error_str}", + model=model, + llm_provider="together_ai", + response=getattr(original_exception, "response", None), + litellm_debug_info=extra_information, + ) elif ( "error" in error_response and "API key doesn't match expected format." in error_response["error"] @@ -2151,6 +2340,19 @@ def _map_azure_exception( response=response, body=getattr(original_exception, "body", None), ) + elif getattr(original_exception, "status_code", None) != 429 and ExceptionCheckers.is_error_str_rate_limit( + error_str, + status_code=getattr(original_exception, "status_code", None), + response=getattr(original_exception, "response", None), + error_body=getattr(original_exception, "body", None), + ): + raise RateLimitError( + message=f"AzureException RateLimitError - {message}", + llm_provider="azure", + model=model, + litellm_debug_info=extra_information, + response=getattr(original_exception, "response", None), + ) elif "invalid_request_error" in error_str and getattr(original_exception, "status_code", None) in (None, 400): raise BadRequestError( message=f"AzureException BadRequestError - {message}", diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 2b1e4a5eed3..b64f49ebe39 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -320,6 +320,52 @@ class TestExceptionCheckers: is True ) + @pytest.mark.parametrize("response_text", ["not-json", '["invalid_request_error"]']) + def test_rate_limit_classifier_falls_back_when_response_is_not_an_object(self, response_text): + """Unusable response JSON must not disable the legacy phrase fallback.""" + + response = httpx.Response( + status_code=400, + request=httpx.Request("POST", "https://example.invalid"), + text=response_text, + ) + assert ( + ExceptionCheckers.is_error_str_rate_limit( + "Provider validation failed: too many requests", + status_code=400, + response=response, + ) + is True + ) + + def test_rate_limit_classifier_leaves_unread_stream_available(self): + response: Final = httpx.Response( + status_code=400, + stream=httpx.ByteStream(b'{"error":{"type":"invalid_request_error"}}'), + ) + assert ( + ExceptionCheckers.is_error_str_rate_limit("too many requests", status_code=400, response=response) is True + ) + assert response.is_stream_consumed is False + assert response.read() == b'{"error":{"type":"invalid_request_error"}}' + + def test_rate_limit_provider_specific_phrases(self): + """Test that common vendor rate-limit and quota exhaustion phrases are properly recognized.""" + vendor_phrases: Final = ( + "GeminiException - Resource has been exhausted (e.g. check quota)", + "VertexAIException - resource_exhausted: quota exceeded for default tier", + "OpenAIException - insufficient_quota: You exceeded your current quota", + "BedrockException - provisioned throughput exceeded for model", + "DatabricksException - request_rate_too_large", + "TogetherAIException - concurrency limit reached: max 20 concurrent requests", + "GroqException - tokens per minute exceeded (TPM limit)", + "AnthropicException - capacity temporarily exceeded", + ) + for phrase in vendor_phrases: + assert ExceptionCheckers.is_error_str_rate_limit(phrase, status_code=400) is True, ( + f"Failed to recognize rate limit in phrase: {phrase}" + ) + def test_is_azure_content_policy_violation_error_with_policy_violation_text(self): """Test detection of Azure content policy violation with explicit policy violation text""" @@ -1582,6 +1628,48 @@ PROVIDERS_THAT_RECOGNISE_A_CONTENT_POLICY_BLOCK = ( ) +def test_anthropic_usage_limit_under_http_400_is_a_rate_limit(quiet_exception_mapping): + from litellm.llms.anthropic.common_utils import AnthropicError + + body: Final = { + "type": "error", + "error": { + "type": "invalid_request_error", + "message": ( + "You have reached your specified API usage limits. You will regain access on 2026-09-01 at 00:00 UTC." + ), + }, + } + original: Final = AnthropicError(status_code=400, message=json.dumps(body)) + with pytest.raises(litellm.RateLimitError) as raised: + exception_type(model="test-model", original_exception=original, custom_llm_provider="anthropic") + + assert raised.value.llm_provider == "anthropic" + assert raised.value.status_code == 429 + + +@pytest.mark.parametrize( + "message", + ( + "Invalid request: You have reached your specified API usage limits.", + "Invalid value: too many requests", + ), +) +def test_anthropic_usage_limit_echo_keeps_bad_request(message, quiet_exception_mapping): + from litellm.llms.anthropic.common_utils import AnthropicError + + body: Final = { + "type": "error", + "error": {"type": "invalid_request_error", "message": message}, + "request": {"message": "You have reached your specified API usage limits."}, + } + original: Final = AnthropicError(status_code=400, message=json.dumps(body)) + with pytest.raises(litellm.BadRequestError) as raised: + exception_type(model="test-model", original_exception=original, custom_llm_provider="anthropic") + + assert raised.value.status_code == 400 + + class _UpstreamErrorWithMessage(_UpstreamHTTPError): def __init__(self, message: str, status_code: int) -> None: super().__init__(status_code=status_code) @@ -1590,6 +1678,129 @@ class _UpstreamErrorWithMessage(_UpstreamHTTPError): self.response = httpx.Response(status_code=status_code, request=self.request, text=message) +@pytest.mark.parametrize( + ("provider", "error_message"), + [ + ("gemini", "GeminiException - resource has been exhausted (e.g. check quota)"), + ("vertex_ai", "VertexAIException - resource_exhausted: quota exceeded for default tier"), + ("bedrock", "BedrockException - provisioned throughput exceeded for model"), + ("azure", "AzureException - request_rate_too_large"), + ("anthropic", "AnthropicException - capacity temporarily exceeded"), + ], +) +def test_provider_rate_limit_phrases_reach_dispatch_boundary(provider, error_message, quiet_exception_mapping): + """Vendor quota messages must reach the provider mapper as RateLimitError.""" + with pytest.raises(litellm.RateLimitError) as raised: + exception_type( + model="test-model", + original_exception=_UpstreamErrorWithMessage(error_message, 400), + custom_llm_provider=provider, + ) + + assert raised.value.llm_provider == provider + + +@pytest.mark.parametrize( + ("provider", "error_message"), + [ + ( + "gemini", + '{"error": {"code": 400, "status": "RESOURCE_EXHAUSTED", "message": "quota exceeded"}}', + ), + ( + "vertex_ai", + '{"error": {"code": 400, "status": "RESOURCE_EXHAUSTED", "message": "quota exceeded"}}', + ), + ( + "bedrock", + '{"__type": "ProvisionedThroughputExceededException", "message": "provisioned throughput exceeded"}', + ), + ( + "azure", + '{"error": {"code": "429", "type": "rate_limit_error", "message": "too many requests"}}', + ), + ( + "anthropic", + 'AnthropicException - b\'{"type": "error", "error": {"type": "rate_limit_error", "message": "too many requests"}}\'', + ), + ], +) +def test_structured_provider_rate_limit_reaches_dispatch_boundary(provider, error_message, quiet_exception_mapping): + """Structured provider throttling signals remain RateLimitError on HTTP 400.""" + with pytest.raises(litellm.RateLimitError) as raised: + exception_type( + model="test-model", + original_exception=_UpstreamErrorWithMessage(error_message, 400), + custom_llm_provider=provider, + ) + + assert raised.value.llm_provider == provider + + +@pytest.mark.parametrize( + ("provider", "error_message", "error_body"), + [ + ( + "gemini", + '{"error": {"code": 400, "status": "INVALID_ARGUMENT", "message": "Invalid value: too many requests"}}', + None, + ), + ( + "vertex_ai", + '{"error": {"code": 400, "status": "INVALID_ARGUMENT", "message": "Invalid request: Quota exceeded for the echoed field"}}', + None, + ), + ( + "bedrock", + '{"__type": "ValidationException", "message": "Invalid request: too many requests"}', + {"__type": "ValidationException", "status": {"echo": "too many requests"}}, + ), + ( + "azure", + '{"error": {"code": "invalid_request_error", "type": "invalid_request_error", "message": "Invalid request: too many requests"}}', + None, + ), + ( + "anthropic", + '{"type": "error", "error": {"type": "invalid_request_error", "message": "Invalid request: too many requests"}}', + None, + ), + ( + "openai", + "{'error': {'type': 'invalid_request_error', 'message': 'Invalid request: too many requests'}}", + { + "error": { + "type": "invalid_request_error", + "message": "Invalid request: too many requests", + } + }, + ), + ( + "together_ai", + "{'error': {'message': 'Invalid request: too many requests'}, 'error_type': 'validation'}", + {"error_type": "validation", "error": "Invalid request: too many requests"}, + ), + ], +) +def test_validation_echo_does_not_reach_rate_limit_boundary( + provider, error_message, error_body, quiet_exception_mapping +): + """A structured 400 validation echo must not become RateLimitError.""" + original_exception = _UpstreamErrorWithMessage(error_message, 400) + if error_body is not None: + original_exception.body = error_body + + with pytest.raises(openai.APIError) as raised: + exception_type( + model="test-model", + original_exception=original_exception, + custom_llm_provider=provider, + ) + + assert not isinstance(raised.value, litellm.RateLimitError) + assert raised.value.status_code == 400 + + @pytest.mark.parametrize("provider", PROVIDERS_WITH_A_HANDLER) def test_a_full_context_window_reaches_the_caller_as_the_router_needs_it(provider, quiet_exception_mapping): if provider in PROVIDERS_THAT_RECOGNISE_A_FULL_CONTEXT_WINDOW: