Merge 49cea41cdf into 523b87ec71

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -41,6 +41,100 @@ from ..exceptions import (
     error_response_without_credentials,
 )
 
+_VALIDATION_STRUCTURED_MARKERS: Final = frozenset(
+    {
+        "bad_request",
+        "invalid_argument",
+        "invalid_request",
+        "invalid_request_error",
+        "validation",
+        "validation_error",
+        "validation_exception",
+    }
+)
+_STRUCTURED_ERROR_SIGNAL_KEYS: Final = (
+    "code",
+    "error_code",
+    "status",
+    "status_code",
+    "type",
+    "error_type",
+    "__type",
+)
+_ERROR_PAYLOAD_ADAPTER: Final = TypeAdapter(Mapping[str, object])
+
+
+def _normalise_structured_marker(value: object) -> str:
+    """Normalize provider code/type values for marker comparison."""
+    if not isinstance(value, (int, str)):
+        return ""
+    value_string: Final = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", str(value))
+    return re.sub(r"[^a-z0-9]+", "_", value_string.lower()).strip("_")
+
+
+def _parse_structured_error_payload(error_text: str) -> Mapping[str, object] | None:
+    """Parse a JSON error body, including SDK prefixes around the JSON."""
+    start: Final = error_text.find("{")
+    end: Final = error_text.rfind("}")
+    candidates: Final = (error_text.strip(),) + ((error_text[start : end + 1],) if start >= 0 and end > start else ())
+
+    for candidate in candidates:
+        try:
+            return _ERROR_PAYLOAD_ADAPTER.validate_json(candidate)
+        except ValidationError:
+            continue
+    return None
+
+
+def _response_json_payload(response: httpx.Response | None) -> Mapping[str, object] | None:
+    if response is None:
+        return None
+    try:
+        return _ERROR_PAYLOAD_ADAPTER.validate_json(response.content)
+    except (ValidationError, httpx.ResponseNotRead):
+        return None
+
+
+def _error_payload_mapping(payload: object) -> Mapping[str, object] | None:
+    try:
+        return _ERROR_PAYLOAD_ADAPTER.validate_python(payload)
+    except ValidationError:
+        return None
+
+
+def _error_object_has_validation_marker(error_object: Mapping[str, object]) -> bool:
+    return any(
+        _normalise_structured_marker(error_object.get(key)) in _VALIDATION_STRUCTURED_MARKERS
+        for key in _STRUCTURED_ERROR_SIGNAL_KEYS
+    )
+
+
+def _payload_has_validation_marker(payload: Mapping[str, object]) -> bool:
+    nested_error: Final = _error_payload_mapping(payload.get("error"))
+    return _error_object_has_validation_marker(payload) or (
+        nested_error is not None and _error_object_has_validation_marker(nested_error)
+    )
+
+
+def _structured_validation_signal(
+    *,
+    error_str: str,
+    response: httpx.Response | None = None,
+    error_body: object | None = None,
+) -> bool:
+    """Return whether the provider supplies an explicit validation signal."""
+    payloads: Final = tuple(
+        payload
+        for payload in (
+            _error_payload_mapping(error_body),
+            _response_json_payload(response),
+            _parse_structured_error_payload(error_str),
+        )
+        if payload is not None
+    )
+
+    return any(_payload_has_validation_marker(payload) for payload in payloads)
+
 
 class ExceptionCheckers:
     """
@@ -48,7 +142,13 @@ class ExceptionCheckers:
     """
 
     @staticmethod
-    def is_error_str_rate_limit(error_str: str, status_code: int | None = None) -> bool:
+    def is_error_str_rate_limit(
+        error_str: str,
+        status_code: int | None = None,
+        *,
+        response: httpx.Response | None = None,
+        error_body: object | None = None,
+    ) -> bool:
         """
         Check if an error string indicates a rate limit error.
 
@@ -57,7 +157,10 @@ class ExceptionCheckers:
             status_code: The HTTP status the provider returned, when known. Gates only the
                 bare-number branch: providers echo the request back in validation errors and
                 429 is an ordinary token id, so an echoed prompt can put a standalone 429 in
-                the body of a 400. The phrase branches stay ungated (#11455).
+                the body of a 400. The legacy rate-limit phrase stays ungated (#11455).
+                New quota phrases require no structured validation signal.
+            response: The provider response, when available, for structured error fields.
+            error_body: The parsed provider error body, when available.
 
         Returns:
             True if the error indicates a rate limit, False otherwise
@@ -76,13 +179,34 @@ class ExceptionCheckers:
         if re.search(r"rate[\s_\-]*limit", _error_str_lower):
             return True
 
-        #######################################
-        # Mistral API returns this error string
-        #########################################
-        if "service tier capacity exceeded" in _error_str_lower:
-            return True
+        if _structured_validation_signal(
+            error_str=error_str,
+            response=response,
+            error_body=error_body,
+        ):
+            return False
 
-        return False
+        # Common provider rate-limit and quota exhaustion phrases across Gemini, Bedrock, Mistral, Azure, and Anthropic
+        rate_limit_patterns: Final = (
+            "service tier capacity exceeded",
+            "too many requests",
+            "quota exceeded",
+            "quota_exceeded",
+            "insufficient_quota",
+            "exceeded your current quota",
+            "resource_exhausted",
+            "resource has been exhausted",
+            "request_rate_too_large",
+            "provisioned throughput exceeded",
+            "capacity temporarily exceeded",
+            "concurrency limit reached",
+            "concurrent requests limit exceeded",
+            "requests per minute exceeded",
+            "tokens per minute exceeded",
+            "requests per day exceeded",
+            "tokens per day exceeded",
+        )
+        return any(pattern in _error_str_lower for pattern in rate_limit_patterns)
 
     @staticmethod
     def is_error_str_context_window_exceeded(error_str: str) -> bool:
@@ -360,7 +484,10 @@ def _map_openai_exception(
         exception_provider = custom_llm_provider[0].upper() + custom_llm_provider[1:] + "Exception"
 
     if ExceptionCheckers.is_error_str_rate_limit(
-        error_str, status_code=getattr(original_exception, "status_code", None)
+        error_str,
+        status_code=getattr(original_exception, "status_code", None),
+        response=getattr(original_exception, "response", None),
+        error_body=getattr(original_exception, "body", None),
     ):
         raise RateLimitError(
             message=f"RateLimitError: {exception_provider} - {message}",
@@ -614,6 +741,9 @@ def _map_anthropic_exception(
     exception_provider: str,
     extra_information: str,
 ) -> None:
+    error_payload: Final = _parse_structured_error_payload(error_str)
+    structured_error: Final = _error_payload_mapping(error_payload.get("error")) if error_payload is not None else None
+    structured_message: Final = structured_error.get("message") if structured_error is not None else None
     if (
         "prompt is too long" in error_str
         or "prompt: length" in error_str
@@ -630,6 +760,25 @@ def _map_anthropic_exception(
             model=model,
             llm_provider="anthropic",
         )
+    elif getattr(original_exception, "status_code", None) != 429 and (
+        (
+            isinstance(structured_message, str)
+            and structured_message.startswith("You have reached your specified API usage limits.")
+        )
+        or ExceptionCheckers.is_error_str_rate_limit(
+            error_str,
+            status_code=getattr(original_exception, "status_code", None),
+            response=getattr(original_exception, "response", None),
+            error_body=getattr(original_exception, "body", None),
+        )
+    ):
+        raise RateLimitError(
+            message=f"AnthropicException - {error_str}",
+            model=model,
+            llm_provider="anthropic",
+            response=getattr(original_exception, "response", None),
+            litellm_debug_info=extra_information,
+        )
     if "Invalid API Key" in error_str:
         raise AuthenticationError(
             message=f"AnthropicError - {error_str}",
@@ -1007,6 +1156,19 @@ def _map_bedrock_exception(
             llm_provider="bedrock",
             response=getattr(original_exception, "response", None),
         )
+    elif getattr(original_exception, "status_code", None) != 429 and ExceptionCheckers.is_error_str_rate_limit(
+        error_str,
+        status_code=getattr(original_exception, "status_code", None),
+        response=getattr(original_exception, "response", None),
+        error_body=getattr(original_exception, "body", None),
+    ):
+        raise RateLimitError(
+            message=f"BedrockException - {error_str}",
+            model=model,
+            llm_provider="bedrock",
+            response=getattr(original_exception, "response", None),
+            litellm_debug_info=extra_information,
+        )
     elif "Conversation blocks and tool result blocks cannot be provided in the same turn." in error_str:
         raise BadRequestError(
             message=f"BedrockException - {error_str}\n. Enable 'litellm.modify_params=True' (for PROXY do: `litellm_settings::modify_params: True`) to insert a dummy assistant message and fix this error.",
@@ -1354,11 +1516,25 @@ def _map_vertex_exception(
             ),
         )
     elif (
-        "429 Quota exceeded" in error_str
-        or "Quota exceeded for" in error_str
-        or "Resource exhausted" in error_str
-        or "IndexError: list index out of range" in error_str
-        or "429 Unable to submit request because the service is temporarily out of capacity." in error_str
+        getattr(original_exception, "status_code", None) != 429
+        and not _structured_validation_signal(
+            error_str=error_str,
+            response=getattr(original_exception, "response", None),
+            error_body=getattr(original_exception, "body", None),
+        )
+        and (
+            "429 Quota exceeded" in error_str
+            or "Quota exceeded for" in error_str
+            or "Resource exhausted" in error_str
+            or "IndexError: list index out of range" in error_str
+            or "429 Unable to submit request because the service is temporarily out of capacity." in error_str
+            or ExceptionCheckers.is_error_str_rate_limit(
+                error_str,
+                status_code=getattr(original_exception, "status_code", None),
+                response=getattr(original_exception, "response", None),
+                error_body=getattr(original_exception, "body", None),
+            )
+        )
     ):
         raise RateLimitError(
             message=f"litellm.RateLimitError: {custom_llm_provider}Exception - {error_str}",
@@ -1880,6 +2056,19 @@ def _map_together_ai_exception(
             model=model,
             llm_provider="together_ai",
         )
+    elif getattr(original_exception, "status_code", None) != 429 and ExceptionCheckers.is_error_str_rate_limit(
+        error_str,
+        status_code=getattr(original_exception, "status_code", None),
+        response=getattr(original_exception, "response", None),
+        error_body=getattr(original_exception, "body", None),
+    ):
+        raise RateLimitError(
+            message=f"TogetherAIException - {error_str}",
+            model=model,
+            llm_provider="together_ai",
+            response=getattr(original_exception, "response", None),
+            litellm_debug_info=extra_information,
+        )
     elif (
         "error" in error_response
         and "API key doesn't match expected format." in error_response["error"]
@@ -2151,6 +2340,19 @@ def _map_azure_exception(
             response=response,
             body=getattr(original_exception, "body", None),
         )
+    elif getattr(original_exception, "status_code", None) != 429 and ExceptionCheckers.is_error_str_rate_limit(
+        error_str,
+        status_code=getattr(original_exception, "status_code", None),
+        response=getattr(original_exception, "response", None),
+        error_body=getattr(original_exception, "body", None),
+    ):
+        raise RateLimitError(
+            message=f"AzureException RateLimitError - {message}",
+            llm_provider="azure",
+            model=model,
+            litellm_debug_info=extra_information,
+            response=getattr(original_exception, "response", None),
+        )
     elif "invalid_request_error" in error_str and getattr(original_exception, "status_code", None) in (None, 400):
         raise BadRequestError(
             message=f"AzureException BadRequestError - {message}",

tests/unit/litellm_core_utils/test_exception_mapping_utils.py

@@ -320,6 +320,52 @@ class TestExceptionCheckers:
             is True
         )
 
+    @pytest.mark.parametrize("response_text", ["not-json", '["invalid_request_error"]'])
+    def test_rate_limit_classifier_falls_back_when_response_is_not_an_object(self, response_text):
+        """Unusable response JSON must not disable the legacy phrase fallback."""
+
+        response = httpx.Response(
+            status_code=400,
+            request=httpx.Request("POST", "https://example.invalid"),
+            text=response_text,
+        )
+        assert (
+            ExceptionCheckers.is_error_str_rate_limit(
+                "Provider validation failed: too many requests",
+                status_code=400,
+                response=response,
+            )
+            is True
+        )
+
+    def test_rate_limit_classifier_leaves_unread_stream_available(self):
+        response: Final = httpx.Response(
+            status_code=400,
+            stream=httpx.ByteStream(b'{"error":{"type":"invalid_request_error"}}'),
+        )
+        assert (
+            ExceptionCheckers.is_error_str_rate_limit("too many requests", status_code=400, response=response) is True
+        )
+        assert response.is_stream_consumed is False
+        assert response.read() == b'{"error":{"type":"invalid_request_error"}}'
+
+    def test_rate_limit_provider_specific_phrases(self):
+        """Test that common vendor rate-limit and quota exhaustion phrases are properly recognized."""
+        vendor_phrases: Final = (
+            "GeminiException - Resource has been exhausted (e.g. check quota)",
+            "VertexAIException - resource_exhausted: quota exceeded for default tier",
+            "OpenAIException - insufficient_quota: You exceeded your current quota",
+            "BedrockException - provisioned throughput exceeded for model",
+            "DatabricksException - request_rate_too_large",
+            "TogetherAIException - concurrency limit reached: max 20 concurrent requests",
+            "GroqException - tokens per minute exceeded (TPM limit)",
+            "AnthropicException - capacity temporarily exceeded",
+        )
+        for phrase in vendor_phrases:
+            assert ExceptionCheckers.is_error_str_rate_limit(phrase, status_code=400) is True, (
+                f"Failed to recognize rate limit in phrase: {phrase}"
+            )
+
     def test_is_azure_content_policy_violation_error_with_policy_violation_text(self):
         """Test detection of Azure content policy violation with explicit policy violation text"""
 
@@ -1582,6 +1628,48 @@ PROVIDERS_THAT_RECOGNISE_A_CONTENT_POLICY_BLOCK = (
 )
 
 
+def test_anthropic_usage_limit_under_http_400_is_a_rate_limit(quiet_exception_mapping):
+    from litellm.llms.anthropic.common_utils import AnthropicError
+
+    body: Final = {
+        "type": "error",
+        "error": {
+            "type": "invalid_request_error",
+            "message": (
+                "You have reached your specified API usage limits. You will regain access on 2026-09-01 at 00:00 UTC."
+            ),
+        },
+    }
+    original: Final = AnthropicError(status_code=400, message=json.dumps(body))
+    with pytest.raises(litellm.RateLimitError) as raised:
+        exception_type(model="test-model", original_exception=original, custom_llm_provider="anthropic")
+
+    assert raised.value.llm_provider == "anthropic"
+    assert raised.value.status_code == 429
+
+
+@pytest.mark.parametrize(
+    "message",
+    (
+        "Invalid request: You have reached your specified API usage limits.",
+        "Invalid value: too many requests",
+    ),
+)
+def test_anthropic_usage_limit_echo_keeps_bad_request(message, quiet_exception_mapping):
+    from litellm.llms.anthropic.common_utils import AnthropicError
+
+    body: Final = {
+        "type": "error",
+        "error": {"type": "invalid_request_error", "message": message},
+        "request": {"message": "You have reached your specified API usage limits."},
+    }
+    original: Final = AnthropicError(status_code=400, message=json.dumps(body))
+    with pytest.raises(litellm.BadRequestError) as raised:
+        exception_type(model="test-model", original_exception=original, custom_llm_provider="anthropic")
+
+    assert raised.value.status_code == 400
+
+
 class _UpstreamErrorWithMessage(_UpstreamHTTPError):
     def __init__(self, message: str, status_code: int) -> None:
         super().__init__(status_code=status_code)
@@ -1590,6 +1678,129 @@ class _UpstreamErrorWithMessage(_UpstreamHTTPError):
         self.response = httpx.Response(status_code=status_code, request=self.request, text=message)
 
 
+@pytest.mark.parametrize(
+    ("provider", "error_message"),
+    [
+        ("gemini", "GeminiException - resource has been exhausted (e.g. check quota)"),
+        ("vertex_ai", "VertexAIException - resource_exhausted: quota exceeded for default tier"),
+        ("bedrock", "BedrockException - provisioned throughput exceeded for model"),
+        ("azure", "AzureException - request_rate_too_large"),
+        ("anthropic", "AnthropicException - capacity temporarily exceeded"),
+    ],
+)
+def test_provider_rate_limit_phrases_reach_dispatch_boundary(provider, error_message, quiet_exception_mapping):
+    """Vendor quota messages must reach the provider mapper as RateLimitError."""
+    with pytest.raises(litellm.RateLimitError) as raised:
+        exception_type(
+            model="test-model",
+            original_exception=_UpstreamErrorWithMessage(error_message, 400),
+            custom_llm_provider=provider,
+        )
+
+    assert raised.value.llm_provider == provider
+
+
+@pytest.mark.parametrize(
+    ("provider", "error_message"),
+    [
+        (
+            "gemini",
+            '{"error": {"code": 400, "status": "RESOURCE_EXHAUSTED", "message": "quota exceeded"}}',
+        ),
+        (
+            "vertex_ai",
+            '{"error": {"code": 400, "status": "RESOURCE_EXHAUSTED", "message": "quota exceeded"}}',
+        ),
+        (
+            "bedrock",
+            '{"__type": "ProvisionedThroughputExceededException", "message": "provisioned throughput exceeded"}',
+        ),
+        (
+            "azure",
+            '{"error": {"code": "429", "type": "rate_limit_error", "message": "too many requests"}}',
+        ),
+        (
+            "anthropic",
+            'AnthropicException - b\'{"type": "error", "error": {"type": "rate_limit_error", "message": "too many requests"}}\'',
+        ),
+    ],
+)
+def test_structured_provider_rate_limit_reaches_dispatch_boundary(provider, error_message, quiet_exception_mapping):
+    """Structured provider throttling signals remain RateLimitError on HTTP 400."""
+    with pytest.raises(litellm.RateLimitError) as raised:
+        exception_type(
+            model="test-model",
+            original_exception=_UpstreamErrorWithMessage(error_message, 400),
+            custom_llm_provider=provider,
+        )
+
+    assert raised.value.llm_provider == provider
+
+
+@pytest.mark.parametrize(
+    ("provider", "error_message", "error_body"),
+    [
+        (
+            "gemini",
+            '{"error": {"code": 400, "status": "INVALID_ARGUMENT", "message": "Invalid value: too many requests"}}',
+            None,
+        ),
+        (
+            "vertex_ai",
+            '{"error": {"code": 400, "status": "INVALID_ARGUMENT", "message": "Invalid request: Quota exceeded for the echoed field"}}',
+            None,
+        ),
+        (
+            "bedrock",
+            '{"__type": "ValidationException", "message": "Invalid request: too many requests"}',
+            {"__type": "ValidationException", "status": {"echo": "too many requests"}},
+        ),
+        (
+            "azure",
+            '{"error": {"code": "invalid_request_error", "type": "invalid_request_error", "message": "Invalid request: too many requests"}}',
+            None,
+        ),
+        (
+            "anthropic",
+            '{"type": "error", "error": {"type": "invalid_request_error", "message": "Invalid request: too many requests"}}',
+            None,
+        ),
+        (
+            "openai",
+            "{'error': {'type': 'invalid_request_error', 'message': 'Invalid request: too many requests'}}",
+            {
+                "error": {
+                    "type": "invalid_request_error",
+                    "message": "Invalid request: too many requests",
+                }
+            },
+        ),
+        (
+            "together_ai",
+            "{'error': {'message': 'Invalid request: too many requests'}, 'error_type': 'validation'}",
+            {"error_type": "validation", "error": "Invalid request: too many requests"},
+        ),
+    ],
+)
+def test_validation_echo_does_not_reach_rate_limit_boundary(
+    provider, error_message, error_body, quiet_exception_mapping
+):
+    """A structured 400 validation echo must not become RateLimitError."""
+    original_exception = _UpstreamErrorWithMessage(error_message, 400)
+    if error_body is not None:
+        original_exception.body = error_body
+
+    with pytest.raises(openai.APIError) as raised:
+        exception_type(
+            model="test-model",
+            original_exception=original_exception,
+            custom_llm_provider=provider,
+        )
+
+    assert not isinstance(raised.value, litellm.RateLimitError)
+    assert raised.value.status_code == 400
+
+
 @pytest.mark.parametrize("provider", PROVIDERS_WITH_A_HANDLER)
 def test_a_full_context_window_reaches_the_caller_as_the_router_needs_it(provider, quiet_exception_mapping):
     if provider in PROVIDERS_THAT_RECOGNISE_A_FULL_CONTEXT_WINDOW: