From b9d2fd0ee9ec66b39a0c3e33a6bc32299019e2c5 Mon Sep 17 00:00:00 2001 From: Fahima Mokhtari Date: Fri, 14 Aug 2026 19:39:35 +0100 Subject: [PATCH] fix(exception_mapping): bare 429 in an error body no longer outranks the status code (#36705) is_error_str_rate_limit treats any standalone 429 in the stringified exception as a rate limit, and for openai-compatible providers that check runs before the status-code branch. Providers echo the request back in validation errors, so a 400 whose body happens to contain a 429 comes out as RateLimitError. Tokenised prompts hit this routinely, since 429 is an ordinary token id (" that" in several tokenisers) and an echoed prompt_token_ids array is enough: {"error":{"message":"`tools` must not be an empty array", "type":"invalid_request_error","code":400}, "prompt_token_ids":[9906,429,1234]} The mislabel is not cosmetic. RateLimitError tells callers and routers to retry, so a request that cannot succeed gets replayed, and the failure is booked against provider throttling rather than the caller. Against DeepInfra, one recurring 400 ("`tools` must not be an empty array") came back as a rate limit in 77 of 198 occurrences, the split depending only on whether the echoed prompt contained 429. 16482 narrowed '"429" in error_str' to \b429\b after a false positive on 'asbjdad429addad'. Word boundaries cannot separate a real 429 from a token id, so the same class of false positive survives. is_error_str_rate_limit now takes an optional status_code, and the bare-number branch fires only when no explicit status contradicts it. The status is read off an arbitrary exception, so a non-integer is treated as unknown and left to the existing behaviour. The repo has a single call site. The phrase branches are untouched, so a provider reporting a real rate limit in the message text under a non-429 status still maps to RateLimitError (11455). This is not "status code wins". Tests cover the matcher (suppressed under a 400; still detected with no status, None, 429, or a non-integer status; phrase honoured under a 400) and exception_type end to end (400 with 429 in the echoed body -> BadRequestError, real 429 -> RateLimitError). Reverting the source change fails the latter. --- .../exception_mapping_utils.py | 15 +++- .../test_exception_mapping_utils.py | 82 +++++++++++++++++++ 2 files changed, 93 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index bad8e93e0c5..d23466938f2 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -34,12 +34,16 @@ class ExceptionCheckers: """ @staticmethod - def is_error_str_rate_limit(error_str: str) -> bool: + def is_error_str_rate_limit(error_str: str, status_code: int | None = None) -> bool: """ Check if an error string indicates a rate limit error. Args: error_str: The error string to check + status_code: The HTTP status the provider returned, when known. Gates only the + bare-number branch: providers echo the request back in validation errors and + 429 is an ordinary token id, so an echoed prompt can put a standalone 429 in + the body of a 400. The phrase branches stay ungated (#11455). Returns: True if the error indicates a rate limit, False otherwise @@ -47,8 +51,9 @@ class ExceptionCheckers: if not isinstance(error_str, str): return False - # Only treat 429 as a rate limit signal when it appears as a standalone token - if re.search(r"\b429\b", error_str): + # A standalone 429 counts unless the provider's own status says otherwise. The + # status is read off an arbitrary exception, so a non-integer means "unknown". + if re.search(r"\b429\b", error_str) and (not isinstance(status_code, int) or status_code == 429): return True _error_str_lower: Final = error_str.lower() @@ -280,7 +285,9 @@ def _map_openai_exception( else: exception_provider = custom_llm_provider[0].upper() + custom_llm_provider[1:] + "Exception" - if ExceptionCheckers.is_error_str_rate_limit(error_str): + if ExceptionCheckers.is_error_str_rate_limit( + error_str, status_code=getattr(original_exception, "status_code", None) + ): raise RateLimitError( message=f"RateLimitError: {exception_provider} - {message}", model=model, diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index 1fcee1b1c42..d5676aaf288 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -133,6 +133,40 @@ class TestExceptionCheckers: result = ExceptionCheckers.is_error_str_rate_limit(error_str) assert result is True + def test_bare_429_in_body_is_ignored_when_status_code_says_otherwise(self): + """A 429 echoed back inside a 400's body is not a rate limit. + + Word boundaries don't help: 429 is an ordinary token id (" that" in several + tokenisers), so an echoed prompt_token_ids array reads as a standalone 429. + """ + error_str = ( + '{"error":{"message":"`tools` must not be an empty array",' + '"type":"invalid_request_error"},' + '"prompt_token_ids":[9906,429,1234]}' + ) + assert ExceptionCheckers.is_error_str_rate_limit(error_str, status_code=400) is False + + def test_bare_429_still_detected_without_a_status_code(self): + """With no status available, a standalone 429 still counts (unchanged behaviour).""" + + assert ExceptionCheckers.is_error_str_rate_limit("HTTP 429 Too Many Requests") is True + assert ExceptionCheckers.is_error_str_rate_limit("HTTP 429 Too Many Requests", status_code=None) is True + assert ExceptionCheckers.is_error_str_rate_limit("HTTP 429 Too Many Requests", status_code=429) is True + + def test_non_integer_status_code_does_not_suppress_bare_429(self): + """A non-integer status counts as unknown, not as a contradiction.""" + + assert ExceptionCheckers.is_error_str_rate_limit("HTTP 429 Too Many Requests", status_code="not-an-int") is True + + def test_rate_limit_phrase_is_honoured_under_a_non_429_status(self): + """Phrase matching stays ungated: some providers report a real rate limit in + the text under a non-429 status (#11455).""" + + assert ( + ExceptionCheckers.is_error_str_rate_limit("FireworksException - rate limit exceeded", status_code=400) + is True + ) + def test_is_azure_content_policy_violation_error_with_policy_violation_text(self): """Test detection of Azure content policy violation with explicit policy violation text""" @@ -300,6 +334,54 @@ def test_lemonade_context_window_error_mapping(): assert excinfo.value.model == model +def test_openai_compatible_400_with_bare_429_in_body_maps_to_bad_request(): + """A provider 400 whose echoed body contains a 429 must stay a 400. + + ``is_error_str_rate_limit`` runs before the status-code branch for + openai-compatible providers, so a validation error echoing the request back came + out as RateLimitError, which tells the caller to retry a request that cannot + succeed and books the failure against provider throttling. + """ + error_message = ( + '{"error":{"message":"`tools` must not be an empty array",' + '"type":"invalid_request_error","code":400},' + '"prompt_token_ids":[9906,429,1234]}' + ) + original_exception = OpenAIError( + status_code=400, + message=error_message, + headers={}, + ) + + with pytest.raises(litellm.BadRequestError) as excinfo: + exception_type( + model="deepseek-ai/DeepSeek-V3", + original_exception=original_exception, + custom_llm_provider="deepinfra", + ) + + assert excinfo.value.status_code == 400 + assert excinfo.value.llm_provider == "deepinfra" + + +def test_openai_compatible_429_still_maps_to_rate_limit(): + """A real 429 still maps to RateLimitError.""" + original_exception = OpenAIError( + status_code=429, + message='{"error":{"message":"Too Many Requests","type":"rate_limit_error"}}', + headers={}, + ) + + with pytest.raises(litellm.RateLimitError) as excinfo: + exception_type( + model="deepseek-ai/DeepSeek-V3", + original_exception=original_exception, + custom_llm_provider="deepinfra", + ) + + assert excinfo.value.status_code == 429 + + @pytest.mark.parametrize( "error_message", [