From fda99ecb418361442a12fa768558c4fe9d35f5e7 Mon Sep 17 00:00:00 2001 From: Cole McIntosh Date: Thu, 5 Jun 2025 15:47:25 -0600 Subject: [PATCH 1/2] Enhance exception mapping for Fireworks AI: add better handling for 429 status codes and text-based rate limit detection. Update tests to verify correct mapping to RateLimitError for both 429 and related error messages. --- .../exception_mapping_utils.py | 11 ++- tests/local_testing/test_exceptions.py | 68 ++++++++++++++++++- 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index c514ffd12f9..3a78519074f 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -274,7 +274,7 @@ def exception_type( # type: ignore # noqa: PLR0915 + "Exception" ) - if "429" in error_str: + if "429" in error_str or "rate limit" in error_str.lower(): exception_mapping_worked = True raise RateLimitError( message=f"RateLimitError: {exception_provider} - {message}", @@ -451,6 +451,15 @@ def exception_type( # type: ignore # noqa: PLR0915 response=getattr(original_exception, "response", None), litellm_debug_info=extra_information, ) + elif original_exception.status_code == 500: + exception_mapping_worked = True + raise InternalServerError( + message=f"InternalServerError: {exception_provider} - {message}", + model=model, + llm_provider=custom_llm_provider, + response=getattr(original_exception, "response", None), + litellm_debug_info=extra_information, + ) elif original_exception.status_code == 503: exception_mapping_worked = True raise ServiceUnavailableError( diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py index 32f0a4f84af..275172291bb 100644 --- a/tests/local_testing/test_exceptions.py +++ b/tests/local_testing/test_exceptions.py @@ -773,7 +773,7 @@ def test_litellm_predibase_exception(): @pytest.mark.parametrize( - "provider", ["predibase", "vertex_ai_beta", "anthropic", "databricks", "watsonx"] + "provider", ["predibase", "vertex_ai_beta", "anthropic", "databricks", "watsonx", "fireworks_ai"] ) def test_exception_mapping(provider): """ @@ -819,6 +819,72 @@ def test_exception_mapping(provider): pass +def test_fireworks_ai_429_exception_mapping(): + """ + Test that Fireworks AI 429 status codes are properly mapped to RateLimitError + instead of being converted to 400 BadRequestError. + + Related issue: https://github.com/BerriAI/litellm/issues/xxxxx + """ + import litellm + from litellm.llms.fireworks_ai.common_utils import FireworksAIException + + # Create a mock FireworksAIException with 429 status code + mock_exception = FireworksAIException( + status_code=429, + message="Rate limit exceeded", + headers={"retry-after": "60"} + ) + + try: + response = litellm.completion( + model="fireworks_ai/llama-v3p1-8b-instruct", + messages=[{"role": "user", "content": "Hello"}], + mock_response=mock_exception, + ) + pytest.fail("Expected RateLimitError to be raised") + except litellm.RateLimitError as e: + # This is the expected behavior - 429 should map to RateLimitError + assert e.status_code == 429 + assert "Rate limit exceeded" in str(e) + except Exception as e: + pytest.fail(f"Expected RateLimitError but got {type(e).__name__}: {e}") + + +def test_fireworks_ai_rate_limit_text_detection(): + """ + Test that Fireworks AI rate limit errors are detected by text matching when no 429 status code is present. + + This tests the exact scenario from the real failure where Fireworks AI returns: + {"error":{"object":"error","type":"invalid_request_error","message":"rate limit exceeded, please try again later"}} + + This should be detected as a RateLimitError, not a BadRequestError. + """ + import litellm + from litellm.llms.fireworks_ai.common_utils import FireworksAIException + + # Create a mock exception that simulates the exact Fireworks AI response + # with "invalid_request_error" type and "rate limit exceeded" message + mock_exception = FireworksAIException( + status_code=400, # Fireworks AI is returning 400 instead of 429 + message='{"error":{"object":"error","type":"invalid_request_error","message":"rate limit exceeded, please try again later"}}', + headers={} + ) + + try: + response = litellm.completion( + model="fireworks_ai/qwen3-235b-a22b", + messages=[{"role": "user", "content": "Hello"}], + mock_response=mock_exception, + ) + pytest.fail("Expected RateLimitError to be raised") + except litellm.RateLimitError as e: + # This is the expected behavior - should detect "rate limit" text and map to RateLimitError + assert "rate limit exceeded" in str(e).lower() + except Exception as e: + pytest.fail(f"Expected RateLimitError but got {type(e).__name__}: {e}") + + def test_anthropic_tool_calling_exception(): """ Related - https://github.com/BerriAI/litellm/issues/4348 From 08239357cf61f152aab565a150e6260573c82ee4 Mon Sep 17 00:00:00 2001 From: Cole McIntosh Date: Thu, 5 Jun 2025 17:15:53 -0600 Subject: [PATCH 2/2] Add ExceptionCheckers class for improved error string detection Introduce the ExceptionCheckers class to encapsulate methods for checking error conditions in exception strings, specifically for identifying rate limit errors. Update the Fireworks AI exception mapping tests to cover various scenarios, including standard 429 errors and text-based detection, ensuring accurate mapping to RateLimitError. Enhance test coverage for both positive and negative cases of rate limit detection. --- .../exception_mapping_utils.py | 24 +++- tests/local_testing/test_exceptions.py | 133 +++++++++++------- 2 files changed, 104 insertions(+), 53 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 3a78519074f..6f3ef28e9cb 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -24,6 +24,28 @@ from ..exceptions import ( ) +class ExceptionCheckers: + """ + Helper class for checking various error conditions in exception strings. + """ + + @staticmethod + def is_error_str_rate_limit(error_str: str) -> bool: + """ + Check if an error string indicates a rate limit error. + + Args: + error_str: The error string to check + + Returns: + True if the error indicates a rate limit, False otherwise + """ + if not isinstance(error_str, str): + return False + + return "429" in error_str or "rate limit" in error_str.lower() + + def get_error_message(error_obj) -> Optional[str]: """ OpenAI Returns Error message that is nested, this extract the message @@ -274,7 +296,7 @@ def exception_type( # type: ignore # noqa: PLR0915 + "Exception" ) - if "429" in error_str or "rate limit" in error_str.lower(): + if ExceptionCheckers.is_error_str_rate_limit(error_str): exception_mapping_worked = True raise RateLimitError( message=f"RateLimitError: {exception_provider} - {message}", diff --git a/tests/local_testing/test_exceptions.py b/tests/local_testing/test_exceptions.py index 275172291bb..6851325cc1f 100644 --- a/tests/local_testing/test_exceptions.py +++ b/tests/local_testing/test_exceptions.py @@ -819,70 +819,96 @@ def test_exception_mapping(provider): pass -def test_fireworks_ai_429_exception_mapping(): +def test_fireworks_ai_exception_mapping(): """ - Test that Fireworks AI 429 status codes are properly mapped to RateLimitError - instead of being converted to 400 BadRequestError. + Comprehensive test for Fireworks AI exception mapping, including: + 1. Standard 429 rate limit errors + 2. Text-based rate limit detection (the main issue fixed) + 3. Generic 400 errors that should NOT be rate limits + 4. ExceptionCheckers utility function - Related issue: https://github.com/BerriAI/litellm/issues/xxxxx + Related to: https://github.com/BerriAI/litellm/pull/11455 + Based on Fireworks AI documentation: https://docs.fireworks.ai/tools-sdks/python-client/api-reference """ import litellm from litellm.llms.fireworks_ai.common_utils import FireworksAIException + from litellm.litellm_core_utils.exception_mapping_utils import ExceptionCheckers - # Create a mock FireworksAIException with 429 status code - mock_exception = FireworksAIException( - status_code=429, - message="Rate limit exceeded", - headers={"retry-after": "60"} - ) + # Test scenarios covering all important cases + test_scenarios = [ + { + "name": "Standard 429 rate limit with proper status code", + "status_code": 429, + "message": "Rate limit exceeded. Please try again in 60 seconds.", + "expected_exception": litellm.RateLimitError, + }, + { + "name": "Status 400 with rate limit text (the main issue fixed)", + "status_code": 400, + "message": '{"error":{"object":"error","type":"invalid_request_error","message":"rate limit exceeded, please try again later"}}', + "expected_exception": litellm.RateLimitError, + }, + { + "name": "Status 400 with generic invalid request (should NOT be rate limit)", + "status_code": 400, + "message": '{"error":{"type":"invalid_request_error","message":"Invalid parameter value"}}', + "expected_exception": litellm.BadRequestError, + }, + ] - try: - response = litellm.completion( - model="fireworks_ai/llama-v3p1-8b-instruct", - messages=[{"role": "user", "content": "Hello"}], - mock_response=mock_exception, + # Test each scenario + for scenario in test_scenarios: + mock_exception = FireworksAIException( + status_code=scenario["status_code"], + message=scenario["message"], + headers={} ) - pytest.fail("Expected RateLimitError to be raised") - except litellm.RateLimitError as e: - # This is the expected behavior - 429 should map to RateLimitError - assert e.status_code == 429 - assert "Rate limit exceeded" in str(e) - except Exception as e: - pytest.fail(f"Expected RateLimitError but got {type(e).__name__}: {e}") - - -def test_fireworks_ai_rate_limit_text_detection(): - """ - Test that Fireworks AI rate limit errors are detected by text matching when no 429 status code is present. + + try: + response = litellm.completion( + model="fireworks_ai/llama-v3p1-70b-instruct", + messages=[{"role": "user", "content": "Hello"}], + mock_response=mock_exception, + ) + pytest.fail(f"Expected {scenario['expected_exception'].__name__} to be raised") + except scenario["expected_exception"] as e: + if scenario["expected_exception"] == litellm.RateLimitError: + assert "rate limit" in str(e).lower() or "429" in str(e) + except Exception as e: + pytest.fail(f"Expected {scenario['expected_exception'].__name__} but got {type(e).__name__}: {e}") - This tests the exact scenario from the real failure where Fireworks AI returns: - {"error":{"object":"error","type":"invalid_request_error","message":"rate limit exceeded, please try again later"}} + # Test ExceptionCheckers.is_error_str_rate_limit() method directly - This should be detected as a RateLimitError, not a BadRequestError. - """ - import litellm - from litellm.llms.fireworks_ai.common_utils import FireworksAIException + # Test cases that should return True (rate limit detected) + rate_limit_strings = [ + "429 rate limit exceeded", + "Rate limit exceeded, please try again later", + "RATE LIMIT ERROR", + "Error 429: rate limit", + '{"error":{"type":"invalid_request_error","message":"rate limit exceeded, please try again later"}}', + "HTTP 429 Too Many Requests", + ] - # Create a mock exception that simulates the exact Fireworks AI response - # with "invalid_request_error" type and "rate limit exceeded" message - mock_exception = FireworksAIException( - status_code=400, # Fireworks AI is returning 400 instead of 429 - message='{"error":{"object":"error","type":"invalid_request_error","message":"rate limit exceeded, please try again later"}}', - headers={} - ) + for error_str in rate_limit_strings: + assert ExceptionCheckers.is_error_str_rate_limit(error_str), f"Should detect rate limit in: {error_str}" - try: - response = litellm.completion( - model="fireworks_ai/qwen3-235b-a22b", - messages=[{"role": "user", "content": "Hello"}], - mock_response=mock_exception, - ) - pytest.fail("Expected RateLimitError to be raised") - except litellm.RateLimitError as e: - # This is the expected behavior - should detect "rate limit" text and map to RateLimitError - assert "rate limit exceeded" in str(e).lower() - except Exception as e: - pytest.fail(f"Expected RateLimitError but got {type(e).__name__}: {e}") + # Test cases that should return False (not rate limit) + non_rate_limit_strings = [ + "400 Bad Request", + "Authentication failed", + "Invalid model specified", + "Context window exceeded", + "Internal server error", + "", + "Some other error message", + ] + + for error_str in non_rate_limit_strings: + assert not ExceptionCheckers.is_error_str_rate_limit(error_str), f"Should NOT detect rate limit in: {error_str}" + + # Test edge cases + assert not ExceptionCheckers.is_error_str_rate_limit(None) # type: ignore + assert not ExceptionCheckers.is_error_str_rate_limit(42) # type: ignore def test_anthropic_tool_calling_exception(): @@ -1351,3 +1377,6 @@ async def test_exception_bubbling_up(sync_mode, stream_mode, model): assert exc_info.value.code == "invalid_value" assert exc_info.value.param is not None assert exc_info.value.type == "invalid_request_error" + + +