Merge 3d8adc9925 into 40423e6ec0

litellm/router.py

@@ -7133,7 +7133,10 @@ class Router:
             # Check retry policy FIRST, before should_retry_this_error
             # This allows retry policies to override the healthy deployments check
             _retry_policy_applies = False
-            if self.retry_policy is not None or model_group_retry_policy is not None:
+            if (
+                deployment_num_retries is None
+                and (self.retry_policy is not None or model_group_retry_policy is not None)
+            ):
                 # get num_retries from retry policy
                 # Use the model_group captured at the start of the function, or get it from metadata
                 # kwargs.get("model") at this point is the deployment model, not the model_group

litellm/router_utils/get_retry_from_policy.py

@@ -8,6 +8,7 @@ from litellm.exceptions import (
     AuthenticationError,
     BadRequestError,
     ContentPolicyViolationError,
+    InternalServerError,
     RateLimitError,
     Timeout,
 )
@@ -26,6 +27,7 @@ def get_num_retries_from_retry_policy(
     TimeoutErrorRetries: Optional[int] = None
     RateLimitErrorRetries: Optional[int] = None
     ContentPolicyViolationErrorRetries: Optional[int] = None
+    InternalServerErrorRetries: Optional[int] = None
     """
     # if we can find the exception then in the retry policy -> return the number of retries
 
@@ -50,6 +52,8 @@ def get_num_retries_from_retry_policy(
         return retry_policy.ContentPolicyViolationErrorRetries
     if isinstance(exception, BadRequestError) and retry_policy.BadRequestErrorRetries is not None:
         return retry_policy.BadRequestErrorRetries
+    if isinstance(exception, InternalServerError) and retry_policy.InternalServerErrorRetries is not None:
+        return retry_policy.InternalServerErrorRetries
 
 
 def reset_retry_policy() -> RetryPolicy:

tests/router_unit_tests/test_router_helper_utils.py

@@ -1344,6 +1344,11 @@ def test_track_deployment_metrics(model_list):
             "ContentPolicyViolationError",
             7,
         ),
+        (
+            litellm.exceptions.InternalServerError,
+            "InternalServerError",
+            5,
+        ),
     ],
 )
 def test_get_num_retries_from_retry_policy(
@@ -1367,6 +1372,24 @@ def test_get_num_retries_from_retry_policy(
     assert calc_num_retries == num_retries
 
 
+def test_get_num_retries_from_retry_policy_handles_internal_server_error_directly():
+    from litellm.router_utils.get_retry_from_policy import (
+        get_num_retries_from_retry_policy,
+    )
+    from litellm.types.router import RetryPolicy
+
+    calc_num_retries = get_num_retries_from_retry_policy(
+        exception=litellm.InternalServerError(
+            message="test",
+            llm_provider="openai",
+            model="gpt-5-mini",
+        ),
+        retry_policy=RetryPolicy(InternalServerErrorRetries=5),
+    )
+
+    assert calc_num_retries == 5
+
+
 @pytest.mark.parametrize(
     "exception_type, exception_name, allowed_fails",
     [

tests/test_litellm/test_router_per_deployment_num_retries.py

@@ -3,15 +3,17 @@ Unit tests for per-deployment num_retries in litellm_params
 GitHub Issue: #18968 - Per-deployment max_retries/num_retries in litellm_params is not used in retry logic
 """
 
+from unittest.mock import patch
+
 import httpx
 import pytest
 import pytest_asyncio
-from unittest.mock import patch
 
 import litellm
 from litellm import Router
-from litellm.types.router import RetryPolicy
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.router_utils.get_retry_from_policy import get_num_retries_from_retry_policy
+from litellm.types.router import RetryPolicy
 
 
 class TestPerDeploymentNumRetries:
@@ -426,6 +428,18 @@ class TestNoProviderRetryAmplification:
         )
         assert await self._call_and_count(router) == 6
 
+    def test_retry_policy_helper_handles_internal_server_error(self):
+        retry_count = get_num_retries_from_retry_policy(
+            exception=litellm.InternalServerError(
+                message="test error",
+                llm_provider="openai",
+                model="gpt-4",
+            ),
+            retry_policy=RetryPolicy(InternalServerErrorRetries=2),
+        )
+
+        assert retry_count == 2
+
     @pytest.mark.asyncio
     async def test_global_num_retries_not_amplified(self):
         """