Merge pull request #35853 from BerriAI/litellm_retry_policy_503

fix(router): resolve retry_policy by exception hierarchy, add ServiceUnavailableErrorRetries and DefaultRetries

basedpyright-code-budget.json

@@ -57,7 +57,7 @@
     "limit": 5601
   },
   "reportMissingTypeArgument": {
-    "limit": 15285
+    "limit": 15284
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -99,7 +99,7 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44360
+    "limit": 44358
   },
   "reportUnknownLambdaType": {
     "limit": 109
@@ -108,10 +108,10 @@
     "limit": 38309
   },
   "reportUnknownParameterType": {
-    "limit": 19622
+    "limit": 19621
   },
   "reportUnknownVariableType": {
-    "limit": 29846
+    "limit": 29844
   },
   "reportUnnecessaryCast": {
     "limit": 111

litellm/router_utils/get_retry_from_policy.py

@@ -1,55 +1,62 @@
-"""
-Get num retries for an exception.
+"""Resolve how many retries a RetryPolicy grants for a given exception."""
 
-- Account for retry policy by exception type.
-"""
+from collections.abc import Callable, Mapping
+from types import MappingProxyType
+from typing import Final
 
 from litellm.exceptions import (
     AuthenticationError,
     BadRequestError,
     ContentPolicyViolationError,
+    InternalServerError,
     RateLimitError,
+    ServiceUnavailableError,
     Timeout,
 )
 from litellm.types.router import RetryPolicy
 
+_RETRIES_BY_EXCEPTION_TYPE: Final[Mapping[type, Callable[[RetryPolicy], int | None]]] = MappingProxyType(
+    {
+        AuthenticationError: lambda policy: policy.AuthenticationErrorRetries,
+        Timeout: lambda policy: policy.TimeoutErrorRetries,
+        RateLimitError: lambda policy: policy.RateLimitErrorRetries,
+        ContentPolicyViolationError: lambda policy: policy.ContentPolicyViolationErrorRetries,
+        BadRequestError: lambda policy: policy.BadRequestErrorRetries,
+        ServiceUnavailableError: lambda policy: policy.ServiceUnavailableErrorRetries,
+        InternalServerError: lambda policy: policy.InternalServerErrorRetries,
+    }
+)
+
+
+def _resolve_policy(
+    retry_policy: RetryPolicy | Mapping[str, int | None] | None,
+    model_group: str | None,
+    model_group_retry_policy: Mapping[str, RetryPolicy | Mapping[str, int | None]] | None,
+) -> RetryPolicy | None:
+    selected: Final = (
+        model_group_retry_policy[model_group]
+        if model_group_retry_policy is not None and model_group is not None and model_group in model_group_retry_policy
+        else retry_policy
+    )
+    if isinstance(selected, Mapping):
+        return RetryPolicy(**selected)
+    return selected
+
 
 def get_num_retries_from_retry_policy(
     exception: Exception,
-    retry_policy: RetryPolicy | dict | None = None,
+    retry_policy: RetryPolicy | Mapping[str, int | None] | None = None,
     model_group: str | None = None,
-    model_group_retry_policy: dict[str, RetryPolicy] | None = None,
-):
-    """
-    BadRequestErrorRetries: Optional[int] = None
-    AuthenticationErrorRetries: Optional[int] = None
-    TimeoutErrorRetries: Optional[int] = None
-    RateLimitErrorRetries: Optional[int] = None
-    ContentPolicyViolationErrorRetries: Optional[int] = None
-    """
-    # if we can find the exception then in the retry policy -> return the number of retries
-
-    if model_group_retry_policy is not None and model_group is not None and model_group in model_group_retry_policy:
-        retry_policy = model_group_retry_policy.get(model_group, None)
-
-    if retry_policy is None:
+    model_group_retry_policy: Mapping[str, RetryPolicy | Mapping[str, int | None]] | None = None,
+) -> int | None:
+    """Walk the exception's MRO, most specific class first, and return the first configured retry count."""
+    policy: Final = _resolve_policy(retry_policy, model_group, model_group_retry_policy)
+    if policy is None:
         return None
-    if isinstance(retry_policy, dict):
-        retry_policy = RetryPolicy(**retry_policy)
-
-    if isinstance(exception, AuthenticationError) and retry_policy.AuthenticationErrorRetries is not None:
-        return retry_policy.AuthenticationErrorRetries
-    if isinstance(exception, Timeout) and retry_policy.TimeoutErrorRetries is not None:
-        return retry_policy.TimeoutErrorRetries
-    if isinstance(exception, RateLimitError) and retry_policy.RateLimitErrorRetries is not None:
-        return retry_policy.RateLimitErrorRetries
-    if (
-        isinstance(exception, ContentPolicyViolationError)
-        and retry_policy.ContentPolicyViolationErrorRetries is not None
-    ):
-        return retry_policy.ContentPolicyViolationErrorRetries
-    if isinstance(exception, BadRequestError) and retry_policy.BadRequestErrorRetries is not None:
-        return retry_policy.BadRequestErrorRetries
+    configured: Final = (
+        _RETRIES_BY_EXCEPTION_TYPE[cls](policy) for cls in type(exception).__mro__ if cls in _RETRIES_BY_EXCEPTION_TYPE
+    )
+    return next((retries for retries in configured if retries is not None), policy.DefaultRetries)
 
 
 def reset_retry_policy() -> RetryPolicy:

litellm/types/router.py

@@ -104,6 +104,8 @@ class RetryPolicy(BaseModel):
     RateLimitErrorRetries: int | None = None
     ContentPolicyViolationErrorRetries: int | None = None
     InternalServerErrorRetries: int | None = None
+    ServiceUnavailableErrorRetries: int | None = None
+    DefaultRetries: int | None = None
 
 
 OptionalPreCallChecks = list[

ruff-strict-budget.json

@@ -9,7 +9,7 @@
     "limit": 809
   },
   "ANN201": {
-    "limit": 1999
+    "limit": 1998
   },
   "ANN202": {
     "limit": 835

tests/test_litellm/router_utils/test_get_retry_from_policy.py

@@ -0,0 +1,147 @@
+from types import MappingProxyType
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.router_utils.get_retry_from_policy import get_num_retries_from_retry_policy
+from litellm.types.router import RetryPolicy
+
+_EXCEPTION_FOR_FIELD: Final = MappingProxyType(
+    {
+        "BadRequestErrorRetries": litellm.BadRequestError,
+        "AuthenticationErrorRetries": litellm.AuthenticationError,
+        "TimeoutErrorRetries": litellm.Timeout,
+        "RateLimitErrorRetries": litellm.RateLimitError,
+        "ContentPolicyViolationErrorRetries": litellm.ContentPolicyViolationError,
+        "InternalServerErrorRetries": litellm.InternalServerError,
+        "ServiceUnavailableErrorRetries": litellm.ServiceUnavailableError,
+    }
+)
+
+_SPECIFIC_FIELDS: Final = tuple(name for name in RetryPolicy.model_fields if name != "DefaultRetries")
+
+
+def _error(exception_type: type[Exception]) -> Exception:
+    return exception_type(message="boom", llm_provider="openai", model="gpt-5.6")
+
+
+@pytest.mark.parametrize("field", _SPECIFIC_FIELDS)
+def test_every_specific_field_controls_retries_for_its_exception(field: str):
+    exception: Final = _error(_EXCEPTION_FOR_FIELD[field])
+
+    assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(**{field: 0})) == 0
+    assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(**{field: 4})) == 4
+
+
+@pytest.mark.parametrize("field", _SPECIFIC_FIELDS)
+def test_specific_field_does_not_apply_to_unrelated_exceptions(field: str):
+    policy: Final = RetryPolicy(**{field: 0})
+    unrelated: Final = tuple(
+        exception_type
+        for name, exception_type in _EXCEPTION_FOR_FIELD.items()
+        if name != field and not issubclass(exception_type, _EXCEPTION_FOR_FIELD[field])
+    )
+
+    for exception_type in unrelated:
+        assert get_num_retries_from_retry_policy(exception=_error(exception_type), retry_policy=policy) is None
+
+
+def test_subclass_prefers_its_own_field_over_the_parent_field():
+    policy: Final = RetryPolicy(BadRequestErrorRetries=5, ContentPolicyViolationErrorRetries=1)
+
+    assert (
+        get_num_retries_from_retry_policy(exception=_error(litellm.ContentPolicyViolationError), retry_policy=policy)
+        == 1
+    )
+    assert get_num_retries_from_retry_policy(exception=_error(litellm.BadRequestError), retry_policy=policy) == 5
+
+
+def test_subclass_falls_back_to_the_parent_field():
+    policy: Final = RetryPolicy(BadRequestErrorRetries=5)
+
+    assert (
+        get_num_retries_from_retry_policy(exception=_error(litellm.ContentPolicyViolationError), retry_policy=policy)
+        == 5
+    )
+
+
+@pytest.mark.parametrize("exception_type", (litellm.BadGatewayError, litellm.NotFoundError))
+def test_default_retries_covers_exceptions_without_a_specific_field(exception_type: type[Exception]):
+    exception: Final = _error(exception_type)
+
+    assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(DefaultRetries=0)) == 0
+    assert (
+        get_num_retries_from_retry_policy(
+            exception=exception, retry_policy=RetryPolicy(ServiceUnavailableErrorRetries=0)
+        )
+        is None
+    )
+
+
+def test_specific_field_wins_over_default_retries():
+    policy: Final = RetryPolicy(DefaultRetries=0, RateLimitErrorRetries=3)
+
+    assert get_num_retries_from_retry_policy(exception=_error(litellm.RateLimitError), retry_policy=policy) == 3
+    assert get_num_retries_from_retry_policy(exception=_error(litellm.BadGatewayError), retry_policy=policy) == 0
+
+
+def test_default_retries_applies_when_the_specific_field_is_unset():
+    policy: Final = RetryPolicy(DefaultRetries=2)
+
+    assert (
+        get_num_retries_from_retry_policy(exception=_error(litellm.ServiceUnavailableError), retry_policy=policy) == 2
+    )
+
+
+def test_empty_policy_matches_nothing():
+    assert (
+        get_num_retries_from_retry_policy(exception=_error(litellm.ServiceUnavailableError), retry_policy=RetryPolicy())
+        is None
+    )
+    assert (
+        get_num_retries_from_retry_policy(exception=_error(litellm.ServiceUnavailableError), retry_policy=None) is None
+    )
+
+
+def test_dict_policy_is_accepted():
+    assert (
+        get_num_retries_from_retry_policy(
+            exception=_error(litellm.ServiceUnavailableError),
+            retry_policy={"ServiceUnavailableErrorRetries": 0},
+        )
+        == 0
+    )
+
+
+def test_model_group_policy_replaces_the_global_policy():
+    exception: Final = _error(litellm.ServiceUnavailableError)
+    global_policy: Final = RetryPolicy(ServiceUnavailableErrorRetries=5)
+
+    assert (
+        get_num_retries_from_retry_policy(
+            exception=exception,
+            retry_policy=global_policy,
+            model_group="gpt-5.6",
+            model_group_retry_policy={"gpt-5.6": {"ServiceUnavailableErrorRetries": 1}},
+        )
+        == 1
+    )
+    assert (
+        get_num_retries_from_retry_policy(
+            exception=exception,
+            retry_policy=global_policy,
+            model_group="gpt-5.6",
+            model_group_retry_policy={"gpt-5.6": RetryPolicy(RateLimitErrorRetries=1)},
+        )
+        is None
+    )
+    assert (
+        get_num_retries_from_retry_policy(
+            exception=exception,
+            retry_policy=global_policy,
+            model_group="other-group",
+            model_group_retry_policy={"gpt-5.6": RetryPolicy(ServiceUnavailableErrorRetries=1)},
+        )
+        == 5
+    )

tests/test_litellm/test_router.py

@@ -12,6 +12,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
 import httpx
 import openai
 import pytest
+import respx
 
 
 
@@ -567,7 +568,6 @@ async def test_async_router_acancel_batch_does_not_fall_back_across_model_groups
     model string, and the fallback provider is then asked to cancel a batch it never
     issued, which can only answer not-found. The router re-raises the owner's error after
     that wasted round trip, so the pin's observable is the foreign call never happening."""
-    import respx
 
     monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
     router = litellm.Router(
@@ -716,7 +716,6 @@ async def test_async_router_acreate_file_litellm_proxy_sends_target_model_names_
     from io import BytesIO
 
     import httpx
-    import respx
 
     jsonl_file = BytesIO(
         json.dumps({"body": {"model": "chained-batch", "messages": [{"role": "user", "content": "hi"}]}}).encode(
@@ -12893,3 +12892,49 @@ async def test_prompt_management_factory_marks_injection_for_every_deployment(mo
     bucket = captured.get("litellm_metadata") or captured["metadata"]
     assert captured["model_info"]["id"] == "provisional-dep"
     assert bucket["litellm_gateway_injected_cache"] == ""
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "retry_policy,upstream_status,error_type,expected_upstream_calls",
+    [
+        ({"ServiceUnavailableErrorRetries": 0}, 503, litellm.ServiceUnavailableError, 1),
+        ({"ServiceUnavailableErrorRetries": 1}, 503, litellm.ServiceUnavailableError, 2),
+        ({"InternalServerErrorRetries": 0}, 500, litellm.InternalServerError, 1),
+        ({"DefaultRetries": 0}, 502, litellm.BadGatewayError, 1),
+        ({"DefaultRetries": 0, "ServiceUnavailableErrorRetries": 1}, 503, litellm.ServiceUnavailableError, 2),
+        ({"ServiceUnavailableErrorRetries": 0}, 502, litellm.BadGatewayError, 3),
+    ],
+)
+async def test_router_retry_policy_controls_upstream_attempt_count(
+    monkeypatch: pytest.MonkeyPatch, retry_policy, upstream_status, error_type, expected_upstream_calls
+):
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-5.6",
+                "litellm_params": {
+                    "model": "openai/gpt-5.6",
+                    "api_key": "sk-fake",
+                    "api_base": "https://retry-policy.local/v1",
+                },
+            }
+        ],
+        num_retries=2,
+        retry_policy=retry_policy,
+        disable_cooldowns=True,
+    )
+
+    with respx.mock(assert_all_called=True) as respx_mock:
+        upstream = respx_mock.post("https://retry-policy.local/v1/chat/completions").mock(
+            return_value=httpx.Response(
+                upstream_status,
+                headers={"retry-after": "0"},
+                json={"error": {"message": "model is down", "type": "server_error"}},
+            )
+        )
+        with pytest.raises(error_type):
+            await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
+
+    assert upstream.call_count == expected_upstream_calls

tests/test_litellm/test_router_per_deployment_num_retries.py

@@ -415,8 +415,9 @@ class TestNoProviderRetryAmplification:
     @pytest.mark.asyncio
     async def test_retry_policy_configured_does_not_reintroduce_amplification(self):
         """
-        With a retry policy configured alongside a per-deployment ``num_retries=5``, the
-        provider SDK still must not retry: exactly ``6`` upstream requests, not 36.
+        ``InternalServerErrorRetries=2`` overrides the per-deployment ``num_retries=5`` for the
+        500s this upstream returns, and the provider SDK still must not retry on top: exactly
+        ``3`` upstream requests, not 18.
         """
         router = self._router(
             "https://policy.local/v1",
@@ -424,7 +425,7 @@ class TestNoProviderRetryAmplification:
             num_retries=1,
             retry_policy=RetryPolicy(InternalServerErrorRetries=2),
         )
-        assert await self._call_and_count(router) == 6
+        assert await self._call_and_count(router) == 3
 
     @pytest.mark.asyncio
     async def test_global_num_retries_not_amplified(self):

type-discipline-budget.json

@@ -1,6 +1,6 @@
 {
   "LIT001": {
-    "limit": 22328
+    "limit": 22326
   },
   "LIT002": {
     "limit": 26748
@@ -30,7 +30,7 @@
     "limit": 16468
   },
   "LIT011": {
-    "limit": 5514
+    "limit": 5512
   },
   "LIT012": {
     "limit": 4487

ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx

@@ -34,6 +34,8 @@ const retryPolicyMap: Record<string, string> = {
   "RateLimitError (429)": "RateLimitErrorRetries",
   "ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries",
   "InternalServerError (500)": "InternalServerErrorRetries",
+  "ServiceUnavailableError (503)": "ServiceUnavailableErrorRetries",
+  "All other errors": "DefaultRetries",
 };
 
 const isValidRetryCount = (value: number) => Number.isFinite(value) && Number.isInteger(value) && value >= 0;

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -35026,10 +35026,14 @@ export interface components {
             BadRequestErrorRetries?: number | null;
             /** Contentpolicyviolationerrorretries */
             ContentPolicyViolationErrorRetries?: number | null;
+            /** Defaultretries */
+            DefaultRetries?: number | null;
             /** Internalservererrorretries */
             InternalServerErrorRetries?: number | null;
             /** Ratelimiterrorretries */
             RateLimitErrorRetries?: number | null;
+            /** Serviceunavailableerrorretries */
+            ServiceUnavailableErrorRetries?: number | null;
             /** Timeouterrorretries */
             TimeoutErrorRetries?: number | null;
         };