fix(router): add NotFoundErrorRetries to RetryPolicy (gh-36896)

Add a `NotFoundErrorRetries` field to `RetryPolicy` and wire it into
`get_num_retries_from_retry_policy()`. Previously a 404 (litellm.NotFoundError)
could not be pinned to 0 retries and fell back to `num_retries`, so the router
retried 404s. Each retry also triggers an immediate cooldown, so retrying across
the deployment pool cools every deployment and can cascade into a fleet-wide 5xx
storm. Operators can now set `NotFoundErrorRetries: 0` to surface 404s to the
client immediately.

Fixes #36896

litellm/router_utils/get_retry_from_policy.py

@@ -8,6 +8,7 @@ from litellm.exceptions import (
     AuthenticationError,
     BadRequestError,
     ContentPolicyViolationError,
+    NotFoundError,
     RateLimitError,
     Timeout,
 )
@@ -50,6 +51,8 @@ def get_num_retries_from_retry_policy(
         return retry_policy.ContentPolicyViolationErrorRetries
     if isinstance(exception, BadRequestError) and retry_policy.BadRequestErrorRetries is not None:
         return retry_policy.BadRequestErrorRetries
+    if isinstance(exception, NotFoundError) and retry_policy.NotFoundErrorRetries is not None:
+        return retry_policy.NotFoundErrorRetries
 
 
 def reset_retry_policy() -> RetryPolicy:

litellm/types/router.py

@@ -100,6 +100,7 @@ class RetryPolicy(BaseModel):
     RateLimitErrorRetries: int | None = None
     ContentPolicyViolationErrorRetries: int | None = None
     InternalServerErrorRetries: int | None = None
+    NotFoundErrorRetries: int | None = None
 
 
 class UpdateRouterConfig(BaseModel):

tests/router_unit_tests/test_router_helper_utils.py

@@ -1375,6 +1375,48 @@ def test_get_num_retries_from_retry_policy(
     assert calc_num_retries == num_retries
 
 
+def test_get_num_retries_from_retry_policy_notfounderror_zero(model_list):
+    """gh-36896: NotFoundErrorRetries lets operators pin 404s to 0 retries
+    so the router surfaces a 404 to the client immediately instead of retrying
+    it across the deployment pool (which would cool every deployment)."""
+    from litellm.router import RetryPolicy
+    from litellm.router_utils.get_retry_from_policy import (
+        get_num_retries_from_retry_policy,
+    )
+
+    router = Router(
+        model_list=model_list,
+        retry_policy=RetryPolicy(NotFoundErrorRetries=0),
+    )
+    calc_num_retries = router.get_num_retries_from_retry_policy(
+        exception=litellm.exceptions.NotFoundError(
+            message="test", llm_provider="openai", model="gpt-5-mini"
+        )
+    )
+    assert calc_num_retries == 0
+
+
+def test_get_num_retries_from_retry_policy_notfounderror_falls_back_to_num_retries(model_list):
+    """gh-36896: when NotFoundErrorRetries is unset, a 404 still falls back to
+    the router's default num_retries (unchanged behavior)."""
+    from litellm.router import RetryPolicy
+    from litellm.router_utils.get_retry_from_policy import (
+        get_num_retries_from_retry_policy,
+    )
+
+    router = Router(
+        model_list=model_list,
+        retry_policy=RetryPolicy(),
+        num_retries=3,
+    )
+    calc_num_retries = router.get_num_retries_from_retry_policy(
+        exception=litellm.exceptions.NotFoundError(
+            message="test", llm_provider="openai", model="gpt-5-mini"
+        )
+    )
+    assert calc_num_retries is None  # no policy-set count; falls back to num_retries
+
+
 @pytest.mark.parametrize(
     "exception_type, exception_name, allowed_fails",
     [