From 36c53fe051ebc2509de737f184cd27eb301cc9d0 Mon Sep 17 00:00:00 2001
From: Siraj637909 <siraj@example.com>
Date: Fri, 14 Aug 2026 16:38:38 +0530
Subject: [PATCH] fix(router): add NotFoundErrorRetries to RetryPolicy
 (gh-36896)

Add a `NotFoundErrorRetries` field to `RetryPolicy` and wire it into
`get_num_retries_from_retry_policy()`. Previously a 404 (litellm.NotFoundError)
could not be pinned to 0 retries and fell back to `num_retries`, so the router
retried 404s. Each retry also triggers an immediate cooldown, so retrying across
the deployment pool cools every deployment and can cascade into a fleet-wide 5xx
storm. Operators can now set `NotFoundErrorRetries: 0` to surface 404s to the
client immediately.

Fixes #36896
---
 litellm/router_utils/get_retry_from_policy.py |  3 ++
 litellm/types/router.py                       |  1 +
 .../test_router_helper_utils.py               | 42 +++++++++++++++++++
 3 files changed, 46 insertions(+)

diff --git a/litellm/router_utils/get_retry_from_policy.py b/litellm/router_utils/get_retry_from_policy.py
index 7cf55e80e0c..ce6b6c7c122 100644
--- a/litellm/router_utils/get_retry_from_policy.py
+++ b/litellm/router_utils/get_retry_from_policy.py
@@ -8,6 +8,7 @@ from litellm.exceptions import (
     AuthenticationError,
     BadRequestError,
     ContentPolicyViolationError,
+    NotFoundError,
     RateLimitError,
     Timeout,
 )
@@ -50,6 +51,8 @@ def get_num_retries_from_retry_policy(
         return retry_policy.ContentPolicyViolationErrorRetries
     if isinstance(exception, BadRequestError) and retry_policy.BadRequestErrorRetries is not None:
         return retry_policy.BadRequestErrorRetries
+    if isinstance(exception, NotFoundError) and retry_policy.NotFoundErrorRetries is not None:
+        return retry_policy.NotFoundErrorRetries
 
 
 def reset_retry_policy() -> RetryPolicy:
diff --git a/litellm/types/router.py b/litellm/types/router.py
index 217364c48b7..44985a28c66 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -100,6 +100,7 @@ class RetryPolicy(BaseModel):
     RateLimitErrorRetries: int | None = None
     ContentPolicyViolationErrorRetries: int | None = None
     InternalServerErrorRetries: int | None = None
+    NotFoundErrorRetries: int | None = None
 
 
 class UpdateRouterConfig(BaseModel):
diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py
index c883890f5f6..16b898064e2 100644
--- a/tests/router_unit_tests/test_router_helper_utils.py
+++ b/tests/router_unit_tests/test_router_helper_utils.py
@@ -1375,6 +1375,48 @@ def test_get_num_retries_from_retry_policy(
     assert calc_num_retries == num_retries
 
 
+def test_get_num_retries_from_retry_policy_notfounderror_zero(model_list):
+    """gh-36896: NotFoundErrorRetries lets operators pin 404s to 0 retries
+    so the router surfaces a 404 to the client immediately instead of retrying
+    it across the deployment pool (which would cool every deployment)."""
+    from litellm.router import RetryPolicy
+    from litellm.router_utils.get_retry_from_policy import (
+        get_num_retries_from_retry_policy,
+    )
+
+    router = Router(
+        model_list=model_list,
+        retry_policy=RetryPolicy(NotFoundErrorRetries=0),
+    )
+    calc_num_retries = router.get_num_retries_from_retry_policy(
+        exception=litellm.exceptions.NotFoundError(
+            message="test", llm_provider="openai", model="gpt-5-mini"
+        )
+    )
+    assert calc_num_retries == 0
+
+
+def test_get_num_retries_from_retry_policy_notfounderror_falls_back_to_num_retries(model_list):
+    """gh-36896: when NotFoundErrorRetries is unset, a 404 still falls back to
+    the router's default num_retries (unchanged behavior)."""
+    from litellm.router import RetryPolicy
+    from litellm.router_utils.get_retry_from_policy import (
+        get_num_retries_from_retry_policy,
+    )
+
+    router = Router(
+        model_list=model_list,
+        retry_policy=RetryPolicy(),
+        num_retries=3,
+    )
+    calc_num_retries = router.get_num_retries_from_retry_policy(
+        exception=litellm.exceptions.NotFoundError(
+            message="test", llm_provider="openai", model="gpt-5-mini"
+        )
+    )
+    assert calc_num_retries is None  # no policy-set count; falls back to num_retries
+
+
 @pytest.mark.parametrize(
     "exception_type, exception_name, allowed_fails",
     [