Merge 36c53fe051 into e52f05566d

litellm/router_utils/get_retry_from_policy.py

@@ -8,6 +8,7 @@ from litellm.exceptions import (
     AuthenticationError,
     BadRequestError,
     ContentPolicyViolationError,
+    NotFoundError,
     RateLimitError,
     Timeout,
 )
@@ -50,6 +51,8 @@ def get_num_retries_from_retry_policy(
         return retry_policy.ContentPolicyViolationErrorRetries
     if isinstance(exception, BadRequestError) and retry_policy.BadRequestErrorRetries is not None:
         return retry_policy.BadRequestErrorRetries
+    if isinstance(exception, NotFoundError) and retry_policy.NotFoundErrorRetries is not None:
+        return retry_policy.NotFoundErrorRetries
 
 
 def reset_retry_policy() -> RetryPolicy:

litellm/types/router.py

@@ -101,6 +101,7 @@ class RetryPolicy(BaseModel):
     RateLimitErrorRetries: int | None = None
     ContentPolicyViolationErrorRetries: int | None = None
     InternalServerErrorRetries: int | None = None
+    NotFoundErrorRetries: int | None = None
 
 
 class UpdateRouterConfig(BaseModel):

tests/router_unit_tests/test_router_helper_utils.py

@@ -1367,6 +1367,48 @@ def test_get_num_retries_from_retry_policy(
     assert calc_num_retries == num_retries
 
 
+def test_get_num_retries_from_retry_policy_notfounderror_zero(model_list):
+    """gh-36896: NotFoundErrorRetries lets operators pin 404s to 0 retries
+    so the router surfaces a 404 to the client immediately instead of retrying
+    it across the deployment pool (which would cool every deployment)."""
+    from litellm.router import RetryPolicy
+    from litellm.router_utils.get_retry_from_policy import (
+        get_num_retries_from_retry_policy,
+    )
+
+    router = Router(
+        model_list=model_list,
+        retry_policy=RetryPolicy(NotFoundErrorRetries=0),
+    )
+    calc_num_retries = router.get_num_retries_from_retry_policy(
+        exception=litellm.exceptions.NotFoundError(
+            message="test", llm_provider="openai", model="gpt-5-mini"
+        )
+    )
+    assert calc_num_retries == 0
+
+
+def test_get_num_retries_from_retry_policy_notfounderror_falls_back_to_num_retries(model_list):
+    """gh-36896: when NotFoundErrorRetries is unset, a 404 still falls back to
+    the router's default num_retries (unchanged behavior)."""
+    from litellm.router import RetryPolicy
+    from litellm.router_utils.get_retry_from_policy import (
+        get_num_retries_from_retry_policy,
+    )
+
+    router = Router(
+        model_list=model_list,
+        retry_policy=RetryPolicy(),
+        num_retries=3,
+    )
+    calc_num_retries = router.get_num_retries_from_retry_policy(
+        exception=litellm.exceptions.NotFoundError(
+            message="test", llm_provider="openai", model="gpt-5-mini"
+        )
+    )
+    assert calc_num_retries is None  # no policy-set count; falls back to num_retries
+
+
 @pytest.mark.parametrize(
     "exception_type, exception_name, allowed_fails",
     [