From f1b90cbf2bc8afc2a14bb623720b4dd8013bfa8c Mon Sep 17 00:00:00 2001
From: tombii <github@tombii.com>
Date: Tue, 17 Feb 2026 14:04:53 +0100
Subject: [PATCH] fix: Allow NotFoundError retry with multiple deployments for
 order-based routing

**Problem:**
NotFoundError (404) was immediately raised without trying fallback deployments,
even when order-based routing was configured with multiple deployments.

**Root Cause:**
Lines 5285-5286 in router.py unconditionally raised NotFoundError without
checking for alternative deployments. This was added in commit 19c3a82d1bc
to prevent retries on "model not found" errors.

**Why This Is Too Aggressive:**
1. Not all 404s mean "model doesn't exist" - providers can return 404 for:
   - Policy restrictions (e.g., OpenRouter privacy settings blocking free models)
   - Regional availability issues
   - Temporary endpoint unavailability
2. It breaks documented order-based routing behavior
3. Inconsistent with AuthenticationError handling which checks for alternatives

**Solution:**
Check if multiple deployments exist before raising NotFoundError,
similar to AuthenticationError handling (lines 5296-5304).

**Tests Added:**
1. Updated existing test to pass all_deployments parameter
2. Added new test for NotFoundError with multiple deployments to verify retry logic

Fixes #21377

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
---
 litellm/router.py                          | 16 +++++-
 tests/local_testing/test_router_retries.py | 63 +++++++++++++++++++++-
 2 files changed, 76 insertions(+), 3 deletions(-)

diff --git a/litellm/router.py b/litellm/router.py
index 888c97ca0b1..aad482b5d66 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -5283,7 +5283,21 @@ class Router:
                 raise error
 
         if isinstance(error, litellm.NotFoundError):
-            raise error
+            """
+            - if other deployments available -> retry
+            - else -> raise error
+
+            NotFoundError can occur for reasons other than "model doesn't exist":
+            - Provider policy restrictions (e.g., OpenRouter privacy settings)
+            - Regional availability issues
+            - Temporary endpoint unavailability
+
+            Allow fallback to other deployments when multiple are configured.
+            """
+            if (
+                _num_all_deployments <= 1
+            ):  # if there is only 1 deployment for this model group then don't retry
+                raise error  # then raise error
         # Error we should only retry if there are other deployments
         if isinstance(error, openai.RateLimitError):
             if (
diff --git a/tests/local_testing/test_router_retries.py b/tests/local_testing/test_router_retries.py
index 1d72d5914c5..91987331cd9 100644
--- a/tests/local_testing/test_router_retries.py
+++ b/tests/local_testing/test_router_retries.py
@@ -643,6 +643,9 @@ def test_timeout_for_rate_limit_error_with_no_healthy_deployments():
 
 
 def test_no_retry_for_not_found_error_404():
+    """
+    Test that NotFoundError is raised when there is only 1 deployment
+    """
     healthy_deployments = []
 
     router = Router(
@@ -667,15 +670,71 @@ def test_no_retry_for_not_found_error_404():
     )
     try:
         response = router.should_retry_this_error(
-            error=error, healthy_deployments=healthy_deployments
+            error=error,
+            healthy_deployments=healthy_deployments,
+            all_deployments=router.model_list
         )
         pytest.fail(
-            "Should have raised an exception 404 NotFoundError should never be retried, it's typically model_not_found error"
+            "Should have raised an exception - 404 NotFoundError with single deployment should not retry"
         )
     except Exception as e:
         print("got exception", e)
 
 
+def test_retry_for_not_found_error_404_with_multiple_deployments():
+    """
+    Test that NotFoundError allows retry when multiple deployments are available.
+
+    Covers the case where NotFoundError is due to provider-specific issues
+    (e.g., OpenRouter privacy policy, regional restrictions) rather than
+    the model actually not existing.
+    """
+    model_list = [
+        {
+            "model_name": "gpt-oss-20b",
+            "litellm_params": {
+                "model": "openrouter/openai/gpt-oss-20b:free",
+                "api_key": "test-key-1",
+                "order": 1,
+            },
+        },
+        {
+            "model_name": "gpt-oss-20b",
+            "litellm_params": {
+                "model": "openai/openai/gpt-oss-20b",
+                "api_key": "test-key-2",
+                "api_base": "https://integrate.api.nvidia.com/v1",
+                "order": 2,
+            },
+        }
+    ]
+
+    healthy_deployments = [model_list[1]]  # Second deployment is healthy
+
+    router = Router(model_list=model_list, enable_pre_call_checks=True)
+
+    # Simulate NotFoundError from first deployment (e.g., OpenRouter privacy policy)
+    error = litellm.NotFoundError(
+        message='OpenrouterException - {"error":{"message":"No endpoints found matching your data policy (Free model publication)","code":404}}',
+        model="gpt-oss-20b",
+        llm_provider="openrouter",
+    )
+
+    # Should NOT raise - should return True to allow retry with second deployment
+    try:
+        should_retry = router.should_retry_this_error(
+            error=error,
+            healthy_deployments=healthy_deployments,
+            all_deployments=model_list
+        )
+        assert should_retry is True, "should_retry_this_error should return True when multiple deployments exist"
+        print("✓ NotFoundError correctly allows retry with multiple deployments")
+    except Exception as e:
+        pytest.fail(
+            f"NotFoundError should allow retry when multiple deployments are available, but got exception: {e}"
+        )
+
+
 def test_no_retry_for_bad_request_error_400():
     """
     Test that 400 BadRequestError is NOT retried, even if healthy deployments exist.