Merge pull request #42045 from BerriAI/litellm_lit_8201_notfound_retry_policy

fix(router): add NotFoundErrorRetries so a retry policy can pin 404 retries

litellm/router_utils/get_retry_from_policy.py

@@ -1,6 +1,7 @@
 """Resolve how many retries a RetryPolicy grants for a given exception."""
 
 from collections.abc import Callable, Mapping
+from itertools import chain
 from types import MappingProxyType
 from typing import Final
 
@@ -28,6 +29,11 @@ _RETRIES_BY_EXCEPTION_TYPE: Final[Mapping[type, Callable[[RetryPolicy], int | No
 )
 
 
+def _retries_for_a_404_answer(exception: Exception, policy: RetryPolicy) -> int | None:
+    status_code: Final = getattr(exception, "status_code", None)
+    return policy.NotFoundErrorRetries if status_code == 404 else None
+
+
 def _resolve_policy(
     retry_policy: RetryPolicy | Mapping[str, int | None] | None,
     model_group: str | None,
@@ -49,13 +55,14 @@ def get_num_retries_from_retry_policy(
     model_group: str | None = None,
     model_group_retry_policy: Mapping[str, RetryPolicy | Mapping[str, int | None]] | None = None,
 ) -> int | None:
-    """Walk the exception's MRO, most specific class first, and return the first configured retry count."""
+    """Prefer NotFoundErrorRetries for any 404 answer, then walk the exception's MRO most specific class first."""
     policy: Final = _resolve_policy(retry_policy, model_group, model_group_retry_policy)
     if policy is None:
         return None
-    configured: Final = (
+    by_class: Final = (
         _RETRIES_BY_EXCEPTION_TYPE[cls](policy) for cls in type(exception).__mro__ if cls in _RETRIES_BY_EXCEPTION_TYPE
     )
+    configured: Final = chain((_retries_for_a_404_answer(exception, policy),), by_class)
     return next((retries for retries in configured if retries is not None), policy.DefaultRetries)
 
 

litellm/types/router.py

@@ -109,6 +109,7 @@ class RetryPolicy(BaseModel):
     ContentPolicyViolationErrorRetries: int | None = None
     InternalServerErrorRetries: int | None = None
     ServiceUnavailableErrorRetries: int | None = None
+    NotFoundErrorRetries: int | None = None
     DefaultRetries: int | None = None
 
 

tests/test_litellm/router_utils/test_get_retry_from_policy.py

@@ -1,6 +1,7 @@
 from types import MappingProxyType
 from typing import Final
 
+import httpx
 import pytest
 
 import litellm
@@ -16,6 +17,7 @@ _EXCEPTION_FOR_FIELD: Final = MappingProxyType(
         "ContentPolicyViolationErrorRetries": litellm.ContentPolicyViolationError,
         "InternalServerErrorRetries": litellm.InternalServerError,
         "ServiceUnavailableErrorRetries": litellm.ServiceUnavailableError,
+        "NotFoundErrorRetries": litellm.NotFoundError,
     }
 )
 
@@ -26,6 +28,17 @@ def _error(exception_type: type[Exception]) -> Exception:
     return exception_type(message="boom", llm_provider="openai", model="gpt-5.6")
 
 
+def _bad_request_answered_with_404() -> litellm.BadRequestError:
+    upstream: Final = httpx.Response(
+        404, request=httpx.Request("GET", "https://api.openai.com/v1/responses/resp_missing")
+    )
+    exception: Final = litellm.BadRequestError(
+        message="Response with id 'resp_missing' not found.", llm_provider="openai", model="gpt-5.6", response=upstream
+    )
+    assert exception.status_code == 404
+    return exception
+
+
 @pytest.mark.parametrize("field", _SPECIFIC_FIELDS)
 def test_every_specific_field_controls_retries_for_its_exception(field: str):
     exception: Final = _error(_EXCEPTION_FOR_FIELD[field])
@@ -66,7 +79,7 @@ def test_subclass_falls_back_to_the_parent_field():
     )
 
 
-@pytest.mark.parametrize("exception_type", (litellm.BadGatewayError, litellm.NotFoundError))
+@pytest.mark.parametrize("exception_type", (litellm.BadGatewayError,))
 def test_default_retries_covers_exceptions_without_a_specific_field(exception_type: type[Exception]):
     exception: Final = _error(exception_type)
 
@@ -86,6 +99,46 @@ def test_specific_field_wins_over_default_retries():
     assert get_num_retries_from_retry_policy(exception=_error(litellm.BadGatewayError), retry_policy=policy) == 0
 
 
+def test_not_found_retries_governs_a_bad_request_error_answered_with_404():
+    exception: Final = _bad_request_answered_with_404()
+
+    assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(NotFoundErrorRetries=0)) == 0
+    assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(NotFoundErrorRetries=4)) == 4
+
+
+def test_not_found_retries_wins_over_bad_request_and_default_retries_for_a_404():
+    policy: Final = RetryPolicy(NotFoundErrorRetries=0, BadRequestErrorRetries=5, DefaultRetries=3)
+
+    assert get_num_retries_from_retry_policy(exception=_bad_request_answered_with_404(), retry_policy=policy) == 0
+    assert get_num_retries_from_retry_policy(exception=_error(litellm.NotFoundError), retry_policy=policy) == 0
+
+
+def test_a_404_without_not_found_retries_falls_back_to_bad_request_then_default_retries():
+    exception: Final = _bad_request_answered_with_404()
+
+    assert (
+        get_num_retries_from_retry_policy(
+            exception=exception, retry_policy=RetryPolicy(BadRequestErrorRetries=0, DefaultRetries=3)
+        )
+        == 0
+    )
+    assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(DefaultRetries=3)) == 3
+    assert get_num_retries_from_retry_policy(exception=_error(litellm.NotFoundError), retry_policy=RetryPolicy(DefaultRetries=3)) == 3
+
+
+def test_not_found_retries_leaves_a_plain_400_alone():
+    exception: Final = _error(litellm.BadRequestError)
+    assert exception.status_code == 400
+
+    assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(NotFoundErrorRetries=0)) is None
+    assert (
+        get_num_retries_from_retry_policy(
+            exception=exception, retry_policy=RetryPolicy(NotFoundErrorRetries=0, BadRequestErrorRetries=2)
+        )
+        == 2
+    )
+
+
 def test_default_retries_applies_when_the_specific_field_is_unset():
     policy: Final = RetryPolicy(DefaultRetries=2)
 

tests/test_litellm/test_router.py

@@ -15846,6 +15846,53 @@ async def test_router_retry_policy_controls_upstream_attempt_count(
     assert upstream.call_count == expected_upstream_calls
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "error_body,error_type",
+    [
+        ({"message": "model is down", "type": "server_error"}, litellm.NotFoundError),
+        ({"message": "Response with id 'resp_x' not found.", "type": "invalid_request_error"}, litellm.BadRequestError),
+    ],
+)
+@pytest.mark.parametrize(
+    "retry_policy,expected_upstream_calls",
+    [
+        ({"DefaultRetries": 3}, 4),
+        ({"DefaultRetries": 3, "NotFoundErrorRetries": 0}, 1),
+        ({"NotFoundErrorRetries": 2}, 3),
+    ],
+)
+async def test_router_not_found_retries_governs_every_404_shape(
+    monkeypatch: pytest.MonkeyPatch, retry_policy, expected_upstream_calls, error_body, error_type
+):
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-5.6",
+                "litellm_params": {
+                    "model": "openai/gpt-5.6",
+                    "api_key": "sk-fake",
+                    "api_base": "https://retry-policy.local/v1",
+                },
+            }
+        ],
+        num_retries=2,
+        retry_policy=retry_policy,
+        disable_cooldowns=True,
+    )
+
+    with respx.mock(assert_all_called=True) as respx_mock:
+        upstream = respx_mock.post("https://retry-policy.local/v1/chat/completions").mock(
+            return_value=httpx.Response(404, headers={"retry-after": "0"}, json={"error": error_body})
+        )
+        with pytest.raises(error_type) as raised:
+            await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
+
+    assert raised.value.status_code == 404
+    assert upstream.call_count == expected_upstream_calls
+
+
 @pytest.mark.asyncio
 async def test_generic_call_keeps_the_deployment_name_of_an_azure_ai_model_on_an_azure_openai_host(monkeypatch):
     monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)

ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.test.tsx

@@ -43,6 +43,26 @@ describe("ModelRetrySettingsTab", () => {
     expect(screen.getByText(/RateLimitError \(429\)/)).toBeInTheDocument();
     expect(screen.getByText(/ContentPolicyViolationError \(400\)/)).toBeInTheDocument();
     expect(screen.getByText(/InternalServerError \(500\)/)).toBeInTheDocument();
+    expect(screen.getByText(/NotFoundError \(404\)/)).toBeInTheDocument();
+  });
+
+  it("should write the NotFoundError row to NotFoundErrorRetries ahead of the catch-all row", () => {
+    const setGlobalRetryPolicy = vi.fn();
+    render(
+      <ModelRetrySettingsTab
+        {...buildProps({
+          selectedModelGroup: "global",
+          globalRetryPolicy: { DefaultRetries: 3 },
+          setGlobalRetryPolicy,
+        })}
+      />,
+    );
+
+    const notFoundInput = screen.getByRole("spinbutton", { name: /NotFoundError \(404\) retry count$/ });
+    fireEvent.change(notFoundInput, { target: { value: "2" } });
+
+    const updater = setGlobalRetryPolicy.mock.calls.at(-1)![0];
+    expect(updater({ DefaultRetries: 3 })).toMatchObject({ DefaultRetries: 3, NotFoundErrorRetries: 2 });
   });
 
   it("should use defaultRetry when globalRetryPolicy is null (global scope)", () => {

ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx

@@ -35,6 +35,7 @@ const retryPolicyMap: Record<string, string> = {
   "ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries",
   "InternalServerError (500)": "InternalServerErrorRetries",
   "ServiceUnavailableError (503)": "ServiceUnavailableErrorRetries",
+  "NotFoundError (404)": "NotFoundErrorRetries",
   "All other errors": "DefaultRetries",
 };
 

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -36914,6 +36914,8 @@ export interface components {
             DefaultRetries?: number | null;
             /** Internalservererrorretries */
             InternalServerErrorRetries?: number | null;
+            /** Notfounderrorretries */
+            NotFoundErrorRetries?: number | null;
             /** Ratelimiterrorretries */
             RateLimitErrorRetries?: number | null;
             /** Serviceunavailableerrorretries */