From a9ad3eaf9548dea95882bc3b71625b511ebf840b Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 19 Sep 2026 16:25:47 -0700
Subject: [PATCH] fix(router): add NotFoundErrorRetries so a retry policy can
pin 404 retries
RetryPolicy had no field for 404s, so any policy that set DefaultRetries made
the router retry every 404 across the pool, including OpenAI's 404 on a missing
response id, which arrives as a BadRequestError whose status_code is 404
NotFoundErrorRetries now governs every answer whose status code is 404 whatever
exception class the mapping picked, ahead of the class walk and DefaultRetries.
A 404 without it still falls back to BadRequestErrorRetries for the
BadRequestError shape and then to DefaultRetries, so existing policies keep
their behavior until the new field is set. The Admin UI retry settings tab
gains a NotFoundError (404) row above the catch-all row
Fixes #36896
---
litellm/router_utils/get_retry_from_policy.py | 11 +++-
litellm/types/router.py | 1 +
.../test_get_retry_from_policy.py | 55 ++++++++++++++++++-
tests/test_litellm/test_router.py | 47 ++++++++++++++++
.../components/ModelRetrySettingsTab.test.tsx | 20 +++++++
.../components/ModelRetrySettingsTab.tsx | 1 +
ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +
7 files changed, 134 insertions(+), 3 deletions(-)
diff --git a/litellm/router_utils/get_retry_from_policy.py b/litellm/router_utils/get_retry_from_policy.py
index ad4a6b0be99..8771d072434 100644
--- a/litellm/router_utils/get_retry_from_policy.py
+++ b/litellm/router_utils/get_retry_from_policy.py
@@ -1,6 +1,7 @@
"""Resolve how many retries a RetryPolicy grants for a given exception."""
from collections.abc import Callable, Mapping
+from itertools import chain
from types import MappingProxyType
from typing import Final
@@ -28,6 +29,11 @@ _RETRIES_BY_EXCEPTION_TYPE: Final[Mapping[type, Callable[[RetryPolicy], int | No
)
+def _retries_for_a_404_answer(exception: Exception, policy: RetryPolicy) -> int | None:
+ status_code: Final = getattr(exception, "status_code", None)
+ return policy.NotFoundErrorRetries if status_code == 404 else None
+
+
def _resolve_policy(
retry_policy: RetryPolicy | Mapping[str, int | None] | None,
model_group: str | None,
@@ -49,13 +55,14 @@ def get_num_retries_from_retry_policy(
model_group: str | None = None,
model_group_retry_policy: Mapping[str, RetryPolicy | Mapping[str, int | None]] | None = None,
) -> int | None:
- """Walk the exception's MRO, most specific class first, and return the first configured retry count."""
+ """Prefer NotFoundErrorRetries for any 404 answer, then walk the exception's MRO most specific class first."""
policy: Final = _resolve_policy(retry_policy, model_group, model_group_retry_policy)
if policy is None:
return None
- configured: Final = (
+ by_class: Final = (
_RETRIES_BY_EXCEPTION_TYPE[cls](policy) for cls in type(exception).__mro__ if cls in _RETRIES_BY_EXCEPTION_TYPE
)
+ configured: Final = chain((_retries_for_a_404_answer(exception, policy),), by_class)
return next((retries for retries in configured if retries is not None), policy.DefaultRetries)
diff --git a/litellm/types/router.py b/litellm/types/router.py
index adadb053ab2..7fec8be8231 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -109,6 +109,7 @@ class RetryPolicy(BaseModel):
ContentPolicyViolationErrorRetries: int | None = None
InternalServerErrorRetries: int | None = None
ServiceUnavailableErrorRetries: int | None = None
+ NotFoundErrorRetries: int | None = None
DefaultRetries: int | None = None
diff --git a/tests/test_litellm/router_utils/test_get_retry_from_policy.py b/tests/test_litellm/router_utils/test_get_retry_from_policy.py
index df157ea5ff7..1f358f477d4 100644
--- a/tests/test_litellm/router_utils/test_get_retry_from_policy.py
+++ b/tests/test_litellm/router_utils/test_get_retry_from_policy.py
@@ -1,6 +1,7 @@
from types import MappingProxyType
from typing import Final
+import httpx
import pytest
import litellm
@@ -16,6 +17,7 @@ _EXCEPTION_FOR_FIELD: Final = MappingProxyType(
"ContentPolicyViolationErrorRetries": litellm.ContentPolicyViolationError,
"InternalServerErrorRetries": litellm.InternalServerError,
"ServiceUnavailableErrorRetries": litellm.ServiceUnavailableError,
+ "NotFoundErrorRetries": litellm.NotFoundError,
}
)
@@ -26,6 +28,17 @@ def _error(exception_type: type[Exception]) -> Exception:
return exception_type(message="boom", llm_provider="openai", model="gpt-5.6")
+def _bad_request_answered_with_404() -> litellm.BadRequestError:
+ upstream: Final = httpx.Response(
+ 404, request=httpx.Request("GET", "https://api.openai.com/v1/responses/resp_missing")
+ )
+ exception: Final = litellm.BadRequestError(
+ message="Response with id 'resp_missing' not found.", llm_provider="openai", model="gpt-5.6", response=upstream
+ )
+ assert exception.status_code == 404
+ return exception
+
+
@pytest.mark.parametrize("field", _SPECIFIC_FIELDS)
def test_every_specific_field_controls_retries_for_its_exception(field: str):
exception: Final = _error(_EXCEPTION_FOR_FIELD[field])
@@ -66,7 +79,7 @@ def test_subclass_falls_back_to_the_parent_field():
)
-@pytest.mark.parametrize("exception_type", (litellm.BadGatewayError, litellm.NotFoundError))
+@pytest.mark.parametrize("exception_type", (litellm.BadGatewayError,))
def test_default_retries_covers_exceptions_without_a_specific_field(exception_type: type[Exception]):
exception: Final = _error(exception_type)
@@ -86,6 +99,46 @@ def test_specific_field_wins_over_default_retries():
assert get_num_retries_from_retry_policy(exception=_error(litellm.BadGatewayError), retry_policy=policy) == 0
+def test_not_found_retries_governs_a_bad_request_error_answered_with_404():
+ exception: Final = _bad_request_answered_with_404()
+
+ assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(NotFoundErrorRetries=0)) == 0
+ assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(NotFoundErrorRetries=4)) == 4
+
+
+def test_not_found_retries_wins_over_bad_request_and_default_retries_for_a_404():
+ policy: Final = RetryPolicy(NotFoundErrorRetries=0, BadRequestErrorRetries=5, DefaultRetries=3)
+
+ assert get_num_retries_from_retry_policy(exception=_bad_request_answered_with_404(), retry_policy=policy) == 0
+ assert get_num_retries_from_retry_policy(exception=_error(litellm.NotFoundError), retry_policy=policy) == 0
+
+
+def test_a_404_without_not_found_retries_falls_back_to_bad_request_then_default_retries():
+ exception: Final = _bad_request_answered_with_404()
+
+ assert (
+ get_num_retries_from_retry_policy(
+ exception=exception, retry_policy=RetryPolicy(BadRequestErrorRetries=0, DefaultRetries=3)
+ )
+ == 0
+ )
+ assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(DefaultRetries=3)) == 3
+ assert get_num_retries_from_retry_policy(exception=_error(litellm.NotFoundError), retry_policy=RetryPolicy(DefaultRetries=3)) == 3
+
+
+def test_not_found_retries_leaves_a_plain_400_alone():
+ exception: Final = _error(litellm.BadRequestError)
+ assert exception.status_code == 400
+
+ assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(NotFoundErrorRetries=0)) is None
+ assert (
+ get_num_retries_from_retry_policy(
+ exception=exception, retry_policy=RetryPolicy(NotFoundErrorRetries=0, BadRequestErrorRetries=2)
+ )
+ == 2
+ )
+
+
def test_default_retries_applies_when_the_specific_field_is_unset():
policy: Final = RetryPolicy(DefaultRetries=2)
diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index c5ae5d4b151..fde91b25fa5 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -15391,6 +15391,53 @@ async def test_router_retry_policy_controls_upstream_attempt_count(
assert upstream.call_count == expected_upstream_calls
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "error_body,error_type",
+ [
+ ({"message": "model is down", "type": "server_error"}, litellm.NotFoundError),
+ ({"message": "Response with id 'resp_x' not found.", "type": "invalid_request_error"}, litellm.BadRequestError),
+ ],
+)
+@pytest.mark.parametrize(
+ "retry_policy,expected_upstream_calls",
+ [
+ ({"DefaultRetries": 3}, 4),
+ ({"DefaultRetries": 3, "NotFoundErrorRetries": 0}, 1),
+ ({"NotFoundErrorRetries": 2}, 3),
+ ],
+)
+async def test_router_not_found_retries_governs_every_404_shape(
+ monkeypatch: pytest.MonkeyPatch, retry_policy, expected_upstream_calls, error_body, error_type
+):
+ monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+ router = litellm.Router(
+ model_list=[
+ {
+ "model_name": "gpt-5.6",
+ "litellm_params": {
+ "model": "openai/gpt-5.6",
+ "api_key": "sk-fake",
+ "api_base": "https://retry-policy.local/v1",
+ },
+ }
+ ],
+ num_retries=2,
+ retry_policy=retry_policy,
+ disable_cooldowns=True,
+ )
+
+ with respx.mock(assert_all_called=True) as respx_mock:
+ upstream = respx_mock.post("https://retry-policy.local/v1/chat/completions").mock(
+ return_value=httpx.Response(404, headers={"retry-after": "0"}, json={"error": error_body})
+ )
+ with pytest.raises(error_type) as raised:
+ await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
+
+ assert raised.value.status_code == 404
+ assert upstream.call_count == expected_upstream_calls
+
+
@pytest.mark.asyncio
async def test_generic_call_keeps_the_deployment_name_of_an_azure_ai_model_on_an_azure_openai_host(monkeypatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.test.tsx
index 14549420623..838d47ff7e5 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.test.tsx
@@ -43,6 +43,26 @@ describe("ModelRetrySettingsTab", () => {
expect(screen.getByText(/RateLimitError \(429\)/)).toBeInTheDocument();
expect(screen.getByText(/ContentPolicyViolationError \(400\)/)).toBeInTheDocument();
expect(screen.getByText(/InternalServerError \(500\)/)).toBeInTheDocument();
+ expect(screen.getByText(/NotFoundError \(404\)/)).toBeInTheDocument();
+ });
+
+ it("should write the NotFoundError row to NotFoundErrorRetries ahead of the catch-all row", () => {
+ const setGlobalRetryPolicy = vi.fn();
+ render(
+ ,
+ );
+
+ const notFoundInput = screen.getByRole("spinbutton", { name: /NotFoundError \(404\) retry count$/ });
+ fireEvent.change(notFoundInput, { target: { value: "2" } });
+
+ const updater = setGlobalRetryPolicy.mock.calls.at(-1)![0];
+ expect(updater({ DefaultRetries: 3 })).toMatchObject({ DefaultRetries: 3, NotFoundErrorRetries: 2 });
});
it("should use defaultRetry when globalRetryPolicy is null (global scope)", () => {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx
index 069a3f27beb..a61eccbb5a2 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx
@@ -35,6 +35,7 @@ const retryPolicyMap: Record = {
"ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries",
"InternalServerError (500)": "InternalServerErrorRetries",
"ServiceUnavailableError (503)": "ServiceUnavailableErrorRetries",
+ "NotFoundError (404)": "NotFoundErrorRetries",
"All other errors": "DefaultRetries",
};
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 7e725da3f46..94df947eeab 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -36811,6 +36811,8 @@ export interface components {
DefaultRetries?: number | null;
/** Internalservererrorretries */
InternalServerErrorRetries?: number | null;
+ /** Notfounderrorretries */
+ NotFoundErrorRetries?: number | null;
/** Ratelimiterrorretries */
RateLimitErrorRetries?: number | null;
/** Serviceunavailableerrorretries */