diff --git a/litellm/router_utils/get_retry_from_policy.py b/litellm/router_utils/get_retry_from_policy.py index ad4a6b0be99..8771d072434 100644 --- a/litellm/router_utils/get_retry_from_policy.py +++ b/litellm/router_utils/get_retry_from_policy.py @@ -1,6 +1,7 @@ """Resolve how many retries a RetryPolicy grants for a given exception.""" from collections.abc import Callable, Mapping +from itertools import chain from types import MappingProxyType from typing import Final @@ -28,6 +29,11 @@ _RETRIES_BY_EXCEPTION_TYPE: Final[Mapping[type, Callable[[RetryPolicy], int | No ) +def _retries_for_a_404_answer(exception: Exception, policy: RetryPolicy) -> int | None: + status_code: Final = getattr(exception, "status_code", None) + return policy.NotFoundErrorRetries if status_code == 404 else None + + def _resolve_policy( retry_policy: RetryPolicy | Mapping[str, int | None] | None, model_group: str | None, @@ -49,13 +55,14 @@ def get_num_retries_from_retry_policy( model_group: str | None = None, model_group_retry_policy: Mapping[str, RetryPolicy | Mapping[str, int | None]] | None = None, ) -> int | None: - """Walk the exception's MRO, most specific class first, and return the first configured retry count.""" + """Prefer NotFoundErrorRetries for any 404 answer, then walk the exception's MRO most specific class first.""" policy: Final = _resolve_policy(retry_policy, model_group, model_group_retry_policy) if policy is None: return None - configured: Final = ( + by_class: Final = ( _RETRIES_BY_EXCEPTION_TYPE[cls](policy) for cls in type(exception).__mro__ if cls in _RETRIES_BY_EXCEPTION_TYPE ) + configured: Final = chain((_retries_for_a_404_answer(exception, policy),), by_class) return next((retries for retries in configured if retries is not None), policy.DefaultRetries) diff --git a/litellm/types/router.py b/litellm/types/router.py index aef64c09417..a75b4654cab 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -109,6 +109,7 @@ class RetryPolicy(BaseModel): ContentPolicyViolationErrorRetries: int | None = None InternalServerErrorRetries: int | None = None ServiceUnavailableErrorRetries: int | None = None + NotFoundErrorRetries: int | None = None DefaultRetries: int | None = None diff --git a/tests/test_litellm/router_utils/test_get_retry_from_policy.py b/tests/test_litellm/router_utils/test_get_retry_from_policy.py index df157ea5ff7..1f358f477d4 100644 --- a/tests/test_litellm/router_utils/test_get_retry_from_policy.py +++ b/tests/test_litellm/router_utils/test_get_retry_from_policy.py @@ -1,6 +1,7 @@ from types import MappingProxyType from typing import Final +import httpx import pytest import litellm @@ -16,6 +17,7 @@ _EXCEPTION_FOR_FIELD: Final = MappingProxyType( "ContentPolicyViolationErrorRetries": litellm.ContentPolicyViolationError, "InternalServerErrorRetries": litellm.InternalServerError, "ServiceUnavailableErrorRetries": litellm.ServiceUnavailableError, + "NotFoundErrorRetries": litellm.NotFoundError, } ) @@ -26,6 +28,17 @@ def _error(exception_type: type[Exception]) -> Exception: return exception_type(message="boom", llm_provider="openai", model="gpt-5.6") +def _bad_request_answered_with_404() -> litellm.BadRequestError: + upstream: Final = httpx.Response( + 404, request=httpx.Request("GET", "https://api.openai.com/v1/responses/resp_missing") + ) + exception: Final = litellm.BadRequestError( + message="Response with id 'resp_missing' not found.", llm_provider="openai", model="gpt-5.6", response=upstream + ) + assert exception.status_code == 404 + return exception + + @pytest.mark.parametrize("field", _SPECIFIC_FIELDS) def test_every_specific_field_controls_retries_for_its_exception(field: str): exception: Final = _error(_EXCEPTION_FOR_FIELD[field]) @@ -66,7 +79,7 @@ def test_subclass_falls_back_to_the_parent_field(): ) -@pytest.mark.parametrize("exception_type", (litellm.BadGatewayError, litellm.NotFoundError)) +@pytest.mark.parametrize("exception_type", (litellm.BadGatewayError,)) def test_default_retries_covers_exceptions_without_a_specific_field(exception_type: type[Exception]): exception: Final = _error(exception_type) @@ -86,6 +99,46 @@ def test_specific_field_wins_over_default_retries(): assert get_num_retries_from_retry_policy(exception=_error(litellm.BadGatewayError), retry_policy=policy) == 0 +def test_not_found_retries_governs_a_bad_request_error_answered_with_404(): + exception: Final = _bad_request_answered_with_404() + + assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(NotFoundErrorRetries=0)) == 0 + assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(NotFoundErrorRetries=4)) == 4 + + +def test_not_found_retries_wins_over_bad_request_and_default_retries_for_a_404(): + policy: Final = RetryPolicy(NotFoundErrorRetries=0, BadRequestErrorRetries=5, DefaultRetries=3) + + assert get_num_retries_from_retry_policy(exception=_bad_request_answered_with_404(), retry_policy=policy) == 0 + assert get_num_retries_from_retry_policy(exception=_error(litellm.NotFoundError), retry_policy=policy) == 0 + + +def test_a_404_without_not_found_retries_falls_back_to_bad_request_then_default_retries(): + exception: Final = _bad_request_answered_with_404() + + assert ( + get_num_retries_from_retry_policy( + exception=exception, retry_policy=RetryPolicy(BadRequestErrorRetries=0, DefaultRetries=3) + ) + == 0 + ) + assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(DefaultRetries=3)) == 3 + assert get_num_retries_from_retry_policy(exception=_error(litellm.NotFoundError), retry_policy=RetryPolicy(DefaultRetries=3)) == 3 + + +def test_not_found_retries_leaves_a_plain_400_alone(): + exception: Final = _error(litellm.BadRequestError) + assert exception.status_code == 400 + + assert get_num_retries_from_retry_policy(exception=exception, retry_policy=RetryPolicy(NotFoundErrorRetries=0)) is None + assert ( + get_num_retries_from_retry_policy( + exception=exception, retry_policy=RetryPolicy(NotFoundErrorRetries=0, BadRequestErrorRetries=2) + ) + == 2 + ) + + def test_default_retries_applies_when_the_specific_field_is_unset(): policy: Final = RetryPolicy(DefaultRetries=2) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index d62c9af7ee5..8310d30d90e 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -15846,6 +15846,53 @@ async def test_router_retry_policy_controls_upstream_attempt_count( assert upstream.call_count == expected_upstream_calls +@pytest.mark.asyncio +@pytest.mark.parametrize( + "error_body,error_type", + [ + ({"message": "model is down", "type": "server_error"}, litellm.NotFoundError), + ({"message": "Response with id 'resp_x' not found.", "type": "invalid_request_error"}, litellm.BadRequestError), + ], +) +@pytest.mark.parametrize( + "retry_policy,expected_upstream_calls", + [ + ({"DefaultRetries": 3}, 4), + ({"DefaultRetries": 3, "NotFoundErrorRetries": 0}, 1), + ({"NotFoundErrorRetries": 2}, 3), + ], +) +async def test_router_not_found_retries_governs_every_404_shape( + monkeypatch: pytest.MonkeyPatch, retry_policy, expected_upstream_calls, error_body, error_type +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-5.6", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_key": "sk-fake", + "api_base": "https://retry-policy.local/v1", + }, + } + ], + num_retries=2, + retry_policy=retry_policy, + disable_cooldowns=True, + ) + + with respx.mock(assert_all_called=True) as respx_mock: + upstream = respx_mock.post("https://retry-policy.local/v1/chat/completions").mock( + return_value=httpx.Response(404, headers={"retry-after": "0"}, json={"error": error_body}) + ) + with pytest.raises(error_type) as raised: + await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) + + assert raised.value.status_code == 404 + assert upstream.call_count == expected_upstream_calls + + @pytest.mark.asyncio async def test_generic_call_keeps_the_deployment_name_of_an_azure_ai_model_on_an_azure_openai_host(monkeypatch): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.test.tsx index 14549420623..838d47ff7e5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.test.tsx @@ -43,6 +43,26 @@ describe("ModelRetrySettingsTab", () => { expect(screen.getByText(/RateLimitError \(429\)/)).toBeInTheDocument(); expect(screen.getByText(/ContentPolicyViolationError \(400\)/)).toBeInTheDocument(); expect(screen.getByText(/InternalServerError \(500\)/)).toBeInTheDocument(); + expect(screen.getByText(/NotFoundError \(404\)/)).toBeInTheDocument(); + }); + + it("should write the NotFoundError row to NotFoundErrorRetries ahead of the catch-all row", () => { + const setGlobalRetryPolicy = vi.fn(); + render( + , + ); + + const notFoundInput = screen.getByRole("spinbutton", { name: /NotFoundError \(404\) retry count$/ }); + fireEvent.change(notFoundInput, { target: { value: "2" } }); + + const updater = setGlobalRetryPolicy.mock.calls.at(-1)![0]; + expect(updater({ DefaultRetries: 3 })).toMatchObject({ DefaultRetries: 3, NotFoundErrorRetries: 2 }); }); it("should use defaultRetry when globalRetryPolicy is null (global scope)", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx index 069a3f27beb..a61eccbb5a2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx @@ -35,6 +35,7 @@ const retryPolicyMap: Record = { "ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries", "InternalServerError (500)": "InternalServerErrorRetries", "ServiceUnavailableError (503)": "ServiceUnavailableErrorRetries", + "NotFoundError (404)": "NotFoundErrorRetries", "All other errors": "DefaultRetries", }; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 1349dd0d099..e33764c3d1a 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -36914,6 +36914,8 @@ export interface components { DefaultRetries?: number | null; /** Internalservererrorretries */ InternalServerErrorRetries?: number | null; + /** Notfounderrorretries */ + NotFoundErrorRetries?: number | null; /** Ratelimiterrorretries */ RateLimitErrorRetries?: number | null; /** Serviceunavailableerrorretries */