From d7ad5892cae4fae3a2e9ceebc30713f5d4e2e5c4 Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Mon, 12 Oct 2026 15:29:07 +1100 Subject: [PATCH 1/4] fix(router): fail over to a sibling deployment on non-retryable 4xx errors should_retry_this_error raised every non-retryable status except 401/403 straight to the caller, so a 404 model_not_found or 400 from one leg of a multi-deployment group surfaced to the client even when another deployment in the group was healthy. The broken leg was only benched afterwards by cooldown, so the first caller in each cooldown window still saw the failure A non-retryable error now retries when a healthy deployment that has not already refused this request exists. The existing retry skip list (_retry_skipped_deployment_ids) steers the retry away from the deployments that refused, and the retry stops as soon as none is left, so a request every leg rejects is not repeated in place (the concern behind #19216). Single-deployment groups and retry policies behave as before, and the refusing deployment still goes into cooldown --- litellm/router.py | 40 +++-- tests/unit/test_router/test_router.py | 151 +++++++++++++++++- tests/unit/test_router/test_router_retries.py | 61 +++++++ 3 files changed, 236 insertions(+), 16 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index cdede46ef5c..c9c08617ab9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -502,6 +502,7 @@ _SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str]) _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None) _ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) _FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_DEPLOYMENTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _DEPLOYMENT_SELECTED_EVENT: Final = "litellm.request.deployment_selected" @@ -545,6 +546,12 @@ def _as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]: return tuple(item for item in value if isinstance(item, str)) if isinstance(value, tuple) else () +def _has_untried_deployment(deployments: Sequence[Mapping[str, object]], tried_deployment_ids: tuple[str, ...]) -> bool: + return bool(tried_deployment_ids) and any( + _configured_model_info(deployment).get("id") not in tried_deployment_ids for deployment in deployments + ) + + def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]: if silent_model is None: return () @@ -8103,6 +8110,10 @@ class Router: num_retries = _retry_policy_retries _retry_policy_applies = True + first_skipped_ids: Final = self._deployment_ids_to_skip_on_retry( + exception=original_exception, + already_skipped=kwargs.get("_retry_skipped_deployment_ids"), + ) # raises an exception if this error should not be retries # Skip this check if retry policy applies (retry policy takes precedence) if not _retry_policy_applies: @@ -8113,6 +8124,7 @@ class Router: context_window_fallbacks=context_window_fallbacks, regular_fallbacks=fallbacks, content_policy_fallbacks=content_policy_fallbacks, + tried_deployment_ids=first_skipped_ids, ) # Update max_retries after overrides (deployment_num_retries / retry_policy) _metadata["max_retries"] = num_retries @@ -8120,10 +8132,6 @@ class Router: ## LOGGING if num_retries > 0: kwargs = self.log_retry(kwargs=kwargs, e=original_exception) - first_skipped_ids: Final = self._deployment_ids_to_skip_on_retry( - exception=original_exception, - already_skipped=kwargs.get("_retry_skipped_deployment_ids"), - ) if first_skipped_ids: kwargs["_retry_skipped_deployment_ids"] = first_skipped_ids # rebind-ok: the next attempt reads it else: @@ -8178,6 +8186,10 @@ class Router: else: _healthy_deployments = [] + skipped_ids = self._deployment_ids_to_skip_on_retry( + exception=e, + already_skipped=kwargs.get("_retry_skipped_deployment_ids"), + ) # Check if this error is non-retryable (e.g., 400 context # window exceeded). If so, raise immediately instead of # continuing the retry loop. Respect retry policy @@ -8191,14 +8203,11 @@ class Router: context_window_fallbacks=context_window_fallbacks, regular_fallbacks=fallbacks, content_policy_fallbacks=content_policy_fallbacks, + tried_deployment_ids=skipped_ids, ) except Exception: raise e - skipped_ids = self._deployment_ids_to_skip_on_retry( - exception=e, - already_skipped=kwargs.get("_retry_skipped_deployment_ids"), - ) if skipped_ids: kwargs["_retry_skipped_deployment_ids"] = skipped_ids # rebind-ok: the next attempt reads it _timeout = self._time_to_sleep_before_retry( @@ -8270,10 +8279,13 @@ class Router: context_window_fallbacks: list | None = None, content_policy_fallbacks: list | None = None, regular_fallbacks: list | None = None, + tried_deployment_ids: tuple[str, ...] = (), ): """ 1. raise an exception for ContextWindowExceededError if context_window_fallbacks is not None 2. raise an exception for ContentPolicyViolationError if content_policy_fallbacks is not None + 3. raise a non-retryable error (e.g. 400, 404) unless a healthy deployment that has not yet + refused this request (`tried_deployment_ids`) can take the retry 2. raise an exception for RateLimitError if - there are no fallbacks @@ -8295,13 +8307,13 @@ class Router: raise error status_code: Final = getattr(error, "status_code", None) - if status_code is not None and not litellm.should_retry(status_code): - # 401/403 are special cases - allow retry if multiple deployments exist (handled below) - if status_code not in (401, 403): + # 401/403 are special cases - allow retry if multiple deployments exist (handled below) + if status_code is not None and not litellm.should_retry(status_code) and status_code not in (401, 403): + if not _has_untried_deployment( + _DEPLOYMENTS_ADAPTER.validate_python(healthy_deployments or ()), tried_deployment_ids + ): raise error - - if isinstance(error, litellm.NotFoundError): - raise error + return True # Error we should only retry if there are other deployments if isinstance(error, openai.RateLimitError): if ( diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 4a7a23ff57b..e883c6a6bb5 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -18709,6 +18709,153 @@ def _retry_skip_deployment(deployment_id, host, litellm_params=None, model_info= } +_UPSTREAM_404 = { + "message": "The model `gpt-5.6` does not exist or you do not have access to it.", + "type": "invalid_request_error", + "code": "model_not_found", +} + +_UPSTREAM_SUCCESS = { + "id": "chatcmpl-failover", + "object": "chat.completion", + "created": 1, + "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi back"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status_code,upstream_error", [(404, _UPSTREAM_404), (400, _UPSTREAM_400)]) +async def test_router_non_retryable_error_fails_over_to_sibling_deployment( + monkeypatch: pytest.MonkeyPatch, status_code, upstream_error +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = litellm.Router( + model_list=[ + _retry_skip_deployment("rejecting", "rejecting", litellm_params={"weight": 1}), + _retry_skip_deployment("accepting", "accepting", litellm_params={"weight": 0}), + ], + num_retries=2, + ) + + with respx.mock(assert_all_called=True) as respx_mock: + rejecting = respx_mock.post("https://rejecting.local/v1/chat/completions").mock( + return_value=httpx.Response(status_code, json={"error": upstream_error}) + ) + accepting = respx_mock.post("https://accepting.local/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_UPSTREAM_SUCCESS) + ) + response = await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) + + assert (rejecting.call_count, accepting.call_count) == (1, 1) + assert response.choices[0].message.content == "hi back" + assert response._hidden_params["model_id"] == "accepting" + assert response._hidden_params["additional_headers"]["x-litellm-attempted-retries"] == 1 + + +@pytest.mark.asyncio +async def test_router_not_found_failover_still_cools_down_the_refusing_deployment(monkeypatch: pytest.MonkeyPatch): + from litellm.router_utils.cooldown_handlers import async_get_cooldown_deployments + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = litellm.Router( + model_list=[ + _retry_skip_deployment("rejecting", "rejecting", litellm_params={"weight": 1}), + _retry_skip_deployment("accepting", "accepting", litellm_params={"weight": 0}), + ], + num_retries=2, + ) + + with respx.mock(assert_all_called=True) as respx_mock: + respx_mock.post("https://rejecting.local/v1/chat/completions").mock( + return_value=httpx.Response(404, json={"error": _UPSTREAM_404}) + ) + respx_mock.post("https://accepting.local/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_UPSTREAM_SUCCESS) + ) + await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) + + assert await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == [ + "rejecting" + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "status_code,upstream_error,expected_exception", + [(404, _UPSTREAM_404, litellm.NotFoundError), (400, _UPSTREAM_400, litellm.BadRequestError)], +) +async def test_router_non_retryable_error_on_single_deployment_group_is_not_retried( + monkeypatch: pytest.MonkeyPatch, status_code, upstream_error, expected_exception +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = litellm.Router(model_list=[_retry_skip_deployment("only", "only")], num_retries=2) + + with respx.mock(assert_all_called=True) as respx_mock: + only = respx_mock.post("https://only.local/v1/chat/completions").mock( + return_value=httpx.Response(status_code, json={"error": upstream_error}) + ) + with pytest.raises(expected_exception): + await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) + + assert only.call_count == 1 + + +@pytest.mark.asyncio +async def test_router_non_retryable_error_fails_over_past_every_refusing_deployment(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = litellm.Router( + model_list=[ + _retry_skip_deployment("first", "first", litellm_params={"weight": 1}), + _retry_skip_deployment("second", "second", litellm_params={"weight": 1}), + _retry_skip_deployment("accepting", "accepting", litellm_params={"weight": 0}), + ], + num_retries=2, + ) + + with respx.mock(assert_all_called=True) as respx_mock: + refusing = tuple( + respx_mock.post(f"https://{host}.local/v1/chat/completions").mock( + return_value=httpx.Response(404, json={"error": _UPSTREAM_404}) + ) + for host in ("first", "second") + ) + accepting = respx_mock.post("https://accepting.local/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_UPSTREAM_SUCCESS) + ) + response = await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) + + assert tuple(route.call_count for route in (*refusing, accepting)) == (1, 1, 1) + assert response._hidden_params["model_id"] == "accepting" + + +@pytest.mark.asyncio +async def test_router_non_retryable_error_stops_once_every_deployment_refused(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = litellm.Router( + model_list=[ + _retry_skip_deployment("first", "first", litellm_params={"weight": 1}), + _retry_skip_deployment("second", "second", litellm_params={"weight": 0}), + ], + num_retries=3, + disable_cooldowns=True, + ) + + with respx.mock(assert_all_called=True) as respx_mock: + first = respx_mock.post("https://first.local/v1/chat/completions").mock( + return_value=httpx.Response(400, json={"error": _UPSTREAM_400}) + ) + second = respx_mock.post("https://second.local/v1/chat/completions").mock( + return_value=httpx.Response(400, json={"error": _UPSTREAM_400}) + ) + with pytest.raises(litellm.BadRequestError) as exc_info: + await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) + + assert (first.call_count, second.call_count) == (1, 1) + assert exc_info.value.failed_deployment_id == "second" + + @pytest.mark.parametrize( "status_code,failed_deployment_id,already_skipped,expected", [ @@ -19500,7 +19647,7 @@ async def test_router_max_parallel_requests_overflow_is_429_without_cooldown_or_ assert all("capped-deployment" in r.message and "max_parallel_requests=1" in r.message for r in rejected) assert route.call_count == 1 assert sibling_route.call_count == 0 - assert await _async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == [] + assert await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == [] @pytest.mark.asyncio @@ -19619,7 +19766,7 @@ async def test_router_max_parallel_requests_overflow_takes_the_ordinary_429_fall assert len(results) == 3 assert primary.call_count == 1 assert fallback.call_count == 2 - assert await _async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == [] + assert await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == [] @pytest.mark.asyncio diff --git a/tests/unit/test_router/test_router_retries.py b/tests/unit/test_router/test_router_retries.py index a5664c1348f..0e9b1b6fcff 100644 --- a/tests/unit/test_router/test_router_retries.py +++ b/tests/unit/test_router/test_router_retries.py @@ -703,6 +703,67 @@ def test_no_retry_for_unprocessable_entity_error_422(): print("Correctly raised UnprocessableEntityError without retry:", e) +def _not_found() -> litellm.NotFoundError: + return litellm.NotFoundError(message="model not found", model="m", llm_provider="openai") + + +def _bad_request() -> litellm.BadRequestError: + return litellm.BadRequestError(message="bad request", model="m", llm_provider="openai") + + +def _deployments(*deployment_ids: str) -> list[dict[str, dict[str, str]]]: + return [{"model_info": {"id": deployment_id}} for deployment_id in deployment_ids] + + +@pytest.mark.parametrize( + "error,tried_deployment_ids,healthy_deployment_ids", + [ + (_not_found(), ("a",), ("a", "b")), + (_bad_request(), ("a",), ("a", "b")), + (_bad_request(), ("a",), ("b",)), + ], +) +def test_non_retryable_error_is_retried_on_an_untried_healthy_deployment( + error: Exception, tried_deployment_ids: tuple[str, ...], healthy_deployment_ids: tuple[str, ...] +) -> None: + healthy_deployments: Final = _deployments(*healthy_deployment_ids) + + assert ( + _retry_test_router().should_retry_this_error( + error=error, + healthy_deployments=healthy_deployments, + all_deployments=healthy_deployments, + tried_deployment_ids=tried_deployment_ids, + ) + is True + ) + + +@pytest.mark.parametrize( + "error,tried_deployment_ids,healthy_deployment_ids", + [ + (_bad_request(), (), ("a", "b")), + (_bad_request(), ("a", "b"), ("a", "b")), + (_not_found(), ("a",), ("a",)), + (_not_found(), ("a",), ()), + ], +) +def test_non_retryable_error_is_raised_when_no_untried_healthy_deployment_is_left( + error: Exception, tried_deployment_ids: tuple[str, ...], healthy_deployment_ids: tuple[str, ...] +) -> None: + healthy_deployments: Final = _deployments(*healthy_deployment_ids) + + with pytest.raises(type(error)) as exc_info: + _retry_test_router().should_retry_this_error( + error=error, + healthy_deployments=healthy_deployments, + all_deployments=healthy_deployments, + tried_deployment_ids=tried_deployment_ids, + ) + + assert exc_info.value is error + + def test_no_retry_when_no_healthy_deployments(): healthy_deployments = [] From 1080389f64fa69b3f2021be05d26cf5f57d03429 Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Mon, 12 Oct 2026 15:39:29 +1100 Subject: [PATCH 2/4] refactor(router): inline the single-use untried-deployment check The router coverage gate requires every router.py function to be named in a test, and the helper had one caller --- litellm/router.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index c9c08617ab9..dc6130a162c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -546,12 +546,6 @@ def _as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]: return tuple(item for item in value if isinstance(item, str)) if isinstance(value, tuple) else () -def _has_untried_deployment(deployments: Sequence[Mapping[str, object]], tried_deployment_ids: tuple[str, ...]) -> bool: - return bool(tried_deployment_ids) and any( - _configured_model_info(deployment).get("id") not in tried_deployment_ids for deployment in deployments - ) - - def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]: if silent_model is None: return () @@ -8309,9 +8303,11 @@ class Router: status_code: Final = getattr(error, "status_code", None) # 401/403 are special cases - allow retry if multiple deployments exist (handled below) if status_code is not None and not litellm.should_retry(status_code) and status_code not in (401, 403): - if not _has_untried_deployment( - _DEPLOYMENTS_ADAPTER.validate_python(healthy_deployments or ()), tried_deployment_ids - ): + deployments: Final = _DEPLOYMENTS_ADAPTER.validate_python(healthy_deployments or ()) + has_untried_deployment: Final = bool(tried_deployment_ids) and any( + _configured_model_info(deployment).get("id") not in tried_deployment_ids for deployment in deployments + ) + if not has_untried_deployment: raise error return True # Error we should only retry if there are other deployments From fc0783c5e4be00b2c2613d7ddbe9e1be0219a940 Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Mon, 12 Oct 2026 15:54:26 +1100 Subject: [PATCH 3/4] fix(router): never fail a guardrail rejection over to a sibling on retry A guardrail intervention raised by a retry attempt is a 400, so once non-retryable errors could fail over it would be forwarded to an untried sibling deployment instead of reaching the caller. The first attempt already re-raised guardrail interventions; the retry loop now does too --- litellm/router.py | 2 + tests/unit/test_router/test_router_retries.py | 37 ++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index dc6130a162c..86b39ef93e3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8161,6 +8161,8 @@ class Router: return response except Exception as e: + if is_guardrail_intervention(e): + raise # Always track the latest error so we raise the most # recent exception instead of the first one. original_exception = e diff --git a/tests/unit/test_router/test_router_retries.py b/tests/unit/test_router/test_router_retries.py index 0e9b1b6fcff..bc21b3fdab1 100644 --- a/tests/unit/test_router/test_router_retries.py +++ b/tests/unit/test_router/test_router_retries.py @@ -12,6 +12,7 @@ from pydantic import TypeAdapter import litellm from litellm import Router +from litellm.exceptions import GuardrailRaisedException from litellm.integrations.custom_logger import CustomLogger from litellm.router import AllowedFailsPolicy, RetryPolicy import os @@ -629,7 +630,7 @@ def test_no_retry_for_bad_request_error_400(): Test that 400 BadRequestError is NOT retried, even if healthy deployments exist. This tests the fix for GitHub issue #19216. """ - healthy_deployments = ["deployment1", "deployment2"] # Multiple healthy deployments + healthy_deployments = _deployments("deployment1", "deployment2") router = Router( model_list=[ @@ -666,7 +667,7 @@ def test_no_retry_for_unprocessable_entity_error_422(): """ Test that 422 UnprocessableEntityError is NOT retried, even if healthy deployments exist. """ - healthy_deployments = ["deployment1", "deployment2"] # Multiple healthy deployments + healthy_deployments = _deployments("deployment1", "deployment2") router = Router( model_list=[ @@ -764,6 +765,38 @@ def test_non_retryable_error_is_raised_when_no_untried_healthy_deployment_is_lef assert exc_info.value is error +@pytest.mark.asyncio +async def test_guardrail_rejection_on_a_retry_is_not_failed_over_to_a_sibling() -> None: + router: Final = Router( + model_list=[ + { + "model_name": "retry-test-model", + "litellm_params": {"model": "openai/retry-test-model", "api_key": "test-key"}, + "model_info": {"id": deployment_id}, + } + for deployment_id in ("refusing", "guarded", "unguarded") + ], + disable_cooldowns=True, + ) + not_found: Final = _not_found() + not_found.retry_skip_deployment_id = "refusing" + guardrail_rejection: Final = GuardrailRaisedException(guardrail_name="pii", message="blocked", blocked_content=True) + guardrail_rejection.retry_skip_deployment_id = "guarded" + attempts: Final = AsyncMock(side_effect=[not_found, guardrail_rejection, "unguarded answer"]) + + with pytest.raises(GuardrailRaisedException) as exc_info: + await router.async_function_with_retries( + model="retry-test-model", + messages=[{"role": "user", "content": "retry test"}], + original_function=attempts, + num_retries=2, + metadata={}, + ) + + assert exc_info.value is guardrail_rejection + assert attempts.await_count == 2 + + def test_no_retry_when_no_healthy_deployments(): healthy_deployments = [] From ea88c47eadf3c4352b3b841ab6bd9f5300ea2ac6 Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Fri, 16 Oct 2026 08:29:34 +1100 Subject: [PATCH 4/4] test(router): cover sibling failover for a 422 alongside 400 and 404 --- tests/unit/test_router/test_router_retries.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/unit/test_router/test_router_retries.py b/tests/unit/test_router/test_router_retries.py index bc21b3fdab1..55b11780404 100644 --- a/tests/unit/test_router/test_router_retries.py +++ b/tests/unit/test_router/test_router_retries.py @@ -722,6 +722,16 @@ def _deployments(*deployment_ids: str) -> list[dict[str, dict[str, str]]]: (_not_found(), ("a",), ("a", "b")), (_bad_request(), ("a",), ("a", "b")), (_bad_request(), ("a",), ("b",)), + ( + litellm.UnprocessableEntityError( + message="unprocessable", + model="m", + llm_provider="openai", + response=httpx.Response(422, request=httpx.Request("POST", "https://example.invalid/v1")), + ), + ("a",), + ("a", "b"), + ), ], ) def test_non_retryable_error_is_retried_on_an_untried_healthy_deployment(