diff --git a/litellm/router.py b/litellm/router.py index 142b2019e6e..72f5c38b7f3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -565,6 +565,7 @@ _SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str]) _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None) _ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) _FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_DEPLOYMENTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _DEPLOYMENT_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | GenericLiteLLMParams]] = TypeAdapter( Mapping[str, object] | GenericLiteLLMParams @@ -8654,6 +8655,10 @@ class Router: num_retries = _retry_policy_retries _retry_policy_applies = True + first_skipped_ids: Final = self._deployment_ids_to_skip_on_retry( + exception=original_exception, + already_skipped=kwargs.get("_retry_skipped_deployment_ids"), + ) # raises an exception if this error should not be retries # Skip this check if retry policy applies (retry policy takes precedence) if not _retry_policy_applies: @@ -8664,6 +8669,7 @@ class Router: context_window_fallbacks=context_window_fallbacks, regular_fallbacks=fallbacks, content_policy_fallbacks=content_policy_fallbacks, + tried_deployment_ids=first_skipped_ids, ) # Update max_retries after overrides (deployment_num_retries / retry_policy) _metadata["max_retries"] = num_retries @@ -8671,10 +8677,6 @@ class Router: ## LOGGING if num_retries > 0: kwargs = self.log_retry(kwargs=kwargs, e=original_exception) - first_skipped_ids: Final = self._deployment_ids_to_skip_on_retry( - exception=original_exception, - already_skipped=kwargs.get("_retry_skipped_deployment_ids"), - ) if first_skipped_ids: kwargs["_retry_skipped_deployment_ids"] = first_skipped_ids # rebind-ok: the next attempt reads it else: @@ -8710,6 +8712,8 @@ class Router: return response except Exception as e: + if is_guardrail_intervention(e): + raise # Always track the latest error so we raise the most # recent exception instead of the first one. original_exception = e @@ -8733,6 +8737,10 @@ class Router: else: _healthy_deployments = [] + skipped_ids = self._deployment_ids_to_skip_on_retry( + exception=e, + already_skipped=kwargs.get("_retry_skipped_deployment_ids"), + ) # Check if this error is non-retryable (e.g., 400 context # window exceeded). If so, raise immediately instead of # continuing the retry loop. Respect retry policy @@ -8746,14 +8754,11 @@ class Router: context_window_fallbacks=context_window_fallbacks, regular_fallbacks=fallbacks, content_policy_fallbacks=content_policy_fallbacks, + tried_deployment_ids=skipped_ids, ) except Exception: raise e - skipped_ids = self._deployment_ids_to_skip_on_retry( - exception=e, - already_skipped=kwargs.get("_retry_skipped_deployment_ids"), - ) if skipped_ids: kwargs["_retry_skipped_deployment_ids"] = skipped_ids # rebind-ok: the next attempt reads it _timeout = self._time_to_sleep_before_retry( @@ -8825,10 +8830,13 @@ class Router: context_window_fallbacks: list | None = None, content_policy_fallbacks: list | None = None, regular_fallbacks: list | None = None, + tried_deployment_ids: tuple[str, ...] = (), ): """ 1. raise an exception for ContextWindowExceededError if context_window_fallbacks is not None 2. raise an exception for ContentPolicyViolationError if content_policy_fallbacks is not None + 3. raise a non-retryable error (e.g. 400, 404) unless a healthy deployment that has not yet + refused this request (`tried_deployment_ids`) can take the retry 2. raise an exception for RateLimitError if - there are no fallbacks @@ -8850,13 +8858,15 @@ class Router: raise error status_code: Final = getattr(error, "status_code", None) - if status_code is not None and not litellm.should_retry(status_code): - # 401/403 are special cases - allow retry if multiple deployments exist (handled below) - if status_code not in (401, 403): + # 401/403 are special cases - allow retry if multiple deployments exist (handled below) + if status_code is not None and not litellm.should_retry(status_code) and status_code not in (401, 403): + deployments: Final = _DEPLOYMENTS_ADAPTER.validate_python(healthy_deployments or ()) + has_untried_deployment: Final = bool(tried_deployment_ids) and any( + _configured_model_info(deployment).get("id") not in tried_deployment_ids for deployment in deployments + ) + if not has_untried_deployment: raise error - - if isinstance(error, litellm.NotFoundError): - raise error + return True # Error we should only retry if there are other deployments if isinstance(error, openai.RateLimitError): if ( diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 9b1bdbf702e..297105d8a9d 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -20376,6 +20376,153 @@ def _retry_skip_deployment(deployment_id, host, litellm_params=None, model_info= } +_UPSTREAM_404 = { + "message": "The model `gpt-5.6` does not exist or you do not have access to it.", + "type": "invalid_request_error", + "code": "model_not_found", +} + +_UPSTREAM_SUCCESS = { + "id": "chatcmpl-failover", + "object": "chat.completion", + "created": 1, + "model": "gpt-5.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi back"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status_code,upstream_error", [(404, _UPSTREAM_404), (400, _UPSTREAM_400)]) +async def test_router_non_retryable_error_fails_over_to_sibling_deployment( + monkeypatch: pytest.MonkeyPatch, status_code, upstream_error +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = litellm.Router( + model_list=[ + _retry_skip_deployment("rejecting", "rejecting", litellm_params={"weight": 1}), + _retry_skip_deployment("accepting", "accepting", litellm_params={"weight": 0}), + ], + num_retries=2, + ) + + with respx.mock(assert_all_called=True) as respx_mock: + rejecting = respx_mock.post("https://rejecting.local/v1/chat/completions").mock( + return_value=httpx.Response(status_code, json={"error": upstream_error}) + ) + accepting = respx_mock.post("https://accepting.local/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_UPSTREAM_SUCCESS) + ) + response = await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) + + assert (rejecting.call_count, accepting.call_count) == (1, 1) + assert response.choices[0].message.content == "hi back" + assert response._hidden_params["model_id"] == "accepting" + assert response._hidden_params["additional_headers"]["x-litellm-attempted-retries"] == 1 + + +@pytest.mark.asyncio +async def test_router_not_found_failover_still_cools_down_the_refusing_deployment(monkeypatch: pytest.MonkeyPatch): + from litellm.router_utils.cooldown_handlers import async_get_cooldown_deployments + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = litellm.Router( + model_list=[ + _retry_skip_deployment("rejecting", "rejecting", litellm_params={"weight": 1}), + _retry_skip_deployment("accepting", "accepting", litellm_params={"weight": 0}), + ], + num_retries=2, + ) + + with respx.mock(assert_all_called=True) as respx_mock: + respx_mock.post("https://rejecting.local/v1/chat/completions").mock( + return_value=httpx.Response(404, json={"error": _UPSTREAM_404}) + ) + respx_mock.post("https://accepting.local/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_UPSTREAM_SUCCESS) + ) + await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) + + assert await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == [ + "rejecting" + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "status_code,upstream_error,expected_exception", + [(404, _UPSTREAM_404, litellm.NotFoundError), (400, _UPSTREAM_400, litellm.BadRequestError)], +) +async def test_router_non_retryable_error_on_single_deployment_group_is_not_retried( + monkeypatch: pytest.MonkeyPatch, status_code, upstream_error, expected_exception +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = litellm.Router(model_list=[_retry_skip_deployment("only", "only")], num_retries=2) + + with respx.mock(assert_all_called=True) as respx_mock: + only = respx_mock.post("https://only.local/v1/chat/completions").mock( + return_value=httpx.Response(status_code, json={"error": upstream_error}) + ) + with pytest.raises(expected_exception): + await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) + + assert only.call_count == 1 + + +@pytest.mark.asyncio +async def test_router_non_retryable_error_fails_over_past_every_refusing_deployment(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = litellm.Router( + model_list=[ + _retry_skip_deployment("first", "first", litellm_params={"weight": 1}), + _retry_skip_deployment("second", "second", litellm_params={"weight": 1}), + _retry_skip_deployment("accepting", "accepting", litellm_params={"weight": 0}), + ], + num_retries=2, + ) + + with respx.mock(assert_all_called=True) as respx_mock: + refusing = tuple( + respx_mock.post(f"https://{host}.local/v1/chat/completions").mock( + return_value=httpx.Response(404, json={"error": _UPSTREAM_404}) + ) + for host in ("first", "second") + ) + accepting = respx_mock.post("https://accepting.local/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_UPSTREAM_SUCCESS) + ) + response = await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) + + assert tuple(route.call_count for route in (*refusing, accepting)) == (1, 1, 1) + assert response._hidden_params["model_id"] == "accepting" + + +@pytest.mark.asyncio +async def test_router_non_retryable_error_stops_once_every_deployment_refused(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = litellm.Router( + model_list=[ + _retry_skip_deployment("first", "first", litellm_params={"weight": 1}), + _retry_skip_deployment("second", "second", litellm_params={"weight": 0}), + ], + num_retries=3, + disable_cooldowns=True, + ) + + with respx.mock(assert_all_called=True) as respx_mock: + first = respx_mock.post("https://first.local/v1/chat/completions").mock( + return_value=httpx.Response(400, json={"error": _UPSTREAM_400}) + ) + second = respx_mock.post("https://second.local/v1/chat/completions").mock( + return_value=httpx.Response(400, json={"error": _UPSTREAM_400}) + ) + with pytest.raises(litellm.BadRequestError) as exc_info: + await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]) + + assert (first.call_count, second.call_count) == (1, 1) + assert exc_info.value.failed_deployment_id == "second" + + @pytest.mark.parametrize( "status_code,failed_deployment_id,already_skipped,expected", [ @@ -21167,7 +21314,7 @@ async def test_router_max_parallel_requests_overflow_is_429_without_cooldown_or_ assert all("capped-deployment" in r.message and "max_parallel_requests=1" in r.message for r in rejected) assert route.call_count == 1 assert sibling_route.call_count == 0 - assert await _async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == [] + assert await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == [] @pytest.mark.asyncio @@ -21286,7 +21433,7 @@ async def test_router_max_parallel_requests_overflow_takes_the_ordinary_429_fall assert len(results) == 3 assert primary.call_count == 1 assert fallback.call_count == 2 - assert await _async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == [] + assert await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == [] @pytest.mark.asyncio diff --git a/tests/unit/test_router/test_router_retries.py b/tests/unit/test_router/test_router_retries.py index a5664c1348f..55b11780404 100644 --- a/tests/unit/test_router/test_router_retries.py +++ b/tests/unit/test_router/test_router_retries.py @@ -12,6 +12,7 @@ from pydantic import TypeAdapter import litellm from litellm import Router +from litellm.exceptions import GuardrailRaisedException from litellm.integrations.custom_logger import CustomLogger from litellm.router import AllowedFailsPolicy, RetryPolicy import os @@ -629,7 +630,7 @@ def test_no_retry_for_bad_request_error_400(): Test that 400 BadRequestError is NOT retried, even if healthy deployments exist. This tests the fix for GitHub issue #19216. """ - healthy_deployments = ["deployment1", "deployment2"] # Multiple healthy deployments + healthy_deployments = _deployments("deployment1", "deployment2") router = Router( model_list=[ @@ -666,7 +667,7 @@ def test_no_retry_for_unprocessable_entity_error_422(): """ Test that 422 UnprocessableEntityError is NOT retried, even if healthy deployments exist. """ - healthy_deployments = ["deployment1", "deployment2"] # Multiple healthy deployments + healthy_deployments = _deployments("deployment1", "deployment2") router = Router( model_list=[ @@ -703,6 +704,109 @@ def test_no_retry_for_unprocessable_entity_error_422(): print("Correctly raised UnprocessableEntityError without retry:", e) +def _not_found() -> litellm.NotFoundError: + return litellm.NotFoundError(message="model not found", model="m", llm_provider="openai") + + +def _bad_request() -> litellm.BadRequestError: + return litellm.BadRequestError(message="bad request", model="m", llm_provider="openai") + + +def _deployments(*deployment_ids: str) -> list[dict[str, dict[str, str]]]: + return [{"model_info": {"id": deployment_id}} for deployment_id in deployment_ids] + + +@pytest.mark.parametrize( + "error,tried_deployment_ids,healthy_deployment_ids", + [ + (_not_found(), ("a",), ("a", "b")), + (_bad_request(), ("a",), ("a", "b")), + (_bad_request(), ("a",), ("b",)), + ( + litellm.UnprocessableEntityError( + message="unprocessable", + model="m", + llm_provider="openai", + response=httpx.Response(422, request=httpx.Request("POST", "https://example.invalid/v1")), + ), + ("a",), + ("a", "b"), + ), + ], +) +def test_non_retryable_error_is_retried_on_an_untried_healthy_deployment( + error: Exception, tried_deployment_ids: tuple[str, ...], healthy_deployment_ids: tuple[str, ...] +) -> None: + healthy_deployments: Final = _deployments(*healthy_deployment_ids) + + assert ( + _retry_test_router().should_retry_this_error( + error=error, + healthy_deployments=healthy_deployments, + all_deployments=healthy_deployments, + tried_deployment_ids=tried_deployment_ids, + ) + is True + ) + + +@pytest.mark.parametrize( + "error,tried_deployment_ids,healthy_deployment_ids", + [ + (_bad_request(), (), ("a", "b")), + (_bad_request(), ("a", "b"), ("a", "b")), + (_not_found(), ("a",), ("a",)), + (_not_found(), ("a",), ()), + ], +) +def test_non_retryable_error_is_raised_when_no_untried_healthy_deployment_is_left( + error: Exception, tried_deployment_ids: tuple[str, ...], healthy_deployment_ids: tuple[str, ...] +) -> None: + healthy_deployments: Final = _deployments(*healthy_deployment_ids) + + with pytest.raises(type(error)) as exc_info: + _retry_test_router().should_retry_this_error( + error=error, + healthy_deployments=healthy_deployments, + all_deployments=healthy_deployments, + tried_deployment_ids=tried_deployment_ids, + ) + + assert exc_info.value is error + + +@pytest.mark.asyncio +async def test_guardrail_rejection_on_a_retry_is_not_failed_over_to_a_sibling() -> None: + router: Final = Router( + model_list=[ + { + "model_name": "retry-test-model", + "litellm_params": {"model": "openai/retry-test-model", "api_key": "test-key"}, + "model_info": {"id": deployment_id}, + } + for deployment_id in ("refusing", "guarded", "unguarded") + ], + disable_cooldowns=True, + ) + not_found: Final = _not_found() + not_found.retry_skip_deployment_id = "refusing" + guardrail_rejection: Final = GuardrailRaisedException(guardrail_name="pii", message="blocked", blocked_content=True) + guardrail_rejection.retry_skip_deployment_id = "guarded" + attempts: Final = AsyncMock(side_effect=[not_found, guardrail_rejection, "unguarded answer"]) + + with pytest.raises(GuardrailRaisedException) as exc_info: + await router.async_function_with_retries( + model="retry-test-model", + messages=[{"role": "user", "content": "retry test"}], + original_function=attempts, + num_retries=2, + metadata={}, + ) + + assert exc_info.value is guardrail_rejection + assert attempts.await_count == 2 + + def test_no_retry_when_no_healthy_deployments(): healthy_deployments = []