From b5eba5744c0a5e08a357e9c6ca73c00fabda4495 Mon Sep 17 00:00:00 2001 From: Hieu Vu Date: Thu, 29 Oct 2026 15:57:14 +0700 Subject: [PATCH] fix(router): enforce per-request retry limits --- litellm/router.py | 26 ++++++++++-- tests/unit/test_router/test_router.py | 60 +++++++++++++++++++++++++-- 2 files changed, 79 insertions(+), 7 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 02ad6275b82..365dfeddd56 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -124,6 +124,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, get_or_create_metadata_bucket, get_parent_otel_span_from_kwargs, + max_retries_per_request_hit, ) from litellm.litellm_core_utils.coroutine_checker import coroutine_checker from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference @@ -9123,6 +9124,12 @@ class Router: ## ADD RETRY TRACKING TO METADATA - used for spend logs retry tracking _metadata["attempted_retries"] = 0 _metadata["max_retries"] = num_retries # Updated after overrides in exception handler + budget_request_kwargs: Final[Mapping[str, object]] = cast( # cast-ok: kwargs keys are request parameter names + Mapping[str, object], kwargs + ) + initial_budget_exhausted: Final = max_retries_per_request_hit( + budget_request_kwargs, litellm.num_retries_per_request + ) try: self._handle_mock_testing_rate_limit_error(model_group=model_group, kwargs=kwargs) # if the function call is successful, no exception will be raised and we'll break out of the loop @@ -9206,6 +9213,9 @@ class Router: else: raise + if initial_budget_exhausted: + raise original_exception + verbose_router_logger.debug("Retrying request with num_retries: %s", num_retries) # decides how long to sleep before retry retry_after: Final = self._time_to_sleep_before_retry( @@ -9216,9 +9226,13 @@ class Router: all_deployments=_all_deployments, ) - await asyncio.sleep(retry_after) + if not max_retries_per_request_hit(budget_request_kwargs, litellm.num_retries_per_request): + await asyncio.sleep(retry_after) for current_attempt in range(num_retries): + budget_exhausted_before_call: bool = max_retries_per_request_hit( + budget_request_kwargs, litellm.num_retries_per_request + ) try: # Update retry tracking metadata before each retry attempt _metadata["attempted_retries"] = current_attempt + 1 @@ -9282,8 +9296,11 @@ class Router: ) if skipped_ids: kwargs["_retry_skipped_deployment_ids"] = skipped_ids # rebind-ok: the next attempt reads it - if isinstance(e, litellm.UnsupportedEndpointError) and _every_deployment_refused( - _DEPLOYMENTS_ADAPTER.validate_python(_healthy_deployments), skipped_ids + if budget_exhausted_before_call or ( + isinstance(e, litellm.UnsupportedEndpointError) + and _every_deployment_refused( + _DEPLOYMENTS_ADAPTER.validate_python(_healthy_deployments), skipped_ids + ) ): raise _timeout = self._time_to_sleep_before_retry( @@ -9293,7 +9310,8 @@ class Router: healthy_deployments=_healthy_deployments, all_deployments=_all_deployments, ) - await asyncio.sleep(_timeout) + if not max_retries_per_request_hit(budget_request_kwargs, litellm.num_retries_per_request): + await asyncio.sleep(_timeout) if type(original_exception) in litellm.LITELLM_EXCEPTION_TYPES: setattr(original_exception, "max_retries", num_retries) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index b55b3c5efc2..43bf9b85b2b 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -16172,15 +16172,42 @@ async def test_num_retries_per_request_stops_retries_at_caps_above_four(monkeypa records = await _fail_one_proxy_shaped_request(router, "request-1", expected_error=litellm.APIConnectionError) - assert [record["attempted_retries"] for record in records] == [3, 4, 5, 6] + assert [record["attempted_retries"] for record in records] == [2, 3, 4, 5] assert ["Max retries per request hit!" in record["exception_string"] for record in records] == [ False, False, - True, + False, True, ] +@pytest.mark.asyncio +@pytest.mark.parametrize("cap", [0, 2]) +@pytest.mark.parametrize("stream", [False, True]) +async def test_exhausted_request_retry_budget_stops_router_backoff( + monkeypatch: pytest.MonkeyPatch, cap: int, stream: bool +) -> None: + monkeypatch.setattr(litellm, "num_retries_per_request", cap) + router = _always_failing_router(num_retries=15) + metadata: dict = {} + with patch("litellm.router.asyncio.sleep", new_callable=AsyncMock) as sleep: + with pytest.raises(litellm.APIConnectionError, match="Max retries per request hit!"): + await router.acompletion( + model="broken-group", + messages=[{"role": "user", "content": "hi"}], + metadata=metadata, + stream=stream, + ) + + assert len(sleep.await_args_list) == max(cap - 1, 0) + cap_refusals = [ + record + for record in metadata["previous_models"] + if "Max retries per request hit!" in record["exception_string"] + ] + assert len(cap_refusals) == 1 + + def _failing_group_with_healthy_fallback_router(num_retries: int) -> litellm.Router: return litellm.Router( model_list=[ @@ -16238,7 +16265,7 @@ async def test_num_retries_per_request_counts_retries_across_fallback_hops( for record in recorder.breadcrumbs_per_target[0] if record["model_group"] == "healthy-group" and "Max retries per request hit!" in record["exception_string"] ] - assert hop_refusals == [0, 1] + assert hop_refusals == [0] @pytest.mark.asyncio @@ -22343,6 +22370,33 @@ async def test_router_tries_each_deployment_that_cannot_transcribe_once_then_ans assert not respx_mock.calls +@pytest.mark.asyncio +async def test_router_stops_after_retry_reaches_only_unsupported_transcription_deployments( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(simple_shuffle.random, "choice", lambda deployments: deployments[0]) + router: Final = litellm.Router( + model_list=[ + _transcription_deployment("transcriber", "rejecting", "openai/gpt-transcribe", weight=1), + _transcription_deployment("transcriber", "unsupported", "anthropic/claude-sonnet-5-5", weight=0), + ], + num_retries=3, + retry_policy=RetryPolicy(BadRequestErrorRetries=3), + disable_cooldowns=True, + ) + + with respx.mock as respx_mock: + rejecting: Final = respx_mock.post(f"{_TRANSCRIPTION_API_BASE}/audio/transcriptions").mock( + return_value=httpx.Response(400, json={"error": _UPSTREAM_400}) + ) + with pytest.raises(litellm.UnsupportedEndpointError): + await router.atranscription(model="transcriber", file=_TRANSCRIPTION_AUDIO) + + assert rejecting.call_count == 1 + assert router.fail_calls["transcriber"] == 2 + + @pytest.mark.asyncio async def test_router_falls_back_from_a_group_that_cannot_transcribe_when_unsupported_params_do_not( monkeypatch: pytest.MonkeyPatch,