diff --git a/litellm/router.py b/litellm/router.py index c8153b596d6..cb4b81f556e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -92,6 +92,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, get_or_create_metadata_bucket, get_parent_otel_span_from_kwargs, + max_retries_per_request_hit, ) from litellm.litellm_core_utils.coroutine_checker import coroutine_checker from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference @@ -8978,6 +8979,12 @@ class Router: ## ADD RETRY TRACKING TO METADATA - used for spend logs retry tracking _metadata["attempted_retries"] = 0 _metadata["max_retries"] = num_retries # Updated after overrides in exception handler + budget_request_kwargs: Final[Mapping[str, object]] = cast( # cast-ok: kwargs keys are request parameter names + Mapping[str, object], kwargs + ) + initial_budget_exhausted: Final = max_retries_per_request_hit( + budget_request_kwargs, litellm.num_retries_per_request + ) try: self._handle_mock_testing_rate_limit_error(model_group=model_group, kwargs=kwargs) # if the function call is successful, no exception will be raised and we'll break out of the loop @@ -9057,6 +9064,9 @@ class Router: else: raise + if initial_budget_exhausted: + raise original_exception + verbose_router_logger.debug("Retrying request with num_retries: %s", num_retries) # decides how long to sleep before retry retry_after: Final = self._time_to_sleep_before_retry( @@ -9067,9 +9077,13 @@ class Router: all_deployments=_all_deployments, ) - await asyncio.sleep(retry_after) + if not max_retries_per_request_hit(budget_request_kwargs, litellm.num_retries_per_request): + await asyncio.sleep(retry_after) for current_attempt in range(num_retries): + budget_exhausted_before_call: bool = max_retries_per_request_hit( + budget_request_kwargs, litellm.num_retries_per_request + ) try: # Update retry tracking metadata before each retry attempt _metadata["attempted_retries"] = current_attempt + 1 @@ -9133,6 +9147,8 @@ class Router: ) if skipped_ids: kwargs["_retry_skipped_deployment_ids"] = skipped_ids # rebind-ok: the next attempt reads it + if budget_exhausted_before_call: + raise _timeout = self._time_to_sleep_before_retry( e=e, remaining_retries=remaining_retries, @@ -9140,7 +9156,8 @@ class Router: healthy_deployments=_healthy_deployments, all_deployments=_all_deployments, ) - await asyncio.sleep(_timeout) + if not max_retries_per_request_hit(budget_request_kwargs, litellm.num_retries_per_request): + await asyncio.sleep(_timeout) if type(original_exception) in litellm.LITELLM_EXCEPTION_TYPES: setattr(original_exception, "max_retries", num_retries) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index d9b655a0aff..5915bff89c8 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -15561,15 +15561,42 @@ async def test_num_retries_per_request_stops_retries_at_caps_above_four(monkeypa records = await _fail_one_proxy_shaped_request(router, "request-1", expected_error=litellm.APIConnectionError) - assert [record["attempted_retries"] for record in records] == [3, 4, 5, 6] + assert [record["attempted_retries"] for record in records] == [2, 3, 4, 5] assert ["Max retries per request hit!" in record["exception_string"] for record in records] == [ False, False, - True, + False, True, ] +@pytest.mark.asyncio +@pytest.mark.parametrize("cap", [0, 2]) +@pytest.mark.parametrize("stream", [False, True]) +async def test_exhausted_request_retry_budget_stops_router_backoff( + monkeypatch: pytest.MonkeyPatch, cap: int, stream: bool +) -> None: + monkeypatch.setattr(litellm, "num_retries_per_request", cap) + router = _always_failing_router(num_retries=15) + metadata: dict = {} + with patch("litellm.router.asyncio.sleep", new_callable=AsyncMock) as sleep: + with pytest.raises(litellm.APIConnectionError, match="Max retries per request hit!"): + await router.acompletion( + model="broken-group", + messages=[{"role": "user", "content": "hi"}], + metadata=metadata, + stream=stream, + ) + + assert len(sleep.await_args_list) == max(cap - 1, 0) + cap_refusals = [ + record + for record in metadata["previous_models"] + if "Max retries per request hit!" in record["exception_string"] + ] + assert len(cap_refusals) == 1 + + def _failing_group_with_healthy_fallback_router(num_retries: int) -> litellm.Router: return litellm.Router( model_list=[ @@ -15627,7 +15654,7 @@ async def test_num_retries_per_request_counts_retries_across_fallback_hops( for record in recorder.breadcrumbs_per_target[0] if record["model_group"] == "healthy-group" and "Max retries per request hit!" in record["exception_string"] ] - assert hop_refusals == [0, 1] + assert hop_refusals == [0] @pytest.mark.asyncio