Merge b8dcacd8bb into c7c895db45

litellm/router.py

@@ -92,6 +92,7 @@ from litellm.litellm_core_utils.core_helpers import (
     get_metadata_variable_name_from_kwargs,
     get_or_create_metadata_bucket,
     get_parent_otel_span_from_kwargs,
+    max_retries_per_request_hit,
 )
 from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
 from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference
@@ -8978,6 +8979,12 @@ class Router:
         ## ADD RETRY TRACKING TO METADATA - used for spend logs retry tracking
         _metadata["attempted_retries"] = 0
         _metadata["max_retries"] = num_retries  # Updated after overrides in exception handler
+        budget_request_kwargs: Final[Mapping[str, object]] = cast(  # cast-ok: kwargs keys are request parameter names
+            Mapping[str, object], kwargs
+        )
+        initial_budget_exhausted: Final = max_retries_per_request_hit(
+            budget_request_kwargs, litellm.num_retries_per_request
+        )
         try:
             self._handle_mock_testing_rate_limit_error(model_group=model_group, kwargs=kwargs)
             # if the function call is successful, no exception will be raised and we'll break out of the loop
@@ -9057,6 +9064,9 @@ class Router:
             else:
                 raise
 
+            if initial_budget_exhausted:
+                raise original_exception
+
             verbose_router_logger.debug("Retrying request with num_retries: %s", num_retries)
             # decides how long to sleep before retry
             retry_after: Final = self._time_to_sleep_before_retry(
@@ -9067,9 +9077,13 @@ class Router:
                 all_deployments=_all_deployments,
             )
 
-            await asyncio.sleep(retry_after)
+            if not max_retries_per_request_hit(budget_request_kwargs, litellm.num_retries_per_request):
+                await asyncio.sleep(retry_after)
 
             for current_attempt in range(num_retries):
+                budget_exhausted_before_call: bool = max_retries_per_request_hit(
+                    budget_request_kwargs, litellm.num_retries_per_request
+                )
                 try:
                     # Update retry tracking metadata before each retry attempt
                     _metadata["attempted_retries"] = current_attempt + 1
@@ -9133,6 +9147,8 @@ class Router:
                     )
                     if skipped_ids:
                         kwargs["_retry_skipped_deployment_ids"] = skipped_ids  # rebind-ok: the next attempt reads it
+                    if budget_exhausted_before_call:
+                        raise
                     _timeout = self._time_to_sleep_before_retry(
                         e=e,
                         remaining_retries=remaining_retries,
@@ -9140,7 +9156,8 @@ class Router:
                         healthy_deployments=_healthy_deployments,
                         all_deployments=_all_deployments,
                     )
-                    await asyncio.sleep(_timeout)
+                    if not max_retries_per_request_hit(budget_request_kwargs, litellm.num_retries_per_request):
+                        await asyncio.sleep(_timeout)
 
             if type(original_exception) in litellm.LITELLM_EXCEPTION_TYPES:
                 setattr(original_exception, "max_retries", num_retries)

tests/unit/test_router/test_router.py

@@ -15561,15 +15561,42 @@ async def test_num_retries_per_request_stops_retries_at_caps_above_four(monkeypa
 
     records = await _fail_one_proxy_shaped_request(router, "request-1", expected_error=litellm.APIConnectionError)
 
-    assert [record["attempted_retries"] for record in records] == [3, 4, 5, 6]
+    assert [record["attempted_retries"] for record in records] == [2, 3, 4, 5]
     assert ["Max retries per request hit!" in record["exception_string"] for record in records] == [
         False,
         False,
-        True,
+        False,
         True,
     ]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("cap", [0, 2])
+@pytest.mark.parametrize("stream", [False, True])
+async def test_exhausted_request_retry_budget_stops_router_backoff(
+    monkeypatch: pytest.MonkeyPatch, cap: int, stream: bool
+) -> None:
+    monkeypatch.setattr(litellm, "num_retries_per_request", cap)
+    router = _always_failing_router(num_retries=15)
+    metadata: dict = {}
+    with patch("litellm.router.asyncio.sleep", new_callable=AsyncMock) as sleep:
+        with pytest.raises(litellm.APIConnectionError, match="Max retries per request hit!"):
+            await router.acompletion(
+                model="broken-group",
+                messages=[{"role": "user", "content": "hi"}],
+                metadata=metadata,
+                stream=stream,
+            )
+
+    assert len(sleep.await_args_list) == max(cap - 1, 0)
+    cap_refusals = [
+        record
+        for record in metadata["previous_models"]
+        if "Max retries per request hit!" in record["exception_string"]
+    ]
+    assert len(cap_refusals) == 1
+
+
 def _failing_group_with_healthy_fallback_router(num_retries: int) -> litellm.Router:
     return litellm.Router(
         model_list=[
@@ -15627,7 +15654,7 @@ async def test_num_retries_per_request_counts_retries_across_fallback_hops(
         for record in recorder.breadcrumbs_per_target[0]
         if record["model_group"] == "healthy-group" and "Max retries per request hit!" in record["exception_string"]
     ]
-    assert hop_refusals == [0, 1]
+    assert hop_refusals == [0]
 
 
 @pytest.mark.asyncio