Merge b5eba5744c into a392a39627

litellm/router.py

@@ -141,6 +141,7 @@ from litellm.litellm_core_utils.core_helpers import (
     get_metadata_variable_name_from_kwargs,
     get_or_create_metadata_bucket,
     get_parent_otel_span_from_kwargs,
+    max_retries_per_request_hit,
 )
 from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
 from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference
@@ -9426,6 +9427,12 @@ class Router(DeprecatedRouterMembers):
         ## ADD RETRY TRACKING TO METADATA - used for spend logs retry tracking
         _metadata["attempted_retries"] = 0
         _metadata["max_retries"] = num_retries  # Updated after overrides in exception handler
+        budget_request_kwargs: Final[Mapping[str, object]] = cast(  # cast-ok: kwargs keys are request parameter names
+            Mapping[str, object], kwargs
+        )
+        initial_budget_exhausted: Final = max_retries_per_request_hit(
+            budget_request_kwargs, litellm.num_retries_per_request
+        )
         try:
             self._handle_mock_testing_rate_limit_error(model_group=model_group, kwargs=kwargs)
             # if the function call is successful, no exception will be raised and we'll break out of the loop
@@ -9509,6 +9516,9 @@ class Router(DeprecatedRouterMembers):
             else:
                 raise
 
+            if initial_budget_exhausted:
+                raise original_exception
+
             verbose_router_logger.debug("Retrying request with num_retries: %s", num_retries)
             # decides how long to sleep before retry
             retry_after: Final = self._time_to_sleep_before_retry(
@@ -9519,9 +9529,13 @@ class Router(DeprecatedRouterMembers):
                 all_deployments=_all_deployments,
             )
 
-            await asyncio.sleep(retry_after)
+            if not max_retries_per_request_hit(budget_request_kwargs, litellm.num_retries_per_request):
+                await asyncio.sleep(retry_after)
 
             for current_attempt in range(num_retries):
+                budget_exhausted_before_call: bool = max_retries_per_request_hit(
+                    budget_request_kwargs, litellm.num_retries_per_request
+                )
                 try:
                     # Update retry tracking metadata before each retry attempt
                     _metadata["attempted_retries"] = current_attempt + 1
@@ -9585,8 +9599,11 @@ class Router(DeprecatedRouterMembers):
                     )
                     if skipped_ids:
                         kwargs["_retry_skipped_deployment_ids"] = skipped_ids  # rebind-ok: the next attempt reads it
-                    if isinstance(e, litellm.UnsupportedEndpointError) and _every_deployment_refused(
-                        _DEPLOYMENTS_ADAPTER.validate_python(_healthy_deployments), skipped_ids
+                    if budget_exhausted_before_call or (
+                        isinstance(e, litellm.UnsupportedEndpointError)
+                        and _every_deployment_refused(
+                            _DEPLOYMENTS_ADAPTER.validate_python(_healthy_deployments), skipped_ids
+                        )
                     ):
                         raise
                     _timeout = self._time_to_sleep_before_retry(
@@ -9596,7 +9613,8 @@ class Router(DeprecatedRouterMembers):
                         healthy_deployments=_healthy_deployments,
                         all_deployments=_all_deployments,
                     )
-                    await asyncio.sleep(_timeout)
+                    if not max_retries_per_request_hit(budget_request_kwargs, litellm.num_retries_per_request):
+                        await asyncio.sleep(_timeout)
 
             if type(original_exception) in litellm.LITELLM_EXCEPTION_TYPES:
                 setattr(original_exception, "max_retries", num_retries)

tests/unit/test_router/test_router.py

@@ -16283,15 +16283,42 @@ async def test_num_retries_per_request_stops_retries_at_caps_above_four(monkeypa
 
     records = await _fail_one_proxy_shaped_request(router, "request-1", expected_error=litellm.APIConnectionError)
 
-    assert [record["attempted_retries"] for record in records] == [3, 4, 5, 6]
+    assert [record["attempted_retries"] for record in records] == [2, 3, 4, 5]
     assert ["Max retries per request hit!" in record["exception_string"] for record in records] == [
         False,
         False,
-        True,
+        False,
         True,
     ]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("cap", [0, 2])
+@pytest.mark.parametrize("stream", [False, True])
+async def test_exhausted_request_retry_budget_stops_router_backoff(
+    monkeypatch: pytest.MonkeyPatch, cap: int, stream: bool
+) -> None:
+    monkeypatch.setattr(litellm, "num_retries_per_request", cap)
+    router = _always_failing_router(num_retries=15)
+    metadata: dict = {}
+    with patch("litellm.router.asyncio.sleep", new_callable=AsyncMock) as sleep:
+        with pytest.raises(litellm.APIConnectionError, match="Max retries per request hit!"):
+            await router.acompletion(
+                model="broken-group",
+                messages=[{"role": "user", "content": "hi"}],
+                metadata=metadata,
+                stream=stream,
+            )
+
+    assert len(sleep.await_args_list) == max(cap - 1, 0)
+    cap_refusals = [
+        record
+        for record in metadata["previous_models"]
+        if "Max retries per request hit!" in record["exception_string"]
+    ]
+    assert len(cap_refusals) == 1
+
+
 def _failing_group_with_healthy_fallback_router(num_retries: int) -> litellm.Router:
     return litellm.Router(
         model_list=[
@@ -16349,7 +16376,7 @@ async def test_num_retries_per_request_counts_retries_across_fallback_hops(
         for record in recorder.breadcrumbs_per_target[0]
         if record["model_group"] == "healthy-group" and "Max retries per request hit!" in record["exception_string"]
     ]
-    assert hop_refusals == [0, 1]
+    assert hop_refusals == [0]
 
 
 @pytest.mark.asyncio
@@ -22483,6 +22510,33 @@ async def test_router_tries_each_deployment_that_cannot_transcribe_once_then_ans
     assert not respx_mock.calls
 
 
+@pytest.mark.asyncio
+async def test_router_stops_after_retry_reaches_only_unsupported_transcription_deployments(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    monkeypatch.setattr(simple_shuffle.random, "choice", lambda deployments: deployments[0])
+    router: Final = litellm.Router(
+        model_list=[
+            _transcription_deployment("transcriber", "rejecting", "openai/gpt-transcribe", weight=1),
+            _transcription_deployment("transcriber", "unsupported", "anthropic/claude-sonnet-5-5", weight=0),
+        ],
+        num_retries=3,
+        retry_policy=RetryPolicy(BadRequestErrorRetries=3),
+        disable_cooldowns=True,
+    )
+
+    with respx.mock as respx_mock:
+        rejecting: Final = respx_mock.post(f"{_TRANSCRIPTION_API_BASE}/audio/transcriptions").mock(
+            return_value=httpx.Response(400, json={"error": _UPSTREAM_400})
+        )
+        with pytest.raises(litellm.UnsupportedEndpointError):
+            await router.atranscription(model="transcriber", file=_TRANSCRIPTION_AUDIO)
+
+    assert rejecting.call_count == 1
+    assert router.fail_calls["transcriber"] == 2
+
+
 @pytest.mark.asyncio
 async def test_router_falls_back_from_a_group_that_cannot_transcribe_when_unsupported_params_do_not(
     monkeypatch: pytest.MonkeyPatch,