feat: add retry_with_backoff() utility with exponential backoff and jitter

litellm/__init__.py

@@ -1171,7 +1171,7 @@ _key_management_settings: KeyManagementSettings = KeyManagementSettings()
 
 # client must be imported immediately as it's used as a decorator at function definition time
 from .utils import client
-
+from .utils import retry_with_backoff  
 # Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py
 # (which imports tiktoken) at import time
 

litellm/utils.py

@@ -51,6 +51,10 @@ from tokenizers import Tokenizer
 import litellm
 import litellm.litellm_core_utils
 
+
+from typing import Callable, Any, Tuple, Type
+
+
 # audio_utils.utils is lazy-loaded - only imported when needed for transcription calls
 import litellm.litellm_core_utils.json_validation_rule
 from litellm._internal_context import is_internal_call
@@ -9722,3 +9726,81 @@ def __getattr__(name: str) -> Any:
         return handler_func(name)
 
     raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+
+# ─────────────────────────────────────────────
+# Retry with Exponential Backoff Utility
+# Handles RateLimitError and ServiceUnavailableError
+# automatically with jitter to avoid thundering herd
+# ─────────────────────────────────────────────
+
+def retry_with_backoff(
+   fn: Callable,
+   max_retries: int = 3,
+   base_delay: float = 1.0,
+   max_delay: float = 60.0,
+   backoff_factor: float = 2.0,
+   retry_on: Optional[Tuple[Type[Exception], ...]] = None,
+) -> Any:
+   """
+   Retries a callable with exponential backoff and jitter.
+
+   Args:
+       fn:             The callable to retry (e.g. lambda: litellm.completion(...))
+       max_retries:    Maximum number of retry attempts (default: 3)
+       base_delay:     Initial delay in seconds (default: 1.0)
+       max_delay:      Maximum delay cap in seconds (default: 60.0)
+       backoff_factor: Multiplier per retry (default: 2.0)
+       retry_on:       Tuple of exception types to retry on.
+                       Defaults to (RateLimitError, ServiceUnavailableError)
+
+   Returns:
+       The return value of fn() on success.
+
+   Raises:
+       The last exception if all retries are exhausted.
+
+   Example:
+       import litellm
+       from litellm.utils import retry_with_backoff
+
+       response = retry_with_backoff(
+           lambda: litellm.completion(
+               model="gpt-4o",
+               messages=[{"role": "user", "content": "Hello"}]
+           ),
+           max_retries=5,
+           base_delay=2.0,
+       )
+   """
+
+   if retry_on is None:
+       retry_on = (RateLimitError, ServiceUnavailableError)
+
+   last_exception = None
+
+   for attempt in range(max_retries + 1):
+       try:
+           return fn()
+
+       except retry_on as e:
+           last_exception = e
+
+           if attempt == max_retries:
+               # Exhausted all retries
+               raise last_exception
+
+           # Exponential backoff with full jitter
+           delay = min(
+               base_delay * (backoff_factor ** attempt) + random.uniform(0, 1),
+               max_delay,
+           )
+
+           verbose_logger.warning(
+               f"[LiteLLM] Attempt {attempt + 1}/{max_retries} failed "
+               f"({type(e).__name__}). Retrying in {delay:.2f}s..."
+           )
+           time.sleep(delay)
+
+   raise last_exception

tests/test_litellm/test_utils.py

@@ -4107,3 +4107,83 @@ class TestValidateAndFixThinkingParam:
         validate_and_fix_thinking_param(thinking=thinking)
         assert "budgetTokens" in thinking
         assert "budget_tokens" not in thinking
+
+
+# ── Tests for retry_with_backoff ──────────────────────
+from litellm.utils import retry_with_backoff
+from litellm.exceptions import RateLimitError
+
+
+class TestRetryWithBackoff:
+    """Tests for the retry_with_backoff utility function."""
+
+    def test_retry_succeeds_on_first_try(self):
+        """Should return immediately with no retries."""
+        result = retry_with_backoff(lambda: "success")
+        assert result == "success"
+
+    def test_retry_succeeds_after_failures(self):
+        """Should retry and succeed after initial failures."""
+        attempts = {"count": 0}
+
+        def flaky_fn():
+            attempts["count"] += 1
+            if attempts["count"] < 3:
+                raise RateLimitError(
+                    message="rate limited",
+                    model="gpt-4o",
+                    llm_provider="openai",
+                )
+            return "recovered"
+
+        result = retry_with_backoff(flaky_fn, max_retries=3, base_delay=0.01)
+        assert result == "recovered"
+        assert attempts["count"] == 3
+
+    def test_retry_raises_after_max_retries(self):
+        """Should raise the exception after exhausting retries."""
+
+        def always_fails():
+            raise RateLimitError(
+                message="always rate limited",
+                model="gpt-4o",
+                llm_provider="openai",
+            )
+
+        with pytest.raises(RateLimitError):
+            retry_with_backoff(always_fails, max_retries=2, base_delay=0.01)
+
+    def test_retry_custom_exception(self):
+        """Should only retry on specified exception types."""
+
+        def raises_value_error():
+            raise ValueError("not a rate limit")
+
+        with pytest.raises(ValueError):
+            retry_with_backoff(
+                raises_value_error,
+                max_retries=3,
+                base_delay=0.01,
+                retry_on=(RateLimitError,),  # ValueError not included
+            )
+
+    def test_retry_respects_max_delay(self):
+        """Delay should never exceed max_delay."""
+        import time
+
+        attempts = {"count": 0, "start": time.time()}
+
+        def fails_twice():
+            attempts["count"] += 1
+            if attempts["count"] < 3:
+                raise RateLimitError("rate limited", "gpt-4o", "openai")
+            return "done"
+
+        retry_with_backoff(
+            fails_twice,
+            max_retries=3,
+            base_delay=0.01,
+            max_delay=0.05,
+        )
+        elapsed = time.time() - attempts["start"]
+        assert elapsed < 1.0  # Should be fast with tiny delays