diff --git a/litellm/__init__.py b/litellm/__init__.py index e1b367fb234..032b6a47c77 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1171,7 +1171,7 @@ _key_management_settings: KeyManagementSettings = KeyManagementSettings() # client must be imported immediately as it's used as a decorator at function definition time from .utils import client - +from .utils import retry_with_backoff # Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py # (which imports tiktoken) at import time diff --git a/litellm/utils.py b/litellm/utils.py index cefd348078b..751ec80ed83 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -51,6 +51,10 @@ from tokenizers import Tokenizer import litellm import litellm.litellm_core_utils + +from typing import Callable, Any, Tuple, Type + + # audio_utils.utils is lazy-loaded - only imported when needed for transcription calls import litellm.litellm_core_utils.json_validation_rule from litellm._internal_context import is_internal_call @@ -9722,3 +9726,81 @@ def __getattr__(name: str) -> Any: return handler_func(name) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + + +# ───────────────────────────────────────────── +# Retry with Exponential Backoff Utility +# Handles RateLimitError and ServiceUnavailableError +# automatically with jitter to avoid thundering herd +# ───────────────────────────────────────────── + +def retry_with_backoff( + fn: Callable, + max_retries: int = 3, + base_delay: float = 1.0, + max_delay: float = 60.0, + backoff_factor: float = 2.0, + retry_on: Optional[Tuple[Type[Exception], ...]] = None, +) -> Any: + """ + Retries a callable with exponential backoff and jitter. + + Args: + fn: The callable to retry (e.g. lambda: litellm.completion(...)) + max_retries: Maximum number of retry attempts (default: 3) + base_delay: Initial delay in seconds (default: 1.0) + max_delay: Maximum delay cap in seconds (default: 60.0) + backoff_factor: Multiplier per retry (default: 2.0) + retry_on: Tuple of exception types to retry on. + Defaults to (RateLimitError, ServiceUnavailableError) + + Returns: + The return value of fn() on success. + + Raises: + The last exception if all retries are exhausted. + + Example: + import litellm + from litellm.utils import retry_with_backoff + + response = retry_with_backoff( + lambda: litellm.completion( + model="gpt-4o", + messages=[{"role": "user", "content": "Hello"}] + ), + max_retries=5, + base_delay=2.0, + ) + """ + + if retry_on is None: + retry_on = (RateLimitError, ServiceUnavailableError) + + last_exception = None + + for attempt in range(max_retries + 1): + try: + return fn() + + except retry_on as e: + last_exception = e + + if attempt == max_retries: + # Exhausted all retries + raise last_exception + + # Exponential backoff with full jitter + delay = min( + base_delay * (backoff_factor ** attempt) + random.uniform(0, 1), + max_delay, + ) + + verbose_logger.warning( + f"[LiteLLM] Attempt {attempt + 1}/{max_retries} failed " + f"({type(e).__name__}). Retrying in {delay:.2f}s..." + ) + time.sleep(delay) + + raise last_exception \ No newline at end of file diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index bc60375f906..c97f92be240 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -4107,3 +4107,83 @@ class TestValidateAndFixThinkingParam: validate_and_fix_thinking_param(thinking=thinking) assert "budgetTokens" in thinking assert "budget_tokens" not in thinking + + +# ── Tests for retry_with_backoff ────────────────────── +from litellm.utils import retry_with_backoff +from litellm.exceptions import RateLimitError + + +class TestRetryWithBackoff: + """Tests for the retry_with_backoff utility function.""" + + def test_retry_succeeds_on_first_try(self): + """Should return immediately with no retries.""" + result = retry_with_backoff(lambda: "success") + assert result == "success" + + def test_retry_succeeds_after_failures(self): + """Should retry and succeed after initial failures.""" + attempts = {"count": 0} + + def flaky_fn(): + attempts["count"] += 1 + if attempts["count"] < 3: + raise RateLimitError( + message="rate limited", + model="gpt-4o", + llm_provider="openai", + ) + return "recovered" + + result = retry_with_backoff(flaky_fn, max_retries=3, base_delay=0.01) + assert result == "recovered" + assert attempts["count"] == 3 + + def test_retry_raises_after_max_retries(self): + """Should raise the exception after exhausting retries.""" + + def always_fails(): + raise RateLimitError( + message="always rate limited", + model="gpt-4o", + llm_provider="openai", + ) + + with pytest.raises(RateLimitError): + retry_with_backoff(always_fails, max_retries=2, base_delay=0.01) + + def test_retry_custom_exception(self): + """Should only retry on specified exception types.""" + + def raises_value_error(): + raise ValueError("not a rate limit") + + with pytest.raises(ValueError): + retry_with_backoff( + raises_value_error, + max_retries=3, + base_delay=0.01, + retry_on=(RateLimitError,), # ValueError not included + ) + + def test_retry_respects_max_delay(self): + """Delay should never exceed max_delay.""" + import time + + attempts = {"count": 0, "start": time.time()} + + def fails_twice(): + attempts["count"] += 1 + if attempts["count"] < 3: + raise RateLimitError("rate limited", "gpt-4o", "openai") + return "done" + + retry_with_backoff( + fails_twice, + max_retries=3, + base_delay=0.01, + max_delay=0.05, + ) + elapsed = time.time() - attempts["start"] + assert elapsed < 1.0 # Should be fast with tiny delays \ No newline at end of file