diff --git a/litellm/utils.py b/litellm/utils.py index 420f85ac7f6..b5ddce479f3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5639,22 +5639,23 @@ def _calculate_retry_after( min_timeout: int = 0, ) -> Union[float, int]: retry_after = _get_retry_after_from_exception_header(response_headers) + + # Add some jitter (default JITTER is 0.75 - so upto 0.75s) + jitter = JITTER * random.random() # If the API asks us to wait a certain amount of time (and it's a reasonable amount), just do what it says. if retry_after is not None and 0 < retry_after <= 60: - return retry_after + return retry_after + jitter - initial_retry_delay = INITIAL_RETRY_DELAY - max_retry_delay = MAX_RETRY_DELAY - nb_retries = max_retries - remaining_retries + # Calculate exponential backoff + num_retries = max_retries - remaining_retries + sleep_seconds = INITIAL_RETRY_DELAY * pow(2.0, num_retries) - # Apply exponential backoff, but not more than the max. - sleep_seconds = min(initial_retry_delay * pow(2.0, nb_retries), max_retry_delay) - - # Apply some jitter, plus-or-minus half a second. - jitter = JITTER * random.random() - timeout = sleep_seconds * jitter - return timeout if timeout >= min_timeout else min_timeout + # Make sure sleep_seconds is boxed between min_timeout and MAX_RETRY_DELAY + sleep_seconds = max(sleep_seconds, min_timeout) + sleep_seconds = min(sleep_seconds, MAX_RETRY_DELAY) + + return sleep_seconds + jitter # custom prompt helper function