From 52f097eb9d8c1a664f3fe760f0df3c049ed72d4c Mon Sep 17 00:00:00 2001 From: Pathikrit Bhowmick Date: Tue, 29 Jul 2025 18:38:08 -0400 Subject: [PATCH] BUGFIX: Jitter should be added not multiplied (#12877) (#12901) * Jitter should be added not multiplied This fixes a bug mentioned in https://github.com/BerriAI/litellm/issues/12877 `JITTER=0.75` is multiplied by `random.random()` so `sleep_seconds*jitter` is a tiny number that is always less than `min_timeout`. jitter should be added not multiplied * Add jitter to min_timeout case also * Cleanup jitter logic * Always apply jitter --- litellm/utils.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 420f85ac7f6..b5ddce479f3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5639,22 +5639,23 @@ def _calculate_retry_after( min_timeout: int = 0, ) -> Union[float, int]: retry_after = _get_retry_after_from_exception_header(response_headers) + + # Add some jitter (default JITTER is 0.75 - so upto 0.75s) + jitter = JITTER * random.random() # If the API asks us to wait a certain amount of time (and it's a reasonable amount), just do what it says. if retry_after is not None and 0 < retry_after <= 60: - return retry_after + return retry_after + jitter - initial_retry_delay = INITIAL_RETRY_DELAY - max_retry_delay = MAX_RETRY_DELAY - nb_retries = max_retries - remaining_retries + # Calculate exponential backoff + num_retries = max_retries - remaining_retries + sleep_seconds = INITIAL_RETRY_DELAY * pow(2.0, num_retries) - # Apply exponential backoff, but not more than the max. - sleep_seconds = min(initial_retry_delay * pow(2.0, nb_retries), max_retry_delay) - - # Apply some jitter, plus-or-minus half a second. - jitter = JITTER * random.random() - timeout = sleep_seconds * jitter - return timeout if timeout >= min_timeout else min_timeout + # Make sure sleep_seconds is boxed between min_timeout and MAX_RETRY_DELAY + sleep_seconds = max(sleep_seconds, min_timeout) + sleep_seconds = min(sleep_seconds, MAX_RETRY_DELAY) + + return sleep_seconds + jitter # custom prompt helper function