From c1b7a4b2a351706793337bb6e05be4c4dbdb1183 Mon Sep 17 00:00:00 2001 From: sharziki Date: Sat, 16 May 2026 13:29:09 -0400 Subject: [PATCH] fix(proxy): randomize deadlock retry backoff in spend tracking Replace fixed exponential backoff (2**i) with randomized backoff (random.uniform(2**i, 2**(i+1))) in update_end_user_spend and update_spend_logs to prevent correlated retries from re-deadlocking. When two transactions deadlock, they are cancelled at the same time. With fixed backoff, they also retry at the same time, making repeated deadlocks more likely and eventually exceeding the retry limit. Randomized backoff spreads out retry times, matching the pattern already used in db_spend_update_writer.py. Fixes #27989 --- litellm/proxy/utils.py | 7 +- .../spend_tracking/test_randomized_backoff.py | 136 ++++++++++++++++++ 2 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/proxy/spend_tracking/test_randomized_backoff.py diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 559d5c99b9d..c93eb865123 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -4,6 +4,7 @@ import hashlib import inspect import json import os +import random import smtplib import sys import threading @@ -5002,7 +5003,9 @@ class ProxyUpdateSpend: e=e, start_time=start_time, proxy_logging_obj=proxy_logging_obj ) # Optionally, sleep for a bit before retrying - await asyncio.sleep(2**i) # Exponential backoff + await asyncio.sleep( + random.uniform(2**i, 2 ** (i + 1)) + ) # Randomized exponential backoff except Exception as e: _raise_failed_update_spend_exception( e=e, start_time=start_time, proxy_logging_obj=proxy_logging_obj @@ -5096,7 +5099,7 @@ class ProxyUpdateSpend: ) if i >= n_retry_times: raise - await asyncio.sleep(2**i) + await asyncio.sleep(random.uniform(2**i, 2 ** (i + 1))) except Exception as e: # Logs already removed from queue at start - don't put them back # This matches the original behavior where logs are removed even on error diff --git a/tests/test_litellm/proxy/spend_tracking/test_randomized_backoff.py b/tests/test_litellm/proxy/spend_tracking/test_randomized_backoff.py new file mode 100644 index 00000000000..4223b21cc02 --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_randomized_backoff.py @@ -0,0 +1,136 @@ +""" +Tests for randomized exponential backoff in spend tracking retry loops. + +Verifies that update_end_user_spend and update_spend_logs use +random.uniform(2**i, 2**(i+1)) instead of a fixed 2**i backoff, +which prevents correlated retries from re-deadlocking. + +See: https://github.com/BerriAI/litellm/issues/27989 +""" + +import asyncio +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +sys.path.insert(0, os.path.abspath("../../../..")) + +from litellm.proxy.utils import PrismaClient, ProxyLogging, update_spend + + +class MockPrismaClient: + def __init__(self): + self.db = AsyncMock() + self.db.litellm_spendlogs = AsyncMock() + self.db.litellm_spendlogs.create_many = AsyncMock() + self.spend_log_transactions = [] + self.daily_user_spend_transactions = {} + self._spend_log_transactions_lock = asyncio.Lock() + + def jsonify_object(self, obj): + return obj + + def add_spend_log_transaction_to_daily_user_transaction(self, payload): + pass + + +def create_mock_proxy_logging(): + proxy_logging_obj = MagicMock() + proxy_logging_obj.failure_handler = AsyncMock() + proxy_logging_obj.db_spend_update_writer = AsyncMock() + proxy_logging_obj.db_spend_update_writer.db_update_spend_transaction_handler = ( + AsyncMock() + ) + return proxy_logging_obj + + +@pytest.mark.asyncio +async def test_update_spend_logs_backoff_is_randomized(): + """ + Test that backoff sleep times vary between retries due to randomization. + + With fixed backoff (2**i), consecutive retries from different concurrent + callers sleep the same duration, causing them to retry simultaneously and + re-deadlock. With random.uniform(2**i, 2**(i+1)), each retry sleeps a + different random duration within the range. + """ + prisma_client = MockPrismaClient() + proxy_logging_obj = create_mock_proxy_logging() + + prisma_client.spend_log_transactions = [{"id": "1", "spend": 10}] + + create_many_mock = AsyncMock( + side_effect=[ + httpx.ConnectError("deadlock detected"), + httpx.ConnectError("deadlock detected"), + None, + ] + ) + prisma_client.db.litellm_spendlogs.create_many = create_many_mock + + sleep_times = [] + + async def mock_sleep(seconds): + sleep_times.append(seconds) + + with patch("asyncio.sleep", mock_sleep): + await update_spend(prisma_client, None, proxy_logging_obj) + + assert len(sleep_times) == 2 + + # Verify each sleep time falls within the randomized range + # i=0: random.uniform(2**0, 2**1) -> [1.0, 2.0) + assert ( + 1.0 <= sleep_times[0] <= 2.0 + ), f"First backoff {sleep_times[0]} not in [1.0, 2.0]" + # i=1: random.uniform(2**1, 2**2) -> [2.0, 4.0) + assert ( + 2.0 <= sleep_times[1] <= 4.0 + ), f"Second backoff {sleep_times[1]} not in [2.0, 4.0]" + + +@pytest.mark.asyncio +async def test_update_spend_logs_backoff_not_fixed(): + """ + Test that backoff values are NOT the exact fixed values 2**i. + + Run multiple trials and verify that at least some sleep values differ + from the fixed backoff pattern, confirming randomization is active. + """ + first_backoffs = [] + + for _ in range(10): + prisma_client = MockPrismaClient() + proxy_logging_obj = create_mock_proxy_logging() + + prisma_client.spend_log_transactions = [{"id": "1", "spend": 10}] + + create_many_mock = AsyncMock( + side_effect=[ + httpx.ConnectError("deadlock detected"), + None, + ] + ) + prisma_client.db.litellm_spendlogs.create_many = create_many_mock + + sleep_times = [] + + async def mock_sleep(seconds): + sleep_times.append(seconds) + + with patch("asyncio.sleep", mock_sleep): + await update_spend(prisma_client, None, proxy_logging_obj) + + assert len(sleep_times) == 1 + first_backoffs.append(sleep_times[0]) + + # With randomized backoff, not all 10 values should be identical + # (the probability of 10 identical random.uniform values is negligible) + unique_values = set(first_backoffs) + assert len(unique_values) > 1, ( + f"All 10 backoff values were identical ({first_backoffs[0]}), " + "suggesting fixed rather than randomized backoff" + )