From 726f94688f18c20453e7b41ae6b01d25e220a17b Mon Sep 17 00:00:00 2001 From: Etienne Chabert Date: Fri, 30 Oct 2026 11:15:40 +0100 Subject: [PATCH 1/2] fix(proxy): keep a window's spend floor in step with the budget reset The budget reset job zeroed a window's spend counter but left the per-pod floor cache holding the ended period's spend. A request in the next 5 seconds on a pod that had cached that floor raised the counter straight back to it, so a busy key or team carried the whole ended period into the new one The reset now writes the post-reset value into the floor cache along with the counter and broadcasts both to every pod, as the manual spend resets already do --- .../proxy/common_utils/reset_budget_job.py | 11 +++ .../common_utils/test_reset_budget_job.py | 68 +++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 59fd00cec43..146a09cd152 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -33,6 +33,7 @@ from litellm.proxy._types import ( LiteLLM_UserTable, LiteLLM_VerificationToken, ) +from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import publish_auth_cache_invalidation from litellm.proxy.common_utils.timezone_utils import ( BudgetResetSettings, compute_budget_reset_at, @@ -1478,13 +1479,23 @@ class ResetBudgetJob: reset_at: Final = datetime.fromisoformat(reset_at_str.replace("Z", "+00:00")).replace(tzinfo=None) if reset_at > now: return False + from litellm.proxy.proxy_server import SPEND_DB_FLOOR_CACHE_TTL_SECONDS + new_value: Final = await ResetBudgetJob._window_carried_spend(window, counter_key, spend_counter_cache) + floor_key: Final = f"spend_db_floor:{counter_key}" spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=new_value) + spend_counter_cache.in_memory_cache.set_cache( + key=floor_key, value=new_value, ttl=SPEND_DB_FLOOR_CACHE_TTL_SECONDS + ) if spend_counter_cache.redis_cache is not None: try: await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=new_value) except Exception as redis_err: verbose_proxy_logger.warning("Failed to reset Redis counter %s: %s", counter_key, redis_err) + await publish_auth_cache_invalidation(cache_key=counter_key, new_value=new_value) + await publish_auth_cache_invalidation( + cache_key=floor_key, new_value=new_value, ttl=SPEND_DB_FLOOR_CACHE_TTL_SECONDS + ) budget_duration: Final = window["budget_duration"] next_reset_at: Final = compute_budget_reset_at(budget_duration=budget_duration, settings=reset_settings) window["reset_at"] = next_reset_at.isoformat() diff --git a/tests/unit/proxy/common_utils/test_reset_budget_job.py b/tests/unit/proxy/common_utils/test_reset_budget_job.py index 183dd7919c1..c923f36707f 100644 --- a/tests/unit/proxy/common_utils/test_reset_budget_job.py +++ b/tests/unit/proxy/common_utils/test_reset_budget_job.py @@ -1073,6 +1073,8 @@ def _make_reset_budget_windows_job( fake_module = types.ModuleType("litellm.proxy.proxy_server") fake_module.spend_counter_cache = spend_counter_cache + fake_module.SPEND_DB_FLOOR_CACHE_TTL_SECONDS = 5 + fake_module.redis_usage_cache = None monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", fake_module) job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) @@ -1289,6 +1291,60 @@ def test_reset_budget_windows_skips_unexpired_key_window(monkeypatch): prisma_client.db.litellm_verificationtoken.update.assert_not_awaited() +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("table", "entity_type", "counter_prefix", "id_column"), + [ + ('"LiteLLM_VerificationToken"', "Key", "spend:key", "token"), + ('"LiteLLM_TeamTable"', "Team", "spend:team", "team_id"), + ], + ids=["key", "team"], +) +async def test_a_window_floor_read_just_before_the_reset_does_not_bring_back_the_ended_period( + table: str, entity_type: str, counter_prefix: str, id_column: str, monkeypatch +) -> None: + from litellm.caching.dual_cache import DualCache + from litellm.proxy import proxy_server + + ended_reset_at: Final = datetime(2026, 10, 26, tzinfo=timezone.utc) + window_start: Final = ended_reset_at - timedelta(days=7) + entity_id: Final = f"floored-{entity_type.lower()}" + counter_key: Final = f"{counter_prefix}:{entity_id}:window:7d" + windowed_row: Final = { + id_column: entity_id, + "budget_limits": [{"budget_duration": "7d", "max_budget": 100.0, "reset_at": ended_reset_at.isoformat()}], + } + prisma_client: Final = MagicMock() + prisma_client.db.query_raw = AsyncMock(side_effect=lambda query, *_: [windowed_row] if table in query else []) + prisma_client.db.litellm_budgetwindowspend.find_unique = AsyncMock( + return_value=types.SimpleNamespace(window_start=window_start, spend=27.0) + ) + prisma_client.db.execute_raw = AsyncMock(return_value=1) + prisma_client.db.litellm_verificationtoken.update = AsyncMock(return_value=None) + prisma_client.db.litellm_teamtable.update = AsyncMock(return_value=None) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "spend_counter_cache", DualCache()) + proxy_server.spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=28.0) + + async def window_spend() -> float: + return await proxy_server.get_current_spend( + counter_key=counter_key, + fallback_spend=0.0, + max_budget=100.0, + window_entity_type=entity_type, + window_entity_id=entity_id, + window_duration="7d", + window_start=window_start, + ) + + floored_before_the_reset: Final = await window_spend() + await ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client).reset_budget_windows() + + assert (floored_before_the_reset, await window_spend()) == (28.0, 0.0), ( + "the floor read cached just before the reset raised the zeroed counter back to the ended period's spend" + ) + + def test_reset_budget_windows_resets_expired_team_window(monkeypatch): """Same as the key test, but for teams.""" now = datetime.utcnow() @@ -1377,6 +1433,8 @@ def test_reset_budget_windows_query_error_does_not_break_team_path(monkeypatch): spend_counter_cache.redis_cache = None fake_module = types.ModuleType("litellm.proxy.proxy_server") fake_module.spend_counter_cache = spend_counter_cache + fake_module.SPEND_DB_FLOOR_CACHE_TTL_SECONDS = 5 + fake_module.redis_usage_cache = None monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", fake_module) job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) @@ -1424,6 +1482,8 @@ def _make_counter_invalidation_job(monkeypatch): fake_module = types.ModuleType("litellm.proxy.proxy_server") fake_module.spend_counter_cache = spend_counter_cache + fake_module.SPEND_DB_FLOOR_CACHE_TTL_SECONDS = 5 + fake_module.redis_usage_cache = None fake_module.user_api_key_cache = user_api_key_cache monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", fake_module) @@ -2627,6 +2687,8 @@ def _make_leader_election_job(monkeypatch, pod_lock_manager): spend_counter_cache.redis_cache = None fake_module = types.ModuleType("litellm.proxy.proxy_server") fake_module.spend_counter_cache = spend_counter_cache + fake_module.SPEND_DB_FLOOR_CACHE_TTL_SECONDS = 5 + fake_module.redis_usage_cache = None monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", fake_module) job = ResetBudgetJob( @@ -2760,6 +2822,8 @@ def _paginating_window_job(monkeypatch, pages_by_table: Dict[str, List[List[Dict spend_counter_cache.redis_cache = None fake_module = types.ModuleType("litellm.proxy.proxy_server") fake_module.spend_counter_cache = spend_counter_cache + fake_module.SPEND_DB_FLOOR_CACHE_TTL_SECONDS = 5 + fake_module.redis_usage_cache = None monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", fake_module) job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) @@ -2830,6 +2894,8 @@ def test_reset_budget_windows_survives_one_table_failing(monkeypatch): spend_counter_cache.redis_cache = None fake_module = types.ModuleType("litellm.proxy.proxy_server") fake_module.spend_counter_cache = spend_counter_cache + fake_module.SPEND_DB_FLOOR_CACHE_TTL_SECONDS = 5 + fake_module.redis_usage_cache = None monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", fake_module) job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) @@ -2901,6 +2967,8 @@ def _cursor_paginating_window_job(monkeypatch, key_rows: List[Dict[str, Any]]): spend_counter_cache.redis_cache = None fake_module = types.ModuleType("litellm.proxy.proxy_server") fake_module.spend_counter_cache = spend_counter_cache + fake_module.SPEND_DB_FLOOR_CACHE_TTL_SECONDS = 5 + fake_module.redis_usage_cache = None monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", fake_module) job = ResetBudgetJob(proxy_logging_obj=MagicMock(), prisma_client=prisma_client) From 885e5342536a92db005ca8c92918d97045d94722 Mon Sep 17 00:00:00 2001 From: Etienne Chabert Date: Fri, 30 Oct 2026 12:14:19 +0100 Subject: [PATCH 2/2] refactor(proxy): type the budget window the reset job rolls --- litellm/proxy/common_utils/reset_budget_job.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 146a09cd152..22f203aa848 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -1463,7 +1463,7 @@ class ResetBudgetJob: @staticmethod @with_service_target(SPEND_COUNTERS_TARGET) async def _reset_expired_window( - window: dict, + window: dict[str, object], counter_key: str, spend_counter_cache: DualCache, now: datetime, @@ -1474,7 +1474,7 @@ class ResetBudgetJob: ) -> bool: """Reset a single budget window if expired. Returns True if the window was reset.""" reset_at_str: Final = window.get("reset_at") - if not reset_at_str: + if not isinstance(reset_at_str, str) or not reset_at_str: return False reset_at: Final = datetime.fromisoformat(reset_at_str.replace("Z", "+00:00")).replace(tzinfo=None) if reset_at > now: @@ -1496,7 +1496,7 @@ class ResetBudgetJob: await publish_auth_cache_invalidation( cache_key=floor_key, new_value=new_value, ttl=SPEND_DB_FLOOR_CACHE_TTL_SECONDS ) - budget_duration: Final = window["budget_duration"] + budget_duration: Final = str(window["budget_duration"]) next_reset_at: Final = compute_budget_reset_at(budget_duration=budget_duration, settings=reset_settings) window["reset_at"] = next_reset_at.isoformat() await ResetBudgetJob._roll_window_spend_row(