From 779d9d5dbe7d626e6fe77fbdee55baf74bcc4705 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Sat, 24 Oct 2026 01:46:05 -0700 Subject: [PATCH] fix(proxy): keep unflushed end-user charges when a wiped counter reseeds from the row A cold end-user counter seeds from LiteLLM_EndUserTable, which lags the batch writer. After Redis loses the counter, the budget check now floors on the worker's cached spend while the row is still at or above the spend that cache was loaded with, so an over-budget customer is refused again inside the flush window, and a reset that lowered the row still wins --- litellm/proxy/auth/auth_checks.py | 25 ++++++++++++- litellm/proxy/proxy_server.py | 9 +++++ .../spend/test_end_user_budget_enforcement.py | 26 ++++++++++++++ tests/unit/proxy/auth/test_auth_checks.py | 36 +++++++++++++++++++ 4 files changed, 95 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 8887d86570a..64e5f6be78d 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -1901,13 +1901,17 @@ async def check_end_user_budget( if end_user_budget is None: return - from litellm.proxy.proxy_server import get_current_spend + from litellm.proxy.proxy_server import get_current_spend, user_api_key_cache + loaded_spend: Final = user_api_key_cache.get_cache( + key=_end_user_loaded_spend_key(end_user_obj.user_id), local_only=True + ) end_user_spend: Final = await get_current_spend( counter_key=f"spend:end_user:{end_user_obj.user_id}", fallback_spend=end_user_obj.spend or 0.0, max_budget=end_user_budget, fallback_authoritative=True, + fallback_loaded_spend=float(loaded_spend) if isinstance(loaded_spend, (int, float)) else None, ) if end_user_spend > end_user_budget: raise litellm.BudgetExceededError( @@ -2119,6 +2123,23 @@ async def _end_user_is_known_unrestricted( return registry is not None and end_user_id not in registry +def _end_user_loaded_spend_key(end_user_id: str) -> str: + return f"end_user_loaded_spend:{end_user_id}" + + +def _remember_end_user_loaded_spend( + user_api_key_cache: UserApiKeyCache, end_user_id: str, spend: float, *, keep_existing: bool +) -> None: + """Record, on this worker only, the DB spend its cached end-user object was loaded with. Spend this worker + charges later is added to the cached object, never to this value.""" + key: Final = _end_user_loaded_spend_key(end_user_id) + if keep_existing and user_api_key_cache.get_cache(key=key, local_only=True) is not None: + return + user_api_key_cache.set_cache( + key=key, value=spend, local_only=True, ttl=get_management_object_ttl(user_api_key_cache) + ) + + @log_db_metrics @with_service_target(AUTH_OBJECTS_TARGET) async def get_end_user_object( @@ -2166,6 +2187,7 @@ async def get_end_user_object( model_type=LiteLLM_EndUserTable, ) if cached_user_obj is not None: + _remember_end_user_loaded_spend(user_api_key_cache, end_user_id, cached_user_obj.spend, keep_existing=True) return await _apply_default_budget_to_end_user( end_user_obj=cached_user_obj, prisma_client=prisma_client, @@ -2206,6 +2228,7 @@ async def get_end_user_object( model_type=LiteLLM_EndUserTable, ttl=get_management_object_ttl(user_api_key_cache), ) + _remember_end_user_loaded_spend(user_api_key_cache, end_user_id, end_user_row.spend, keep_existing=False) if key_end_user_budget_id is None: return end_user_row diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 37ebbefe4b2..682777657b8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3166,6 +3166,7 @@ async def get_current_spend( window_duration: str | None = None, window_start: datetime | None = None, fallback_authoritative: bool = False, + fallback_loaded_spend: float | None = None, ) -> float: """ Read current spend from the cross-pod spend counter. @@ -3195,6 +3196,11 @@ async def get_current_spend( skipped for healthy primary counters (counter at or above recorded spend) and cached in-process for a few seconds, so a persistently stale counter drives at most one read per counter per window rather than one per request. + + ``fallback_loaded_spend`` is the DB spend ``fallback_spend`` was loaded + from before this worker added its own charges to it. While the row is still + at or above that value, ``fallback_spend`` floors the read, so charges the + batch writer has not flushed yet are not lost with a wiped counter. """ current, verified = await _read_spend_counter_estimate(counter_key=counter_key, fallback_spend=fallback_spend) if fallback_authoritative: @@ -3222,6 +3228,9 @@ async def get_current_spend( verified = True if authoritative > current: await _repair_stale_spend_counter(counter_key=counter_key, db_spend=authoritative) + if fallback_loaded_spend is not None and fallback_loaded_spend <= authoritative: + return max(current, authoritative, fallback_spend) + if authoritative > current: return authoritative elif fallback_spend > current: # nothing to read (tag counters, an end user without a row or a DB client, a diff --git a/tests/integration/spend/test_end_user_budget_enforcement.py b/tests/integration/spend/test_end_user_budget_enforcement.py index b04add28685..e6112c6952e 100644 --- a/tests/integration/spend/test_end_user_budget_enforcement.py +++ b/tests/integration/spend/test_end_user_budget_enforcement.py @@ -791,3 +791,29 @@ def test_a_cold_end_user_counter_seeds_from_a_row_reset_behind_the_cached_custom ) assert float(str(charged[0]["spend"])) == pytest.approx(_CHAT_COST) assert _counter(cache, end_user) == pytest.approx(_CHAT_COST) + + +def test_a_cold_end_user_counter_keeps_spend_the_row_has_not_caught_up_to(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.25) + key: Final = scenario.key(models=[model]) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + call_cost: Final = 20 * 0.001 + 20 * 0.25 + _new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + served: Final = _chat(gateway, model, key, end_user) + assert served.status_code == 200, served.text + eventually( + lambda: _end_user_row(end_user), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) == pytest.approx(_COLD_SEED_SPEND + call_cost), + seconds=70, + ) + _set_end_user_row_spend(end_user, _COLD_SEED_SPEND) + cache.delete(f"spend:end_user:{end_user}") + + denied: Final = _chat(gateway, model, key, end_user) + + assert _refused_spend(denied, end_user) == pytest.approx(_COLD_SEED_SPEND + call_cost) diff --git a/tests/unit/proxy/auth/test_auth_checks.py b/tests/unit/proxy/auth/test_auth_checks.py index e1afa81d372..47e6ba4ead8 100644 --- a/tests/unit/proxy/auth/test_auth_checks.py +++ b/tests/unit/proxy/auth/test_auth_checks.py @@ -538,6 +538,42 @@ async def test_check_end_user_budget(customer_spend, customer_budget): assert exc_info.value.max_budget == customer_budget +@pytest.mark.parametrize("row_spend, refused", [(6.0, True), (0.0, False)]) +@pytest.mark.asyncio +async def test_check_end_user_budget_keeps_unflushed_charges_after_the_counter_is_wiped( + monkeypatch: pytest.MonkeyPatch, row_spend: float, refused: bool +) -> None: + import litellm.proxy.proxy_server as ps + from litellm.proxy.auth.auth_checks import check_end_user_budget + + end_user_id: Final = "customer-42" + prisma: Final = MagicMock() + prisma.db.litellm_endusertable.find_unique = AsyncMock(return_value=MagicMock(spend=row_spend)) + cache: Final = UserApiKeyCache() + await cache.async_set_cache( + key=f"end_user_id:{end_user_id}", + value=LiteLLM_EndUserTable( + user_id=end_user_id, spend=6.0, litellm_budget_table=LiteLLM_BudgetTable(max_budget=10.0), blocked=False + ), + model_type=LiteLLM_EndUserTable, + ) + loaded: Final = await get_end_user_object(end_user_id=end_user_id, prisma_client=prisma, user_api_key_cache=cache) + assert loaded is not None and loaded.spend == 6.0 + ps._add_spend_to_locally_cached_object(cache, f"end_user_id:{end_user_id}", LiteLLM_EndUserTable, 5.0) + charged: Final = await get_end_user_object(end_user_id=end_user_id, prisma_client=prisma, user_api_key_cache=cache) + assert charged is not None and charged.spend == 11.0 + monkeypatch.setattr(ps, "prisma_client", prisma) + monkeypatch.setattr(ps, "spend_counter_cache", DualCache()) + monkeypatch.setattr(ps, "user_api_key_cache", cache) + + if not refused: + await check_end_user_budget(end_user_obj=charged, route="/v1/chat/completions") + return + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await check_end_user_budget(end_user_obj=charged, route="/v1/chat/completions") + assert exc_info.value.current_cost == 11.0 + + @pytest.mark.parametrize( "model, expect_to_work", [