diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 8887d86570a..64e5f6be78d 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -1901,13 +1901,17 @@ async def check_end_user_budget( if end_user_budget is None: return - from litellm.proxy.proxy_server import get_current_spend + from litellm.proxy.proxy_server import get_current_spend, user_api_key_cache + loaded_spend: Final = user_api_key_cache.get_cache( + key=_end_user_loaded_spend_key(end_user_obj.user_id), local_only=True + ) end_user_spend: Final = await get_current_spend( counter_key=f"spend:end_user:{end_user_obj.user_id}", fallback_spend=end_user_obj.spend or 0.0, max_budget=end_user_budget, fallback_authoritative=True, + fallback_loaded_spend=float(loaded_spend) if isinstance(loaded_spend, (int, float)) else None, ) if end_user_spend > end_user_budget: raise litellm.BudgetExceededError( @@ -2119,6 +2123,23 @@ async def _end_user_is_known_unrestricted( return registry is not None and end_user_id not in registry +def _end_user_loaded_spend_key(end_user_id: str) -> str: + return f"end_user_loaded_spend:{end_user_id}" + + +def _remember_end_user_loaded_spend( + user_api_key_cache: UserApiKeyCache, end_user_id: str, spend: float, *, keep_existing: bool +) -> None: + """Record, on this worker only, the DB spend its cached end-user object was loaded with. Spend this worker + charges later is added to the cached object, never to this value.""" + key: Final = _end_user_loaded_spend_key(end_user_id) + if keep_existing and user_api_key_cache.get_cache(key=key, local_only=True) is not None: + return + user_api_key_cache.set_cache( + key=key, value=spend, local_only=True, ttl=get_management_object_ttl(user_api_key_cache) + ) + + @log_db_metrics @with_service_target(AUTH_OBJECTS_TARGET) async def get_end_user_object( @@ -2166,6 +2187,7 @@ async def get_end_user_object( model_type=LiteLLM_EndUserTable, ) if cached_user_obj is not None: + _remember_end_user_loaded_spend(user_api_key_cache, end_user_id, cached_user_obj.spend, keep_existing=True) return await _apply_default_budget_to_end_user( end_user_obj=cached_user_obj, prisma_client=prisma_client, @@ -2206,6 +2228,7 @@ async def get_end_user_object( model_type=LiteLLM_EndUserTable, ttl=get_management_object_ttl(user_api_key_cache), ) + _remember_end_user_loaded_spend(user_api_key_cache, end_user_id, end_user_row.spend, keep_existing=False) if key_end_user_budget_id is None: return end_user_row diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 37ebbefe4b2..682777657b8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3166,6 +3166,7 @@ async def get_current_spend( window_duration: str | None = None, window_start: datetime | None = None, fallback_authoritative: bool = False, + fallback_loaded_spend: float | None = None, ) -> float: """ Read current spend from the cross-pod spend counter. @@ -3195,6 +3196,11 @@ async def get_current_spend( skipped for healthy primary counters (counter at or above recorded spend) and cached in-process for a few seconds, so a persistently stale counter drives at most one read per counter per window rather than one per request. + + ``fallback_loaded_spend`` is the DB spend ``fallback_spend`` was loaded + from before this worker added its own charges to it. While the row is still + at or above that value, ``fallback_spend`` floors the read, so charges the + batch writer has not flushed yet are not lost with a wiped counter. """ current, verified = await _read_spend_counter_estimate(counter_key=counter_key, fallback_spend=fallback_spend) if fallback_authoritative: @@ -3222,6 +3228,9 @@ async def get_current_spend( verified = True if authoritative > current: await _repair_stale_spend_counter(counter_key=counter_key, db_spend=authoritative) + if fallback_loaded_spend is not None and fallback_loaded_spend <= authoritative: + return max(current, authoritative, fallback_spend) + if authoritative > current: return authoritative elif fallback_spend > current: # nothing to read (tag counters, an end user without a row or a DB client, a diff --git a/tests/integration/spend/test_end_user_budget_enforcement.py b/tests/integration/spend/test_end_user_budget_enforcement.py index b04add28685..e6112c6952e 100644 --- a/tests/integration/spend/test_end_user_budget_enforcement.py +++ b/tests/integration/spend/test_end_user_budget_enforcement.py @@ -791,3 +791,29 @@ def test_a_cold_end_user_counter_seeds_from_a_row_reset_behind_the_cached_custom ) assert float(str(charged[0]["spend"])) == pytest.approx(_CHAT_COST) assert _counter(cache, end_user) == pytest.approx(_CHAT_COST) + + +def test_a_cold_end_user_counter_keeps_spend_the_row_has_not_caught_up_to(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.25) + key: Final = scenario.key(models=[model]) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + call_cost: Final = 20 * 0.001 + 20 * 0.25 + _new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + served: Final = _chat(gateway, model, key, end_user) + assert served.status_code == 200, served.text + eventually( + lambda: _end_user_row(end_user), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) == pytest.approx(_COLD_SEED_SPEND + call_cost), + seconds=70, + ) + _set_end_user_row_spend(end_user, _COLD_SEED_SPEND) + cache.delete(f"spend:end_user:{end_user}") + + denied: Final = _chat(gateway, model, key, end_user) + + assert _refused_spend(denied, end_user) == pytest.approx(_COLD_SEED_SPEND + call_cost) diff --git a/tests/unit/proxy/auth/test_auth_checks.py b/tests/unit/proxy/auth/test_auth_checks.py index e1afa81d372..47e6ba4ead8 100644 --- a/tests/unit/proxy/auth/test_auth_checks.py +++ b/tests/unit/proxy/auth/test_auth_checks.py @@ -538,6 +538,42 @@ async def test_check_end_user_budget(customer_spend, customer_budget): assert exc_info.value.max_budget == customer_budget +@pytest.mark.parametrize("row_spend, refused", [(6.0, True), (0.0, False)]) +@pytest.mark.asyncio +async def test_check_end_user_budget_keeps_unflushed_charges_after_the_counter_is_wiped( + monkeypatch: pytest.MonkeyPatch, row_spend: float, refused: bool +) -> None: + import litellm.proxy.proxy_server as ps + from litellm.proxy.auth.auth_checks import check_end_user_budget + + end_user_id: Final = "customer-42" + prisma: Final = MagicMock() + prisma.db.litellm_endusertable.find_unique = AsyncMock(return_value=MagicMock(spend=row_spend)) + cache: Final = UserApiKeyCache() + await cache.async_set_cache( + key=f"end_user_id:{end_user_id}", + value=LiteLLM_EndUserTable( + user_id=end_user_id, spend=6.0, litellm_budget_table=LiteLLM_BudgetTable(max_budget=10.0), blocked=False + ), + model_type=LiteLLM_EndUserTable, + ) + loaded: Final = await get_end_user_object(end_user_id=end_user_id, prisma_client=prisma, user_api_key_cache=cache) + assert loaded is not None and loaded.spend == 6.0 + ps._add_spend_to_locally_cached_object(cache, f"end_user_id:{end_user_id}", LiteLLM_EndUserTable, 5.0) + charged: Final = await get_end_user_object(end_user_id=end_user_id, prisma_client=prisma, user_api_key_cache=cache) + assert charged is not None and charged.spend == 11.0 + monkeypatch.setattr(ps, "prisma_client", prisma) + monkeypatch.setattr(ps, "spend_counter_cache", DualCache()) + monkeypatch.setattr(ps, "user_api_key_cache", cache) + + if not refused: + await check_end_user_budget(end_user_obj=charged, route="/v1/chat/completions") + return + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await check_end_user_budget(end_user_obj=charged, route="/v1/chat/completions") + assert exc_info.value.current_cost == 11.0 + + @pytest.mark.parametrize( "model, expect_to_work", [