From ee606be1dcb512f724f42c10f9f0a706c6abdeed Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:37:13 -0700 Subject: [PATCH 1/4] fix(proxy): seed cold end-user spend counters from the end-user row Overlapping cold starts each added the cached end-user spend to the shared counter, so a burst multiplied it and refused a customer still under budget End-user counters now seed once from LiteLLM_EndUserTable through the same set-if-absent path the other entity counters use Fixes #47062 --- .../proxy/common_utils/reset_budget_job.py | 14 ++-- litellm/proxy/db/spend_counter_reseed.py | 33 ++------ litellm/proxy/proxy_server.py | 6 +- .../spend/test_end_user_budget_enforcement.py | 79 +++++++++++++++++++ .../test_user_api_key_auth_request_flow.py | 1 + .../proxy/db/test_spend_counter_reseed.py | 48 ++--------- .../proxy/proxy_server/test_spend_counters.py | 44 +++++++++++ ...test_proxy_server_endpoints_and_startup.py | 21 ++--- 8 files changed, 155 insertions(+), 91 deletions(-) diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index ccf2232e379..53a1e07973b 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -636,13 +636,13 @@ class ResetBudgetJob: async def _invalidate_user_api_key_cache_entry(cache_key: str) -> None: """Drop a stale management-cache entry so the next read fetches from DB. - Tags and end-users are not reseeded by ``SpendCounterReseed.from_db``; - for those, when the spend counter expires the budget check falls back - to ``cached_obj.spend``. Keys, orgs, and team memberships are reseeded - from the DB, but auth still may consult ``user_api_key_cache`` objects - whose ``.spend`` field can lag a cross-pod DB reset. Deleting the cache - entry forces the next auth-time fetch to reload the zeroed row from - Postgres. + Tags are not reseeded by ``SpendCounterReseed.from_db``; for those, + when the spend counter expires the budget check falls back to + ``cached_obj.spend``. Keys, orgs, team memberships and end users are + reseeded from the DB, but auth still may consult ``user_api_key_cache`` + objects whose ``.spend`` field can lag a cross-pod DB reset. Deleting + the cache entry forces the next auth-time fetch to reload the zeroed + row from Postgres. """ try: from litellm.proxy.proxy_server import user_api_key_cache diff --git a/litellm/proxy/db/spend_counter_reseed.py b/litellm/proxy/db/spend_counter_reseed.py index 721999b1868..0df46593c21 100644 --- a/litellm/proxy/db/spend_counter_reseed.py +++ b/litellm/proxy/db/spend_counter_reseed.py @@ -45,8 +45,6 @@ from litellm.repositories.verification_token_repository import ( ) if TYPE_CHECKING: - from prisma.types import LiteLLM_EndUserTableWhereUniqueInput - from litellm.caching.dual_cache import DualCache from litellm.proxy.utils import PrismaClient @@ -84,14 +82,11 @@ class SpendCounterReseed: spend:user:{user_id} -> LiteLLM_UserTable.spend spend:org:{org_id} -> LiteLLM_OrganizationTable.spend spend:project:{project_id} -> LiteLLM_ProjectTable.spend + spend:end_user:{end_user_id} -> LiteLLM_EndUserTable.spend - End-user and tag spend counters intentionally do not reseed here. Their - auth paths already load the corresponding objects via get_end_user_object() - and get_tag_objects_batch(); callers pass those values as fallback_spend. - end_user_from_db is the one end-user read, used only as the budget floor when - a counter sits below that cached spend: a worker that did not run the budget - reset still caches the pre-reset end-user object, and LiteLLM_EndUserTable - is the row the reset zeroed. + Tag spend counters intentionally do not reseed here. Their auth path already + loads the tag objects via get_tag_objects_batch(); callers pass those values + as fallback_spend. """ _locks: ClassVar["OrderedDict[str, asyncio.Lock]"] = OrderedDict() @@ -180,24 +175,12 @@ class SpendCounterReseed: return await ProjectRepository(prisma_client).table.find_unique( where={"project_id": counter_key[len("spend:project:") :]} ) + if counter_key.startswith(END_USER_COUNTER_PREFIX): + return await EndUserRepository(prisma_client).table.find_unique( + where={"user_id": counter_key[len(END_USER_COUNTER_PREFIX) :]} + ) return None - @staticmethod - async def end_user_from_db(prisma_client: Optional["PrismaClient"], counter_key: str) -> float | None: - if prisma_client is None or not counter_key.startswith(END_USER_COUNTER_PREFIX): - return None - where: Final[LiteLLM_EndUserTableWhereUniqueInput] = {"user_id": counter_key[len(END_USER_COUNTER_PREFIX) :]} - try: - row: Final = await bounded_db_lookup( - EndUserRepository(prisma_client).table.find_unique(where=where), name="end_user_spend" - ) - except Exception: # noqa: BLE001 # a failed floor read falls back to the cached spend, like from_db - verbose_proxy_logger.exception("SpendCounterReseed.end_user_from_db: failed for %s", counter_key) - return None - if row is None: - return None - return float(row.spend or 0.0) - @staticmethod def _is_key_or_team_window_counter(counter_key: str) -> bool: for prefix in ("spend:key:", "spend:team:"): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c6d3e12a378..37ebbefe4b2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -615,7 +615,7 @@ from litellm.proxy.db.request_error_tracking import ( flush_request_errors, request_error_accumulator, ) -from litellm.proxy.db.spend_counter_reseed import END_USER_COUNTER_PREFIX, SpendCounterReseed +from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed from litellm.proxy.discovery_endpoints import ( agent_skills_discovery_router, ui_discovery_endpoints_router, @@ -3290,7 +3290,7 @@ async def reseed_spend_counter_from_db(counter_key: str) -> bool: the read-time floor (_authoritative_floor_spend) converges to the true total as the buffer flushes. The point is to restore enforcement to a real floor rather than leave the counter deleted and unenforced (the prior fail-open). - Counters with no DB row (window/end-user/tag) are left untouched rather than + Counters with no DB row (window/tag) are left untouched rather than deleted, so enforcement keeps reading whatever value they hold. """ db_spend: Final = await SpendCounterReseed.from_db(prisma_client=prisma_client, counter_key=counter_key) @@ -3307,8 +3307,6 @@ async def _floor_spend_from_db( window_duration: str | None, window_start: datetime | None, ) -> float | None: - if counter_key.startswith(END_USER_COUNTER_PREFIX): - return await SpendCounterReseed.end_user_from_db(prisma_client=prisma_client, counter_key=counter_key) entity_spend: Final = await SpendCounterReseed.from_db(prisma_client=prisma_client, counter_key=counter_key) if entity_spend is not None: return entity_spend diff --git a/tests/integration/spend/test_end_user_budget_enforcement.py b/tests/integration/spend/test_end_user_budget_enforcement.py index 02c8aee8f5e..dff5c893848 100644 --- a/tests/integration/spend/test_end_user_budget_enforcement.py +++ b/tests/integration/spend/test_end_user_budget_enforcement.py @@ -1,7 +1,9 @@ import json +import os import re import uuid from collections.abc import Callable, Mapping +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from typing import Final @@ -12,6 +14,7 @@ from integration._support.database import read_rows from integration._support.upstream import delete_scenario, register_scenario from integration.cost_calculation.cost_tracking_case import JsonResponse from pydantic import BaseModel, JsonValue +from redis import Redis PROVIDER_AUTHORIZATION: Final = "Bearer integration-provider-key" CANNED_CONTENT: Final = "Hello! This is a mock response from the fake OpenAI endpoint." @@ -639,3 +642,79 @@ def test_streamed_call_is_refused_as_json_once_over_budget(gateway: Gateway) -> assert "data:" not in denied.text, denied.text _assert_budget_refusal(denied, end_user, 0.05) assert _observed_requests(upstream) == [] + + +_COLD_BURST: Final = 40 +_COLD_SEED_SPEND: Final = 6.0 +_COLD_BUDGET: Final = 10.0 +_CHAT_COST: Final = 20 * 0.001 + 20 * 0.002 + + +def _bounded_chat(gateway: Gateway, model: str, key: str, end_user: str) -> httpx.Response: + return gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "max_tokens": 20, + "messages": [{"role": "user", "content": f"cold counter {uuid.uuid4().hex}"}], + "user": end_user, + }, + key=key, + ) + + +def _serves_model(gateway: Gateway, model: str) -> bool: + response: Final = gateway.request("GET", "/model/info") + return response.status_code == 200 and any( + object_value(entry)["model_name"] == model for entry in object_value(response.json())["data"] + ) + + +def _counter(cache: Redis, end_user: str) -> float | None: + raw: Final = cache.get(f"spend:end_user:{end_user}") + return None if raw is None else float(raw) + + +def test_overlapping_cold_starts_seed_the_end_user_counter_once(gateway: Gateway, peer: Gateway) -> None: + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + eventually(lambda: _serves_model(peer, model), bool, seconds=30) + key: Final = scenario.key(models=[model]) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + _new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + assert _customer_info(gateway, end_user).spend == _COLD_SEED_SPEND + assert _counter(cache, end_user) is None + upstream.delete("/__observations").raise_for_status() + + proxies: Final = (gateway, peer) + with ThreadPoolExecutor(max_workers=_COLD_BURST) as pool: + burst: Final = tuple( + pool.map(lambda index: _bounded_chat(proxies[index % 2], model, key, end_user), range(_COLD_BURST)) + ) + refused: Final = [response.text for response in burst if response.status_code != 200] + assert refused == [], refused + follow_up: Final = _bounded_chat(gateway, model, key, end_user) + assert follow_up.status_code == 200, follow_up.text + assert len(_observed_requests(upstream)) == _COLD_BURST + 1 + + expected_spend: Final = _COLD_SEED_SPEND + (_COLD_BURST + 1) * _CHAT_COST + charged: Final = eventually( + lambda: _end_user_row(end_user), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) >= expected_spend - 1e-6, + seconds=70, + ) + assert float(str(charged[0]["spend"])) == pytest.approx(expected_spend) + assert _customer_info(gateway, end_user).spend == pytest.approx(expected_spend) + counted: Final = eventually( + lambda: _counter(cache, end_user), + lambda value: value is not None and value == pytest.approx(expected_spend), + seconds=30, + return_last_on_timeout=True, + ) + assert counted == pytest.approx(expected_spend) diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 4cdd7fefd28..3f87e0b8654 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -5740,6 +5740,7 @@ def _unrestricted_end_user_prisma(spend: float): end_user_row = MagicMock() end_user_row.user_id = "customer-1" end_user_row.dict = lambda: {"user_id": "customer-1", "blocked": False, "spend": spend} + end_user_row.spend = spend mock_prisma = MagicMock() mock_prisma.db.litellm_endusertable.find_many = AsyncMock(return_value=[]) diff --git a/tests/unit/proxy/db/test_spend_counter_reseed.py b/tests/unit/proxy/db/test_spend_counter_reseed.py index ab931277313..336059da9b4 100644 --- a/tests/unit/proxy/db/test_spend_counter_reseed.py +++ b/tests/unit/proxy/db/test_spend_counter_reseed.py @@ -367,49 +367,23 @@ async def test_cold_reseed_preserves_concurrent_local_increment( @pytest.mark.asyncio -async def test_end_user_from_db_reads_the_end_user_row_by_user_id(): - prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=0.0)) +async def test_from_db_reads_the_end_user_row_by_user_id(): + prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=12.5)) - result: Final = await SpendCounterReseed.end_user_from_db( - prisma_client=prisma, counter_key="spend:end_user:customer-42" - ) - - assert result == 0.0 + assert await SpendCounterReseed.from_db(prisma_client=prisma, counter_key="spend:end_user:customer-42") == 12.5 assert prisma.db.litellm_endusertable.where_clauses == [{"user_id": "customer-42"}] @pytest.mark.asyncio -async def test_end_user_from_db_returns_the_recorded_spend(): - prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=12.5)) - +async def test_from_db_returns_none_for_a_missing_or_unreadable_end_user_row(): assert ( - await SpendCounterReseed.end_user_from_db(prisma_client=prisma, counter_key="spend:end_user:customer-42") - == 12.5 - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("counter_key", ["spend:key:hashed", "spend:team:t1", "spend:tag:t1"]) -async def test_end_user_from_db_ignores_other_counter_kinds_without_touching_the_db(counter_key): - prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="x", spend=5.0)) - - assert await SpendCounterReseed.end_user_from_db(prisma_client=prisma, counter_key=counter_key) is None - assert prisma.db.litellm_endusertable.where_clauses == [] - - -@pytest.mark.asyncio -async def test_end_user_from_db_returns_none_without_a_row_a_client_or_on_db_error(): - assert ( - await SpendCounterReseed.end_user_from_db(prisma_client=None, counter_key="spend:end_user:customer-42") is None - ) - assert ( - await SpendCounterReseed.end_user_from_db( + await SpendCounterReseed.from_db( prisma_client=_FakePrismaClient(end_user_row=None), counter_key="spend:end_user:customer-42" ) is None ) assert ( - await SpendCounterReseed.end_user_from_db( + await SpendCounterReseed.from_db( prisma_client=_FakePrismaClient(end_user_error=RuntimeError("db down")), counter_key="spend:end_user:customer-42", ) @@ -470,13 +444,3 @@ async def test_from_db_deadline_covers_the_wait_for_a_gate_slot(monkeypatch: pyt find_unique.assert_not_called() finally: db_lookup_stall_tracker.clear() - - -@pytest.mark.asyncio -async def test_from_db_still_never_reads_the_end_user_row(): - """A cold end-user counter keeps seeding from the cached end-user object the auth - path already loaded; the row is read only as the budget floor.""" - prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=5.0)) - - assert await SpendCounterReseed.from_db(prisma_client=prisma, counter_key="spend:end_user:customer-42") is None - assert prisma.db.litellm_endusertable.where_clauses == [] diff --git a/tests/unit/proxy/proxy_server/test_spend_counters.py b/tests/unit/proxy/proxy_server/test_spend_counters.py index d273ef8f42f..fdd05db52de 100644 --- a/tests/unit/proxy/proxy_server/test_spend_counters.py +++ b/tests/unit/proxy/proxy_server/test_spend_counters.py @@ -50,6 +50,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_loca from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, end_user_cache_key, tag_cache_key from litellm.proxy.management_endpoints.team_endpoints import refresh_cached_team +from litellm.proxy.spend_tracking.spend_counter_batch import spend_counter_batch_scope from .conftest import normalize @@ -1357,6 +1358,49 @@ async def test_ensure_spend_counter_initialized_cold_seeds_from_source_cache( } +class _FakeRedisMeetingAtTheWarmCheck(FakeAsyncRedis): + """Holds each request's MGET until every request has made one, so all of them find the counter missing before + any of them seeds it.""" + + def __init__(self, server: FakeServer, callers: int) -> None: + super().__init__(server=server) + self._warm_checks: Final = asyncio.Barrier(callers) + + async def mget(self, keys: list[str], *args: str) -> list[bytes | None]: + values: Final = await super().mget(keys, *args) + await self._warm_checks.wait() + return values + + +@pytest.mark.asyncio +async def test_overlapping_cold_end_user_seeds_write_the_end_user_row_once(monkeypatch): + counter_key: Final = "spend:end_user:customer-1" + callers: Final = 3 + redis: Final = _FakeRedisMeetingAtTheWarmCheck(FakeServer(), callers) + auth_cache: Final = UserApiKeyCache(in_memory_cache=InMemoryCache()) + auth_cache.set_cache( + key=end_user_cache_key("customer-1"), + value=LiteLLM_EndUserTable(user_id="customer-1", blocked=False, spend=6.0), + local_only=True, + ) + prisma: Final = MagicMock() + prisma.db.litellm_endusertable.find_unique = AsyncMock(return_value=MagicMock(spend=7.5)) + counter_cache: Final = ps.DualCache(redis_cache=_RedisCacheOnClient(redis)) + monkeypatch.setattr(ps, "spend_counter_cache", counter_cache) + monkeypatch.setattr(ps, "user_api_key_cache", auth_cache) + monkeypatch.setattr(ps, "prisma_client", prisma) + + async def _request() -> None: + with spend_counter_batch_scope(counter_cache.redis_cache, counter_keys=frozenset((counter_key,))): + await ps.ensure_spend_counter_initialized( + counter_key=counter_key, source_cache_key=end_user_cache_key("customer-1") + ) + + await asyncio.wait_for(asyncio.gather(*(_request() for _ in range(callers))), timeout=5) + + assert float(await redis.get(counter_key)) == 7.5 + + # --------------------------------------------------------------------------- # _get_source_cache_base_spend # --------------------------------------------------------------------------- diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 14196d4a1ac..f0c1a3438d6 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -8849,36 +8849,31 @@ async def test_primary_spend_counter_redis_concurrent_seed_does_not_double_seed( @pytest.mark.asyncio async def test_reseed_spend_from_db_user_and_org_prefixes(): - """User and org counters reseed from their own DB tables. + """User, end-user and org counters reseed from their own DB tables. - End-user and tag counters use the already fetched auth objects passed as - fallback_spend, so this reseed helper must not add extra per-request DB - reads for them. + Tag counters use the already fetched auth objects passed as fallback_spend, + so this reseed helper must not add extra per-request DB reads for them. """ from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed user_row = MagicMock() user_row.spend = 17.0 + end_user_row = MagicMock() + end_user_row.spend = 6.0 org_row = MagicMock() org_row.spend = 305.0 fake_prisma = MagicMock() fake_prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=user_row) - fake_prisma.db.litellm_endusertable.find_unique = AsyncMock() + fake_prisma.db.litellm_endusertable.find_unique = AsyncMock(return_value=end_user_row) fake_prisma.db.litellm_tagtable.find_unique = AsyncMock() fake_prisma.db.litellm_organizationtable.find_unique = AsyncMock(return_value=org_row) assert await SpendCounterReseed.from_db(fake_prisma, "spend:user:alice") == 17.0 fake_prisma.db.litellm_usertable.find_unique.assert_awaited_once_with(where={"user_id": "alice"}) - assert ( - await SpendCounterReseed.from_db( - fake_prisma, - "spend:end_user:customer-1", - ) - is None - ) - fake_prisma.db.litellm_endusertable.find_unique.assert_not_awaited() + assert await SpendCounterReseed.from_db(fake_prisma, "spend:end_user:customer-1") == 6.0 + fake_prisma.db.litellm_endusertable.find_unique.assert_awaited_once_with(where={"user_id": "customer-1"}) assert await SpendCounterReseed.from_db(fake_prisma, "spend:tag:paid-tag") is None fake_prisma.db.litellm_tagtable.find_unique.assert_not_awaited() From 1d571b71b52d4ddce33660da7bce35bc156c937e Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:12:13 -0700 Subject: [PATCH 2/4] fix(proxy): read the seeded counter after waiting on its seed lock A request that waited while another seeded a cold counter kept its batched miss and read the DB again; a caller that waited now re-reads Redis first --- litellm/proxy/db/spend_counter_reseed.py | 7 +++++-- tests/unit/proxy/proxy_server/test_spend_counters.py | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/db/spend_counter_reseed.py b/litellm/proxy/db/spend_counter_reseed.py index 0df46593c21..139dec82238 100644 --- a/litellm/proxy/db/spend_counter_reseed.py +++ b/litellm/proxy/db/spend_counter_reseed.py @@ -218,14 +218,17 @@ class SpendCounterReseed: when the DB read succeeds, or None when the DB is unavailable. """ lock: Final = await SpendCounterReseed._get_lock(counter_key) + waited_for_seed: Final = lock.locked() async with lock: batched: Final = await SpendCounterReseed._read_active_batch(counter_key) if batched is not None and batched[0] is not None: return batched[0] # Re-check after acquiring the lock. Skip in-memory on a clean - # Redis miss - in-memory is per-pod-stale. + # Redis miss - in-memory is per-pod-stale. The request's batched + # read can predate the lock, so a caller that waited on it re-reads + # Redis for the value the holder seeded. redis_clean_miss = batched is not None - if spend_counter_cache.redis_cache is not None and not redis_clean_miss: + if spend_counter_cache.redis_cache is not None and (waited_for_seed or not redis_clean_miss): try: val = await spend_counter_cache.redis_cache.async_get_cache(key=counter_key) if val is not None: diff --git a/tests/unit/proxy/proxy_server/test_spend_counters.py b/tests/unit/proxy/proxy_server/test_spend_counters.py index fdd05db52de..a811a4b483b 100644 --- a/tests/unit/proxy/proxy_server/test_spend_counters.py +++ b/tests/unit/proxy/proxy_server/test_spend_counters.py @@ -1373,7 +1373,7 @@ class _FakeRedisMeetingAtTheWarmCheck(FakeAsyncRedis): @pytest.mark.asyncio -async def test_overlapping_cold_end_user_seeds_write_the_end_user_row_once(monkeypatch): +async def test_overlapping_cold_end_user_seeds_read_and_write_the_end_user_row_once(monkeypatch: pytest.MonkeyPatch): counter_key: Final = "spend:end_user:customer-1" callers: Final = 3 redis: Final = _FakeRedisMeetingAtTheWarmCheck(FakeServer(), callers) @@ -1399,6 +1399,7 @@ async def test_overlapping_cold_end_user_seeds_write_the_end_user_row_once(monke await asyncio.wait_for(asyncio.gather(*(_request() for _ in range(callers))), timeout=5) assert float(await redis.get(counter_key)) == 7.5 + prisma.db.litellm_endusertable.find_unique.assert_awaited_once() # --------------------------------------------------------------------------- From ad4ab0e6c1794ee7d7273bf6156d45e2d3461c2e Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:06:41 -0700 Subject: [PATCH 3/4] test(proxy): cover cold end-user seeding on all three endpoints and behind a stale cached customer The overlapping cold-start burst now runs on chat completions, responses and messages, and two new cells pin that a cold counter seeds from the end-user row when the row was raised past the budget or reset behind the cached customer --- .../spend/test_end_user_budget_enforcement.py | 89 +++++++++++++++++-- 1 file changed, 81 insertions(+), 8 deletions(-) diff --git a/tests/integration/spend/test_end_user_budget_enforcement.py b/tests/integration/spend/test_end_user_budget_enforcement.py index dff5c893848..b04add28685 100644 --- a/tests/integration/spend/test_end_user_budget_enforcement.py +++ b/tests/integration/spend/test_end_user_budget_enforcement.py @@ -10,7 +10,7 @@ from typing import Final import httpx import pytest from integration._support.client import Gateway, Scenario, eventually, object_value, string_value -from integration._support.database import read_rows +from integration._support.database import read_rows, write_rows from integration._support.upstream import delete_scenario, register_scenario from integration.cost_calculation.cost_tracking_case import JsonResponse from pydantic import BaseModel, JsonValue @@ -676,15 +676,17 @@ def _counter(cache: Redis, end_user: str) -> float | None: return None if raw is None else float(raw) -def test_overlapping_cold_starts_seed_the_end_user_counter_once(gateway: Gateway, peer: Gateway) -> None: +@pytest.mark.parametrize("spelling", ("user", "responses_user", "messages_metadata_user_id")) +def test_overlapping_cold_starts_seed_the_end_user_counter_once(gateway: Gateway, peer: Gateway, spelling: str) -> None: + chosen: Final = SPELLINGS[spelling] with ( gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache, ): - model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) - eventually(lambda: _serves_model(peer, model), bool, seconds=30) - key: Final = scenario.key(models=[model]) + deployment: Final = _deployment(scenario, chosen) + eventually(lambda: _serves_model(peer, deployment.model), bool, seconds=30) + key: Final = scenario.key(models=[deployment.model]) end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" _new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND) scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) @@ -695,12 +697,15 @@ def test_overlapping_cold_starts_seed_the_end_user_counter_once(gateway: Gateway proxies: Final = (gateway, peer) with ThreadPoolExecutor(max_workers=_COLD_BURST) as pool: burst: Final = tuple( - pool.map(lambda index: _bounded_chat(proxies[index % 2], model, key, end_user), range(_COLD_BURST)) + pool.map( + lambda index: _send(proxies[index % 2], chosen, deployment, key, end_user, f"cold {index}"), + range(_COLD_BURST), + ) ) refused: Final = [response.text for response in burst if response.status_code != 200] assert refused == [], refused - follow_up: Final = _bounded_chat(gateway, model, key, end_user) - assert follow_up.status_code == 200, follow_up.text + follow_up: Final = _send(gateway, chosen, deployment, key, end_user, "cold follow-up") + _assert_served(chosen, follow_up) assert len(_observed_requests(upstream)) == _COLD_BURST + 1 expected_spend: Final = _COLD_SEED_SPEND + (_COLD_BURST + 1) * _CHAT_COST @@ -718,3 +723,71 @@ def test_overlapping_cold_starts_seed_the_end_user_counter_once(gateway: Gateway return_last_on_timeout=True, ) assert counted == pytest.approx(expected_spend) + + +def _set_end_user_row_spend(end_user: str, spend: float) -> None: + write_rows('UPDATE "LiteLLM_EndUserTable" SET spend = %s WHERE user_id = %s', (str(spend), end_user)) + + +def _refused_spend(denied: httpx.Response, end_user: str) -> float: + error: Final = _denial_error(denied) + assert error["type"] == "budget_exceeded", denied.text + matched: Final = re.fullmatch( + rf"ExceededBudget: End User={re.escape(end_user)} over budget\. Spend=(\S+), Budget=10\.0", + string_value(error["message"]), + ) + assert matched is not None, denied.text + return float(matched.group(1)) + + +def test_a_cold_end_user_counter_seeds_from_a_row_raised_behind_the_cached_customer(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model]) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + _new_customer(gateway, end_user, max_budget=_COLD_BUDGET) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + served: Final = _bounded_chat(gateway, model, key, end_user) + assert served.status_code == 200, served.text + eventually( + lambda: _end_user_row(end_user), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) == pytest.approx(_CHAT_COST), + seconds=70, + ) + _set_end_user_row_spend(end_user, _COLD_BUDGET + 1) + cache.delete(f"spend:end_user:{end_user}") + + denied: Final = _bounded_chat(gateway, model, key, end_user) + + assert _refused_spend(denied, end_user) == pytest.approx(_COLD_BUDGET + 1) + assert _counter(cache, end_user) == pytest.approx(_COLD_BUDGET + 1) + + +def test_a_cold_end_user_counter_seeds_from_a_row_reset_behind_the_cached_customer(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model]) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + _new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_BUDGET + 2) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + denied: Final = _bounded_chat(gateway, model, key, end_user) + assert _refused_spend(denied, end_user) == pytest.approx(_COLD_BUDGET + 2) + _set_end_user_row_spend(end_user, 0.0) + cache.delete(f"spend:end_user:{end_user}") + + served: Final = _bounded_chat(gateway, model, key, end_user) + + assert served.status_code == 200, served.text + charged: Final = eventually( + lambda: _end_user_row(end_user), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) > 0, + seconds=70, + ) + assert float(str(charged[0]["spend"])) == pytest.approx(_CHAT_COST) + assert _counter(cache, end_user) == pytest.approx(_CHAT_COST) From 779d9d5dbe7d626e6fe77fbdee55baf74bcc4705 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Sat, 24 Oct 2026 01:46:05 -0700 Subject: [PATCH 4/4] fix(proxy): keep unflushed end-user charges when a wiped counter reseeds from the row A cold end-user counter seeds from LiteLLM_EndUserTable, which lags the batch writer. After Redis loses the counter, the budget check now floors on the worker's cached spend while the row is still at or above the spend that cache was loaded with, so an over-budget customer is refused again inside the flush window, and a reset that lowered the row still wins --- litellm/proxy/auth/auth_checks.py | 25 ++++++++++++- litellm/proxy/proxy_server.py | 9 +++++ .../spend/test_end_user_budget_enforcement.py | 26 ++++++++++++++ tests/unit/proxy/auth/test_auth_checks.py | 36 +++++++++++++++++++ 4 files changed, 95 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 8887d86570a..64e5f6be78d 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -1901,13 +1901,17 @@ async def check_end_user_budget( if end_user_budget is None: return - from litellm.proxy.proxy_server import get_current_spend + from litellm.proxy.proxy_server import get_current_spend, user_api_key_cache + loaded_spend: Final = user_api_key_cache.get_cache( + key=_end_user_loaded_spend_key(end_user_obj.user_id), local_only=True + ) end_user_spend: Final = await get_current_spend( counter_key=f"spend:end_user:{end_user_obj.user_id}", fallback_spend=end_user_obj.spend or 0.0, max_budget=end_user_budget, fallback_authoritative=True, + fallback_loaded_spend=float(loaded_spend) if isinstance(loaded_spend, (int, float)) else None, ) if end_user_spend > end_user_budget: raise litellm.BudgetExceededError( @@ -2119,6 +2123,23 @@ async def _end_user_is_known_unrestricted( return registry is not None and end_user_id not in registry +def _end_user_loaded_spend_key(end_user_id: str) -> str: + return f"end_user_loaded_spend:{end_user_id}" + + +def _remember_end_user_loaded_spend( + user_api_key_cache: UserApiKeyCache, end_user_id: str, spend: float, *, keep_existing: bool +) -> None: + """Record, on this worker only, the DB spend its cached end-user object was loaded with. Spend this worker + charges later is added to the cached object, never to this value.""" + key: Final = _end_user_loaded_spend_key(end_user_id) + if keep_existing and user_api_key_cache.get_cache(key=key, local_only=True) is not None: + return + user_api_key_cache.set_cache( + key=key, value=spend, local_only=True, ttl=get_management_object_ttl(user_api_key_cache) + ) + + @log_db_metrics @with_service_target(AUTH_OBJECTS_TARGET) async def get_end_user_object( @@ -2166,6 +2187,7 @@ async def get_end_user_object( model_type=LiteLLM_EndUserTable, ) if cached_user_obj is not None: + _remember_end_user_loaded_spend(user_api_key_cache, end_user_id, cached_user_obj.spend, keep_existing=True) return await _apply_default_budget_to_end_user( end_user_obj=cached_user_obj, prisma_client=prisma_client, @@ -2206,6 +2228,7 @@ async def get_end_user_object( model_type=LiteLLM_EndUserTable, ttl=get_management_object_ttl(user_api_key_cache), ) + _remember_end_user_loaded_spend(user_api_key_cache, end_user_id, end_user_row.spend, keep_existing=False) if key_end_user_budget_id is None: return end_user_row diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 37ebbefe4b2..682777657b8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3166,6 +3166,7 @@ async def get_current_spend( window_duration: str | None = None, window_start: datetime | None = None, fallback_authoritative: bool = False, + fallback_loaded_spend: float | None = None, ) -> float: """ Read current spend from the cross-pod spend counter. @@ -3195,6 +3196,11 @@ async def get_current_spend( skipped for healthy primary counters (counter at or above recorded spend) and cached in-process for a few seconds, so a persistently stale counter drives at most one read per counter per window rather than one per request. + + ``fallback_loaded_spend`` is the DB spend ``fallback_spend`` was loaded + from before this worker added its own charges to it. While the row is still + at or above that value, ``fallback_spend`` floors the read, so charges the + batch writer has not flushed yet are not lost with a wiped counter. """ current, verified = await _read_spend_counter_estimate(counter_key=counter_key, fallback_spend=fallback_spend) if fallback_authoritative: @@ -3222,6 +3228,9 @@ async def get_current_spend( verified = True if authoritative > current: await _repair_stale_spend_counter(counter_key=counter_key, db_spend=authoritative) + if fallback_loaded_spend is not None and fallback_loaded_spend <= authoritative: + return max(current, authoritative, fallback_spend) + if authoritative > current: return authoritative elif fallback_spend > current: # nothing to read (tag counters, an end user without a row or a DB client, a diff --git a/tests/integration/spend/test_end_user_budget_enforcement.py b/tests/integration/spend/test_end_user_budget_enforcement.py index b04add28685..e6112c6952e 100644 --- a/tests/integration/spend/test_end_user_budget_enforcement.py +++ b/tests/integration/spend/test_end_user_budget_enforcement.py @@ -791,3 +791,29 @@ def test_a_cold_end_user_counter_seeds_from_a_row_reset_behind_the_cached_custom ) assert float(str(charged[0]["spend"])) == pytest.approx(_CHAT_COST) assert _counter(cache, end_user) == pytest.approx(_CHAT_COST) + + +def test_a_cold_end_user_counter_keeps_spend_the_row_has_not_caught_up_to(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.25) + key: Final = scenario.key(models=[model]) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + call_cost: Final = 20 * 0.001 + 20 * 0.25 + _new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + served: Final = _chat(gateway, model, key, end_user) + assert served.status_code == 200, served.text + eventually( + lambda: _end_user_row(end_user), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) == pytest.approx(_COLD_SEED_SPEND + call_cost), + seconds=70, + ) + _set_end_user_row_spend(end_user, _COLD_SEED_SPEND) + cache.delete(f"spend:end_user:{end_user}") + + denied: Final = _chat(gateway, model, key, end_user) + + assert _refused_spend(denied, end_user) == pytest.approx(_COLD_SEED_SPEND + call_cost) diff --git a/tests/unit/proxy/auth/test_auth_checks.py b/tests/unit/proxy/auth/test_auth_checks.py index e1afa81d372..47e6ba4ead8 100644 --- a/tests/unit/proxy/auth/test_auth_checks.py +++ b/tests/unit/proxy/auth/test_auth_checks.py @@ -538,6 +538,42 @@ async def test_check_end_user_budget(customer_spend, customer_budget): assert exc_info.value.max_budget == customer_budget +@pytest.mark.parametrize("row_spend, refused", [(6.0, True), (0.0, False)]) +@pytest.mark.asyncio +async def test_check_end_user_budget_keeps_unflushed_charges_after_the_counter_is_wiped( + monkeypatch: pytest.MonkeyPatch, row_spend: float, refused: bool +) -> None: + import litellm.proxy.proxy_server as ps + from litellm.proxy.auth.auth_checks import check_end_user_budget + + end_user_id: Final = "customer-42" + prisma: Final = MagicMock() + prisma.db.litellm_endusertable.find_unique = AsyncMock(return_value=MagicMock(spend=row_spend)) + cache: Final = UserApiKeyCache() + await cache.async_set_cache( + key=f"end_user_id:{end_user_id}", + value=LiteLLM_EndUserTable( + user_id=end_user_id, spend=6.0, litellm_budget_table=LiteLLM_BudgetTable(max_budget=10.0), blocked=False + ), + model_type=LiteLLM_EndUserTable, + ) + loaded: Final = await get_end_user_object(end_user_id=end_user_id, prisma_client=prisma, user_api_key_cache=cache) + assert loaded is not None and loaded.spend == 6.0 + ps._add_spend_to_locally_cached_object(cache, f"end_user_id:{end_user_id}", LiteLLM_EndUserTable, 5.0) + charged: Final = await get_end_user_object(end_user_id=end_user_id, prisma_client=prisma, user_api_key_cache=cache) + assert charged is not None and charged.spend == 11.0 + monkeypatch.setattr(ps, "prisma_client", prisma) + monkeypatch.setattr(ps, "spend_counter_cache", DualCache()) + monkeypatch.setattr(ps, "user_api_key_cache", cache) + + if not refused: + await check_end_user_budget(end_user_obj=charged, route="/v1/chat/completions") + return + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await check_end_user_budget(end_user_obj=charged, route="/v1/chat/completions") + assert exc_info.value.current_cost == 11.0 + + @pytest.mark.parametrize( "model, expect_to_work", [