diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 79e385c376a..14fbde4a565 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -1915,13 +1915,17 @@ async def check_end_user_budget( if end_user_budget is None: return - from litellm.proxy.proxy_server import get_current_spend + from litellm.proxy.proxy_server import get_current_spend, user_api_key_cache + loaded_spend: Final = user_api_key_cache.get_cache( + key=_end_user_loaded_spend_key(end_user_obj.user_id), local_only=True + ) end_user_spend: Final = await get_current_spend( counter_key=f"spend:end_user:{end_user_obj.user_id}", fallback_spend=end_user_obj.spend or 0.0, max_budget=end_user_budget, fallback_authoritative=True, + fallback_loaded_spend=float(loaded_spend) if isinstance(loaded_spend, (int, float)) else None, ) if end_user_spend > end_user_budget: raise litellm.BudgetExceededError( @@ -2133,6 +2137,23 @@ async def _end_user_is_known_unrestricted( return registry is not None and end_user_id not in registry +def _end_user_loaded_spend_key(end_user_id: str) -> str: + return f"end_user_loaded_spend:{end_user_id}" + + +def _remember_end_user_loaded_spend( + user_api_key_cache: UserApiKeyCache, end_user_id: str, spend: float, *, keep_existing: bool +) -> None: + """Record, on this worker only, the DB spend its cached end-user object was loaded with. Spend this worker + charges later is added to the cached object, never to this value.""" + key: Final = _end_user_loaded_spend_key(end_user_id) + if keep_existing and user_api_key_cache.get_cache(key=key, local_only=True) is not None: + return + user_api_key_cache.set_cache( + key=key, value=spend, local_only=True, ttl=get_management_object_ttl(user_api_key_cache) + ) + + @log_db_metrics @with_service_target(AUTH_OBJECTS_TARGET) async def get_end_user_object( @@ -2180,6 +2201,7 @@ async def get_end_user_object( model_type=LiteLLM_EndUserTable, ) if cached_user_obj is not None: + _remember_end_user_loaded_spend(user_api_key_cache, end_user_id, cached_user_obj.spend, keep_existing=True) return await _apply_default_budget_to_end_user( end_user_obj=cached_user_obj, prisma_client=prisma_client, @@ -2220,6 +2242,7 @@ async def get_end_user_object( model_type=LiteLLM_EndUserTable, ttl=get_management_object_ttl(user_api_key_cache), ) + _remember_end_user_loaded_spend(user_api_key_cache, end_user_id, end_user_row.spend, keep_existing=False) if key_end_user_budget_id is None: return end_user_row diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 59fd00cec43..6e247e1dfdf 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -637,13 +637,13 @@ class ResetBudgetJob: async def _invalidate_user_api_key_cache_entry(cache_key: str) -> None: """Drop a stale management-cache entry so the next read fetches from DB. - Tags and end-users are not reseeded by ``SpendCounterReseed.from_db``; - for those, when the spend counter expires the budget check falls back - to ``cached_obj.spend``. Keys, orgs, and team memberships are reseeded - from the DB, but auth still may consult ``user_api_key_cache`` objects - whose ``.spend`` field can lag a cross-pod DB reset. Deleting the cache - entry forces the next auth-time fetch to reload the zeroed row from - Postgres. + Tags are not reseeded by ``SpendCounterReseed.from_db``; for those, + when the spend counter expires the budget check falls back to + ``cached_obj.spend``. Keys, orgs, team memberships and end users are + reseeded from the DB, but auth still may consult ``user_api_key_cache`` + objects whose ``.spend`` field can lag a cross-pod DB reset. Deleting + the cache entry forces the next auth-time fetch to reload the zeroed + row from Postgres. """ try: from litellm.proxy.proxy_server import user_api_key_cache diff --git a/litellm/proxy/db/spend_counter_reseed.py b/litellm/proxy/db/spend_counter_reseed.py index 721999b1868..139dec82238 100644 --- a/litellm/proxy/db/spend_counter_reseed.py +++ b/litellm/proxy/db/spend_counter_reseed.py @@ -45,8 +45,6 @@ from litellm.repositories.verification_token_repository import ( ) if TYPE_CHECKING: - from prisma.types import LiteLLM_EndUserTableWhereUniqueInput - from litellm.caching.dual_cache import DualCache from litellm.proxy.utils import PrismaClient @@ -84,14 +82,11 @@ class SpendCounterReseed: spend:user:{user_id} -> LiteLLM_UserTable.spend spend:org:{org_id} -> LiteLLM_OrganizationTable.spend spend:project:{project_id} -> LiteLLM_ProjectTable.spend + spend:end_user:{end_user_id} -> LiteLLM_EndUserTable.spend - End-user and tag spend counters intentionally do not reseed here. Their - auth paths already load the corresponding objects via get_end_user_object() - and get_tag_objects_batch(); callers pass those values as fallback_spend. - end_user_from_db is the one end-user read, used only as the budget floor when - a counter sits below that cached spend: a worker that did not run the budget - reset still caches the pre-reset end-user object, and LiteLLM_EndUserTable - is the row the reset zeroed. + Tag spend counters intentionally do not reseed here. Their auth path already + loads the tag objects via get_tag_objects_batch(); callers pass those values + as fallback_spend. """ _locks: ClassVar["OrderedDict[str, asyncio.Lock]"] = OrderedDict() @@ -180,24 +175,12 @@ class SpendCounterReseed: return await ProjectRepository(prisma_client).table.find_unique( where={"project_id": counter_key[len("spend:project:") :]} ) + if counter_key.startswith(END_USER_COUNTER_PREFIX): + return await EndUserRepository(prisma_client).table.find_unique( + where={"user_id": counter_key[len(END_USER_COUNTER_PREFIX) :]} + ) return None - @staticmethod - async def end_user_from_db(prisma_client: Optional["PrismaClient"], counter_key: str) -> float | None: - if prisma_client is None or not counter_key.startswith(END_USER_COUNTER_PREFIX): - return None - where: Final[LiteLLM_EndUserTableWhereUniqueInput] = {"user_id": counter_key[len(END_USER_COUNTER_PREFIX) :]} - try: - row: Final = await bounded_db_lookup( - EndUserRepository(prisma_client).table.find_unique(where=where), name="end_user_spend" - ) - except Exception: # noqa: BLE001 # a failed floor read falls back to the cached spend, like from_db - verbose_proxy_logger.exception("SpendCounterReseed.end_user_from_db: failed for %s", counter_key) - return None - if row is None: - return None - return float(row.spend or 0.0) - @staticmethod def _is_key_or_team_window_counter(counter_key: str) -> bool: for prefix in ("spend:key:", "spend:team:"): @@ -235,14 +218,17 @@ class SpendCounterReseed: when the DB read succeeds, or None when the DB is unavailable. """ lock: Final = await SpendCounterReseed._get_lock(counter_key) + waited_for_seed: Final = lock.locked() async with lock: batched: Final = await SpendCounterReseed._read_active_batch(counter_key) if batched is not None and batched[0] is not None: return batched[0] # Re-check after acquiring the lock. Skip in-memory on a clean - # Redis miss - in-memory is per-pod-stale. + # Redis miss - in-memory is per-pod-stale. The request's batched + # read can predate the lock, so a caller that waited on it re-reads + # Redis for the value the holder seeded. redis_clean_miss = batched is not None - if spend_counter_cache.redis_cache is not None and not redis_clean_miss: + if spend_counter_cache.redis_cache is not None and (waited_for_seed or not redis_clean_miss): try: val = await spend_counter_cache.redis_cache.async_get_cache(key=counter_key) if val is not None: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index df948b69a67..0d25e38ea87 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -619,7 +619,7 @@ from litellm.proxy.db.request_error_tracking import ( flush_request_errors, request_error_accumulator, ) -from litellm.proxy.db.spend_counter_reseed import END_USER_COUNTER_PREFIX, SpendCounterReseed +from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed from litellm.proxy.discovery_endpoints import ( agent_skills_discovery_router, ui_discovery_endpoints_router, @@ -3194,6 +3194,7 @@ async def get_current_spend( window_duration: str | None = None, window_start: datetime | None = None, fallback_authoritative: bool = False, + fallback_loaded_spend: float | None = None, ) -> float: """ Read current spend from the cross-pod spend counter. @@ -3223,6 +3224,11 @@ async def get_current_spend( skipped for healthy primary counters (counter at or above recorded spend) and cached in-process for a few seconds, so a persistently stale counter drives at most one read per counter per window rather than one per request. + + ``fallback_loaded_spend`` is the DB spend ``fallback_spend`` was loaded + from before this worker added its own charges to it. While the row is still + at or above that value, ``fallback_spend`` floors the read, so charges the + batch writer has not flushed yet are not lost with a wiped counter. """ current, verified = await _read_spend_counter_estimate(counter_key=counter_key, fallback_spend=fallback_spend) if fallback_authoritative: @@ -3250,6 +3256,9 @@ async def get_current_spend( verified = True if authoritative > current: await _repair_stale_spend_counter(counter_key=counter_key, db_spend=authoritative) + if fallback_loaded_spend is not None and fallback_loaded_spend <= authoritative: + return max(current, authoritative, fallback_spend) + if authoritative > current: return authoritative elif fallback_spend > current: # nothing to read (tag counters, an end user without a row or a DB client, a @@ -3318,7 +3327,7 @@ async def reseed_spend_counter_from_db(counter_key: str) -> bool: the read-time floor (_authoritative_floor_spend) converges to the true total as the buffer flushes. The point is to restore enforcement to a real floor rather than leave the counter deleted and unenforced (the prior fail-open). - Counters with no DB row (window/end-user/tag) are left untouched rather than + Counters with no DB row (window/tag) are left untouched rather than deleted, so enforcement keeps reading whatever value they hold. """ db_spend: Final = await SpendCounterReseed.from_db(prisma_client=prisma_client, counter_key=counter_key) @@ -3335,8 +3344,6 @@ async def _floor_spend_from_db( window_duration: str | None, window_start: datetime | None, ) -> float | None: - if counter_key.startswith(END_USER_COUNTER_PREFIX): - return await SpendCounterReseed.end_user_from_db(prisma_client=prisma_client, counter_key=counter_key) entity_spend: Final = await SpendCounterReseed.from_db(prisma_client=prisma_client, counter_key=counter_key) if entity_spend is not None: return entity_spend diff --git a/tests/integration/spend/test_end_user_budget_enforcement.py b/tests/integration/spend/test_end_user_budget_enforcement.py index 02c8aee8f5e..e6112c6952e 100644 --- a/tests/integration/spend/test_end_user_budget_enforcement.py +++ b/tests/integration/spend/test_end_user_budget_enforcement.py @@ -1,17 +1,20 @@ import json +import os import re import uuid from collections.abc import Callable, Mapping +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from typing import Final import httpx import pytest from integration._support.client import Gateway, Scenario, eventually, object_value, string_value -from integration._support.database import read_rows +from integration._support.database import read_rows, write_rows from integration._support.upstream import delete_scenario, register_scenario from integration.cost_calculation.cost_tracking_case import JsonResponse from pydantic import BaseModel, JsonValue +from redis import Redis PROVIDER_AUTHORIZATION: Final = "Bearer integration-provider-key" CANNED_CONTENT: Final = "Hello! This is a mock response from the fake OpenAI endpoint." @@ -639,3 +642,178 @@ def test_streamed_call_is_refused_as_json_once_over_budget(gateway: Gateway) -> assert "data:" not in denied.text, denied.text _assert_budget_refusal(denied, end_user, 0.05) assert _observed_requests(upstream) == [] + + +_COLD_BURST: Final = 40 +_COLD_SEED_SPEND: Final = 6.0 +_COLD_BUDGET: Final = 10.0 +_CHAT_COST: Final = 20 * 0.001 + 20 * 0.002 + + +def _bounded_chat(gateway: Gateway, model: str, key: str, end_user: str) -> httpx.Response: + return gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "max_tokens": 20, + "messages": [{"role": "user", "content": f"cold counter {uuid.uuid4().hex}"}], + "user": end_user, + }, + key=key, + ) + + +def _serves_model(gateway: Gateway, model: str) -> bool: + response: Final = gateway.request("GET", "/model/info") + return response.status_code == 200 and any( + object_value(entry)["model_name"] == model for entry in object_value(response.json())["data"] + ) + + +def _counter(cache: Redis, end_user: str) -> float | None: + raw: Final = cache.get(f"spend:end_user:{end_user}") + return None if raw is None else float(raw) + + +@pytest.mark.parametrize("spelling", ("user", "responses_user", "messages_metadata_user_id")) +def test_overlapping_cold_starts_seed_the_end_user_counter_once(gateway: Gateway, peer: Gateway, spelling: str) -> None: + chosen: Final = SPELLINGS[spelling] + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache, + ): + deployment: Final = _deployment(scenario, chosen) + eventually(lambda: _serves_model(peer, deployment.model), bool, seconds=30) + key: Final = scenario.key(models=[deployment.model]) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + _new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + assert _customer_info(gateway, end_user).spend == _COLD_SEED_SPEND + assert _counter(cache, end_user) is None + upstream.delete("/__observations").raise_for_status() + + proxies: Final = (gateway, peer) + with ThreadPoolExecutor(max_workers=_COLD_BURST) as pool: + burst: Final = tuple( + pool.map( + lambda index: _send(proxies[index % 2], chosen, deployment, key, end_user, f"cold {index}"), + range(_COLD_BURST), + ) + ) + refused: Final = [response.text for response in burst if response.status_code != 200] + assert refused == [], refused + follow_up: Final = _send(gateway, chosen, deployment, key, end_user, "cold follow-up") + _assert_served(chosen, follow_up) + assert len(_observed_requests(upstream)) == _COLD_BURST + 1 + + expected_spend: Final = _COLD_SEED_SPEND + (_COLD_BURST + 1) * _CHAT_COST + charged: Final = eventually( + lambda: _end_user_row(end_user), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) >= expected_spend - 1e-6, + seconds=70, + ) + assert float(str(charged[0]["spend"])) == pytest.approx(expected_spend) + assert _customer_info(gateway, end_user).spend == pytest.approx(expected_spend) + counted: Final = eventually( + lambda: _counter(cache, end_user), + lambda value: value is not None and value == pytest.approx(expected_spend), + seconds=30, + return_last_on_timeout=True, + ) + assert counted == pytest.approx(expected_spend) + + +def _set_end_user_row_spend(end_user: str, spend: float) -> None: + write_rows('UPDATE "LiteLLM_EndUserTable" SET spend = %s WHERE user_id = %s', (str(spend), end_user)) + + +def _refused_spend(denied: httpx.Response, end_user: str) -> float: + error: Final = _denial_error(denied) + assert error["type"] == "budget_exceeded", denied.text + matched: Final = re.fullmatch( + rf"ExceededBudget: End User={re.escape(end_user)} over budget\. Spend=(\S+), Budget=10\.0", + string_value(error["message"]), + ) + assert matched is not None, denied.text + return float(matched.group(1)) + + +def test_a_cold_end_user_counter_seeds_from_a_row_raised_behind_the_cached_customer(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model]) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + _new_customer(gateway, end_user, max_budget=_COLD_BUDGET) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + served: Final = _bounded_chat(gateway, model, key, end_user) + assert served.status_code == 200, served.text + eventually( + lambda: _end_user_row(end_user), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) == pytest.approx(_CHAT_COST), + seconds=70, + ) + _set_end_user_row_spend(end_user, _COLD_BUDGET + 1) + cache.delete(f"spend:end_user:{end_user}") + + denied: Final = _bounded_chat(gateway, model, key, end_user) + + assert _refused_spend(denied, end_user) == pytest.approx(_COLD_BUDGET + 1) + assert _counter(cache, end_user) == pytest.approx(_COLD_BUDGET + 1) + + +def test_a_cold_end_user_counter_seeds_from_a_row_reset_behind_the_cached_customer(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002) + key: Final = scenario.key(models=[model]) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + _new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_BUDGET + 2) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + denied: Final = _bounded_chat(gateway, model, key, end_user) + assert _refused_spend(denied, end_user) == pytest.approx(_COLD_BUDGET + 2) + _set_end_user_row_spend(end_user, 0.0) + cache.delete(f"spend:end_user:{end_user}") + + served: Final = _bounded_chat(gateway, model, key, end_user) + + assert served.status_code == 200, served.text + charged: Final = eventually( + lambda: _end_user_row(end_user), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) > 0, + seconds=70, + ) + assert float(str(charged[0]["spend"])) == pytest.approx(_CHAT_COST) + assert _counter(cache, end_user) == pytest.approx(_CHAT_COST) + + +def test_a_cold_end_user_counter_keeps_spend_the_row_has_not_caught_up_to(gateway: Gateway) -> None: + with ( + gateway.scenario() as scenario, + Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache, + ): + model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.25) + key: Final = scenario.key(models=[model]) + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + call_cost: Final = 20 * 0.001 + 20 * 0.25 + _new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND) + scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]}) + served: Final = _chat(gateway, model, key, end_user) + assert served.status_code == 200, served.text + eventually( + lambda: _end_user_row(end_user), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) == pytest.approx(_COLD_SEED_SPEND + call_cost), + seconds=70, + ) + _set_end_user_row_spend(end_user, _COLD_SEED_SPEND) + cache.delete(f"spend:end_user:{end_user}") + + denied: Final = _chat(gateway, model, key, end_user) + + assert _refused_spend(denied, end_user) == pytest.approx(_COLD_SEED_SPEND + call_cost) diff --git a/tests/unit/proxy/auth/test_auth_checks.py b/tests/unit/proxy/auth/test_auth_checks.py index e1afa81d372..47e6ba4ead8 100644 --- a/tests/unit/proxy/auth/test_auth_checks.py +++ b/tests/unit/proxy/auth/test_auth_checks.py @@ -538,6 +538,42 @@ async def test_check_end_user_budget(customer_spend, customer_budget): assert exc_info.value.max_budget == customer_budget +@pytest.mark.parametrize("row_spend, refused", [(6.0, True), (0.0, False)]) +@pytest.mark.asyncio +async def test_check_end_user_budget_keeps_unflushed_charges_after_the_counter_is_wiped( + monkeypatch: pytest.MonkeyPatch, row_spend: float, refused: bool +) -> None: + import litellm.proxy.proxy_server as ps + from litellm.proxy.auth.auth_checks import check_end_user_budget + + end_user_id: Final = "customer-42" + prisma: Final = MagicMock() + prisma.db.litellm_endusertable.find_unique = AsyncMock(return_value=MagicMock(spend=row_spend)) + cache: Final = UserApiKeyCache() + await cache.async_set_cache( + key=f"end_user_id:{end_user_id}", + value=LiteLLM_EndUserTable( + user_id=end_user_id, spend=6.0, litellm_budget_table=LiteLLM_BudgetTable(max_budget=10.0), blocked=False + ), + model_type=LiteLLM_EndUserTable, + ) + loaded: Final = await get_end_user_object(end_user_id=end_user_id, prisma_client=prisma, user_api_key_cache=cache) + assert loaded is not None and loaded.spend == 6.0 + ps._add_spend_to_locally_cached_object(cache, f"end_user_id:{end_user_id}", LiteLLM_EndUserTable, 5.0) + charged: Final = await get_end_user_object(end_user_id=end_user_id, prisma_client=prisma, user_api_key_cache=cache) + assert charged is not None and charged.spend == 11.0 + monkeypatch.setattr(ps, "prisma_client", prisma) + monkeypatch.setattr(ps, "spend_counter_cache", DualCache()) + monkeypatch.setattr(ps, "user_api_key_cache", cache) + + if not refused: + await check_end_user_budget(end_user_obj=charged, route="/v1/chat/completions") + return + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await check_end_user_budget(end_user_obj=charged, route="/v1/chat/completions") + assert exc_info.value.current_cost == 11.0 + + @pytest.mark.parametrize( "model, expect_to_work", [ diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 108f7ce2983..d3ae11ef26b 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -5740,6 +5740,7 @@ def _unrestricted_end_user_prisma(spend: float): end_user_row = MagicMock() end_user_row.user_id = "customer-1" end_user_row.dict = lambda: {"user_id": "customer-1", "blocked": False, "spend": spend} + end_user_row.spend = spend mock_prisma = MagicMock() mock_prisma.db.litellm_endusertable.find_many = AsyncMock(return_value=[]) diff --git a/tests/unit/proxy/db/test_spend_counter_reseed.py b/tests/unit/proxy/db/test_spend_counter_reseed.py index ab931277313..336059da9b4 100644 --- a/tests/unit/proxy/db/test_spend_counter_reseed.py +++ b/tests/unit/proxy/db/test_spend_counter_reseed.py @@ -367,49 +367,23 @@ async def test_cold_reseed_preserves_concurrent_local_increment( @pytest.mark.asyncio -async def test_end_user_from_db_reads_the_end_user_row_by_user_id(): - prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=0.0)) +async def test_from_db_reads_the_end_user_row_by_user_id(): + prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=12.5)) - result: Final = await SpendCounterReseed.end_user_from_db( - prisma_client=prisma, counter_key="spend:end_user:customer-42" - ) - - assert result == 0.0 + assert await SpendCounterReseed.from_db(prisma_client=prisma, counter_key="spend:end_user:customer-42") == 12.5 assert prisma.db.litellm_endusertable.where_clauses == [{"user_id": "customer-42"}] @pytest.mark.asyncio -async def test_end_user_from_db_returns_the_recorded_spend(): - prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=12.5)) - +async def test_from_db_returns_none_for_a_missing_or_unreadable_end_user_row(): assert ( - await SpendCounterReseed.end_user_from_db(prisma_client=prisma, counter_key="spend:end_user:customer-42") - == 12.5 - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("counter_key", ["spend:key:hashed", "spend:team:t1", "spend:tag:t1"]) -async def test_end_user_from_db_ignores_other_counter_kinds_without_touching_the_db(counter_key): - prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="x", spend=5.0)) - - assert await SpendCounterReseed.end_user_from_db(prisma_client=prisma, counter_key=counter_key) is None - assert prisma.db.litellm_endusertable.where_clauses == [] - - -@pytest.mark.asyncio -async def test_end_user_from_db_returns_none_without_a_row_a_client_or_on_db_error(): - assert ( - await SpendCounterReseed.end_user_from_db(prisma_client=None, counter_key="spend:end_user:customer-42") is None - ) - assert ( - await SpendCounterReseed.end_user_from_db( + await SpendCounterReseed.from_db( prisma_client=_FakePrismaClient(end_user_row=None), counter_key="spend:end_user:customer-42" ) is None ) assert ( - await SpendCounterReseed.end_user_from_db( + await SpendCounterReseed.from_db( prisma_client=_FakePrismaClient(end_user_error=RuntimeError("db down")), counter_key="spend:end_user:customer-42", ) @@ -470,13 +444,3 @@ async def test_from_db_deadline_covers_the_wait_for_a_gate_slot(monkeypatch: pyt find_unique.assert_not_called() finally: db_lookup_stall_tracker.clear() - - -@pytest.mark.asyncio -async def test_from_db_still_never_reads_the_end_user_row(): - """A cold end-user counter keeps seeding from the cached end-user object the auth - path already loaded; the row is read only as the budget floor.""" - prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=5.0)) - - assert await SpendCounterReseed.from_db(prisma_client=prisma, counter_key="spend:end_user:customer-42") is None - assert prisma.db.litellm_endusertable.where_clauses == [] diff --git a/tests/unit/proxy/proxy_server/test_spend_counters.py b/tests/unit/proxy/proxy_server/test_spend_counters.py index d273ef8f42f..a811a4b483b 100644 --- a/tests/unit/proxy/proxy_server/test_spend_counters.py +++ b/tests/unit/proxy/proxy_server/test_spend_counters.py @@ -50,6 +50,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_loca from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, end_user_cache_key, tag_cache_key from litellm.proxy.management_endpoints.team_endpoints import refresh_cached_team +from litellm.proxy.spend_tracking.spend_counter_batch import spend_counter_batch_scope from .conftest import normalize @@ -1357,6 +1358,50 @@ async def test_ensure_spend_counter_initialized_cold_seeds_from_source_cache( } +class _FakeRedisMeetingAtTheWarmCheck(FakeAsyncRedis): + """Holds each request's MGET until every request has made one, so all of them find the counter missing before + any of them seeds it.""" + + def __init__(self, server: FakeServer, callers: int) -> None: + super().__init__(server=server) + self._warm_checks: Final = asyncio.Barrier(callers) + + async def mget(self, keys: list[str], *args: str) -> list[bytes | None]: + values: Final = await super().mget(keys, *args) + await self._warm_checks.wait() + return values + + +@pytest.mark.asyncio +async def test_overlapping_cold_end_user_seeds_read_and_write_the_end_user_row_once(monkeypatch: pytest.MonkeyPatch): + counter_key: Final = "spend:end_user:customer-1" + callers: Final = 3 + redis: Final = _FakeRedisMeetingAtTheWarmCheck(FakeServer(), callers) + auth_cache: Final = UserApiKeyCache(in_memory_cache=InMemoryCache()) + auth_cache.set_cache( + key=end_user_cache_key("customer-1"), + value=LiteLLM_EndUserTable(user_id="customer-1", blocked=False, spend=6.0), + local_only=True, + ) + prisma: Final = MagicMock() + prisma.db.litellm_endusertable.find_unique = AsyncMock(return_value=MagicMock(spend=7.5)) + counter_cache: Final = ps.DualCache(redis_cache=_RedisCacheOnClient(redis)) + monkeypatch.setattr(ps, "spend_counter_cache", counter_cache) + monkeypatch.setattr(ps, "user_api_key_cache", auth_cache) + monkeypatch.setattr(ps, "prisma_client", prisma) + + async def _request() -> None: + with spend_counter_batch_scope(counter_cache.redis_cache, counter_keys=frozenset((counter_key,))): + await ps.ensure_spend_counter_initialized( + counter_key=counter_key, source_cache_key=end_user_cache_key("customer-1") + ) + + await asyncio.wait_for(asyncio.gather(*(_request() for _ in range(callers))), timeout=5) + + assert float(await redis.get(counter_key)) == 7.5 + prisma.db.litellm_endusertable.find_unique.assert_awaited_once() + + # --------------------------------------------------------------------------- # _get_source_cache_base_spend # --------------------------------------------------------------------------- diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 14196d4a1ac..f0c1a3438d6 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -8849,36 +8849,31 @@ async def test_primary_spend_counter_redis_concurrent_seed_does_not_double_seed( @pytest.mark.asyncio async def test_reseed_spend_from_db_user_and_org_prefixes(): - """User and org counters reseed from their own DB tables. + """User, end-user and org counters reseed from their own DB tables. - End-user and tag counters use the already fetched auth objects passed as - fallback_spend, so this reseed helper must not add extra per-request DB - reads for them. + Tag counters use the already fetched auth objects passed as fallback_spend, + so this reseed helper must not add extra per-request DB reads for them. """ from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed user_row = MagicMock() user_row.spend = 17.0 + end_user_row = MagicMock() + end_user_row.spend = 6.0 org_row = MagicMock() org_row.spend = 305.0 fake_prisma = MagicMock() fake_prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=user_row) - fake_prisma.db.litellm_endusertable.find_unique = AsyncMock() + fake_prisma.db.litellm_endusertable.find_unique = AsyncMock(return_value=end_user_row) fake_prisma.db.litellm_tagtable.find_unique = AsyncMock() fake_prisma.db.litellm_organizationtable.find_unique = AsyncMock(return_value=org_row) assert await SpendCounterReseed.from_db(fake_prisma, "spend:user:alice") == 17.0 fake_prisma.db.litellm_usertable.find_unique.assert_awaited_once_with(where={"user_id": "alice"}) - assert ( - await SpendCounterReseed.from_db( - fake_prisma, - "spend:end_user:customer-1", - ) - is None - ) - fake_prisma.db.litellm_endusertable.find_unique.assert_not_awaited() + assert await SpendCounterReseed.from_db(fake_prisma, "spend:end_user:customer-1") == 6.0 + fake_prisma.db.litellm_endusertable.find_unique.assert_awaited_once_with(where={"user_id": "customer-1"}) assert await SpendCounterReseed.from_db(fake_prisma, "spend:tag:paid-tag") is None fake_prisma.db.litellm_tagtable.find_unique.assert_not_awaited()