This commit is contained in:
yucheng-berri 2026-10-25 01:44:06 +00:00 • committed by GitHub
commit df98796841
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 330 additions and 95 deletions

View file

@ -1915,13 +1915,17 @@ async def check_end_user_budget(
if end_user_budget is None:
return
from litellm.proxy.proxy_server import get_current_spend
from litellm.proxy.proxy_server import get_current_spend, user_api_key_cache
loaded_spend: Final = user_api_key_cache.get_cache(
key=_end_user_loaded_spend_key(end_user_obj.user_id), local_only=True
)
end_user_spend: Final = await get_current_spend(
counter_key=f"spend:end_user:{end_user_obj.user_id}",
fallback_spend=end_user_obj.spend or 0.0,
max_budget=end_user_budget,
fallback_authoritative=True,
fallback_loaded_spend=float(loaded_spend) if isinstance(loaded_spend, (int, float)) else None,
)
if end_user_spend > end_user_budget:
raise litellm.BudgetExceededError(
@ -2133,6 +2137,23 @@ async def _end_user_is_known_unrestricted(
return registry is not None and end_user_id not in registry
def _end_user_loaded_spend_key(end_user_id: str) -> str:
return f"end_user_loaded_spend:{end_user_id}"
def _remember_end_user_loaded_spend(
user_api_key_cache: UserApiKeyCache, end_user_id: str, spend: float, *, keep_existing: bool
) -> None:
"""Record, on this worker only, the DB spend its cached end-user object was loaded with. Spend this worker
charges later is added to the cached object, never to this value."""
key: Final = _end_user_loaded_spend_key(end_user_id)
if keep_existing and user_api_key_cache.get_cache(key=key, local_only=True) is not None:
return
user_api_key_cache.set_cache(
key=key, value=spend, local_only=True, ttl=get_management_object_ttl(user_api_key_cache)
)
@log_db_metrics
@with_service_target(AUTH_OBJECTS_TARGET)
async def get_end_user_object(
@ -2180,6 +2201,7 @@ async def get_end_user_object(
model_type=LiteLLM_EndUserTable,
)
if cached_user_obj is not None:
_remember_end_user_loaded_spend(user_api_key_cache, end_user_id, cached_user_obj.spend, keep_existing=True)
return await _apply_default_budget_to_end_user(
end_user_obj=cached_user_obj,
prisma_client=prisma_client,
@ -2220,6 +2242,7 @@ async def get_end_user_object(
model_type=LiteLLM_EndUserTable,
ttl=get_management_object_ttl(user_api_key_cache),
)
_remember_end_user_loaded_spend(user_api_key_cache, end_user_id, end_user_row.spend, keep_existing=False)
if key_end_user_budget_id is None:
return end_user_row

View file

@ -637,13 +637,13 @@ class ResetBudgetJob:
async def _invalidate_user_api_key_cache_entry(cache_key: str) -> None:
"""Drop a stale management-cache entry so the next read fetches from DB.
Tags and end-users are not reseeded by ``SpendCounterReseed.from_db``;
for those, when the spend counter expires the budget check falls back
to ``cached_obj.spend``. Keys, orgs, and team memberships are reseeded
from the DB, but auth still may consult ``user_api_key_cache`` objects
whose ``.spend`` field can lag a cross-pod DB reset. Deleting the cache
entry forces the next auth-time fetch to reload the zeroed row from
Postgres.
Tags are not reseeded by ``SpendCounterReseed.from_db``; for those,
when the spend counter expires the budget check falls back to
``cached_obj.spend``. Keys, orgs, team memberships and end users are
reseeded from the DB, but auth still may consult ``user_api_key_cache``
objects whose ``.spend`` field can lag a cross-pod DB reset. Deleting
the cache entry forces the next auth-time fetch to reload the zeroed
row from Postgres.
"""
try:
from litellm.proxy.proxy_server import user_api_key_cache

View file

@ -45,8 +45,6 @@ from litellm.repositories.verification_token_repository import (
)
if TYPE_CHECKING:
from prisma.types import LiteLLM_EndUserTableWhereUniqueInput
from litellm.caching.dual_cache import DualCache
from litellm.proxy.utils import PrismaClient
@ -84,14 +82,11 @@ class SpendCounterReseed:
spend:user:{user_id} -> LiteLLM_UserTable.spend
spend:org:{org_id} -> LiteLLM_OrganizationTable.spend
spend:project:{project_id} -> LiteLLM_ProjectTable.spend
spend:end_user:{end_user_id} -> LiteLLM_EndUserTable.spend
End-user and tag spend counters intentionally do not reseed here. Their
auth paths already load the corresponding objects via get_end_user_object()
and get_tag_objects_batch(); callers pass those values as fallback_spend.
end_user_from_db is the one end-user read, used only as the budget floor when
a counter sits below that cached spend: a worker that did not run the budget
reset still caches the pre-reset end-user object, and LiteLLM_EndUserTable
is the row the reset zeroed.
Tag spend counters intentionally do not reseed here. Their auth path already
loads the tag objects via get_tag_objects_batch(); callers pass those values
as fallback_spend.
"""
_locks: ClassVar["OrderedDict[str, asyncio.Lock]"] = OrderedDict()
@ -180,24 +175,12 @@ class SpendCounterReseed:
return await ProjectRepository(prisma_client).table.find_unique(
where={"project_id": counter_key[len("spend:project:") :]}
)
if counter_key.startswith(END_USER_COUNTER_PREFIX):
return await EndUserRepository(prisma_client).table.find_unique(
where={"user_id": counter_key[len(END_USER_COUNTER_PREFIX) :]}
)
return None
@staticmethod
async def end_user_from_db(prisma_client: Optional["PrismaClient"], counter_key: str) -> float | None:
if prisma_client is None or not counter_key.startswith(END_USER_COUNTER_PREFIX):
return None
where: Final[LiteLLM_EndUserTableWhereUniqueInput] = {"user_id": counter_key[len(END_USER_COUNTER_PREFIX) :]}
try:
row: Final = await bounded_db_lookup(
EndUserRepository(prisma_client).table.find_unique(where=where), name="end_user_spend"
)
except Exception: # noqa: BLE001 # a failed floor read falls back to the cached spend, like from_db
verbose_proxy_logger.exception("SpendCounterReseed.end_user_from_db: failed for %s", counter_key)
return None
if row is None:
return None
return float(row.spend or 0.0)
@staticmethod
def _is_key_or_team_window_counter(counter_key: str) -> bool:
for prefix in ("spend:key:", "spend:team:"):
@ -235,14 +218,17 @@ class SpendCounterReseed:
when the DB read succeeds, or None when the DB is unavailable.
"""
lock: Final = await SpendCounterReseed._get_lock(counter_key)
waited_for_seed: Final = lock.locked()
async with lock:
batched: Final = await SpendCounterReseed._read_active_batch(counter_key)
if batched is not None and batched[0] is not None:
return batched[0]
# Re-check after acquiring the lock. Skip in-memory on a clean
# Redis miss - in-memory is per-pod-stale.
# Redis miss - in-memory is per-pod-stale. The request's batched
# read can predate the lock, so a caller that waited on it re-reads
# Redis for the value the holder seeded.
redis_clean_miss = batched is not None
if spend_counter_cache.redis_cache is not None and not redis_clean_miss:
if spend_counter_cache.redis_cache is not None and (waited_for_seed or not redis_clean_miss):
try:
val = await spend_counter_cache.redis_cache.async_get_cache(key=counter_key)
if val is not None:

View file

@ -619,7 +619,7 @@ from litellm.proxy.db.request_error_tracking import (
flush_request_errors,
request_error_accumulator,
)
from litellm.proxy.db.spend_counter_reseed import END_USER_COUNTER_PREFIX, SpendCounterReseed
from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed
from litellm.proxy.discovery_endpoints import (
agent_skills_discovery_router,
ui_discovery_endpoints_router,
@ -3194,6 +3194,7 @@ async def get_current_spend(
window_duration: str | None = None,
window_start: datetime | None = None,
fallback_authoritative: bool = False,
fallback_loaded_spend: float | None = None,
) -> float:
"""
Read current spend from the cross-pod spend counter.
@ -3223,6 +3224,11 @@ async def get_current_spend(
skipped for healthy primary counters (counter at or above recorded spend)
and cached in-process for a few seconds, so a persistently stale counter
drives at most one read per counter per window rather than one per request.
``fallback_loaded_spend`` is the DB spend ``fallback_spend`` was loaded
from before this worker added its own charges to it. While the row is still
at or above that value, ``fallback_spend`` floors the read, so charges the
batch writer has not flushed yet are not lost with a wiped counter.
"""
current, verified = await _read_spend_counter_estimate(counter_key=counter_key, fallback_spend=fallback_spend)
if fallback_authoritative:
@ -3250,6 +3256,9 @@ async def get_current_spend(
verified = True
if authoritative > current:
await _repair_stale_spend_counter(counter_key=counter_key, db_spend=authoritative)
if fallback_loaded_spend is not None and fallback_loaded_spend <= authoritative:
return max(current, authoritative, fallback_spend)
if authoritative > current:
return authoritative
elif fallback_spend > current:
# nothing to read (tag counters, an end user without a row or a DB client, a
@ -3318,7 +3327,7 @@ async def reseed_spend_counter_from_db(counter_key: str) -> bool:
the read-time floor (_authoritative_floor_spend) converges to the true total
as the buffer flushes. The point is to restore enforcement to a real floor
rather than leave the counter deleted and unenforced (the prior fail-open).
Counters with no DB row (window/end-user/tag) are left untouched rather than
Counters with no DB row (window/tag) are left untouched rather than
deleted, so enforcement keeps reading whatever value they hold.
"""
db_spend: Final = await SpendCounterReseed.from_db(prisma_client=prisma_client, counter_key=counter_key)
@ -3335,8 +3344,6 @@ async def _floor_spend_from_db(
window_duration: str | None,
window_start: datetime | None,
) -> float | None:
if counter_key.startswith(END_USER_COUNTER_PREFIX):
return await SpendCounterReseed.end_user_from_db(prisma_client=prisma_client, counter_key=counter_key)
entity_spend: Final = await SpendCounterReseed.from_db(prisma_client=prisma_client, counter_key=counter_key)
if entity_spend is not None:
return entity_spend

View file

@ -1,17 +1,20 @@
import json
import os
import re
import uuid
from collections.abc import Callable, Mapping
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from typing import Final
import httpx
import pytest
from integration._support.client import Gateway, Scenario, eventually, object_value, string_value
from integration._support.database import read_rows
from integration._support.database import read_rows, write_rows
from integration._support.upstream import delete_scenario, register_scenario
from integration.cost_calculation.cost_tracking_case import JsonResponse
from pydantic import BaseModel, JsonValue
from redis import Redis
PROVIDER_AUTHORIZATION: Final = "Bearer integration-provider-key"
CANNED_CONTENT: Final = "Hello! This is a mock response from the fake OpenAI endpoint."
@ -639,3 +642,178 @@ def test_streamed_call_is_refused_as_json_once_over_budget(gateway: Gateway) ->
assert "data:" not in denied.text, denied.text
_assert_budget_refusal(denied, end_user, 0.05)
assert _observed_requests(upstream) == []
_COLD_BURST: Final = 40
_COLD_SEED_SPEND: Final = 6.0
_COLD_BUDGET: Final = 10.0
_CHAT_COST: Final = 20 * 0.001 + 20 * 0.002
def _bounded_chat(gateway: Gateway, model: str, key: str, end_user: str) -> httpx.Response:
return gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"max_tokens": 20,
"messages": [{"role": "user", "content": f"cold counter {uuid.uuid4().hex}"}],
"user": end_user,
},
key=key,
)
def _serves_model(gateway: Gateway, model: str) -> bool:
response: Final = gateway.request("GET", "/model/info")
return response.status_code == 200 and any(
object_value(entry)["model_name"] == model for entry in object_value(response.json())["data"]
)
def _counter(cache: Redis, end_user: str) -> float | None:
raw: Final = cache.get(f"spend:end_user:{end_user}")
return None if raw is None else float(raw)
@pytest.mark.parametrize("spelling", ("user", "responses_user", "messages_metadata_user_id"))
def test_overlapping_cold_starts_seed_the_end_user_counter_once(gateway: Gateway, peer: Gateway, spelling: str) -> None:
chosen: Final = SPELLINGS[spelling]
with (
gateway.scenario() as scenario,
httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream,
Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache,
):
deployment: Final = _deployment(scenario, chosen)
eventually(lambda: _serves_model(peer, deployment.model), bool, seconds=30)
key: Final = scenario.key(models=[deployment.model])
end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
_new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND)
scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
assert _customer_info(gateway, end_user).spend == _COLD_SEED_SPEND
assert _counter(cache, end_user) is None
upstream.delete("/__observations").raise_for_status()
proxies: Final = (gateway, peer)
with ThreadPoolExecutor(max_workers=_COLD_BURST) as pool:
burst: Final = tuple(
pool.map(
lambda index: _send(proxies[index % 2], chosen, deployment, key, end_user, f"cold {index}"),
range(_COLD_BURST),
)
)
refused: Final = [response.text for response in burst if response.status_code != 200]
assert refused == [], refused
follow_up: Final = _send(gateway, chosen, deployment, key, end_user, "cold follow-up")
_assert_served(chosen, follow_up)
assert len(_observed_requests(upstream)) == _COLD_BURST + 1
expected_spend: Final = _COLD_SEED_SPEND + (_COLD_BURST + 1) * _CHAT_COST
charged: Final = eventually(
lambda: _end_user_row(end_user),
lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) >= expected_spend - 1e-6,
seconds=70,
)
assert float(str(charged[0]["spend"])) == pytest.approx(expected_spend)
assert _customer_info(gateway, end_user).spend == pytest.approx(expected_spend)
counted: Final = eventually(
lambda: _counter(cache, end_user),
lambda value: value is not None and value == pytest.approx(expected_spend),
seconds=30,
return_last_on_timeout=True,
)
assert counted == pytest.approx(expected_spend)
def _set_end_user_row_spend(end_user: str, spend: float) -> None:
write_rows('UPDATE "LiteLLM_EndUserTable" SET spend = %s WHERE user_id = %s', (str(spend), end_user))
def _refused_spend(denied: httpx.Response, end_user: str) -> float:
error: Final = _denial_error(denied)
assert error["type"] == "budget_exceeded", denied.text
matched: Final = re.fullmatch(
rf"ExceededBudget: End User={re.escape(end_user)} over budget\. Spend=(\S+), Budget=10\.0",
string_value(error["message"]),
)
assert matched is not None, denied.text
return float(matched.group(1))
def test_a_cold_end_user_counter_seeds_from_a_row_raised_behind_the_cached_customer(gateway: Gateway) -> None:
with (
gateway.scenario() as scenario,
Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache,
):
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
key: Final = scenario.key(models=[model])
end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
_new_customer(gateway, end_user, max_budget=_COLD_BUDGET)
scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
served: Final = _bounded_chat(gateway, model, key, end_user)
assert served.status_code == 200, served.text
eventually(
lambda: _end_user_row(end_user),
lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) == pytest.approx(_CHAT_COST),
seconds=70,
)
_set_end_user_row_spend(end_user, _COLD_BUDGET + 1)
cache.delete(f"spend:end_user:{end_user}")
denied: Final = _bounded_chat(gateway, model, key, end_user)
assert _refused_spend(denied, end_user) == pytest.approx(_COLD_BUDGET + 1)
assert _counter(cache, end_user) == pytest.approx(_COLD_BUDGET + 1)
def test_a_cold_end_user_counter_seeds_from_a_row_reset_behind_the_cached_customer(gateway: Gateway) -> None:
with (
gateway.scenario() as scenario,
Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache,
):
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
key: Final = scenario.key(models=[model])
end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
_new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_BUDGET + 2)
scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
denied: Final = _bounded_chat(gateway, model, key, end_user)
assert _refused_spend(denied, end_user) == pytest.approx(_COLD_BUDGET + 2)
_set_end_user_row_spend(end_user, 0.0)
cache.delete(f"spend:end_user:{end_user}")
served: Final = _bounded_chat(gateway, model, key, end_user)
assert served.status_code == 200, served.text
charged: Final = eventually(
lambda: _end_user_row(end_user),
lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) > 0,
seconds=70,
)
assert float(str(charged[0]["spend"])) == pytest.approx(_CHAT_COST)
assert _counter(cache, end_user) == pytest.approx(_CHAT_COST)
def test_a_cold_end_user_counter_keeps_spend_the_row_has_not_caught_up_to(gateway: Gateway) -> None:
with (
gateway.scenario() as scenario,
Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache,
):
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.25)
key: Final = scenario.key(models=[model])
end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
call_cost: Final = 20 * 0.001 + 20 * 0.25
_new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND)
scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
served: Final = _chat(gateway, model, key, end_user)
assert served.status_code == 200, served.text
eventually(
lambda: _end_user_row(end_user),
lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) == pytest.approx(_COLD_SEED_SPEND + call_cost),
seconds=70,
)
_set_end_user_row_spend(end_user, _COLD_SEED_SPEND)
cache.delete(f"spend:end_user:{end_user}")
denied: Final = _chat(gateway, model, key, end_user)
assert _refused_spend(denied, end_user) == pytest.approx(_COLD_SEED_SPEND + call_cost)

View file

@ -538,6 +538,42 @@ async def test_check_end_user_budget(customer_spend, customer_budget):
assert exc_info.value.max_budget == customer_budget
@pytest.mark.parametrize("row_spend, refused", [(6.0, True), (0.0, False)])
@pytest.mark.asyncio
async def test_check_end_user_budget_keeps_unflushed_charges_after_the_counter_is_wiped(
monkeypatch: pytest.MonkeyPatch, row_spend: float, refused: bool
) -> None:
import litellm.proxy.proxy_server as ps
from litellm.proxy.auth.auth_checks import check_end_user_budget
end_user_id: Final = "customer-42"
prisma: Final = MagicMock()
prisma.db.litellm_endusertable.find_unique = AsyncMock(return_value=MagicMock(spend=row_spend))
cache: Final = UserApiKeyCache()
await cache.async_set_cache(
key=f"end_user_id:{end_user_id}",
value=LiteLLM_EndUserTable(
user_id=end_user_id, spend=6.0, litellm_budget_table=LiteLLM_BudgetTable(max_budget=10.0), blocked=False
),
model_type=LiteLLM_EndUserTable,
)
loaded: Final = await get_end_user_object(end_user_id=end_user_id, prisma_client=prisma, user_api_key_cache=cache)
assert loaded is not None and loaded.spend == 6.0
ps._add_spend_to_locally_cached_object(cache, f"end_user_id:{end_user_id}", LiteLLM_EndUserTable, 5.0)
charged: Final = await get_end_user_object(end_user_id=end_user_id, prisma_client=prisma, user_api_key_cache=cache)
assert charged is not None and charged.spend == 11.0
monkeypatch.setattr(ps, "prisma_client", prisma)
monkeypatch.setattr(ps, "spend_counter_cache", DualCache())
monkeypatch.setattr(ps, "user_api_key_cache", cache)
if not refused:
await check_end_user_budget(end_user_obj=charged, route="/v1/chat/completions")
return
with pytest.raises(litellm.BudgetExceededError) as exc_info:
await check_end_user_budget(end_user_obj=charged, route="/v1/chat/completions")
assert exc_info.value.current_cost == 11.0
@pytest.mark.parametrize(
"model, expect_to_work",
[

View file

@ -5740,6 +5740,7 @@ def _unrestricted_end_user_prisma(spend: float):
end_user_row = MagicMock()
end_user_row.user_id = "customer-1"
end_user_row.dict = lambda: {"user_id": "customer-1", "blocked": False, "spend": spend}
end_user_row.spend = spend
mock_prisma = MagicMock()
mock_prisma.db.litellm_endusertable.find_many = AsyncMock(return_value=[])

View file

@ -367,49 +367,23 @@ async def test_cold_reseed_preserves_concurrent_local_increment(
@pytest.mark.asyncio
async def test_end_user_from_db_reads_the_end_user_row_by_user_id():
prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=0.0))
async def test_from_db_reads_the_end_user_row_by_user_id():
prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=12.5))
result: Final = await SpendCounterReseed.end_user_from_db(
prisma_client=prisma, counter_key="spend:end_user:customer-42"
)
assert result == 0.0
assert await SpendCounterReseed.from_db(prisma_client=prisma, counter_key="spend:end_user:customer-42") == 12.5
assert prisma.db.litellm_endusertable.where_clauses == [{"user_id": "customer-42"}]
@pytest.mark.asyncio
async def test_end_user_from_db_returns_the_recorded_spend():
prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=12.5))
async def test_from_db_returns_none_for_a_missing_or_unreadable_end_user_row():
assert (
await SpendCounterReseed.end_user_from_db(prisma_client=prisma, counter_key="spend:end_user:customer-42")
== 12.5
)
@pytest.mark.asyncio
@pytest.mark.parametrize("counter_key", ["spend:key:hashed", "spend:team:t1", "spend:tag:t1"])
async def test_end_user_from_db_ignores_other_counter_kinds_without_touching_the_db(counter_key):
prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="x", spend=5.0))
assert await SpendCounterReseed.end_user_from_db(prisma_client=prisma, counter_key=counter_key) is None
assert prisma.db.litellm_endusertable.where_clauses == []
@pytest.mark.asyncio
async def test_end_user_from_db_returns_none_without_a_row_a_client_or_on_db_error():
assert (
await SpendCounterReseed.end_user_from_db(prisma_client=None, counter_key="spend:end_user:customer-42") is None
)
assert (
await SpendCounterReseed.end_user_from_db(
await SpendCounterReseed.from_db(
prisma_client=_FakePrismaClient(end_user_row=None), counter_key="spend:end_user:customer-42"
)
is None
)
assert (
await SpendCounterReseed.end_user_from_db(
await SpendCounterReseed.from_db(
prisma_client=_FakePrismaClient(end_user_error=RuntimeError("db down")),
counter_key="spend:end_user:customer-42",
)
@ -470,13 +444,3 @@ async def test_from_db_deadline_covers_the_wait_for_a_gate_slot(monkeypatch: pyt
find_unique.assert_not_called()
finally:
db_lookup_stall_tracker.clear()
@pytest.mark.asyncio
async def test_from_db_still_never_reads_the_end_user_row():
"""A cold end-user counter keeps seeding from the cached end-user object the auth
path already loaded; the row is read only as the budget floor."""
prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=5.0))
assert await SpendCounterReseed.from_db(prisma_client=prisma, counter_key="spend:end_user:customer-42") is None
assert prisma.db.litellm_endusertable.where_clauses == []

View file

@ -50,6 +50,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_loca
from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, end_user_cache_key, tag_cache_key
from litellm.proxy.management_endpoints.team_endpoints import refresh_cached_team
from litellm.proxy.spend_tracking.spend_counter_batch import spend_counter_batch_scope
from .conftest import normalize
@ -1357,6 +1358,50 @@ async def test_ensure_spend_counter_initialized_cold_seeds_from_source_cache(
}
class _FakeRedisMeetingAtTheWarmCheck(FakeAsyncRedis):
"""Holds each request's MGET until every request has made one, so all of them find the counter missing before
any of them seeds it."""
def __init__(self, server: FakeServer, callers: int) -> None:
super().__init__(server=server)
self._warm_checks: Final = asyncio.Barrier(callers)
async def mget(self, keys: list[str], *args: str) -> list[bytes | None]:
values: Final = await super().mget(keys, *args)
await self._warm_checks.wait()
return values
@pytest.mark.asyncio
async def test_overlapping_cold_end_user_seeds_read_and_write_the_end_user_row_once(monkeypatch: pytest.MonkeyPatch):
counter_key: Final = "spend:end_user:customer-1"
callers: Final = 3
redis: Final = _FakeRedisMeetingAtTheWarmCheck(FakeServer(), callers)
auth_cache: Final = UserApiKeyCache(in_memory_cache=InMemoryCache())
auth_cache.set_cache(
key=end_user_cache_key("customer-1"),
value=LiteLLM_EndUserTable(user_id="customer-1", blocked=False, spend=6.0),
local_only=True,
)
prisma: Final = MagicMock()
prisma.db.litellm_endusertable.find_unique = AsyncMock(return_value=MagicMock(spend=7.5))
counter_cache: Final = ps.DualCache(redis_cache=_RedisCacheOnClient(redis))
monkeypatch.setattr(ps, "spend_counter_cache", counter_cache)
monkeypatch.setattr(ps, "user_api_key_cache", auth_cache)
monkeypatch.setattr(ps, "prisma_client", prisma)
async def _request() -> None:
with spend_counter_batch_scope(counter_cache.redis_cache, counter_keys=frozenset((counter_key,))):
await ps.ensure_spend_counter_initialized(
counter_key=counter_key, source_cache_key=end_user_cache_key("customer-1")
)
await asyncio.wait_for(asyncio.gather(*(_request() for _ in range(callers))), timeout=5)
assert float(await redis.get(counter_key)) == 7.5
prisma.db.litellm_endusertable.find_unique.assert_awaited_once()
# ---------------------------------------------------------------------------
# _get_source_cache_base_spend
# ---------------------------------------------------------------------------

View file

@ -8849,36 +8849,31 @@ async def test_primary_spend_counter_redis_concurrent_seed_does_not_double_seed(
@pytest.mark.asyncio
async def test_reseed_spend_from_db_user_and_org_prefixes():
"""User and org counters reseed from their own DB tables.
"""User, end-user and org counters reseed from their own DB tables.
End-user and tag counters use the already fetched auth objects passed as
fallback_spend, so this reseed helper must not add extra per-request DB
reads for them.
Tag counters use the already fetched auth objects passed as fallback_spend,
so this reseed helper must not add extra per-request DB reads for them.
"""
from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed
user_row = MagicMock()
user_row.spend = 17.0
end_user_row = MagicMock()
end_user_row.spend = 6.0
org_row = MagicMock()
org_row.spend = 305.0
fake_prisma = MagicMock()
fake_prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=user_row)
fake_prisma.db.litellm_endusertable.find_unique = AsyncMock()
fake_prisma.db.litellm_endusertable.find_unique = AsyncMock(return_value=end_user_row)
fake_prisma.db.litellm_tagtable.find_unique = AsyncMock()
fake_prisma.db.litellm_organizationtable.find_unique = AsyncMock(return_value=org_row)
assert await SpendCounterReseed.from_db(fake_prisma, "spend:user:alice") == 17.0
fake_prisma.db.litellm_usertable.find_unique.assert_awaited_once_with(where={"user_id": "alice"})
assert (
await SpendCounterReseed.from_db(
fake_prisma,
"spend:end_user:customer-1",
)
is None
)
fake_prisma.db.litellm_endusertable.find_unique.assert_not_awaited()
assert await SpendCounterReseed.from_db(fake_prisma, "spend:end_user:customer-1") == 6.0
fake_prisma.db.litellm_endusertable.find_unique.assert_awaited_once_with(where={"user_id": "customer-1"})
assert await SpendCounterReseed.from_db(fake_prisma, "spend:tag:paid-tag") is None
fake_prisma.db.litellm_tagtable.find_unique.assert_not_awaited()