mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 779d9d5dbe into de5dc5b238
This commit is contained in:
commit
df98796841
10 changed files with 330 additions and 95 deletions
|
|
@ -1915,13 +1915,17 @@ async def check_end_user_budget(
|
|||
if end_user_budget is None:
|
||||
return
|
||||
|
||||
from litellm.proxy.proxy_server import get_current_spend
|
||||
from litellm.proxy.proxy_server import get_current_spend, user_api_key_cache
|
||||
|
||||
loaded_spend: Final = user_api_key_cache.get_cache(
|
||||
key=_end_user_loaded_spend_key(end_user_obj.user_id), local_only=True
|
||||
)
|
||||
end_user_spend: Final = await get_current_spend(
|
||||
counter_key=f"spend:end_user:{end_user_obj.user_id}",
|
||||
fallback_spend=end_user_obj.spend or 0.0,
|
||||
max_budget=end_user_budget,
|
||||
fallback_authoritative=True,
|
||||
fallback_loaded_spend=float(loaded_spend) if isinstance(loaded_spend, (int, float)) else None,
|
||||
)
|
||||
if end_user_spend > end_user_budget:
|
||||
raise litellm.BudgetExceededError(
|
||||
|
|
@ -2133,6 +2137,23 @@ async def _end_user_is_known_unrestricted(
|
|||
return registry is not None and end_user_id not in registry
|
||||
|
||||
|
||||
def _end_user_loaded_spend_key(end_user_id: str) -> str:
|
||||
return f"end_user_loaded_spend:{end_user_id}"
|
||||
|
||||
|
||||
def _remember_end_user_loaded_spend(
|
||||
user_api_key_cache: UserApiKeyCache, end_user_id: str, spend: float, *, keep_existing: bool
|
||||
) -> None:
|
||||
"""Record, on this worker only, the DB spend its cached end-user object was loaded with. Spend this worker
|
||||
charges later is added to the cached object, never to this value."""
|
||||
key: Final = _end_user_loaded_spend_key(end_user_id)
|
||||
if keep_existing and user_api_key_cache.get_cache(key=key, local_only=True) is not None:
|
||||
return
|
||||
user_api_key_cache.set_cache(
|
||||
key=key, value=spend, local_only=True, ttl=get_management_object_ttl(user_api_key_cache)
|
||||
)
|
||||
|
||||
|
||||
@log_db_metrics
|
||||
@with_service_target(AUTH_OBJECTS_TARGET)
|
||||
async def get_end_user_object(
|
||||
|
|
@ -2180,6 +2201,7 @@ async def get_end_user_object(
|
|||
model_type=LiteLLM_EndUserTable,
|
||||
)
|
||||
if cached_user_obj is not None:
|
||||
_remember_end_user_loaded_spend(user_api_key_cache, end_user_id, cached_user_obj.spend, keep_existing=True)
|
||||
return await _apply_default_budget_to_end_user(
|
||||
end_user_obj=cached_user_obj,
|
||||
prisma_client=prisma_client,
|
||||
|
|
@ -2220,6 +2242,7 @@ async def get_end_user_object(
|
|||
model_type=LiteLLM_EndUserTable,
|
||||
ttl=get_management_object_ttl(user_api_key_cache),
|
||||
)
|
||||
_remember_end_user_loaded_spend(user_api_key_cache, end_user_id, end_user_row.spend, keep_existing=False)
|
||||
|
||||
if key_end_user_budget_id is None:
|
||||
return end_user_row
|
||||
|
|
|
|||
|
|
@ -637,13 +637,13 @@ class ResetBudgetJob:
|
|||
async def _invalidate_user_api_key_cache_entry(cache_key: str) -> None:
|
||||
"""Drop a stale management-cache entry so the next read fetches from DB.
|
||||
|
||||
Tags and end-users are not reseeded by ``SpendCounterReseed.from_db``;
|
||||
for those, when the spend counter expires the budget check falls back
|
||||
to ``cached_obj.spend``. Keys, orgs, and team memberships are reseeded
|
||||
from the DB, but auth still may consult ``user_api_key_cache`` objects
|
||||
whose ``.spend`` field can lag a cross-pod DB reset. Deleting the cache
|
||||
entry forces the next auth-time fetch to reload the zeroed row from
|
||||
Postgres.
|
||||
Tags are not reseeded by ``SpendCounterReseed.from_db``; for those,
|
||||
when the spend counter expires the budget check falls back to
|
||||
``cached_obj.spend``. Keys, orgs, team memberships and end users are
|
||||
reseeded from the DB, but auth still may consult ``user_api_key_cache``
|
||||
objects whose ``.spend`` field can lag a cross-pod DB reset. Deleting
|
||||
the cache entry forces the next auth-time fetch to reload the zeroed
|
||||
row from Postgres.
|
||||
"""
|
||||
try:
|
||||
from litellm.proxy.proxy_server import user_api_key_cache
|
||||
|
|
|
|||
|
|
@ -45,8 +45,6 @@ from litellm.repositories.verification_token_repository import (
|
|||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prisma.types import LiteLLM_EndUserTableWhereUniqueInput
|
||||
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
|
|
@ -84,14 +82,11 @@ class SpendCounterReseed:
|
|||
spend:user:{user_id} -> LiteLLM_UserTable.spend
|
||||
spend:org:{org_id} -> LiteLLM_OrganizationTable.spend
|
||||
spend:project:{project_id} -> LiteLLM_ProjectTable.spend
|
||||
spend:end_user:{end_user_id} -> LiteLLM_EndUserTable.spend
|
||||
|
||||
End-user and tag spend counters intentionally do not reseed here. Their
|
||||
auth paths already load the corresponding objects via get_end_user_object()
|
||||
and get_tag_objects_batch(); callers pass those values as fallback_spend.
|
||||
end_user_from_db is the one end-user read, used only as the budget floor when
|
||||
a counter sits below that cached spend: a worker that did not run the budget
|
||||
reset still caches the pre-reset end-user object, and LiteLLM_EndUserTable
|
||||
is the row the reset zeroed.
|
||||
Tag spend counters intentionally do not reseed here. Their auth path already
|
||||
loads the tag objects via get_tag_objects_batch(); callers pass those values
|
||||
as fallback_spend.
|
||||
"""
|
||||
|
||||
_locks: ClassVar["OrderedDict[str, asyncio.Lock]"] = OrderedDict()
|
||||
|
|
@ -180,24 +175,12 @@ class SpendCounterReseed:
|
|||
return await ProjectRepository(prisma_client).table.find_unique(
|
||||
where={"project_id": counter_key[len("spend:project:") :]}
|
||||
)
|
||||
if counter_key.startswith(END_USER_COUNTER_PREFIX):
|
||||
return await EndUserRepository(prisma_client).table.find_unique(
|
||||
where={"user_id": counter_key[len(END_USER_COUNTER_PREFIX) :]}
|
||||
)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
async def end_user_from_db(prisma_client: Optional["PrismaClient"], counter_key: str) -> float | None:
|
||||
if prisma_client is None or not counter_key.startswith(END_USER_COUNTER_PREFIX):
|
||||
return None
|
||||
where: Final[LiteLLM_EndUserTableWhereUniqueInput] = {"user_id": counter_key[len(END_USER_COUNTER_PREFIX) :]}
|
||||
try:
|
||||
row: Final = await bounded_db_lookup(
|
||||
EndUserRepository(prisma_client).table.find_unique(where=where), name="end_user_spend"
|
||||
)
|
||||
except Exception: # noqa: BLE001 # a failed floor read falls back to the cached spend, like from_db
|
||||
verbose_proxy_logger.exception("SpendCounterReseed.end_user_from_db: failed for %s", counter_key)
|
||||
return None
|
||||
if row is None:
|
||||
return None
|
||||
return float(row.spend or 0.0)
|
||||
|
||||
@staticmethod
|
||||
def _is_key_or_team_window_counter(counter_key: str) -> bool:
|
||||
for prefix in ("spend:key:", "spend:team:"):
|
||||
|
|
@ -235,14 +218,17 @@ class SpendCounterReseed:
|
|||
when the DB read succeeds, or None when the DB is unavailable.
|
||||
"""
|
||||
lock: Final = await SpendCounterReseed._get_lock(counter_key)
|
||||
waited_for_seed: Final = lock.locked()
|
||||
async with lock:
|
||||
batched: Final = await SpendCounterReseed._read_active_batch(counter_key)
|
||||
if batched is not None and batched[0] is not None:
|
||||
return batched[0]
|
||||
# Re-check after acquiring the lock. Skip in-memory on a clean
|
||||
# Redis miss - in-memory is per-pod-stale.
|
||||
# Redis miss - in-memory is per-pod-stale. The request's batched
|
||||
# read can predate the lock, so a caller that waited on it re-reads
|
||||
# Redis for the value the holder seeded.
|
||||
redis_clean_miss = batched is not None
|
||||
if spend_counter_cache.redis_cache is not None and not redis_clean_miss:
|
||||
if spend_counter_cache.redis_cache is not None and (waited_for_seed or not redis_clean_miss):
|
||||
try:
|
||||
val = await spend_counter_cache.redis_cache.async_get_cache(key=counter_key)
|
||||
if val is not None:
|
||||
|
|
|
|||
|
|
@ -619,7 +619,7 @@ from litellm.proxy.db.request_error_tracking import (
|
|||
flush_request_errors,
|
||||
request_error_accumulator,
|
||||
)
|
||||
from litellm.proxy.db.spend_counter_reseed import END_USER_COUNTER_PREFIX, SpendCounterReseed
|
||||
from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed
|
||||
from litellm.proxy.discovery_endpoints import (
|
||||
agent_skills_discovery_router,
|
||||
ui_discovery_endpoints_router,
|
||||
|
|
@ -3194,6 +3194,7 @@ async def get_current_spend(
|
|||
window_duration: str | None = None,
|
||||
window_start: datetime | None = None,
|
||||
fallback_authoritative: bool = False,
|
||||
fallback_loaded_spend: float | None = None,
|
||||
) -> float:
|
||||
"""
|
||||
Read current spend from the cross-pod spend counter.
|
||||
|
|
@ -3223,6 +3224,11 @@ async def get_current_spend(
|
|||
skipped for healthy primary counters (counter at or above recorded spend)
|
||||
and cached in-process for a few seconds, so a persistently stale counter
|
||||
drives at most one read per counter per window rather than one per request.
|
||||
|
||||
``fallback_loaded_spend`` is the DB spend ``fallback_spend`` was loaded
|
||||
from before this worker added its own charges to it. While the row is still
|
||||
at or above that value, ``fallback_spend`` floors the read, so charges the
|
||||
batch writer has not flushed yet are not lost with a wiped counter.
|
||||
"""
|
||||
current, verified = await _read_spend_counter_estimate(counter_key=counter_key, fallback_spend=fallback_spend)
|
||||
if fallback_authoritative:
|
||||
|
|
@ -3250,6 +3256,9 @@ async def get_current_spend(
|
|||
verified = True
|
||||
if authoritative > current:
|
||||
await _repair_stale_spend_counter(counter_key=counter_key, db_spend=authoritative)
|
||||
if fallback_loaded_spend is not None and fallback_loaded_spend <= authoritative:
|
||||
return max(current, authoritative, fallback_spend)
|
||||
if authoritative > current:
|
||||
return authoritative
|
||||
elif fallback_spend > current:
|
||||
# nothing to read (tag counters, an end user without a row or a DB client, a
|
||||
|
|
@ -3318,7 +3327,7 @@ async def reseed_spend_counter_from_db(counter_key: str) -> bool:
|
|||
the read-time floor (_authoritative_floor_spend) converges to the true total
|
||||
as the buffer flushes. The point is to restore enforcement to a real floor
|
||||
rather than leave the counter deleted and unenforced (the prior fail-open).
|
||||
Counters with no DB row (window/end-user/tag) are left untouched rather than
|
||||
Counters with no DB row (window/tag) are left untouched rather than
|
||||
deleted, so enforcement keeps reading whatever value they hold.
|
||||
"""
|
||||
db_spend: Final = await SpendCounterReseed.from_db(prisma_client=prisma_client, counter_key=counter_key)
|
||||
|
|
@ -3335,8 +3344,6 @@ async def _floor_spend_from_db(
|
|||
window_duration: str | None,
|
||||
window_start: datetime | None,
|
||||
) -> float | None:
|
||||
if counter_key.startswith(END_USER_COUNTER_PREFIX):
|
||||
return await SpendCounterReseed.end_user_from_db(prisma_client=prisma_client, counter_key=counter_key)
|
||||
entity_spend: Final = await SpendCounterReseed.from_db(prisma_client=prisma_client, counter_key=counter_key)
|
||||
if entity_spend is not None:
|
||||
return entity_spend
|
||||
|
|
|
|||
|
|
@ -1,17 +1,20 @@
|
|||
import json
|
||||
import os
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Callable, Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.client import Gateway, Scenario, eventually, object_value, string_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.database import read_rows, write_rows
|
||||
from integration._support.upstream import delete_scenario, register_scenario
|
||||
from integration.cost_calculation.cost_tracking_case import JsonResponse
|
||||
from pydantic import BaseModel, JsonValue
|
||||
from redis import Redis
|
||||
|
||||
PROVIDER_AUTHORIZATION: Final = "Bearer integration-provider-key"
|
||||
CANNED_CONTENT: Final = "Hello! This is a mock response from the fake OpenAI endpoint."
|
||||
|
|
@ -639,3 +642,178 @@ def test_streamed_call_is_refused_as_json_once_over_budget(gateway: Gateway) ->
|
|||
assert "data:" not in denied.text, denied.text
|
||||
_assert_budget_refusal(denied, end_user, 0.05)
|
||||
assert _observed_requests(upstream) == []
|
||||
|
||||
|
||||
_COLD_BURST: Final = 40
|
||||
_COLD_SEED_SPEND: Final = 6.0
|
||||
_COLD_BUDGET: Final = 10.0
|
||||
_CHAT_COST: Final = 20 * 0.001 + 20 * 0.002
|
||||
|
||||
|
||||
def _bounded_chat(gateway: Gateway, model: str, key: str, end_user: str) -> httpx.Response:
|
||||
return gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 20,
|
||||
"messages": [{"role": "user", "content": f"cold counter {uuid.uuid4().hex}"}],
|
||||
"user": end_user,
|
||||
},
|
||||
key=key,
|
||||
)
|
||||
|
||||
|
||||
def _serves_model(gateway: Gateway, model: str) -> bool:
|
||||
response: Final = gateway.request("GET", "/model/info")
|
||||
return response.status_code == 200 and any(
|
||||
object_value(entry)["model_name"] == model for entry in object_value(response.json())["data"]
|
||||
)
|
||||
|
||||
|
||||
def _counter(cache: Redis, end_user: str) -> float | None:
|
||||
raw: Final = cache.get(f"spend:end_user:{end_user}")
|
||||
return None if raw is None else float(raw)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("spelling", ("user", "responses_user", "messages_metadata_user_id"))
|
||||
def test_overlapping_cold_starts_seed_the_end_user_counter_once(gateway: Gateway, peer: Gateway, spelling: str) -> None:
|
||||
chosen: Final = SPELLINGS[spelling]
|
||||
with (
|
||||
gateway.scenario() as scenario,
|
||||
httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream,
|
||||
Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache,
|
||||
):
|
||||
deployment: Final = _deployment(scenario, chosen)
|
||||
eventually(lambda: _serves_model(peer, deployment.model), bool, seconds=30)
|
||||
key: Final = scenario.key(models=[deployment.model])
|
||||
end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
|
||||
_new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND)
|
||||
scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
|
||||
assert _customer_info(gateway, end_user).spend == _COLD_SEED_SPEND
|
||||
assert _counter(cache, end_user) is None
|
||||
upstream.delete("/__observations").raise_for_status()
|
||||
|
||||
proxies: Final = (gateway, peer)
|
||||
with ThreadPoolExecutor(max_workers=_COLD_BURST) as pool:
|
||||
burst: Final = tuple(
|
||||
pool.map(
|
||||
lambda index: _send(proxies[index % 2], chosen, deployment, key, end_user, f"cold {index}"),
|
||||
range(_COLD_BURST),
|
||||
)
|
||||
)
|
||||
refused: Final = [response.text for response in burst if response.status_code != 200]
|
||||
assert refused == [], refused
|
||||
follow_up: Final = _send(gateway, chosen, deployment, key, end_user, "cold follow-up")
|
||||
_assert_served(chosen, follow_up)
|
||||
assert len(_observed_requests(upstream)) == _COLD_BURST + 1
|
||||
|
||||
expected_spend: Final = _COLD_SEED_SPEND + (_COLD_BURST + 1) * _CHAT_COST
|
||||
charged: Final = eventually(
|
||||
lambda: _end_user_row(end_user),
|
||||
lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) >= expected_spend - 1e-6,
|
||||
seconds=70,
|
||||
)
|
||||
assert float(str(charged[0]["spend"])) == pytest.approx(expected_spend)
|
||||
assert _customer_info(gateway, end_user).spend == pytest.approx(expected_spend)
|
||||
counted: Final = eventually(
|
||||
lambda: _counter(cache, end_user),
|
||||
lambda value: value is not None and value == pytest.approx(expected_spend),
|
||||
seconds=30,
|
||||
return_last_on_timeout=True,
|
||||
)
|
||||
assert counted == pytest.approx(expected_spend)
|
||||
|
||||
|
||||
def _set_end_user_row_spend(end_user: str, spend: float) -> None:
|
||||
write_rows('UPDATE "LiteLLM_EndUserTable" SET spend = %s WHERE user_id = %s', (str(spend), end_user))
|
||||
|
||||
|
||||
def _refused_spend(denied: httpx.Response, end_user: str) -> float:
|
||||
error: Final = _denial_error(denied)
|
||||
assert error["type"] == "budget_exceeded", denied.text
|
||||
matched: Final = re.fullmatch(
|
||||
rf"ExceededBudget: End User={re.escape(end_user)} over budget\. Spend=(\S+), Budget=10\.0",
|
||||
string_value(error["message"]),
|
||||
)
|
||||
assert matched is not None, denied.text
|
||||
return float(matched.group(1))
|
||||
|
||||
|
||||
def test_a_cold_end_user_counter_seeds_from_a_row_raised_behind_the_cached_customer(gateway: Gateway) -> None:
|
||||
with (
|
||||
gateway.scenario() as scenario,
|
||||
Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache,
|
||||
):
|
||||
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
|
||||
key: Final = scenario.key(models=[model])
|
||||
end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
|
||||
_new_customer(gateway, end_user, max_budget=_COLD_BUDGET)
|
||||
scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
|
||||
served: Final = _bounded_chat(gateway, model, key, end_user)
|
||||
assert served.status_code == 200, served.text
|
||||
eventually(
|
||||
lambda: _end_user_row(end_user),
|
||||
lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) == pytest.approx(_CHAT_COST),
|
||||
seconds=70,
|
||||
)
|
||||
_set_end_user_row_spend(end_user, _COLD_BUDGET + 1)
|
||||
cache.delete(f"spend:end_user:{end_user}")
|
||||
|
||||
denied: Final = _bounded_chat(gateway, model, key, end_user)
|
||||
|
||||
assert _refused_spend(denied, end_user) == pytest.approx(_COLD_BUDGET + 1)
|
||||
assert _counter(cache, end_user) == pytest.approx(_COLD_BUDGET + 1)
|
||||
|
||||
|
||||
def test_a_cold_end_user_counter_seeds_from_a_row_reset_behind_the_cached_customer(gateway: Gateway) -> None:
|
||||
with (
|
||||
gateway.scenario() as scenario,
|
||||
Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache,
|
||||
):
|
||||
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.002)
|
||||
key: Final = scenario.key(models=[model])
|
||||
end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
|
||||
_new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_BUDGET + 2)
|
||||
scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
|
||||
denied: Final = _bounded_chat(gateway, model, key, end_user)
|
||||
assert _refused_spend(denied, end_user) == pytest.approx(_COLD_BUDGET + 2)
|
||||
_set_end_user_row_spend(end_user, 0.0)
|
||||
cache.delete(f"spend:end_user:{end_user}")
|
||||
|
||||
served: Final = _bounded_chat(gateway, model, key, end_user)
|
||||
|
||||
assert served.status_code == 200, served.text
|
||||
charged: Final = eventually(
|
||||
lambda: _end_user_row(end_user),
|
||||
lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) > 0,
|
||||
seconds=70,
|
||||
)
|
||||
assert float(str(charged[0]["spend"])) == pytest.approx(_CHAT_COST)
|
||||
assert _counter(cache, end_user) == pytest.approx(_CHAT_COST)
|
||||
|
||||
|
||||
def test_a_cold_end_user_counter_keeps_spend_the_row_has_not_caught_up_to(gateway: Gateway) -> None:
|
||||
with (
|
||||
gateway.scenario() as scenario,
|
||||
Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache,
|
||||
):
|
||||
model: Final = scenario.model(input_cost_per_token=0.001, output_cost_per_token=0.25)
|
||||
key: Final = scenario.key(models=[model])
|
||||
end_user: Final = f"integration-end-user-{uuid.uuid4().hex}"
|
||||
call_cost: Final = 20 * 0.001 + 20 * 0.25
|
||||
_new_customer(gateway, end_user, max_budget=_COLD_BUDGET, spend=_COLD_SEED_SPEND)
|
||||
scenario.cleanups.callback(gateway.request, "POST", "/customer/delete", {"user_ids": [end_user]})
|
||||
served: Final = _chat(gateway, model, key, end_user)
|
||||
assert served.status_code == 200, served.text
|
||||
eventually(
|
||||
lambda: _end_user_row(end_user),
|
||||
lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) == pytest.approx(_COLD_SEED_SPEND + call_cost),
|
||||
seconds=70,
|
||||
)
|
||||
_set_end_user_row_spend(end_user, _COLD_SEED_SPEND)
|
||||
cache.delete(f"spend:end_user:{end_user}")
|
||||
|
||||
denied: Final = _chat(gateway, model, key, end_user)
|
||||
|
||||
assert _refused_spend(denied, end_user) == pytest.approx(_COLD_SEED_SPEND + call_cost)
|
||||
|
|
|
|||
|
|
@ -538,6 +538,42 @@ async def test_check_end_user_budget(customer_spend, customer_budget):
|
|||
assert exc_info.value.max_budget == customer_budget
|
||||
|
||||
|
||||
@pytest.mark.parametrize("row_spend, refused", [(6.0, True), (0.0, False)])
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_end_user_budget_keeps_unflushed_charges_after_the_counter_is_wiped(
|
||||
monkeypatch: pytest.MonkeyPatch, row_spend: float, refused: bool
|
||||
) -> None:
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy.auth.auth_checks import check_end_user_budget
|
||||
|
||||
end_user_id: Final = "customer-42"
|
||||
prisma: Final = MagicMock()
|
||||
prisma.db.litellm_endusertable.find_unique = AsyncMock(return_value=MagicMock(spend=row_spend))
|
||||
cache: Final = UserApiKeyCache()
|
||||
await cache.async_set_cache(
|
||||
key=f"end_user_id:{end_user_id}",
|
||||
value=LiteLLM_EndUserTable(
|
||||
user_id=end_user_id, spend=6.0, litellm_budget_table=LiteLLM_BudgetTable(max_budget=10.0), blocked=False
|
||||
),
|
||||
model_type=LiteLLM_EndUserTable,
|
||||
)
|
||||
loaded: Final = await get_end_user_object(end_user_id=end_user_id, prisma_client=prisma, user_api_key_cache=cache)
|
||||
assert loaded is not None and loaded.spend == 6.0
|
||||
ps._add_spend_to_locally_cached_object(cache, f"end_user_id:{end_user_id}", LiteLLM_EndUserTable, 5.0)
|
||||
charged: Final = await get_end_user_object(end_user_id=end_user_id, prisma_client=prisma, user_api_key_cache=cache)
|
||||
assert charged is not None and charged.spend == 11.0
|
||||
monkeypatch.setattr(ps, "prisma_client", prisma)
|
||||
monkeypatch.setattr(ps, "spend_counter_cache", DualCache())
|
||||
monkeypatch.setattr(ps, "user_api_key_cache", cache)
|
||||
|
||||
if not refused:
|
||||
await check_end_user_budget(end_user_obj=charged, route="/v1/chat/completions")
|
||||
return
|
||||
with pytest.raises(litellm.BudgetExceededError) as exc_info:
|
||||
await check_end_user_budget(end_user_obj=charged, route="/v1/chat/completions")
|
||||
assert exc_info.value.current_cost == 11.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, expect_to_work",
|
||||
[
|
||||
|
|
|
|||
|
|
@ -5740,6 +5740,7 @@ def _unrestricted_end_user_prisma(spend: float):
|
|||
end_user_row = MagicMock()
|
||||
end_user_row.user_id = "customer-1"
|
||||
end_user_row.dict = lambda: {"user_id": "customer-1", "blocked": False, "spend": spend}
|
||||
end_user_row.spend = spend
|
||||
|
||||
mock_prisma = MagicMock()
|
||||
mock_prisma.db.litellm_endusertable.find_many = AsyncMock(return_value=[])
|
||||
|
|
|
|||
|
|
@ -367,49 +367,23 @@ async def test_cold_reseed_preserves_concurrent_local_increment(
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_end_user_from_db_reads_the_end_user_row_by_user_id():
|
||||
prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=0.0))
|
||||
async def test_from_db_reads_the_end_user_row_by_user_id():
|
||||
prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=12.5))
|
||||
|
||||
result: Final = await SpendCounterReseed.end_user_from_db(
|
||||
prisma_client=prisma, counter_key="spend:end_user:customer-42"
|
||||
)
|
||||
|
||||
assert result == 0.0
|
||||
assert await SpendCounterReseed.from_db(prisma_client=prisma, counter_key="spend:end_user:customer-42") == 12.5
|
||||
assert prisma.db.litellm_endusertable.where_clauses == [{"user_id": "customer-42"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_end_user_from_db_returns_the_recorded_spend():
|
||||
prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=12.5))
|
||||
|
||||
async def test_from_db_returns_none_for_a_missing_or_unreadable_end_user_row():
|
||||
assert (
|
||||
await SpendCounterReseed.end_user_from_db(prisma_client=prisma, counter_key="spend:end_user:customer-42")
|
||||
== 12.5
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("counter_key", ["spend:key:hashed", "spend:team:t1", "spend:tag:t1"])
|
||||
async def test_end_user_from_db_ignores_other_counter_kinds_without_touching_the_db(counter_key):
|
||||
prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="x", spend=5.0))
|
||||
|
||||
assert await SpendCounterReseed.end_user_from_db(prisma_client=prisma, counter_key=counter_key) is None
|
||||
assert prisma.db.litellm_endusertable.where_clauses == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_end_user_from_db_returns_none_without_a_row_a_client_or_on_db_error():
|
||||
assert (
|
||||
await SpendCounterReseed.end_user_from_db(prisma_client=None, counter_key="spend:end_user:customer-42") is None
|
||||
)
|
||||
assert (
|
||||
await SpendCounterReseed.end_user_from_db(
|
||||
await SpendCounterReseed.from_db(
|
||||
prisma_client=_FakePrismaClient(end_user_row=None), counter_key="spend:end_user:customer-42"
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert (
|
||||
await SpendCounterReseed.end_user_from_db(
|
||||
await SpendCounterReseed.from_db(
|
||||
prisma_client=_FakePrismaClient(end_user_error=RuntimeError("db down")),
|
||||
counter_key="spend:end_user:customer-42",
|
||||
)
|
||||
|
|
@ -470,13 +444,3 @@ async def test_from_db_deadline_covers_the_wait_for_a_gate_slot(monkeypatch: pyt
|
|||
find_unique.assert_not_called()
|
||||
finally:
|
||||
db_lookup_stall_tracker.clear()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_from_db_still_never_reads_the_end_user_row():
|
||||
"""A cold end-user counter keeps seeding from the cached end-user object the auth
|
||||
path already loaded; the row is read only as the budget floor."""
|
||||
prisma: Final = _FakePrismaClient(end_user_row=SimpleNamespace(user_id="customer-42", spend=5.0))
|
||||
|
||||
assert await SpendCounterReseed.from_db(prisma_client=prisma, counter_key="spend:end_user:customer-42") is None
|
||||
assert prisma.db.litellm_endusertable.where_clauses == []
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_loca
|
|||
from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, end_user_cache_key, tag_cache_key
|
||||
from litellm.proxy.management_endpoints.team_endpoints import refresh_cached_team
|
||||
from litellm.proxy.spend_tracking.spend_counter_batch import spend_counter_batch_scope
|
||||
|
||||
from .conftest import normalize
|
||||
|
||||
|
|
@ -1357,6 +1358,50 @@ async def test_ensure_spend_counter_initialized_cold_seeds_from_source_cache(
|
|||
}
|
||||
|
||||
|
||||
class _FakeRedisMeetingAtTheWarmCheck(FakeAsyncRedis):
|
||||
"""Holds each request's MGET until every request has made one, so all of them find the counter missing before
|
||||
any of them seeds it."""
|
||||
|
||||
def __init__(self, server: FakeServer, callers: int) -> None:
|
||||
super().__init__(server=server)
|
||||
self._warm_checks: Final = asyncio.Barrier(callers)
|
||||
|
||||
async def mget(self, keys: list[str], *args: str) -> list[bytes | None]:
|
||||
values: Final = await super().mget(keys, *args)
|
||||
await self._warm_checks.wait()
|
||||
return values
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_overlapping_cold_end_user_seeds_read_and_write_the_end_user_row_once(monkeypatch: pytest.MonkeyPatch):
|
||||
counter_key: Final = "spend:end_user:customer-1"
|
||||
callers: Final = 3
|
||||
redis: Final = _FakeRedisMeetingAtTheWarmCheck(FakeServer(), callers)
|
||||
auth_cache: Final = UserApiKeyCache(in_memory_cache=InMemoryCache())
|
||||
auth_cache.set_cache(
|
||||
key=end_user_cache_key("customer-1"),
|
||||
value=LiteLLM_EndUserTable(user_id="customer-1", blocked=False, spend=6.0),
|
||||
local_only=True,
|
||||
)
|
||||
prisma: Final = MagicMock()
|
||||
prisma.db.litellm_endusertable.find_unique = AsyncMock(return_value=MagicMock(spend=7.5))
|
||||
counter_cache: Final = ps.DualCache(redis_cache=_RedisCacheOnClient(redis))
|
||||
monkeypatch.setattr(ps, "spend_counter_cache", counter_cache)
|
||||
monkeypatch.setattr(ps, "user_api_key_cache", auth_cache)
|
||||
monkeypatch.setattr(ps, "prisma_client", prisma)
|
||||
|
||||
async def _request() -> None:
|
||||
with spend_counter_batch_scope(counter_cache.redis_cache, counter_keys=frozenset((counter_key,))):
|
||||
await ps.ensure_spend_counter_initialized(
|
||||
counter_key=counter_key, source_cache_key=end_user_cache_key("customer-1")
|
||||
)
|
||||
|
||||
await asyncio.wait_for(asyncio.gather(*(_request() for _ in range(callers))), timeout=5)
|
||||
|
||||
assert float(await redis.get(counter_key)) == 7.5
|
||||
prisma.db.litellm_endusertable.find_unique.assert_awaited_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _get_source_cache_base_spend
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -8849,36 +8849,31 @@ async def test_primary_spend_counter_redis_concurrent_seed_does_not_double_seed(
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reseed_spend_from_db_user_and_org_prefixes():
|
||||
"""User and org counters reseed from their own DB tables.
|
||||
"""User, end-user and org counters reseed from their own DB tables.
|
||||
|
||||
End-user and tag counters use the already fetched auth objects passed as
|
||||
fallback_spend, so this reseed helper must not add extra per-request DB
|
||||
reads for them.
|
||||
Tag counters use the already fetched auth objects passed as fallback_spend,
|
||||
so this reseed helper must not add extra per-request DB reads for them.
|
||||
"""
|
||||
from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed
|
||||
|
||||
user_row = MagicMock()
|
||||
user_row.spend = 17.0
|
||||
end_user_row = MagicMock()
|
||||
end_user_row.spend = 6.0
|
||||
org_row = MagicMock()
|
||||
org_row.spend = 305.0
|
||||
|
||||
fake_prisma = MagicMock()
|
||||
fake_prisma.db.litellm_usertable.find_unique = AsyncMock(return_value=user_row)
|
||||
fake_prisma.db.litellm_endusertable.find_unique = AsyncMock()
|
||||
fake_prisma.db.litellm_endusertable.find_unique = AsyncMock(return_value=end_user_row)
|
||||
fake_prisma.db.litellm_tagtable.find_unique = AsyncMock()
|
||||
fake_prisma.db.litellm_organizationtable.find_unique = AsyncMock(return_value=org_row)
|
||||
|
||||
assert await SpendCounterReseed.from_db(fake_prisma, "spend:user:alice") == 17.0
|
||||
fake_prisma.db.litellm_usertable.find_unique.assert_awaited_once_with(where={"user_id": "alice"})
|
||||
|
||||
assert (
|
||||
await SpendCounterReseed.from_db(
|
||||
fake_prisma,
|
||||
"spend:end_user:customer-1",
|
||||
)
|
||||
is None
|
||||
)
|
||||
fake_prisma.db.litellm_endusertable.find_unique.assert_not_awaited()
|
||||
assert await SpendCounterReseed.from_db(fake_prisma, "spend:end_user:customer-1") == 6.0
|
||||
fake_prisma.db.litellm_endusertable.find_unique.assert_awaited_once_with(where={"user_id": "customer-1"})
|
||||
|
||||
assert await SpendCounterReseed.from_db(fake_prisma, "spend:tag:paid-tag") is None
|
||||
fake_prisma.db.litellm_tagtable.find_unique.assert_not_awaited()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue