mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-20 10:02:54 +00:00
fix(spend): batch live counter resets and restore integration regression
This commit is contained in:
parent
7ba37ba387
commit
69354fa1e7
3 changed files with 27 additions and 39 deletions
|
|
@ -1,4 +1,5 @@
|
|||
#### SPEND MANAGEMENT #####
|
||||
import asyncio
|
||||
import collections
|
||||
import json
|
||||
import os
|
||||
|
|
@ -297,29 +298,26 @@ def _verification_token_table(prisma_client: PrismaClient) -> _VerificationToken
|
|||
return VerificationTokenRepository(prisma_client).table
|
||||
|
||||
|
||||
async def _reset_spend_counters(prisma_client: PrismaClient) -> None:
|
||||
"""Zero every live spend counter a budget check reads.
|
||||
_SPEND_COUNTER_RESET_BATCH_SIZE: Final = 100
|
||||
|
||||
Budget enforcement reads the Redis spend counters first, so zeroing only
|
||||
the DB rows leaves an exhausted key refused with the stale pre-reset
|
||||
counter. Mirrors /key/{key}/reset_spend, setting each counter to 0 (with
|
||||
floor marker and broadcast) rather than deleting it.
|
||||
"""
|
||||
|
||||
async def _reset_spend_counters(prisma_client: PrismaClient) -> None:
|
||||
from litellm.proxy.management_endpoints.key_management_endpoints import (
|
||||
set_spend_counter_with_floor_and_broadcast,
|
||||
)
|
||||
|
||||
key_rows: Final = await _verification_token_table(prisma_client).find_many(where={})
|
||||
for row in key_rows:
|
||||
token = row.model_dump().get("token", "")
|
||||
if isinstance(token, str) and token:
|
||||
await set_spend_counter_with_floor_and_broadcast(counter_key=f"spend:key:{token}", value=0.0)
|
||||
|
||||
team_rows: Final = await _team_table(prisma_client).find_many(where={})
|
||||
for row in team_rows:
|
||||
team_id = row.model_dump().get("team_id", "")
|
||||
if isinstance(team_id, str) and team_id:
|
||||
await set_spend_counter_with_floor_and_broadcast(counter_key=f"spend:team:{team_id}", value=0.0)
|
||||
for rows, identifier_field, prefix in (
|
||||
(await _verification_token_table(prisma_client).find_many(where={}), "token", "spend:key:"),
|
||||
(await _team_table(prisma_client).find_many(where={}), "team_id", "spend:team:"),
|
||||
):
|
||||
for start in range(0, len(rows), _SPEND_COUNTER_RESET_BATCH_SIZE):
|
||||
await asyncio.gather(
|
||||
*(
|
||||
set_spend_counter_with_floor_and_broadcast(counter_key=f"{prefix}{identifier}", value=0.0)
|
||||
for row in rows[start : start + _SPEND_COUNTER_RESET_BATCH_SIZE]
|
||||
if isinstance(identifier := row.model_dump().get(identifier_field), str) and identifier
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _spend_logs_daily_summary_sql(
|
||||
|
|
|
|||
|
|
@ -132,10 +132,6 @@ def test_global_spend_reset_zeroes_token_and_team_spend_but_keeps_logs(gateway:
|
|||
|
||||
|
||||
def test_reset_exhausted_key_is_served_again(gateway: Gateway) -> None:
|
||||
pytest.skip(
|
||||
"BUG: /global/spend/reset zeroes DB spend but leaves Redis spend:key counters, "
|
||||
"so an exhausted key stays refused with budget_exceeded"
|
||||
)
|
||||
with (
|
||||
gateway.scenario() as scenario,
|
||||
httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream,
|
||||
|
|
|
|||
|
|
@ -8182,7 +8182,8 @@ async def test_global_spend_refresh_runs_the_refresh_on_the_writer_when_a_reader
|
|||
]
|
||||
|
||||
|
||||
def test_global_spend_reset_zeroes_live_spend_counters(monkeypatch):
|
||||
@pytest.mark.parametrize("key_count", [0, 2, 201])
|
||||
def test_global_spend_reset_zeroes_live_spend_counters(monkeypatch, key_count: int):
|
||||
"""Regression test for https://github.com/BerriAI/litellm/issues/46363
|
||||
|
||||
Budget enforcement reads the Redis spend counters before the DB rows, so
|
||||
|
|
@ -8212,7 +8213,9 @@ def test_global_spend_reset_zeroes_live_spend_counters(monkeypatch):
|
|||
|
||||
class MockDB:
|
||||
def __init__(self):
|
||||
self.litellm_verificationtoken = MockTable([Row({"token": "key-1"}), Row({"token": "key-2"})])
|
||||
self.litellm_verificationtoken = MockTable(
|
||||
[Row({"token": f"key-{index}"}) for index in range(1, key_count + 1)]
|
||||
)
|
||||
self.litellm_teamtable = MockTable([Row({"team_id": "team-1"})])
|
||||
|
||||
class MockPrismaClient:
|
||||
|
|
@ -8230,22 +8233,13 @@ def test_global_spend_reset_zeroes_live_spend_counters(monkeypatch):
|
|||
mock_set_counter,
|
||||
)
|
||||
|
||||
async def run_reset():
|
||||
import litellm.proxy.proxy_server as ps
|
||||
|
||||
ps.prisma_client = MockPrismaClient()
|
||||
try:
|
||||
return await global_spend_reset()
|
||||
finally:
|
||||
ps.prisma_client = None
|
||||
|
||||
response: Final = asyncio.run(run_reset())
|
||||
monkeypatch.setattr(ps, "prisma_client", MockPrismaClient())
|
||||
response: Final = asyncio.run(global_spend_reset())
|
||||
assert response == {
|
||||
"message": "Spend for all API Keys and Teams reset successfully",
|
||||
"status": "success",
|
||||
}
|
||||
assert sorted(reset_calls) == [
|
||||
("spend:key:key-1", 0.0),
|
||||
("spend:key:key-2", 0.0),
|
||||
("spend:team:team-1", 0.0),
|
||||
], reset_calls
|
||||
assert sorted(reset_calls) == sorted(
|
||||
[(f"spend:key:key-{index}", 0.0) for index in range(1, key_count + 1)]
|
||||
+ [("spend:team:team-1", 0.0)]
|
||||
), reset_calls
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue