diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index f7acc8180a0..4f3cf729c9a 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1,4 +1,5 @@ #### SPEND MANAGEMENT ##### +import asyncio import collections import json import os @@ -297,29 +298,26 @@ def _verification_token_table(prisma_client: PrismaClient) -> _VerificationToken return VerificationTokenRepository(prisma_client).table -async def _reset_spend_counters(prisma_client: PrismaClient) -> None: - """Zero every live spend counter a budget check reads. +_SPEND_COUNTER_RESET_BATCH_SIZE: Final = 100 - Budget enforcement reads the Redis spend counters first, so zeroing only - the DB rows leaves an exhausted key refused with the stale pre-reset - counter. Mirrors /key/{key}/reset_spend, setting each counter to 0 (with - floor marker and broadcast) rather than deleting it. - """ + +async def _reset_spend_counters(prisma_client: PrismaClient) -> None: from litellm.proxy.management_endpoints.key_management_endpoints import ( set_spend_counter_with_floor_and_broadcast, ) - key_rows: Final = await _verification_token_table(prisma_client).find_many(where={}) - for row in key_rows: - token = row.model_dump().get("token", "") - if isinstance(token, str) and token: - await set_spend_counter_with_floor_and_broadcast(counter_key=f"spend:key:{token}", value=0.0) - - team_rows: Final = await _team_table(prisma_client).find_many(where={}) - for row in team_rows: - team_id = row.model_dump().get("team_id", "") - if isinstance(team_id, str) and team_id: - await set_spend_counter_with_floor_and_broadcast(counter_key=f"spend:team:{team_id}", value=0.0) + for rows, identifier_field, prefix in ( + (await _verification_token_table(prisma_client).find_many(where={}), "token", "spend:key:"), + (await _team_table(prisma_client).find_many(where={}), "team_id", "spend:team:"), + ): + for start in range(0, len(rows), _SPEND_COUNTER_RESET_BATCH_SIZE): + await asyncio.gather( + *( + set_spend_counter_with_floor_and_broadcast(counter_key=f"{prefix}{identifier}", value=0.0) + for row in rows[start : start + _SPEND_COUNTER_RESET_BATCH_SIZE] + if isinstance(identifier := row.model_dump().get(identifier_field), str) and identifier + ) + ) def _spend_logs_daily_summary_sql( diff --git a/tests/integration/spend/test_global_spend_reset.py b/tests/integration/spend/test_global_spend_reset.py index c400d90e184..19241c27579 100644 --- a/tests/integration/spend/test_global_spend_reset.py +++ b/tests/integration/spend/test_global_spend_reset.py @@ -132,10 +132,6 @@ def test_global_spend_reset_zeroes_token_and_team_spend_but_keeps_logs(gateway: def test_reset_exhausted_key_is_served_again(gateway: Gateway) -> None: - pytest.skip( - "BUG: /global/spend/reset zeroes DB spend but leaves Redis spend:key counters, " - "so an exhausted key stays refused with budget_exceeded" - ) with ( gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index b4102dee1fd..40b66b04997 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -8182,7 +8182,8 @@ async def test_global_spend_refresh_runs_the_refresh_on_the_writer_when_a_reader ] -def test_global_spend_reset_zeroes_live_spend_counters(monkeypatch): +@pytest.mark.parametrize("key_count", [0, 2, 201]) +def test_global_spend_reset_zeroes_live_spend_counters(monkeypatch, key_count: int): """Regression test for https://github.com/BerriAI/litellm/issues/46363 Budget enforcement reads the Redis spend counters before the DB rows, so @@ -8212,7 +8213,9 @@ def test_global_spend_reset_zeroes_live_spend_counters(monkeypatch): class MockDB: def __init__(self): - self.litellm_verificationtoken = MockTable([Row({"token": "key-1"}), Row({"token": "key-2"})]) + self.litellm_verificationtoken = MockTable( + [Row({"token": f"key-{index}"}) for index in range(1, key_count + 1)] + ) self.litellm_teamtable = MockTable([Row({"team_id": "team-1"})]) class MockPrismaClient: @@ -8230,22 +8233,13 @@ def test_global_spend_reset_zeroes_live_spend_counters(monkeypatch): mock_set_counter, ) - async def run_reset(): - import litellm.proxy.proxy_server as ps - - ps.prisma_client = MockPrismaClient() - try: - return await global_spend_reset() - finally: - ps.prisma_client = None - - response: Final = asyncio.run(run_reset()) + monkeypatch.setattr(ps, "prisma_client", MockPrismaClient()) + response: Final = asyncio.run(global_spend_reset()) assert response == { "message": "Spend for all API Keys and Teams reset successfully", "status": "success", } - assert sorted(reset_calls) == [ - ("spend:key:key-1", 0.0), - ("spend:key:key-2", 0.0), - ("spend:team:team-1", 0.0), - ], reset_calls + assert sorted(reset_calls) == sorted( + [(f"spend:key:key-{index}", 0.0) for index in range(1, key_count + 1)] + + [("spend:team:team-1", 0.0)] + ), reset_calls