diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index f7b72b263d4..677d1422fe1 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -6610,7 +6610,7 @@ def _validate_reset_spend_value(reset_to: object, key_in_db: LiteLLM_Verificatio @with_service_target(SPEND_COUNTERS_TARGET) -async def _set_spend_counter_with_floor_and_broadcast(counter_key: str, value: float) -> None: +async def set_spend_counter_with_floor_and_broadcast(counter_key: str, value: float) -> None: """ Set a Redis-backed spend counter to `value`, mirror it into the short-lived spend_db_floor marker `_authoritative_floor_spend` reads, and broadcast both @@ -6746,7 +6746,7 @@ async def _reset_key_budget_windows( duration = window.get("budget_duration") if isinstance(duration, str) and duration: counter_key = f"spend:key:{hashed_api_key}:window:{duration}" - await _set_spend_counter_with_floor_and_broadcast(counter_key=counter_key, value=0.0) + await set_spend_counter_with_floor_and_broadcast(counter_key=counter_key, value=0.0) @router.post( @@ -6820,7 +6820,7 @@ async def reset_key_spend_fn( # amount for every enforcement check immediately instead of the stale # pre-reset value. _counter_key: Final = f"spend:key:{hashed_api_key}" - await _set_spend_counter_with_floor_and_broadcast(counter_key=_counter_key, value=reset_to) + await set_spend_counter_with_floor_and_broadcast(counter_key=_counter_key, value=reset_to) await _reset_key_budget_windows( prisma_client=prisma_client, hashed_api_key=hashed_api_key, diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 3c65fc0bf06..e103c557eac 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1,4 +1,5 @@ #### SPEND MANAGEMENT ##### +import asyncio import collections import json import os @@ -280,6 +281,8 @@ class _TeamTable(Protocol): class _VerificationTokenTable(Protocol): """The subset of the Prisma verification token table API this module uses.""" + async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_SupportsModelDump]: ... + async def update_many(self, *, data: Mapping[str, float], where: Mapping[str, object]) -> int: ... @@ -295,6 +298,28 @@ def _verification_token_table(prisma_client: PrismaClient) -> _VerificationToken return VerificationTokenRepository(prisma_client).table +_SPEND_COUNTER_RESET_BATCH_SIZE: Final = 100 + + +async def _reset_spend_counters(prisma_client: PrismaClient) -> None: + from litellm.proxy.management_endpoints.key_management_endpoints import ( + set_spend_counter_with_floor_and_broadcast, + ) + + for rows, identifier_field, prefix in ( + (await _verification_token_table(prisma_client).find_many(where={}), "token", "spend:key:"), + (await _team_table(prisma_client).find_many(where={}), "team_id", "spend:team:"), + ): + for start in range(0, len(rows), _SPEND_COUNTER_RESET_BATCH_SIZE): + await asyncio.gather( + *( + set_spend_counter_with_floor_and_broadcast(counter_key=f"{prefix}{identifier}", value=0.0) + for row in rows[start : start + _SPEND_COUNTER_RESET_BATCH_SIZE] + if isinstance(identifier := row.model_dump().get(identifier_field), str) and identifier + ) + ) + + def _spend_logs_daily_summary_sql( *, start_date_iso: str, @@ -3721,6 +3746,8 @@ async def global_spend_reset(): await _verification_token_table(prisma_client).update_many(data={"spend": 0.0}, where={}) await _team_table(prisma_client).update_many(data={"spend": 0.0}, where={}) + await _reset_spend_counters(prisma_client) + return { "message": "Spend for all API Keys and Teams reset successfully", "status": "success", diff --git a/osv-scanner.toml b/osv-scanner.toml index 33e2f614edc..9ea1fe14ddf 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -8,15 +8,25 @@ id = "GHSA-v4x8-3wr5-3wq8" ignoreUntil = 2026-11-20 reason = "the only fixed filelock is 4.2.1, a new major that requires Python >= 3.11 while litellm still supports 3.10, and the advisory is in ReadWriteLock, which litellm never uses (only FileLock); bump filelock and remove this entry once 3.10 is dropped or a 3.x fix ships" +[[IgnoredVulns]] +id = "GHSA-567q-5qvw-pf92" +ignoreUntil = 2026-11-20 +reason = "Windows-only local DoS (CVSS 2.5) in filelock soft locks, fixed only in 4.2.1, a new major that requires Python >= 3.11 while litellm still supports 3.10 and pins filelock<4.0; litellm only uses FileLock, which takes an OS lock rather than a soft lock; bump filelock and remove this entry once 3.10 is dropped or a 3.x fix ships" + +[[IgnoredVulns]] +id = "GHSA-9vhv-jxww-xfm5" +ignoreUntil = 2026-11-20 +reason = "Windows-only local DoS (CVSS 3.3) in filelock ReadWriteLock, fixed only in 4.2.1, a new major that requires Python >= 3.11 while litellm still supports 3.10 and pins filelock<4.0; litellm never uses ReadWriteLock (only FileLock); bump filelock and remove this entry once 3.10 is dropped or a 3.x fix ships" + [[IgnoredVulns]] id = "GHSA-vfj7-8cjw-p6xm" ignoreUntil = 2026-11-03 reason = "braces has no fixed release published (3.0.3, 2024-05-21, is latest and last_affected); dev-only via knip > fast-glob > micromatch; remove this entry once one exists" [[IgnoredVulns]] -id = "GHSA-6q48-6c7w-9gw2" -ignoreUntil = 2026-10-23 -reason = "certifi 2026.10.19 (published 2026-10-20) is inside the 3-day uv exclude-newer window until 2026-10-23 02:24 UTC; bump certifi and remove this entry once it clears" +id = "GHSA-xcx6-jx8p-x3h5" +ignoreUntil = 2026-11-24 +reason = "@azure/msal-node is fixed only in 7.1.0 (published 2026-10-13), but it is dev-only in vscode-extension via @vscode/vsce > @azure/identity, and the latest @azure/identity (4.14.0) still requires msal-node ^6.0.0, so the lock cannot move without forcing a major override; the advisory is in the auth_code flow, which extension packaging does not run; bump and remove this entry once @azure/identity accepts msal-node >= 7.1.0" [[PackageOverrides]] name = "pillow" diff --git a/tests/integration/spend/test_global_spend_reset.py b/tests/integration/spend/test_global_spend_reset.py index c400d90e184..19241c27579 100644 --- a/tests/integration/spend/test_global_spend_reset.py +++ b/tests/integration/spend/test_global_spend_reset.py @@ -132,10 +132,6 @@ def test_global_spend_reset_zeroes_token_and_team_spend_but_keeps_logs(gateway: def test_reset_exhausted_key_is_served_again(gateway: Gateway) -> None: - pytest.skip( - "BUG: /global/spend/reset zeroes DB spend but leaves Redis spend:key counters, " - "so an exhausted key stays refused with budget_exceeded" - ) with ( gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 8d53817f51e..5672f478e29 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -8268,3 +8268,66 @@ async def test_global_spend_refresh_runs_the_refresh_on_the_writer_when_a_reader assert [statement for statement in journal.statements if statement[1].startswith("REFRESH")] == [ ("writer", 'REFRESH MATERIALIZED VIEW "MonthlyGlobalSpend";') ] + + +@pytest.mark.parametrize("key_count", [0, 2, 201]) +def test_global_spend_reset_zeroes_live_spend_counters(monkeypatch, key_count: int): + """Regression test for https://github.com/BerriAI/litellm/issues/46363 + + Budget enforcement reads the Redis spend counters before the DB rows, so + a global reset that only zeroes the DB leaves an exhausted key refused + with the stale pre-reset counter. The reset must zero every live + spend:key: and spend:team: counter, not just the rows. + """ + import litellm.proxy.management_endpoints.key_management_endpoints as kme + from litellm.proxy.spend_tracking.spend_management_endpoints import global_spend_reset + + class Row: + def __init__(self, data): + self._data = data + + def model_dump(self): + return self._data + + class MockTable: + def __init__(self, rows): + self.rows = rows + + async def find_many(self, *args, **kwargs): + return self.rows + + async def update_many(self, *args, **kwargs): + return len(self.rows) + + class MockDB: + def __init__(self): + self.litellm_verificationtoken = MockTable( + [Row({"token": f"key-{index}"}) for index in range(1, key_count + 1)] + ) + self.litellm_teamtable = MockTable([Row({"team_id": "team-1"})]) + + class MockPrismaClient: + def __init__(self): + self.db = MockDB() + + reset_calls: list[tuple[str, float]] = [] + + async def mock_set_counter(counter_key: str, value: float) -> None: + reset_calls.append((counter_key, value)) + + monkeypatch.setattr( + kme, + "set_spend_counter_with_floor_and_broadcast", + mock_set_counter, + ) + + monkeypatch.setattr(ps, "prisma_client", MockPrismaClient()) + response: Final = asyncio.run(global_spend_reset()) + assert response == { + "message": "Spend for all API Keys and Teams reset successfully", + "status": "success", + } + assert sorted(reset_calls) == sorted( + [(f"spend:key:key-{index}", 0.0) for index in range(1, key_count + 1)] + + [("spend:team:team-1", 0.0)] + ), reset_calls