diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py index 35e74418628..a9f41255772 100644 --- a/litellm/proxy/common_utils/reset_budget_job.py +++ b/litellm/proxy/common_utils/reset_budget_job.py @@ -258,6 +258,7 @@ class _BudgetCascade: budgets: tuple[LiteLLM_BudgetTableFull, ...] = () budget_ids: tuple[str, ...] = () budget_resets: tuple[tuple[str, datetime], ...] = () + budget_quarantines: tuple[str, ...] = () endusers: tuple[_EndUserRow, ...] = () counter_resets: tuple[tuple[str, float], ...] = () cache_keys: tuple[str, ...] = () @@ -299,14 +300,70 @@ def _as_utc(moment: datetime) -> datetime: def _count_advanced(reset_ats: Iterable[object], cutoff: datetime) -> int: """How many rows the write actually moved past the due cutoff. - A budget_duration of "0s" (or one the parser cannot read) resolves to the - current time, so the row is written and stays due. Counting it as progress - would re-read the same chunk until the per-run cap on every tick. + A budget_duration of "0s" resolves to the current time, so the row is written + and stays due. Counting it as progress would re-read the same chunk until + the per-run cap on every tick. """ utc_cutoff: Final = _as_utc(cutoff) return sum(1 for reset_at in reset_ats if isinstance(reset_at, datetime) and _as_utc(reset_at) > utc_cutoff) +def _try_compute_budget_reset_at(duration: object, settings: BudgetResetSettings) -> datetime | None: + if not isinstance(duration, str): + return None + try: + return compute_budget_reset_at(budget_duration=duration, settings=settings) + except ValueError: + return None + + +def _is_duration_quarantine( + item: LiteLLM_TeamTable | LiteLLM_UserTable | LiteLLM_VerificationToken, +) -> bool: + return item.budget_duration is None and item.budget_reset_at is None + + +def _count_rows_leaving_due_set( + items: Iterable[LiteLLM_TeamTable | LiteLLM_UserTable | LiteLLM_VerificationToken], + cutoff: datetime, +) -> int: + return sum( + 1 + for item in items + if _is_duration_quarantine(item) + or (isinstance(item.budget_reset_at, datetime) and _as_utc(item.budget_reset_at) > _as_utc(cutoff)) + ) + + +@dataclass(frozen=True, slots=True) +class _AdvanceWindow: + budget_id: str + reset_at: datetime + + +@dataclass(frozen=True, slots=True) +class _QuarantineWindow: + budget_id: str + + +def _plan_budget_window( + budget: LiteLLM_BudgetTableFull, settings: BudgetResetSettings +) -> _AdvanceWindow | _QuarantineWindow | None: + budget_id: Final = budget.budget_id + duration: Final = budget.budget_duration + if budget_id is None or duration is None: + return None + reset_at: Final = _try_compute_budget_reset_at(duration, settings) + if reset_at is None: + verbose_proxy_logger.warning( + "Reset budget job: clearing unparseable budget_duration %r on budget %s so it leaves the due set", + duration, + budget_id, + ) + return _QuarantineWindow(budget_id) + return _AdvanceWindow(budget_id, reset_at) + + def _phase_is_drained(outcome: _ChunkOutcome) -> bool: """A short chunk means the due rows ran out. A full chunk that advanced nothing would be re-read unchanged forever, so it ends the phase too and @@ -626,9 +683,15 @@ class ResetBudgetJob: own schedule via reset_budget_for_litellm_keys(), so sweeping them here would reset them twice. """ - budget_ids: Final = tuple(b.budget_id for b in budgets_to_reset if b.budget_id is not None) + window_plans: Final = tuple(_plan_budget_window(b, self.reset_settings) for b in budgets_to_reset) + budget_resets: Final = tuple((p.budget_id, p.reset_at) for p in window_plans if isinstance(p, _AdvanceWindow)) + budget_quarantines: Final = tuple(p.budget_id for p in window_plans if isinstance(p, _QuarantineWindow)) + quarantine_ids: Final = frozenset(budget_quarantines) + budget_ids: Final = tuple( + b.budget_id for b in budgets_to_reset if b.budget_id is not None and b.budget_id not in quarantine_ids + ) if not budget_ids: - return _EMPTY_CASCADE + return _BudgetCascade(budgets=tuple(budgets_to_reset), budget_quarantines=budget_quarantines) team_memberships: Final[tuple[_TeamMembershipRow, ...]] = await self._fetch_linked_rows( table=TeamMembershipRepository(self.prisma_client).table, @@ -659,7 +722,9 @@ class ResetBudgetJob: { # mutable-ok: MappingProxyType wraps a one-shot dict comprehension b.budget_id: cap for b in budgets_to_reset - if b.budget_id is not None and (cap := _rollover_cap(b.max_budget)) is not None + if b.budget_id is not None + and b.budget_id not in quarantine_ids + and (cap := _rollover_cap(b.max_budget)) is not None } if _rollover_enabled() else {} # mutable-ok: empty sentinel immediately frozen by MappingProxyType @@ -668,14 +733,8 @@ class ResetBudgetJob: return _BudgetCascade( budgets=tuple(budgets_to_reset), budget_ids=budget_ids, - budget_resets=tuple( - ( - b.budget_id, - compute_budget_reset_at(budget_duration=b.budget_duration, settings=self.reset_settings), - ) - for b in budgets_to_reset - if b.budget_id is not None and b.budget_duration is not None - ), + budget_resets=budget_resets, + budget_quarantines=budget_quarantines, endusers=endusers, counter_resets=( *( @@ -710,7 +769,7 @@ class ResetBudgetJob: batching both means a mid-cascade failure persists nothing and the rows stay due for the next run. """ - if not cascade.budget_ids: + if not cascade.budget_ids and not cascade.budget_quarantines: return await self._with_db_write_retry( @@ -728,6 +787,8 @@ class ResetBudgetJob: _queue_enduser_resets(uow.endusers, cascade) for budget_id, budget_reset_at in cascade.budget_resets: uow.budgets.queue_window_advance(budget_id=budget_id, budget_reset_at=budget_reset_at) + for budget_id in cascade.budget_quarantines: + uow.budgets.queue_window_quarantine(budget_id=budget_id) async def _invalidate_budget_cascade_caches(self, cascade: _BudgetCascade) -> None: for counter_key, new_spend in cascade.counter_resets: @@ -762,7 +823,8 @@ class ResetBudgetJob: advanced=_count_advanced( (reset_at for _, reset_at in cascade.budget_resets), cutoff=datetime.now(timezone.utc), - ), + ) + + len(cascade.budget_quarantines), ) async def reset_budget_for_litellm_budget_table(self) -> None: @@ -863,6 +925,9 @@ class ResetBudgetJob: for k in updated_keys: if k.token is None: continue + if _is_duration_quarantine(k): + uow.keys.queue_duration_quarantine(token=k.token) + continue uow.keys.queue_spend_reset( token=k.token, budget_reset_at=k.budget_reset_at, @@ -885,6 +950,9 @@ class ResetBudgetJob: async def _write_user_reset_updates_once(self, updated_users: list[LiteLLM_UserTable]) -> None: async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow: for u in updated_users: + if _is_duration_quarantine(u): + uow.users.queue_duration_quarantine(user_id=u.user_id) + continue uow.users.queue_spend_reset( user_id=u.user_id, budget_reset_at=u.budget_reset_at, @@ -907,6 +975,9 @@ class ResetBudgetJob: async def _write_team_reset_updates_once(self, updated_teams: list[LiteLLM_TeamTable]) -> None: async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow: for t in updated_teams: + if _is_duration_quarantine(t): + uow.teams.queue_duration_quarantine(team_id=t.team_id) + continue uow.teams.queue_spend_reset( team_id=t.team_id, budget_reset_at=t.budget_reset_at, @@ -985,6 +1056,8 @@ class ResetBudgetJob: if updated_keys: await self._write_key_reset_updates(updated_keys=updated_keys) for k in updated_keys: + if _is_duration_quarantine(k): + continue token = getattr(k, "token", None) if token: await self._invalidate_spend_counter(f"spend:key:{token}", new_spend=k.spend or 0.0) @@ -992,10 +1065,7 @@ class ResetBudgetJob: end_time = time.time() outcome: Final = _ChunkOutcome( fetched=len(keys_to_reset) if keys_to_reset else 0, - advanced=_count_advanced( - (k.budget_reset_at for k in updated_keys), - cutoff=datetime.now(timezone.utc), - ), + advanced=_count_rows_leaving_due_set(updated_keys, cutoff=datetime.now(timezone.utc)), ) if len(failed_keys) > 0: self._emit_phase_failure( @@ -1090,6 +1160,8 @@ class ResetBudgetJob: if updated_users: await self._write_user_reset_updates(updated_users=updated_users) for u in updated_users: + if _is_duration_quarantine(u): + continue user_id = getattr(u, "user_id", None) if user_id: await self._invalidate_spend_counter(f"spend:user:{user_id}", new_spend=u.spend or 0.0) @@ -1099,10 +1171,7 @@ class ResetBudgetJob: end_time = time.time() outcome: Final = _ChunkOutcome( fetched=len(users_to_reset) if users_to_reset else 0, - advanced=_count_advanced( - (u.budget_reset_at for u in updated_users), - cutoff=datetime.now(timezone.utc), - ), + advanced=_count_rows_leaving_due_set(updated_users, cutoff=datetime.now(timezone.utc)), ) if len(failed_users) > 0: self._emit_phase_failure( @@ -1199,6 +1268,8 @@ class ResetBudgetJob: if updated_teams: await self._write_team_reset_updates(updated_teams=updated_teams) for t in updated_teams: + if _is_duration_quarantine(t): + continue team_id = getattr(t, "team_id", None) if team_id: await self._invalidate_spend_counter(f"spend:team:{team_id}", new_spend=t.spend or 0.0) @@ -1206,10 +1277,7 @@ class ResetBudgetJob: end_time = time.time() outcome: Final = _ChunkOutcome( fetched=len(teams_to_reset) if teams_to_reset else 0, - advanced=_count_advanced( - (t.budget_reset_at for t in updated_teams), - cutoff=datetime.now(timezone.utc), - ), + advanced=_count_rows_leaving_due_set(updated_teams, cutoff=datetime.now(timezone.utc)), ) if len(failed_teams) > 0: self._emit_phase_failure( @@ -1277,6 +1345,15 @@ class ResetBudgetJob: reset_at: Final = datetime.fromisoformat(reset_at_str.replace("Z", "+00:00")).replace(tzinfo=None) if reset_at > now: return False + budget_duration: Final = window.get("budget_duration") + next_reset_at: Final = _try_compute_budget_reset_at(budget_duration, reset_settings) + if next_reset_at is None or not isinstance(budget_duration, str): + verbose_proxy_logger.warning( + "Reset budget job: clearing unparseable budget_duration %r on a window so it is not retried", + budget_duration, + ) + window["reset_at"] = None + return True new_value: Final = await ResetBudgetJob._window_carried_spend(window, counter_key, spend_counter_cache) spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=new_value) if spend_counter_cache.redis_cache is not None: @@ -1284,8 +1361,6 @@ class ResetBudgetJob: await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=new_value) except Exception as redis_err: verbose_proxy_logger.warning("Failed to reset Redis counter %s: %s", counter_key, redis_err) - budget_duration: Final = window["budget_duration"] - next_reset_at: Final = compute_budget_reset_at(budget_duration=budget_duration, settings=reset_settings) window["reset_at"] = next_reset_at.isoformat() await ResetBudgetJob._roll_window_spend_row( prisma_client=prisma_client, @@ -1462,11 +1537,20 @@ class ResetBudgetJob: still holds the pre-reset value, admitting requests past the cap. """ try: + duration: Final = item.budget_duration + if duration is not None: + reset_at: Final = _try_compute_budget_reset_at(duration, reset_settings) + if reset_at is None: + verbose_proxy_logger.warning( + "Reset budget job: clearing unparseable budget_duration %r on %s so it leaves the due set", + duration, + item_type, + ) + item.budget_duration = None + item.budget_reset_at = None + return item + item.budget_reset_at = reset_at item.spend = _carried_spend(item.spend, _rollover_cap(item.max_budget)) if _rollover_enabled() else 0.0 - if hasattr(item, "budget_duration") and item.budget_duration is not None: - item.budget_reset_at = compute_budget_reset_at( - budget_duration=item.budget_duration, settings=reset_settings - ) return item except Exception as e: verbose_proxy_logger.exception("Error resetting budget for %s: %s. Item: %s", item_type, e, item) diff --git a/litellm/repositories/unit_of_work.py b/litellm/repositories/unit_of_work.py index a497d0580db..0abf806b170 100644 --- a/litellm/repositories/unit_of_work.py +++ b/litellm/repositories/unit_of_work.py @@ -33,6 +33,13 @@ def _spend_reset_data(budget_reset_at: datetime | None, spend_decrement: float | return {"spend": spend, "budget_reset_at": budget_reset_at} # mutable-ok: prisma update payload must be a dict +def _duration_quarantine_data() -> Mapping[str, object]: + return { + "budget_duration": None, + "budget_reset_at": None, + } # mutable-ok: prisma update payload must be a dict + + @dataclass(frozen=True, slots=True) class KeySpendResetWrites: table: BatchTable @@ -45,6 +52,12 @@ class KeySpendResetWrites: data=_spend_reset_data(budget_reset_at, spend_decrement), ) + def queue_duration_quarantine(self, token: str) -> None: + self.table.update( + where={"token": token}, # mutable-ok: prisma where filter must be a dict + data=_duration_quarantine_data(), + ) + @dataclass(frozen=True, slots=True) class UserSpendResetWrites: @@ -58,6 +71,12 @@ class UserSpendResetWrites: data=_spend_reset_data(budget_reset_at, spend_decrement), ) + def queue_duration_quarantine(self, user_id: str) -> None: + self.table.update( + where={"user_id": user_id}, # mutable-ok: prisma where filter must be a dict + data=_duration_quarantine_data(), + ) + @dataclass(frozen=True, slots=True) class TeamSpendResetWrites: @@ -71,6 +90,12 @@ class TeamSpendResetWrites: data=_spend_reset_data(budget_reset_at, spend_decrement), ) + def queue_duration_quarantine(self, team_id: str) -> None: + self.table.update( + where={"team_id": team_id}, # mutable-ok: prisma where filter must be a dict + data=_duration_quarantine_data(), + ) + @dataclass(frozen=True, slots=True) class LinkedSpendResetWrites: @@ -97,6 +122,9 @@ class BudgetWindowWrites: no-op row count instead of a P2025 that aborts the whole chunk.""" self.table.update_many(where={"budget_id": budget_id}, data={"budget_reset_at": budget_reset_at}) + def queue_window_quarantine(self, budget_id: str) -> None: + self.table.update_many(where={"budget_id": budget_id}, data=_duration_quarantine_data()) + @dataclass(frozen=True, slots=True) class SpendResetUnitOfWork: diff --git a/tests/test_litellm/litellm_core_utils/test_duration_parser.py b/tests/test_litellm/litellm_core_utils/test_duration_parser.py index 94c429530ee..19ed5bc15ad 100644 --- a/tests/test_litellm/litellm_core_utils/test_duration_parser.py +++ b/tests/test_litellm/litellm_core_utils/test_duration_parser.py @@ -376,7 +376,7 @@ class TestWordFormBudgetDurations(unittest.TestCase): def test_unparseable_duration_raises_instead_of_midnight_fallback(self): base_time = datetime(2023, 5, 15, 15, 0, 0, tzinfo=timezone.utc) - for duration in ("", " ", "garbage", "1x"): + for duration in ("", " ", "garbage", "1x", "1dinvalid"): with self.assertRaises(ValueError) as ctx: get_next_standardized_reset_time(duration, base_time, "UTC") self.assertIn("Invalid budget_duration", str(ctx.exception)) diff --git a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py index 560953f0b51..24765ca6ed2 100644 --- a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py +++ b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py @@ -1158,6 +1158,36 @@ def test_reset_budget_windows_handles_string_budget_limits(monkeypatch): prisma_client.db.litellm_verificationtoken.update.assert_awaited_once() +def test_reset_budget_windows_clears_unparseable_duration_and_still_resets_later_rows(monkeypatch): + """An expired window with a duration the new parser rejects must not raise + and stall the keyset walk. Clearing reset_at persists so the next tick + skips it and later keys still reset.""" + now = datetime.utcnow() + expired = (now - timedelta(minutes=5)).isoformat() + "Z" + key_rows = [ + {"token": "sk-bad", "budget_limits": [{"budget_duration": "1dinvalid", "reset_at": expired}]}, + {"token": "sk-good", "budget_limits": [{"budget_duration": "1d", "reset_at": expired}]}, + ] + job, prisma_client, spend_counter_cache = _make_reset_budget_windows_job( + monkeypatch, key_rows=key_rows, team_rows=[] + ) + + asyncio.run(job.reset_budget_windows()) + + assert prisma_client.db.litellm_verificationtoken.update.await_count == 2 + first, second = prisma_client.db.litellm_verificationtoken.update.await_args_list + assert first.kwargs["where"] == {"token": "sk-bad"} + bad_windows = json.loads(first.kwargs["data"]["budget_limits"]) + assert bad_windows[0]["reset_at"] is None + assert second.kwargs["where"] == {"token": "sk-good"} + good_windows = json.loads(second.kwargs["data"]["budget_limits"]) + new_reset_at = datetime.fromisoformat(good_windows[0]["reset_at"].replace("Z", "+00:00")).replace(tzinfo=None) + assert new_reset_at > now + spend_counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:key:sk-good:window:1d", value=0.0) + touched_keys = [call.kwargs.get("key") for call in spend_counter_cache.in_memory_cache.set_cache.call_args_list] + assert "spend:key:sk-bad:window:1dinvalid" not in touched_keys + + def test_reset_budget_windows_skips_row_with_empty_budget_limits(monkeypatch): """A row whose `budget_limits` comes back as an empty/falsy payload (shouldn't happen given the WHERE filter, but we guard anyway) must not @@ -2179,6 +2209,92 @@ def test_budget_table_reset_stops_when_the_new_window_is_not_in_the_future(monke assert len(_batch_writes(client, "budget", op="update_many")) == 2 +_QUARANTINE_DATA = {"budget_duration": None, "budget_reset_at": None} + + +@pytest.mark.parametrize( + "phase, table_name, row_factory, id_field", + [ + ("reset_budget_for_litellm_keys", "key", _key_row, "token"), + ("reset_budget_for_litellm_users", "user", _user_row, "user_id"), + ("reset_budget_for_litellm_teams", "team", _team_row, "team_id"), + ], + ids=["keys", "users", "teams"], +) +def test_reset_quarantines_unparseable_budget_duration_and_keeps_paging( + monkeypatch, phase, table_name, row_factory, id_field +): + """A due row whose duration the new parser rejects used to raise, stay due, + and refill every batch. Clearing duration and reset_at takes it out of the + due query so later tenants can reset.""" + monkeypatch.setattr(reset_budget_job_module, "RESET_BUDGET_JOB_BATCH_SIZE", 2) + client, job = _chunked_job( + { + table_name: [ + [row_factory("bad1", budget_duration="1dinvalid"), row_factory("bad2", budget_duration="garbage")], + [row_factory("good1"), row_factory("good2")], + ] + } + ) + + asyncio.run(getattr(job, phase)()) + + assert client.fetches_by_table[table_name] == 2 + writes = _batch_writes(client, table_name, op="update") + quarantined = [w for w in writes if w["data"] == _QUARANTINE_DATA] + reset = [w for w in writes if "spend" in w["data"]] + assert [w["where"][id_field] for w in quarantined] == ["bad1", "bad2"] + assert [w["where"][id_field] for w in reset] == ["good1", "good2"] + assert all("spend" not in w["data"] for w in quarantined) + + +def test_budget_table_reset_quarantines_unparseable_durations_and_keeps_paging(monkeypatch): + """The cascade used to raise while building the next window, commit nothing, + and leave the invalid tiers at the head of every later fetch.""" + monkeypatch.setattr(reset_budget_job_module, "RESET_BUDGET_JOB_BATCH_SIZE", 2) + client, job = _chunked_job( + { + "budget": [ + [_budget_row("bad1", budget_duration="1dinvalid"), _budget_row("bad2", budget_duration="garbage")], + [_budget_row("good1"), _budget_row("good2")], + ] + } + ) + + asyncio.run(job.reset_budget_for_litellm_budget_table()) + + assert client.fetches_by_table["budget"] == 2 + writes = _batch_writes(client, "budget", op="update_many") + quarantined = [w for w in writes if w["data"] == _QUARANTINE_DATA] + advanced = [w for w in writes if w["data"] != _QUARANTINE_DATA] + assert [w["where"]["budget_id"] for w in quarantined] == ["bad1", "bad2"] + assert [w["where"]["budget_id"] for w in advanced] == ["good1", "good2"] + assert all("spend" not in w["data"] for w in quarantined) + + +def test_budget_table_reset_does_not_sweep_dependents_of_an_unparseable_tier( + reset_budget_job, mock_prisma_client, monkeypatch +): + """Quarantine is not a scheduled reset: linked spend stays put.""" + _make_counter_invalidation_job(monkeypatch) + mock_prisma_client.data["budget"] = [_budget_row(budget_id="bad-tier", budget_duration="1dinvalid")] + mock_prisma_client.db.litellm_teammembership.set_find_many_results( + [type("Membership", (), {"user_id": "alice", "team_id": "team-x", "budget_id": "bad-tier", "spend": 9.0})()] + ) + + asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table()) + + assert _batch_writes(mock_prisma_client, "team_membership") == [] + assert _batch_writes(mock_prisma_client, "budget", op="update_many") == [ + { + "table": "budget", + "op": "update_many", + "where": {"budget_id": "bad-tier"}, + "data": _QUARANTINE_DATA, + } + ] + + class PoisonRow: """A row the in-memory reset cannot write, like the DataError rows in #27730.""" diff --git a/tests/test_litellm/repositories/test_unit_of_work.py b/tests/test_litellm/repositories/test_unit_of_work.py index 1a76b537e95..c346c4c054d 100644 --- a/tests/test_litellm/repositories/test_unit_of_work.py +++ b/tests/test_litellm/repositories/test_unit_of_work.py @@ -59,6 +59,37 @@ async def test_updates_across_tables_share_one_batch_and_commit_once(): ] +async def test_duration_quarantine_clears_duration_and_reset_without_touching_spend(): + batch = FakeBatch() + + async with spend_reset_unit_of_work(lambda: batch) as uow: + uow.keys.queue_duration_quarantine(token="tok-bad") + uow.users.queue_duration_quarantine(user_id="user-bad") + uow.teams.queue_duration_quarantine(team_id="team-bad") + + cleared = {"budget_duration": None, "budget_reset_at": None} + assert batch.calls == [ + ("litellm_verificationtoken", {"token": "tok-bad"}, cleared), + ("litellm_usertable", {"user_id": "user-bad"}, cleared), + ("litellm_teamtable", {"team_id": "team-bad"}, cleared), + ] + + +async def test_budget_window_quarantine_clears_duration_and_reset_without_touching_spend(): + batch = FakeBatch() + + async with budget_cascade_unit_of_work(lambda: batch) as uow: + uow.budgets.queue_window_quarantine(budget_id="budget-bad") + + assert batch.calls == [ + ( + "litellm_budgettable.update_many", + {"budget_id": "budget-bad"}, + {"budget_duration": None, "budget_reset_at": None}, + ) + ] + + async def test_raising_inside_block_skips_commit(): batch = FakeBatch()