fix(proxy): quarantine unparseable budget_duration rows in the reset job

Malformed durations no longer stay due after a failed parse, so they cannot
fill a reset batch and starve later tenants.

Co-authored-by: Zsanz3 <Zsanz3@users.noreply.github.com>
This commit is contained in:
Cursor Agent 2026-09-16 20:18:51 +00:00
parent c110a3ae31
commit fc739ac14e
No known key found for this signature in database
5 changed files with 294 additions and 35 deletions

View file

@ -258,6 +258,7 @@ class _BudgetCascade:
budgets: tuple[LiteLLM_BudgetTableFull, ...] = ()
budget_ids: tuple[str, ...] = ()
budget_resets: tuple[tuple[str, datetime], ...] = ()
budget_quarantines: tuple[str, ...] = ()
endusers: tuple[_EndUserRow, ...] = ()
counter_resets: tuple[tuple[str, float], ...] = ()
cache_keys: tuple[str, ...] = ()
@ -299,14 +300,70 @@ def _as_utc(moment: datetime) -> datetime:
def _count_advanced(reset_ats: Iterable[object], cutoff: datetime) -> int:
"""How many rows the write actually moved past the due cutoff.
A budget_duration of "0s" (or one the parser cannot read) resolves to the
current time, so the row is written and stays due. Counting it as progress
would re-read the same chunk until the per-run cap on every tick.
A budget_duration of "0s" resolves to the current time, so the row is written
and stays due. Counting it as progress would re-read the same chunk until
the per-run cap on every tick.
"""
utc_cutoff: Final = _as_utc(cutoff)
return sum(1 for reset_at in reset_ats if isinstance(reset_at, datetime) and _as_utc(reset_at) > utc_cutoff)
def _try_compute_budget_reset_at(duration: object, settings: BudgetResetSettings) -> datetime | None:
if not isinstance(duration, str):
return None
try:
return compute_budget_reset_at(budget_duration=duration, settings=settings)
except ValueError:
return None
def _is_duration_quarantine(
item: LiteLLM_TeamTable | LiteLLM_UserTable | LiteLLM_VerificationToken,
) -> bool:
return item.budget_duration is None and item.budget_reset_at is None
def _count_rows_leaving_due_set(
items: Iterable[LiteLLM_TeamTable | LiteLLM_UserTable | LiteLLM_VerificationToken],
cutoff: datetime,
) -> int:
return sum(
1
for item in items
if _is_duration_quarantine(item)
or (isinstance(item.budget_reset_at, datetime) and _as_utc(item.budget_reset_at) > _as_utc(cutoff))
)
@dataclass(frozen=True, slots=True)
class _AdvanceWindow:
budget_id: str
reset_at: datetime
@dataclass(frozen=True, slots=True)
class _QuarantineWindow:
budget_id: str
def _plan_budget_window(
budget: LiteLLM_BudgetTableFull, settings: BudgetResetSettings
) -> _AdvanceWindow | _QuarantineWindow | None:
budget_id: Final = budget.budget_id
duration: Final = budget.budget_duration
if budget_id is None or duration is None:
return None
reset_at: Final = _try_compute_budget_reset_at(duration, settings)
if reset_at is None:
verbose_proxy_logger.warning(
"Reset budget job: clearing unparseable budget_duration %r on budget %s so it leaves the due set",
duration,
budget_id,
)
return _QuarantineWindow(budget_id)
return _AdvanceWindow(budget_id, reset_at)
def _phase_is_drained(outcome: _ChunkOutcome) -> bool:
"""A short chunk means the due rows ran out. A full chunk that advanced
nothing would be re-read unchanged forever, so it ends the phase too and
@ -626,9 +683,15 @@ class ResetBudgetJob:
own schedule via reset_budget_for_litellm_keys(), so sweeping them here
would reset them twice.
"""
budget_ids: Final = tuple(b.budget_id for b in budgets_to_reset if b.budget_id is not None)
window_plans: Final = tuple(_plan_budget_window(b, self.reset_settings) for b in budgets_to_reset)
budget_resets: Final = tuple((p.budget_id, p.reset_at) for p in window_plans if isinstance(p, _AdvanceWindow))
budget_quarantines: Final = tuple(p.budget_id for p in window_plans if isinstance(p, _QuarantineWindow))
quarantine_ids: Final = frozenset(budget_quarantines)
budget_ids: Final = tuple(
b.budget_id for b in budgets_to_reset if b.budget_id is not None and b.budget_id not in quarantine_ids
)
if not budget_ids:
return _EMPTY_CASCADE
return _BudgetCascade(budgets=tuple(budgets_to_reset), budget_quarantines=budget_quarantines)
team_memberships: Final[tuple[_TeamMembershipRow, ...]] = await self._fetch_linked_rows(
table=TeamMembershipRepository(self.prisma_client).table,
@ -659,7 +722,9 @@ class ResetBudgetJob:
{ # mutable-ok: MappingProxyType wraps a one-shot dict comprehension
b.budget_id: cap
for b in budgets_to_reset
if b.budget_id is not None and (cap := _rollover_cap(b.max_budget)) is not None
if b.budget_id is not None
and b.budget_id not in quarantine_ids
and (cap := _rollover_cap(b.max_budget)) is not None
}
if _rollover_enabled()
else {} # mutable-ok: empty sentinel immediately frozen by MappingProxyType
@ -668,14 +733,8 @@ class ResetBudgetJob:
return _BudgetCascade(
budgets=tuple(budgets_to_reset),
budget_ids=budget_ids,
budget_resets=tuple(
(
b.budget_id,
compute_budget_reset_at(budget_duration=b.budget_duration, settings=self.reset_settings),
)
for b in budgets_to_reset
if b.budget_id is not None and b.budget_duration is not None
),
budget_resets=budget_resets,
budget_quarantines=budget_quarantines,
endusers=endusers,
counter_resets=(
*(
@ -710,7 +769,7 @@ class ResetBudgetJob:
batching both means a mid-cascade failure persists nothing and the rows
stay due for the next run.
"""
if not cascade.budget_ids:
if not cascade.budget_ids and not cascade.budget_quarantines:
return
await self._with_db_write_retry(
@ -728,6 +787,8 @@ class ResetBudgetJob:
_queue_enduser_resets(uow.endusers, cascade)
for budget_id, budget_reset_at in cascade.budget_resets:
uow.budgets.queue_window_advance(budget_id=budget_id, budget_reset_at=budget_reset_at)
for budget_id in cascade.budget_quarantines:
uow.budgets.queue_window_quarantine(budget_id=budget_id)
async def _invalidate_budget_cascade_caches(self, cascade: _BudgetCascade) -> None:
for counter_key, new_spend in cascade.counter_resets:
@ -762,7 +823,8 @@ class ResetBudgetJob:
advanced=_count_advanced(
(reset_at for _, reset_at in cascade.budget_resets),
cutoff=datetime.now(timezone.utc),
),
)
+ len(cascade.budget_quarantines),
)
async def reset_budget_for_litellm_budget_table(self) -> None:
@ -863,6 +925,9 @@ class ResetBudgetJob:
for k in updated_keys:
if k.token is None:
continue
if _is_duration_quarantine(k):
uow.keys.queue_duration_quarantine(token=k.token)
continue
uow.keys.queue_spend_reset(
token=k.token,
budget_reset_at=k.budget_reset_at,
@ -885,6 +950,9 @@ class ResetBudgetJob:
async def _write_user_reset_updates_once(self, updated_users: list[LiteLLM_UserTable]) -> None:
async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow:
for u in updated_users:
if _is_duration_quarantine(u):
uow.users.queue_duration_quarantine(user_id=u.user_id)
continue
uow.users.queue_spend_reset(
user_id=u.user_id,
budget_reset_at=u.budget_reset_at,
@ -907,6 +975,9 @@ class ResetBudgetJob:
async def _write_team_reset_updates_once(self, updated_teams: list[LiteLLM_TeamTable]) -> None:
async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow:
for t in updated_teams:
if _is_duration_quarantine(t):
uow.teams.queue_duration_quarantine(team_id=t.team_id)
continue
uow.teams.queue_spend_reset(
team_id=t.team_id,
budget_reset_at=t.budget_reset_at,
@ -985,6 +1056,8 @@ class ResetBudgetJob:
if updated_keys:
await self._write_key_reset_updates(updated_keys=updated_keys)
for k in updated_keys:
if _is_duration_quarantine(k):
continue
token = getattr(k, "token", None)
if token:
await self._invalidate_spend_counter(f"spend:key:{token}", new_spend=k.spend or 0.0)
@ -992,10 +1065,7 @@ class ResetBudgetJob:
end_time = time.time()
outcome: Final = _ChunkOutcome(
fetched=len(keys_to_reset) if keys_to_reset else 0,
advanced=_count_advanced(
(k.budget_reset_at for k in updated_keys),
cutoff=datetime.now(timezone.utc),
),
advanced=_count_rows_leaving_due_set(updated_keys, cutoff=datetime.now(timezone.utc)),
)
if len(failed_keys) > 0:
self._emit_phase_failure(
@ -1090,6 +1160,8 @@ class ResetBudgetJob:
if updated_users:
await self._write_user_reset_updates(updated_users=updated_users)
for u in updated_users:
if _is_duration_quarantine(u):
continue
user_id = getattr(u, "user_id", None)
if user_id:
await self._invalidate_spend_counter(f"spend:user:{user_id}", new_spend=u.spend or 0.0)
@ -1099,10 +1171,7 @@ class ResetBudgetJob:
end_time = time.time()
outcome: Final = _ChunkOutcome(
fetched=len(users_to_reset) if users_to_reset else 0,
advanced=_count_advanced(
(u.budget_reset_at for u in updated_users),
cutoff=datetime.now(timezone.utc),
),
advanced=_count_rows_leaving_due_set(updated_users, cutoff=datetime.now(timezone.utc)),
)
if len(failed_users) > 0:
self._emit_phase_failure(
@ -1199,6 +1268,8 @@ class ResetBudgetJob:
if updated_teams:
await self._write_team_reset_updates(updated_teams=updated_teams)
for t in updated_teams:
if _is_duration_quarantine(t):
continue
team_id = getattr(t, "team_id", None)
if team_id:
await self._invalidate_spend_counter(f"spend:team:{team_id}", new_spend=t.spend or 0.0)
@ -1206,10 +1277,7 @@ class ResetBudgetJob:
end_time = time.time()
outcome: Final = _ChunkOutcome(
fetched=len(teams_to_reset) if teams_to_reset else 0,
advanced=_count_advanced(
(t.budget_reset_at for t in updated_teams),
cutoff=datetime.now(timezone.utc),
),
advanced=_count_rows_leaving_due_set(updated_teams, cutoff=datetime.now(timezone.utc)),
)
if len(failed_teams) > 0:
self._emit_phase_failure(
@ -1277,6 +1345,15 @@ class ResetBudgetJob:
reset_at: Final = datetime.fromisoformat(reset_at_str.replace("Z", "+00:00")).replace(tzinfo=None)
if reset_at > now:
return False
budget_duration: Final = window.get("budget_duration")
next_reset_at: Final = _try_compute_budget_reset_at(budget_duration, reset_settings)
if next_reset_at is None or not isinstance(budget_duration, str):
verbose_proxy_logger.warning(
"Reset budget job: clearing unparseable budget_duration %r on a window so it is not retried",
budget_duration,
)
window["reset_at"] = None
return True
new_value: Final = await ResetBudgetJob._window_carried_spend(window, counter_key, spend_counter_cache)
spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=new_value)
if spend_counter_cache.redis_cache is not None:
@ -1284,8 +1361,6 @@ class ResetBudgetJob:
await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=new_value)
except Exception as redis_err:
verbose_proxy_logger.warning("Failed to reset Redis counter %s: %s", counter_key, redis_err)
budget_duration: Final = window["budget_duration"]
next_reset_at: Final = compute_budget_reset_at(budget_duration=budget_duration, settings=reset_settings)
window["reset_at"] = next_reset_at.isoformat()
await ResetBudgetJob._roll_window_spend_row(
prisma_client=prisma_client,
@ -1462,11 +1537,20 @@ class ResetBudgetJob:
still holds the pre-reset value, admitting requests past the cap.
"""
try:
duration: Final = item.budget_duration
if duration is not None:
reset_at: Final = _try_compute_budget_reset_at(duration, reset_settings)
if reset_at is None:
verbose_proxy_logger.warning(
"Reset budget job: clearing unparseable budget_duration %r on %s so it leaves the due set",
duration,
item_type,
)
item.budget_duration = None
item.budget_reset_at = None
return item
item.budget_reset_at = reset_at
item.spend = _carried_spend(item.spend, _rollover_cap(item.max_budget)) if _rollover_enabled() else 0.0
if hasattr(item, "budget_duration") and item.budget_duration is not None:
item.budget_reset_at = compute_budget_reset_at(
budget_duration=item.budget_duration, settings=reset_settings
)
return item
except Exception as e:
verbose_proxy_logger.exception("Error resetting budget for %s: %s. Item: %s", item_type, e, item)

View file

@ -33,6 +33,13 @@ def _spend_reset_data(budget_reset_at: datetime | None, spend_decrement: float |
return {"spend": spend, "budget_reset_at": budget_reset_at} # mutable-ok: prisma update payload must be a dict
def _duration_quarantine_data() -> Mapping[str, object]:
return {
"budget_duration": None,
"budget_reset_at": None,
} # mutable-ok: prisma update payload must be a dict
@dataclass(frozen=True, slots=True)
class KeySpendResetWrites:
table: BatchTable
@ -45,6 +52,12 @@ class KeySpendResetWrites:
data=_spend_reset_data(budget_reset_at, spend_decrement),
)
def queue_duration_quarantine(self, token: str) -> None:
self.table.update(
where={"token": token}, # mutable-ok: prisma where filter must be a dict
data=_duration_quarantine_data(),
)
@dataclass(frozen=True, slots=True)
class UserSpendResetWrites:
@ -58,6 +71,12 @@ class UserSpendResetWrites:
data=_spend_reset_data(budget_reset_at, spend_decrement),
)
def queue_duration_quarantine(self, user_id: str) -> None:
self.table.update(
where={"user_id": user_id}, # mutable-ok: prisma where filter must be a dict
data=_duration_quarantine_data(),
)
@dataclass(frozen=True, slots=True)
class TeamSpendResetWrites:
@ -71,6 +90,12 @@ class TeamSpendResetWrites:
data=_spend_reset_data(budget_reset_at, spend_decrement),
)
def queue_duration_quarantine(self, team_id: str) -> None:
self.table.update(
where={"team_id": team_id}, # mutable-ok: prisma where filter must be a dict
data=_duration_quarantine_data(),
)
@dataclass(frozen=True, slots=True)
class LinkedSpendResetWrites:
@ -97,6 +122,9 @@ class BudgetWindowWrites:
no-op row count instead of a P2025 that aborts the whole chunk."""
self.table.update_many(where={"budget_id": budget_id}, data={"budget_reset_at": budget_reset_at})
def queue_window_quarantine(self, budget_id: str) -> None:
self.table.update_many(where={"budget_id": budget_id}, data=_duration_quarantine_data())
@dataclass(frozen=True, slots=True)
class SpendResetUnitOfWork:

View file

@ -376,7 +376,7 @@ class TestWordFormBudgetDurations(unittest.TestCase):
def test_unparseable_duration_raises_instead_of_midnight_fallback(self):
base_time = datetime(2023, 5, 15, 15, 0, 0, tzinfo=timezone.utc)
for duration in ("", " ", "garbage", "1x"):
for duration in ("", " ", "garbage", "1x", "1dinvalid"):
with self.assertRaises(ValueError) as ctx:
get_next_standardized_reset_time(duration, base_time, "UTC")
self.assertIn("Invalid budget_duration", str(ctx.exception))

View file

@ -1158,6 +1158,36 @@ def test_reset_budget_windows_handles_string_budget_limits(monkeypatch):
prisma_client.db.litellm_verificationtoken.update.assert_awaited_once()
def test_reset_budget_windows_clears_unparseable_duration_and_still_resets_later_rows(monkeypatch):
"""An expired window with a duration the new parser rejects must not raise
and stall the keyset walk. Clearing reset_at persists so the next tick
skips it and later keys still reset."""
now = datetime.utcnow()
expired = (now - timedelta(minutes=5)).isoformat() + "Z"
key_rows = [
{"token": "sk-bad", "budget_limits": [{"budget_duration": "1dinvalid", "reset_at": expired}]},
{"token": "sk-good", "budget_limits": [{"budget_duration": "1d", "reset_at": expired}]},
]
job, prisma_client, spend_counter_cache = _make_reset_budget_windows_job(
monkeypatch, key_rows=key_rows, team_rows=[]
)
asyncio.run(job.reset_budget_windows())
assert prisma_client.db.litellm_verificationtoken.update.await_count == 2
first, second = prisma_client.db.litellm_verificationtoken.update.await_args_list
assert first.kwargs["where"] == {"token": "sk-bad"}
bad_windows = json.loads(first.kwargs["data"]["budget_limits"])
assert bad_windows[0]["reset_at"] is None
assert second.kwargs["where"] == {"token": "sk-good"}
good_windows = json.loads(second.kwargs["data"]["budget_limits"])
new_reset_at = datetime.fromisoformat(good_windows[0]["reset_at"].replace("Z", "+00:00")).replace(tzinfo=None)
assert new_reset_at > now
spend_counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:key:sk-good:window:1d", value=0.0)
touched_keys = [call.kwargs.get("key") for call in spend_counter_cache.in_memory_cache.set_cache.call_args_list]
assert "spend:key:sk-bad:window:1dinvalid" not in touched_keys
def test_reset_budget_windows_skips_row_with_empty_budget_limits(monkeypatch):
"""A row whose `budget_limits` comes back as an empty/falsy payload
(shouldn't happen given the WHERE filter, but we guard anyway) must not
@ -2179,6 +2209,92 @@ def test_budget_table_reset_stops_when_the_new_window_is_not_in_the_future(monke
assert len(_batch_writes(client, "budget", op="update_many")) == 2
_QUARANTINE_DATA = {"budget_duration": None, "budget_reset_at": None}
@pytest.mark.parametrize(
"phase, table_name, row_factory, id_field",
[
("reset_budget_for_litellm_keys", "key", _key_row, "token"),
("reset_budget_for_litellm_users", "user", _user_row, "user_id"),
("reset_budget_for_litellm_teams", "team", _team_row, "team_id"),
],
ids=["keys", "users", "teams"],
)
def test_reset_quarantines_unparseable_budget_duration_and_keeps_paging(
monkeypatch, phase, table_name, row_factory, id_field
):
"""A due row whose duration the new parser rejects used to raise, stay due,
and refill every batch. Clearing duration and reset_at takes it out of the
due query so later tenants can reset."""
monkeypatch.setattr(reset_budget_job_module, "RESET_BUDGET_JOB_BATCH_SIZE", 2)
client, job = _chunked_job(
{
table_name: [
[row_factory("bad1", budget_duration="1dinvalid"), row_factory("bad2", budget_duration="garbage")],
[row_factory("good1"), row_factory("good2")],
]
}
)
asyncio.run(getattr(job, phase)())
assert client.fetches_by_table[table_name] == 2
writes = _batch_writes(client, table_name, op="update")
quarantined = [w for w in writes if w["data"] == _QUARANTINE_DATA]
reset = [w for w in writes if "spend" in w["data"]]
assert [w["where"][id_field] for w in quarantined] == ["bad1", "bad2"]
assert [w["where"][id_field] for w in reset] == ["good1", "good2"]
assert all("spend" not in w["data"] for w in quarantined)
def test_budget_table_reset_quarantines_unparseable_durations_and_keeps_paging(monkeypatch):
"""The cascade used to raise while building the next window, commit nothing,
and leave the invalid tiers at the head of every later fetch."""
monkeypatch.setattr(reset_budget_job_module, "RESET_BUDGET_JOB_BATCH_SIZE", 2)
client, job = _chunked_job(
{
"budget": [
[_budget_row("bad1", budget_duration="1dinvalid"), _budget_row("bad2", budget_duration="garbage")],
[_budget_row("good1"), _budget_row("good2")],
]
}
)
asyncio.run(job.reset_budget_for_litellm_budget_table())
assert client.fetches_by_table["budget"] == 2
writes = _batch_writes(client, "budget", op="update_many")
quarantined = [w for w in writes if w["data"] == _QUARANTINE_DATA]
advanced = [w for w in writes if w["data"] != _QUARANTINE_DATA]
assert [w["where"]["budget_id"] for w in quarantined] == ["bad1", "bad2"]
assert [w["where"]["budget_id"] for w in advanced] == ["good1", "good2"]
assert all("spend" not in w["data"] for w in quarantined)
def test_budget_table_reset_does_not_sweep_dependents_of_an_unparseable_tier(
reset_budget_job, mock_prisma_client, monkeypatch
):
"""Quarantine is not a scheduled reset: linked spend stays put."""
_make_counter_invalidation_job(monkeypatch)
mock_prisma_client.data["budget"] = [_budget_row(budget_id="bad-tier", budget_duration="1dinvalid")]
mock_prisma_client.db.litellm_teammembership.set_find_many_results(
[type("Membership", (), {"user_id": "alice", "team_id": "team-x", "budget_id": "bad-tier", "spend": 9.0})()]
)
asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table())
assert _batch_writes(mock_prisma_client, "team_membership") == []
assert _batch_writes(mock_prisma_client, "budget", op="update_many") == [
{
"table": "budget",
"op": "update_many",
"where": {"budget_id": "bad-tier"},
"data": _QUARANTINE_DATA,
}
]
class PoisonRow:
"""A row the in-memory reset cannot write, like the DataError rows in #27730."""

View file

@ -59,6 +59,37 @@ async def test_updates_across_tables_share_one_batch_and_commit_once():
]
async def test_duration_quarantine_clears_duration_and_reset_without_touching_spend():
batch = FakeBatch()
async with spend_reset_unit_of_work(lambda: batch) as uow:
uow.keys.queue_duration_quarantine(token="tok-bad")
uow.users.queue_duration_quarantine(user_id="user-bad")
uow.teams.queue_duration_quarantine(team_id="team-bad")
cleared = {"budget_duration": None, "budget_reset_at": None}
assert batch.calls == [
("litellm_verificationtoken", {"token": "tok-bad"}, cleared),
("litellm_usertable", {"user_id": "user-bad"}, cleared),
("litellm_teamtable", {"team_id": "team-bad"}, cleared),
]
async def test_budget_window_quarantine_clears_duration_and_reset_without_touching_spend():
batch = FakeBatch()
async with budget_cascade_unit_of_work(lambda: batch) as uow:
uow.budgets.queue_window_quarantine(budget_id="budget-bad")
assert batch.calls == [
(
"litellm_budgettable.update_many",
{"budget_id": "budget-bad"},
{"budget_duration": None, "budget_reset_at": None},
)
]
async def test_raising_inside_block_skips_commit():
batch = FakeBatch()