mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(proxy): quarantine unparseable budget_duration rows in the reset job
Malformed durations no longer stay due after a failed parse, so they cannot fill a reset batch and starve later tenants. Co-authored-by: Zsanz3 <Zsanz3@users.noreply.github.com>
This commit is contained in:
parent
c110a3ae31
commit
fc739ac14e
5 changed files with 294 additions and 35 deletions
|
|
@ -258,6 +258,7 @@ class _BudgetCascade:
|
|||
budgets: tuple[LiteLLM_BudgetTableFull, ...] = ()
|
||||
budget_ids: tuple[str, ...] = ()
|
||||
budget_resets: tuple[tuple[str, datetime], ...] = ()
|
||||
budget_quarantines: tuple[str, ...] = ()
|
||||
endusers: tuple[_EndUserRow, ...] = ()
|
||||
counter_resets: tuple[tuple[str, float], ...] = ()
|
||||
cache_keys: tuple[str, ...] = ()
|
||||
|
|
@ -299,14 +300,70 @@ def _as_utc(moment: datetime) -> datetime:
|
|||
def _count_advanced(reset_ats: Iterable[object], cutoff: datetime) -> int:
|
||||
"""How many rows the write actually moved past the due cutoff.
|
||||
|
||||
A budget_duration of "0s" (or one the parser cannot read) resolves to the
|
||||
current time, so the row is written and stays due. Counting it as progress
|
||||
would re-read the same chunk until the per-run cap on every tick.
|
||||
A budget_duration of "0s" resolves to the current time, so the row is written
|
||||
and stays due. Counting it as progress would re-read the same chunk until
|
||||
the per-run cap on every tick.
|
||||
"""
|
||||
utc_cutoff: Final = _as_utc(cutoff)
|
||||
return sum(1 for reset_at in reset_ats if isinstance(reset_at, datetime) and _as_utc(reset_at) > utc_cutoff)
|
||||
|
||||
|
||||
def _try_compute_budget_reset_at(duration: object, settings: BudgetResetSettings) -> datetime | None:
|
||||
if not isinstance(duration, str):
|
||||
return None
|
||||
try:
|
||||
return compute_budget_reset_at(budget_duration=duration, settings=settings)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _is_duration_quarantine(
|
||||
item: LiteLLM_TeamTable | LiteLLM_UserTable | LiteLLM_VerificationToken,
|
||||
) -> bool:
|
||||
return item.budget_duration is None and item.budget_reset_at is None
|
||||
|
||||
|
||||
def _count_rows_leaving_due_set(
|
||||
items: Iterable[LiteLLM_TeamTable | LiteLLM_UserTable | LiteLLM_VerificationToken],
|
||||
cutoff: datetime,
|
||||
) -> int:
|
||||
return sum(
|
||||
1
|
||||
for item in items
|
||||
if _is_duration_quarantine(item)
|
||||
or (isinstance(item.budget_reset_at, datetime) and _as_utc(item.budget_reset_at) > _as_utc(cutoff))
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _AdvanceWindow:
|
||||
budget_id: str
|
||||
reset_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _QuarantineWindow:
|
||||
budget_id: str
|
||||
|
||||
|
||||
def _plan_budget_window(
|
||||
budget: LiteLLM_BudgetTableFull, settings: BudgetResetSettings
|
||||
) -> _AdvanceWindow | _QuarantineWindow | None:
|
||||
budget_id: Final = budget.budget_id
|
||||
duration: Final = budget.budget_duration
|
||||
if budget_id is None or duration is None:
|
||||
return None
|
||||
reset_at: Final = _try_compute_budget_reset_at(duration, settings)
|
||||
if reset_at is None:
|
||||
verbose_proxy_logger.warning(
|
||||
"Reset budget job: clearing unparseable budget_duration %r on budget %s so it leaves the due set",
|
||||
duration,
|
||||
budget_id,
|
||||
)
|
||||
return _QuarantineWindow(budget_id)
|
||||
return _AdvanceWindow(budget_id, reset_at)
|
||||
|
||||
|
||||
def _phase_is_drained(outcome: _ChunkOutcome) -> bool:
|
||||
"""A short chunk means the due rows ran out. A full chunk that advanced
|
||||
nothing would be re-read unchanged forever, so it ends the phase too and
|
||||
|
|
@ -626,9 +683,15 @@ class ResetBudgetJob:
|
|||
own schedule via reset_budget_for_litellm_keys(), so sweeping them here
|
||||
would reset them twice.
|
||||
"""
|
||||
budget_ids: Final = tuple(b.budget_id for b in budgets_to_reset if b.budget_id is not None)
|
||||
window_plans: Final = tuple(_plan_budget_window(b, self.reset_settings) for b in budgets_to_reset)
|
||||
budget_resets: Final = tuple((p.budget_id, p.reset_at) for p in window_plans if isinstance(p, _AdvanceWindow))
|
||||
budget_quarantines: Final = tuple(p.budget_id for p in window_plans if isinstance(p, _QuarantineWindow))
|
||||
quarantine_ids: Final = frozenset(budget_quarantines)
|
||||
budget_ids: Final = tuple(
|
||||
b.budget_id for b in budgets_to_reset if b.budget_id is not None and b.budget_id not in quarantine_ids
|
||||
)
|
||||
if not budget_ids:
|
||||
return _EMPTY_CASCADE
|
||||
return _BudgetCascade(budgets=tuple(budgets_to_reset), budget_quarantines=budget_quarantines)
|
||||
|
||||
team_memberships: Final[tuple[_TeamMembershipRow, ...]] = await self._fetch_linked_rows(
|
||||
table=TeamMembershipRepository(self.prisma_client).table,
|
||||
|
|
@ -659,7 +722,9 @@ class ResetBudgetJob:
|
|||
{ # mutable-ok: MappingProxyType wraps a one-shot dict comprehension
|
||||
b.budget_id: cap
|
||||
for b in budgets_to_reset
|
||||
if b.budget_id is not None and (cap := _rollover_cap(b.max_budget)) is not None
|
||||
if b.budget_id is not None
|
||||
and b.budget_id not in quarantine_ids
|
||||
and (cap := _rollover_cap(b.max_budget)) is not None
|
||||
}
|
||||
if _rollover_enabled()
|
||||
else {} # mutable-ok: empty sentinel immediately frozen by MappingProxyType
|
||||
|
|
@ -668,14 +733,8 @@ class ResetBudgetJob:
|
|||
return _BudgetCascade(
|
||||
budgets=tuple(budgets_to_reset),
|
||||
budget_ids=budget_ids,
|
||||
budget_resets=tuple(
|
||||
(
|
||||
b.budget_id,
|
||||
compute_budget_reset_at(budget_duration=b.budget_duration, settings=self.reset_settings),
|
||||
)
|
||||
for b in budgets_to_reset
|
||||
if b.budget_id is not None and b.budget_duration is not None
|
||||
),
|
||||
budget_resets=budget_resets,
|
||||
budget_quarantines=budget_quarantines,
|
||||
endusers=endusers,
|
||||
counter_resets=(
|
||||
*(
|
||||
|
|
@ -710,7 +769,7 @@ class ResetBudgetJob:
|
|||
batching both means a mid-cascade failure persists nothing and the rows
|
||||
stay due for the next run.
|
||||
"""
|
||||
if not cascade.budget_ids:
|
||||
if not cascade.budget_ids and not cascade.budget_quarantines:
|
||||
return
|
||||
|
||||
await self._with_db_write_retry(
|
||||
|
|
@ -728,6 +787,8 @@ class ResetBudgetJob:
|
|||
_queue_enduser_resets(uow.endusers, cascade)
|
||||
for budget_id, budget_reset_at in cascade.budget_resets:
|
||||
uow.budgets.queue_window_advance(budget_id=budget_id, budget_reset_at=budget_reset_at)
|
||||
for budget_id in cascade.budget_quarantines:
|
||||
uow.budgets.queue_window_quarantine(budget_id=budget_id)
|
||||
|
||||
async def _invalidate_budget_cascade_caches(self, cascade: _BudgetCascade) -> None:
|
||||
for counter_key, new_spend in cascade.counter_resets:
|
||||
|
|
@ -762,7 +823,8 @@ class ResetBudgetJob:
|
|||
advanced=_count_advanced(
|
||||
(reset_at for _, reset_at in cascade.budget_resets),
|
||||
cutoff=datetime.now(timezone.utc),
|
||||
),
|
||||
)
|
||||
+ len(cascade.budget_quarantines),
|
||||
)
|
||||
|
||||
async def reset_budget_for_litellm_budget_table(self) -> None:
|
||||
|
|
@ -863,6 +925,9 @@ class ResetBudgetJob:
|
|||
for k in updated_keys:
|
||||
if k.token is None:
|
||||
continue
|
||||
if _is_duration_quarantine(k):
|
||||
uow.keys.queue_duration_quarantine(token=k.token)
|
||||
continue
|
||||
uow.keys.queue_spend_reset(
|
||||
token=k.token,
|
||||
budget_reset_at=k.budget_reset_at,
|
||||
|
|
@ -885,6 +950,9 @@ class ResetBudgetJob:
|
|||
async def _write_user_reset_updates_once(self, updated_users: list[LiteLLM_UserTable]) -> None:
|
||||
async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow:
|
||||
for u in updated_users:
|
||||
if _is_duration_quarantine(u):
|
||||
uow.users.queue_duration_quarantine(user_id=u.user_id)
|
||||
continue
|
||||
uow.users.queue_spend_reset(
|
||||
user_id=u.user_id,
|
||||
budget_reset_at=u.budget_reset_at,
|
||||
|
|
@ -907,6 +975,9 @@ class ResetBudgetJob:
|
|||
async def _write_team_reset_updates_once(self, updated_teams: list[LiteLLM_TeamTable]) -> None:
|
||||
async with spend_reset_unit_of_work(self.prisma_client.db.batch_) as uow:
|
||||
for t in updated_teams:
|
||||
if _is_duration_quarantine(t):
|
||||
uow.teams.queue_duration_quarantine(team_id=t.team_id)
|
||||
continue
|
||||
uow.teams.queue_spend_reset(
|
||||
team_id=t.team_id,
|
||||
budget_reset_at=t.budget_reset_at,
|
||||
|
|
@ -985,6 +1056,8 @@ class ResetBudgetJob:
|
|||
if updated_keys:
|
||||
await self._write_key_reset_updates(updated_keys=updated_keys)
|
||||
for k in updated_keys:
|
||||
if _is_duration_quarantine(k):
|
||||
continue
|
||||
token = getattr(k, "token", None)
|
||||
if token:
|
||||
await self._invalidate_spend_counter(f"spend:key:{token}", new_spend=k.spend or 0.0)
|
||||
|
|
@ -992,10 +1065,7 @@ class ResetBudgetJob:
|
|||
end_time = time.time()
|
||||
outcome: Final = _ChunkOutcome(
|
||||
fetched=len(keys_to_reset) if keys_to_reset else 0,
|
||||
advanced=_count_advanced(
|
||||
(k.budget_reset_at for k in updated_keys),
|
||||
cutoff=datetime.now(timezone.utc),
|
||||
),
|
||||
advanced=_count_rows_leaving_due_set(updated_keys, cutoff=datetime.now(timezone.utc)),
|
||||
)
|
||||
if len(failed_keys) > 0:
|
||||
self._emit_phase_failure(
|
||||
|
|
@ -1090,6 +1160,8 @@ class ResetBudgetJob:
|
|||
if updated_users:
|
||||
await self._write_user_reset_updates(updated_users=updated_users)
|
||||
for u in updated_users:
|
||||
if _is_duration_quarantine(u):
|
||||
continue
|
||||
user_id = getattr(u, "user_id", None)
|
||||
if user_id:
|
||||
await self._invalidate_spend_counter(f"spend:user:{user_id}", new_spend=u.spend or 0.0)
|
||||
|
|
@ -1099,10 +1171,7 @@ class ResetBudgetJob:
|
|||
end_time = time.time()
|
||||
outcome: Final = _ChunkOutcome(
|
||||
fetched=len(users_to_reset) if users_to_reset else 0,
|
||||
advanced=_count_advanced(
|
||||
(u.budget_reset_at for u in updated_users),
|
||||
cutoff=datetime.now(timezone.utc),
|
||||
),
|
||||
advanced=_count_rows_leaving_due_set(updated_users, cutoff=datetime.now(timezone.utc)),
|
||||
)
|
||||
if len(failed_users) > 0:
|
||||
self._emit_phase_failure(
|
||||
|
|
@ -1199,6 +1268,8 @@ class ResetBudgetJob:
|
|||
if updated_teams:
|
||||
await self._write_team_reset_updates(updated_teams=updated_teams)
|
||||
for t in updated_teams:
|
||||
if _is_duration_quarantine(t):
|
||||
continue
|
||||
team_id = getattr(t, "team_id", None)
|
||||
if team_id:
|
||||
await self._invalidate_spend_counter(f"spend:team:{team_id}", new_spend=t.spend or 0.0)
|
||||
|
|
@ -1206,10 +1277,7 @@ class ResetBudgetJob:
|
|||
end_time = time.time()
|
||||
outcome: Final = _ChunkOutcome(
|
||||
fetched=len(teams_to_reset) if teams_to_reset else 0,
|
||||
advanced=_count_advanced(
|
||||
(t.budget_reset_at for t in updated_teams),
|
||||
cutoff=datetime.now(timezone.utc),
|
||||
),
|
||||
advanced=_count_rows_leaving_due_set(updated_teams, cutoff=datetime.now(timezone.utc)),
|
||||
)
|
||||
if len(failed_teams) > 0:
|
||||
self._emit_phase_failure(
|
||||
|
|
@ -1277,6 +1345,15 @@ class ResetBudgetJob:
|
|||
reset_at: Final = datetime.fromisoformat(reset_at_str.replace("Z", "+00:00")).replace(tzinfo=None)
|
||||
if reset_at > now:
|
||||
return False
|
||||
budget_duration: Final = window.get("budget_duration")
|
||||
next_reset_at: Final = _try_compute_budget_reset_at(budget_duration, reset_settings)
|
||||
if next_reset_at is None or not isinstance(budget_duration, str):
|
||||
verbose_proxy_logger.warning(
|
||||
"Reset budget job: clearing unparseable budget_duration %r on a window so it is not retried",
|
||||
budget_duration,
|
||||
)
|
||||
window["reset_at"] = None
|
||||
return True
|
||||
new_value: Final = await ResetBudgetJob._window_carried_spend(window, counter_key, spend_counter_cache)
|
||||
spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=new_value)
|
||||
if spend_counter_cache.redis_cache is not None:
|
||||
|
|
@ -1284,8 +1361,6 @@ class ResetBudgetJob:
|
|||
await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=new_value)
|
||||
except Exception as redis_err:
|
||||
verbose_proxy_logger.warning("Failed to reset Redis counter %s: %s", counter_key, redis_err)
|
||||
budget_duration: Final = window["budget_duration"]
|
||||
next_reset_at: Final = compute_budget_reset_at(budget_duration=budget_duration, settings=reset_settings)
|
||||
window["reset_at"] = next_reset_at.isoformat()
|
||||
await ResetBudgetJob._roll_window_spend_row(
|
||||
prisma_client=prisma_client,
|
||||
|
|
@ -1462,11 +1537,20 @@ class ResetBudgetJob:
|
|||
still holds the pre-reset value, admitting requests past the cap.
|
||||
"""
|
||||
try:
|
||||
duration: Final = item.budget_duration
|
||||
if duration is not None:
|
||||
reset_at: Final = _try_compute_budget_reset_at(duration, reset_settings)
|
||||
if reset_at is None:
|
||||
verbose_proxy_logger.warning(
|
||||
"Reset budget job: clearing unparseable budget_duration %r on %s so it leaves the due set",
|
||||
duration,
|
||||
item_type,
|
||||
)
|
||||
item.budget_duration = None
|
||||
item.budget_reset_at = None
|
||||
return item
|
||||
item.budget_reset_at = reset_at
|
||||
item.spend = _carried_spend(item.spend, _rollover_cap(item.max_budget)) if _rollover_enabled() else 0.0
|
||||
if hasattr(item, "budget_duration") and item.budget_duration is not None:
|
||||
item.budget_reset_at = compute_budget_reset_at(
|
||||
budget_duration=item.budget_duration, settings=reset_settings
|
||||
)
|
||||
return item
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error resetting budget for %s: %s. Item: %s", item_type, e, item)
|
||||
|
|
|
|||
|
|
@ -33,6 +33,13 @@ def _spend_reset_data(budget_reset_at: datetime | None, spend_decrement: float |
|
|||
return {"spend": spend, "budget_reset_at": budget_reset_at} # mutable-ok: prisma update payload must be a dict
|
||||
|
||||
|
||||
def _duration_quarantine_data() -> Mapping[str, object]:
|
||||
return {
|
||||
"budget_duration": None,
|
||||
"budget_reset_at": None,
|
||||
} # mutable-ok: prisma update payload must be a dict
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class KeySpendResetWrites:
|
||||
table: BatchTable
|
||||
|
|
@ -45,6 +52,12 @@ class KeySpendResetWrites:
|
|||
data=_spend_reset_data(budget_reset_at, spend_decrement),
|
||||
)
|
||||
|
||||
def queue_duration_quarantine(self, token: str) -> None:
|
||||
self.table.update(
|
||||
where={"token": token}, # mutable-ok: prisma where filter must be a dict
|
||||
data=_duration_quarantine_data(),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UserSpendResetWrites:
|
||||
|
|
@ -58,6 +71,12 @@ class UserSpendResetWrites:
|
|||
data=_spend_reset_data(budget_reset_at, spend_decrement),
|
||||
)
|
||||
|
||||
def queue_duration_quarantine(self, user_id: str) -> None:
|
||||
self.table.update(
|
||||
where={"user_id": user_id}, # mutable-ok: prisma where filter must be a dict
|
||||
data=_duration_quarantine_data(),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TeamSpendResetWrites:
|
||||
|
|
@ -71,6 +90,12 @@ class TeamSpendResetWrites:
|
|||
data=_spend_reset_data(budget_reset_at, spend_decrement),
|
||||
)
|
||||
|
||||
def queue_duration_quarantine(self, team_id: str) -> None:
|
||||
self.table.update(
|
||||
where={"team_id": team_id}, # mutable-ok: prisma where filter must be a dict
|
||||
data=_duration_quarantine_data(),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LinkedSpendResetWrites:
|
||||
|
|
@ -97,6 +122,9 @@ class BudgetWindowWrites:
|
|||
no-op row count instead of a P2025 that aborts the whole chunk."""
|
||||
self.table.update_many(where={"budget_id": budget_id}, data={"budget_reset_at": budget_reset_at})
|
||||
|
||||
def queue_window_quarantine(self, budget_id: str) -> None:
|
||||
self.table.update_many(where={"budget_id": budget_id}, data=_duration_quarantine_data())
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SpendResetUnitOfWork:
|
||||
|
|
|
|||
|
|
@ -376,7 +376,7 @@ class TestWordFormBudgetDurations(unittest.TestCase):
|
|||
|
||||
def test_unparseable_duration_raises_instead_of_midnight_fallback(self):
|
||||
base_time = datetime(2023, 5, 15, 15, 0, 0, tzinfo=timezone.utc)
|
||||
for duration in ("", " ", "garbage", "1x"):
|
||||
for duration in ("", " ", "garbage", "1x", "1dinvalid"):
|
||||
with self.assertRaises(ValueError) as ctx:
|
||||
get_next_standardized_reset_time(duration, base_time, "UTC")
|
||||
self.assertIn("Invalid budget_duration", str(ctx.exception))
|
||||
|
|
|
|||
|
|
@ -1158,6 +1158,36 @@ def test_reset_budget_windows_handles_string_budget_limits(monkeypatch):
|
|||
prisma_client.db.litellm_verificationtoken.update.assert_awaited_once()
|
||||
|
||||
|
||||
def test_reset_budget_windows_clears_unparseable_duration_and_still_resets_later_rows(monkeypatch):
|
||||
"""An expired window with a duration the new parser rejects must not raise
|
||||
and stall the keyset walk. Clearing reset_at persists so the next tick
|
||||
skips it and later keys still reset."""
|
||||
now = datetime.utcnow()
|
||||
expired = (now - timedelta(minutes=5)).isoformat() + "Z"
|
||||
key_rows = [
|
||||
{"token": "sk-bad", "budget_limits": [{"budget_duration": "1dinvalid", "reset_at": expired}]},
|
||||
{"token": "sk-good", "budget_limits": [{"budget_duration": "1d", "reset_at": expired}]},
|
||||
]
|
||||
job, prisma_client, spend_counter_cache = _make_reset_budget_windows_job(
|
||||
monkeypatch, key_rows=key_rows, team_rows=[]
|
||||
)
|
||||
|
||||
asyncio.run(job.reset_budget_windows())
|
||||
|
||||
assert prisma_client.db.litellm_verificationtoken.update.await_count == 2
|
||||
first, second = prisma_client.db.litellm_verificationtoken.update.await_args_list
|
||||
assert first.kwargs["where"] == {"token": "sk-bad"}
|
||||
bad_windows = json.loads(first.kwargs["data"]["budget_limits"])
|
||||
assert bad_windows[0]["reset_at"] is None
|
||||
assert second.kwargs["where"] == {"token": "sk-good"}
|
||||
good_windows = json.loads(second.kwargs["data"]["budget_limits"])
|
||||
new_reset_at = datetime.fromisoformat(good_windows[0]["reset_at"].replace("Z", "+00:00")).replace(tzinfo=None)
|
||||
assert new_reset_at > now
|
||||
spend_counter_cache.in_memory_cache.set_cache.assert_any_call(key="spend:key:sk-good:window:1d", value=0.0)
|
||||
touched_keys = [call.kwargs.get("key") for call in spend_counter_cache.in_memory_cache.set_cache.call_args_list]
|
||||
assert "spend:key:sk-bad:window:1dinvalid" not in touched_keys
|
||||
|
||||
|
||||
def test_reset_budget_windows_skips_row_with_empty_budget_limits(monkeypatch):
|
||||
"""A row whose `budget_limits` comes back as an empty/falsy payload
|
||||
(shouldn't happen given the WHERE filter, but we guard anyway) must not
|
||||
|
|
@ -2179,6 +2209,92 @@ def test_budget_table_reset_stops_when_the_new_window_is_not_in_the_future(monke
|
|||
assert len(_batch_writes(client, "budget", op="update_many")) == 2
|
||||
|
||||
|
||||
_QUARANTINE_DATA = {"budget_duration": None, "budget_reset_at": None}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"phase, table_name, row_factory, id_field",
|
||||
[
|
||||
("reset_budget_for_litellm_keys", "key", _key_row, "token"),
|
||||
("reset_budget_for_litellm_users", "user", _user_row, "user_id"),
|
||||
("reset_budget_for_litellm_teams", "team", _team_row, "team_id"),
|
||||
],
|
||||
ids=["keys", "users", "teams"],
|
||||
)
|
||||
def test_reset_quarantines_unparseable_budget_duration_and_keeps_paging(
|
||||
monkeypatch, phase, table_name, row_factory, id_field
|
||||
):
|
||||
"""A due row whose duration the new parser rejects used to raise, stay due,
|
||||
and refill every batch. Clearing duration and reset_at takes it out of the
|
||||
due query so later tenants can reset."""
|
||||
monkeypatch.setattr(reset_budget_job_module, "RESET_BUDGET_JOB_BATCH_SIZE", 2)
|
||||
client, job = _chunked_job(
|
||||
{
|
||||
table_name: [
|
||||
[row_factory("bad1", budget_duration="1dinvalid"), row_factory("bad2", budget_duration="garbage")],
|
||||
[row_factory("good1"), row_factory("good2")],
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
asyncio.run(getattr(job, phase)())
|
||||
|
||||
assert client.fetches_by_table[table_name] == 2
|
||||
writes = _batch_writes(client, table_name, op="update")
|
||||
quarantined = [w for w in writes if w["data"] == _QUARANTINE_DATA]
|
||||
reset = [w for w in writes if "spend" in w["data"]]
|
||||
assert [w["where"][id_field] for w in quarantined] == ["bad1", "bad2"]
|
||||
assert [w["where"][id_field] for w in reset] == ["good1", "good2"]
|
||||
assert all("spend" not in w["data"] for w in quarantined)
|
||||
|
||||
|
||||
def test_budget_table_reset_quarantines_unparseable_durations_and_keeps_paging(monkeypatch):
|
||||
"""The cascade used to raise while building the next window, commit nothing,
|
||||
and leave the invalid tiers at the head of every later fetch."""
|
||||
monkeypatch.setattr(reset_budget_job_module, "RESET_BUDGET_JOB_BATCH_SIZE", 2)
|
||||
client, job = _chunked_job(
|
||||
{
|
||||
"budget": [
|
||||
[_budget_row("bad1", budget_duration="1dinvalid"), _budget_row("bad2", budget_duration="garbage")],
|
||||
[_budget_row("good1"), _budget_row("good2")],
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
asyncio.run(job.reset_budget_for_litellm_budget_table())
|
||||
|
||||
assert client.fetches_by_table["budget"] == 2
|
||||
writes = _batch_writes(client, "budget", op="update_many")
|
||||
quarantined = [w for w in writes if w["data"] == _QUARANTINE_DATA]
|
||||
advanced = [w for w in writes if w["data"] != _QUARANTINE_DATA]
|
||||
assert [w["where"]["budget_id"] for w in quarantined] == ["bad1", "bad2"]
|
||||
assert [w["where"]["budget_id"] for w in advanced] == ["good1", "good2"]
|
||||
assert all("spend" not in w["data"] for w in quarantined)
|
||||
|
||||
|
||||
def test_budget_table_reset_does_not_sweep_dependents_of_an_unparseable_tier(
|
||||
reset_budget_job, mock_prisma_client, monkeypatch
|
||||
):
|
||||
"""Quarantine is not a scheduled reset: linked spend stays put."""
|
||||
_make_counter_invalidation_job(monkeypatch)
|
||||
mock_prisma_client.data["budget"] = [_budget_row(budget_id="bad-tier", budget_duration="1dinvalid")]
|
||||
mock_prisma_client.db.litellm_teammembership.set_find_many_results(
|
||||
[type("Membership", (), {"user_id": "alice", "team_id": "team-x", "budget_id": "bad-tier", "spend": 9.0})()]
|
||||
)
|
||||
|
||||
asyncio.run(reset_budget_job.reset_budget_for_litellm_budget_table())
|
||||
|
||||
assert _batch_writes(mock_prisma_client, "team_membership") == []
|
||||
assert _batch_writes(mock_prisma_client, "budget", op="update_many") == [
|
||||
{
|
||||
"table": "budget",
|
||||
"op": "update_many",
|
||||
"where": {"budget_id": "bad-tier"},
|
||||
"data": _QUARANTINE_DATA,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
class PoisonRow:
|
||||
"""A row the in-memory reset cannot write, like the DataError rows in #27730."""
|
||||
|
||||
|
|
|
|||
|
|
@ -59,6 +59,37 @@ async def test_updates_across_tables_share_one_batch_and_commit_once():
|
|||
]
|
||||
|
||||
|
||||
async def test_duration_quarantine_clears_duration_and_reset_without_touching_spend():
|
||||
batch = FakeBatch()
|
||||
|
||||
async with spend_reset_unit_of_work(lambda: batch) as uow:
|
||||
uow.keys.queue_duration_quarantine(token="tok-bad")
|
||||
uow.users.queue_duration_quarantine(user_id="user-bad")
|
||||
uow.teams.queue_duration_quarantine(team_id="team-bad")
|
||||
|
||||
cleared = {"budget_duration": None, "budget_reset_at": None}
|
||||
assert batch.calls == [
|
||||
("litellm_verificationtoken", {"token": "tok-bad"}, cleared),
|
||||
("litellm_usertable", {"user_id": "user-bad"}, cleared),
|
||||
("litellm_teamtable", {"team_id": "team-bad"}, cleared),
|
||||
]
|
||||
|
||||
|
||||
async def test_budget_window_quarantine_clears_duration_and_reset_without_touching_spend():
|
||||
batch = FakeBatch()
|
||||
|
||||
async with budget_cascade_unit_of_work(lambda: batch) as uow:
|
||||
uow.budgets.queue_window_quarantine(budget_id="budget-bad")
|
||||
|
||||
assert batch.calls == [
|
||||
(
|
||||
"litellm_budgettable.update_many",
|
||||
{"budget_id": "budget-bad"},
|
||||
{"budget_duration": None, "budget_reset_at": None},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
async def test_raising_inside_block_skips_commit():
|
||||
batch = FakeBatch()
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue