diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index 06ab1544fc0..c04eb88ead5 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -1,9 +1,9 @@ """Once-daily evaluation, polled hourly by one database-elected worker.""" import asyncio -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from datetime import date, datetime, time, timedelta, timezone -from itertools import groupby, product +from itertools import chain, groupby, product from typing import ( TYPE_CHECKING, Final, @@ -22,6 +22,7 @@ from litellm.types.integrations.slack_alerting import AlertType from litellm.types.proxy.usage_alerts import ( UsageAlert, UsageAlertChannel, + UsageAlertEvaluationStatus, UsageAlertMetric, UsageAlertNotification, UsageAlertObservation, @@ -179,9 +180,9 @@ class UsageAlertJob: async def _rules(self) -> tuple[UsageAlertRule, ...]: pages: Final[list[UsageAlertRule]] = [] # mutable-ok: collect configuration pages offset = 0 # rebind-ok: configuration pagination cursor - while page := await self._store.list_enabled_rules(limit=100, offset=offset): + while page := await self._store.list_enabled_rules(limit=500, offset=offset): pages.extend(page) - if len(page) < 100: + if len(page) < 500: return tuple(pages) offset += len(page) return tuple(pages) @@ -192,7 +193,10 @@ class UsageAlertJob: self._defaults_registered = True target: Final = completed_day(now) rules: Final = await self._rules() - due: Final = tuple([rule for rule in rules if not await self._store.is_day_complete(rule, target)]) + complete: Final[set[str]] = set() # mutable-ok: batched completion lookup, bounded by configuration size + for offset in range(0, len(rules), 500): + complete.update(await self._store.completed_rule_ids(rules[offset : offset + 500], target)) + due: Final = tuple(rule for rule in rules if rule.id not in complete) for scoped in (tuple(rule for rule in due if rule.scope_type == scope) for scope in _SCOPES): if scoped: await self._scope(scoped, target, rules) @@ -209,37 +213,71 @@ class UsageAlertJob: rule for rule in rules if coverage.after_grace and (rule.direction == "increase" or coverage.hours == 24) ) suppressed: Final = {rule.id: _suppressed_entities(rule, enabled) for rule in eligible} - for rule in rules: - if rule not in eligible: - await self._store.record_evaluation_status(rule, target, "insufficient_coverage") + eligible_ids: Final = frozenset(rule.id for rule in eligible) + await self._record_evaluations( + tuple((rule, "insufficient_coverage") for rule in rules if rule.id not in eligible_ids), + target, + completed=False, + ) if not eligible: return + by_entity: Final = { + entity: tuple(group) + for entity, group in groupby( + sorted(eligible, key=lambda rule: rule.scope_id or ""), lambda rule: rule.scope_id + ) + } evaluated: Final[set[str]] = set() # mutable-ok: at most one entry per configured rule across streaming pages async for rows in self._metrics.batches(scope, target, max(rule.baseline_days for rule in eligible)): - evaluated.update(await self._page(eligible, rows, target, enabled)) - for rule in eligible: - await self._store.finish_day(rule, target) - await self._store.record_evaluation_status( - rule, target, "complete" if rule.id in evaluated or suppressed[rule.id] else "insufficient_history" - ) + evaluated.update(await self._page(by_entity, rows, target, suppressed)) + await self._record_evaluations( + tuple( + (rule, "complete" if rule.id in evaluated or suppressed[rule.id] else "insufficient_history") + for rule in eligible + ), + target, + completed=True, + ) + + async def _record_evaluations( + self, + evaluations: tuple[tuple[UsageAlertRule, UsageAlertEvaluationStatus], ...], + target: date, + *, + completed: bool, + ) -> None: + for offset in range(0, len(evaluations), 500): + await self._store.record_evaluations(evaluations[offset : offset + 500], target, completed=completed) async def _page( self, - rules: tuple[UsageAlertRule, ...], + rules: Mapping[str | None, tuple[UsageAlertRule, ...]], rows: tuple[DailyAlertMetrics, ...], target: date, - enabled: tuple[UsageAlertRule, ...], + suppressed: Mapping[str, frozenset[str | None]], ) -> tuple[str, ...]: - entities: Final = ( - (None,) if rules[0].scope_type == "deployment" else tuple(dict.fromkeys(row.entity_id for row in rows)) - ) - active: Final = await self._store.breached_entities(tuple(rule.id for rule in rules), entities) + by_entity: Final = {entity: tuple(group) for entity, group in groupby(rows, lambda row: row.entity_id)} + applicable: Final = tuple(chain.from_iterable(rules.get(entity, ()) for entity in (None, *by_entity))) + if not applicable: + return () + entities: Final = (None,) if applicable[0].scope_type == "deployment" else tuple(by_entity) + active: Final[dict[str, frozenset[str | None]]] = {} # mutable-ok: bounded active results for this entity page + for offset in range(0, len(applicable), 500): + active.update( + await self._store.breached_entities( + tuple(rule.id for rule in applicable[offset : offset + 500]), entities + ) + ) return tuple( [ rule.id - for rule in rules + for rule in applicable if await self._rule_page( - rule, rows, target, active.get(rule.id, frozenset()), _suppressed_entities(rule, enabled) + rule, + rows if rule.scope_id is None else by_entity[rule.scope_id], + target, + active.get(rule.id, frozenset()), + suppressed[rule.id], ) ] ) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index deffb85429e..76205a20f55 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -558,6 +558,59 @@ class UsageAlertsRepository: self._checkpoint(rule, day), ) + async def completed_rule_ids(self, rules: tuple[UsageAlertRule, ...], day: date) -> frozenset[str]: + if len(rules) > 500: + raise ValueError("A checkpoint batch must contain at most 500 rules") + rows: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python( + await self.database.query_raw( + """SELECT input.rule_id AS id FROM jsonb_to_recordset($1::jsonb) + AS input(rule_id text, checkpoint_id text) + JOIN "LiteLLM_UsageAlertCheckpoint" c ON c.id=input.checkpoint_id WHERE c.completed""", + json.dumps(tuple({"rule_id": rule.id, "checkpoint_id": self._checkpoint(rule, day)} for rule in rules)), + ) + ) + return frozenset(row.id for row in rows) + + async def record_evaluations( + self, + evaluations: tuple[tuple[UsageAlertRule, UsageAlertEvaluationStatus], ...], + day: date, + *, + completed: bool, + ) -> None: + if not evaluations: + return + if len(evaluations) > 500: + raise ValueError("An evaluation batch must contain at most 500 rules") + await self.database.query_raw( + """WITH input AS ( + SELECT * FROM jsonb_to_recordset($1::jsonb) + AS i(rule_id text,checkpoint_id text,revision timestamptz,status text) + ), checkpoints AS ( + INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,completed) + SELECT checkpoint_id,true FROM input WHERE $3::boolean + ON CONFLICT(id) DO NOTHING + ) UPDATE "LiteLLM_UsageAlertRule" r SET record=r.record || jsonb_build_object( + 'last_evaluated_day',$2::text,'evaluation_status',input.status) + FROM input WHERE r.id=input.rule_id AND NOT r.deleted + AND (r.record->>'updated_at')::timestamptz=input.revision + AND (r.record->>'last_evaluated_day' IS DISTINCT FROM $2::text + OR r.record->>'evaluation_status' IS DISTINCT FROM input.status) RETURNING r.id""", + json.dumps( + tuple( + { + "rule_id": rule.id, + "checkpoint_id": self._checkpoint(rule, day), + "revision": rule.updated_at.isoformat(), + "status": status, + } + for rule, status in evaluations + ) + ), + day.isoformat(), + completed, + ) + async def ensure_default_rules(self, rules: tuple[UsageAlertRuleInput, ...]) -> None: for rule in rules: await self._ensure_default_rule(rule) diff --git a/tests/integration/database/test_usage_alert_evaluation.py b/tests/integration/database/test_usage_alert_evaluation.py new file mode 100644 index 00000000000..b6f5c46bdbb --- /dev/null +++ b/tests/integration/database/test_usage_alert_evaluation.py @@ -0,0 +1,35 @@ +from datetime import date +from typing import Final + +import pytest +from integration._support.actors import ManagementDatabase + +from litellm.repositories.usage_alerts import UsageAlertsRepository +from litellm.types.proxy.usage_alerts import UsageAlertRuleInput + + +@pytest.mark.asyncio(loop_scope="session") +async def test_batch_checkpoints_preserve_rule_revisions_and_incomplete_status(db: ManagementDatabase) -> None: + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + definition: Final = UsageAlertRuleInput( + name="Daily evaluation", scope_type="team", scope_id="team-example", metric="spend", direction="increase" + ) + rule: Final = await repository.save_rule(definition, "qa-admin") + assert rule is not None + day: Final = date(2026, 11, 4) + try: + assert not await repository.completed_rule_ids((rule,), day) + await repository.record_evaluations(((rule, "complete"),), day, completed=True) + assert await repository.completed_rule_ids((rule,), day) == frozenset({rule.id}) + edited: Final = await repository.save_rule(definition.model_copy(update={"multiplier": 4}), "qa-admin", rule) + assert edited is not None + assert not await repository.completed_rule_ids((edited,), day) + await repository.record_evaluations(((rule, "insufficient_coverage"),), day, completed=False) + assert await repository.get_rule(rule.id) == edited + await repository.record_evaluations(((edited, "insufficient_coverage"),), day, completed=False) + incomplete: Final = await repository.get_rule(rule.id) + assert incomplete is not None and incomplete.evaluation_status == "insufficient_coverage" + assert incomplete.last_evaluated_day == day + assert not await repository.completed_rule_ids((edited,), day) + finally: + await repository.delete_rule(rule.id)