perf: batch daily alert checkpoints and index scoped rules

This commit is contained in:
moe-berri 2026-11-05 12:02:29 +08:00 • committed by ishaan-j-berri
parent 008593cd91
commit 2fe5bad503
3 changed files with 148 additions and 22 deletions

View file

@ -1,9 +1,9 @@
"""Once-daily evaluation, polled hourly by one database-elected worker."""
import asyncio
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from datetime import date, datetime, time, timedelta, timezone
from itertools import groupby, product
from itertools import chain, groupby, product
from typing import (
TYPE_CHECKING,
Final,
@ -22,6 +22,7 @@ from litellm.types.integrations.slack_alerting import AlertType
from litellm.types.proxy.usage_alerts import (
UsageAlert,
UsageAlertChannel,
UsageAlertEvaluationStatus,
UsageAlertMetric,
UsageAlertNotification,
UsageAlertObservation,
@ -179,9 +180,9 @@ class UsageAlertJob:
async def _rules(self) -> tuple[UsageAlertRule, ...]:
pages: Final[list[UsageAlertRule]] = [] # mutable-ok: collect configuration pages
offset = 0 # rebind-ok: configuration pagination cursor
while page := await self._store.list_enabled_rules(limit=100, offset=offset):
while page := await self._store.list_enabled_rules(limit=500, offset=offset):
pages.extend(page)
if len(page) < 100:
if len(page) < 500:
return tuple(pages)
offset += len(page)
return tuple(pages)
@ -192,7 +193,10 @@ class UsageAlertJob:
self._defaults_registered = True
target: Final = completed_day(now)
rules: Final = await self._rules()
due: Final = tuple([rule for rule in rules if not await self._store.is_day_complete(rule, target)])
complete: Final[set[str]] = set() # mutable-ok: batched completion lookup, bounded by configuration size
for offset in range(0, len(rules), 500):
complete.update(await self._store.completed_rule_ids(rules[offset : offset + 500], target))
due: Final = tuple(rule for rule in rules if rule.id not in complete)
for scoped in (tuple(rule for rule in due if rule.scope_type == scope) for scope in _SCOPES):
if scoped:
await self._scope(scoped, target, rules)
@ -209,37 +213,71 @@ class UsageAlertJob:
rule for rule in rules if coverage.after_grace and (rule.direction == "increase" or coverage.hours == 24)
)
suppressed: Final = {rule.id: _suppressed_entities(rule, enabled) for rule in eligible}
for rule in rules:
if rule not in eligible:
await self._store.record_evaluation_status(rule, target, "insufficient_coverage")
eligible_ids: Final = frozenset(rule.id for rule in eligible)
await self._record_evaluations(
tuple((rule, "insufficient_coverage") for rule in rules if rule.id not in eligible_ids),
target,
completed=False,
)
if not eligible:
return
by_entity: Final = {
entity: tuple(group)
for entity, group in groupby(
sorted(eligible, key=lambda rule: rule.scope_id or ""), lambda rule: rule.scope_id
)
}
evaluated: Final[set[str]] = set() # mutable-ok: at most one entry per configured rule across streaming pages
async for rows in self._metrics.batches(scope, target, max(rule.baseline_days for rule in eligible)):
evaluated.update(await self._page(eligible, rows, target, enabled))
for rule in eligible:
await self._store.finish_day(rule, target)
await self._store.record_evaluation_status(
rule, target, "complete" if rule.id in evaluated or suppressed[rule.id] else "insufficient_history"
)
evaluated.update(await self._page(by_entity, rows, target, suppressed))
await self._record_evaluations(
tuple(
(rule, "complete" if rule.id in evaluated or suppressed[rule.id] else "insufficient_history")
for rule in eligible
),
target,
completed=True,
)
async def _record_evaluations(
self,
evaluations: tuple[tuple[UsageAlertRule, UsageAlertEvaluationStatus], ...],
target: date,
*,
completed: bool,
) -> None:
for offset in range(0, len(evaluations), 500):
await self._store.record_evaluations(evaluations[offset : offset + 500], target, completed=completed)
async def _page(
self,
rules: tuple[UsageAlertRule, ...],
rules: Mapping[str | None, tuple[UsageAlertRule, ...]],
rows: tuple[DailyAlertMetrics, ...],
target: date,
enabled: tuple[UsageAlertRule, ...],
suppressed: Mapping[str, frozenset[str | None]],
) -> tuple[str, ...]:
entities: Final = (
(None,) if rules[0].scope_type == "deployment" else tuple(dict.fromkeys(row.entity_id for row in rows))
)
active: Final = await self._store.breached_entities(tuple(rule.id for rule in rules), entities)
by_entity: Final = {entity: tuple(group) for entity, group in groupby(rows, lambda row: row.entity_id)}
applicable: Final = tuple(chain.from_iterable(rules.get(entity, ()) for entity in (None, *by_entity)))
if not applicable:
return ()
entities: Final = (None,) if applicable[0].scope_type == "deployment" else tuple(by_entity)
active: Final[dict[str, frozenset[str | None]]] = {} # mutable-ok: bounded active results for this entity page
for offset in range(0, len(applicable), 500):
active.update(
await self._store.breached_entities(
tuple(rule.id for rule in applicable[offset : offset + 500]), entities
)
)
return tuple(
[
rule.id
for rule in rules
for rule in applicable
if await self._rule_page(
rule, rows, target, active.get(rule.id, frozenset()), _suppressed_entities(rule, enabled)
rule,
rows if rule.scope_id is None else by_entity[rule.scope_id],
target,
active.get(rule.id, frozenset()),
suppressed[rule.id],
)
]
)

View file

@ -558,6 +558,59 @@ class UsageAlertsRepository:
self._checkpoint(rule, day),
)
async def completed_rule_ids(self, rules: tuple[UsageAlertRule, ...], day: date) -> frozenset[str]:
if len(rules) > 500:
raise ValueError("A checkpoint batch must contain at most 500 rules")
rows: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python(
await self.database.query_raw(
"""SELECT input.rule_id AS id FROM jsonb_to_recordset($1::jsonb)
AS input(rule_id text, checkpoint_id text)
JOIN "LiteLLM_UsageAlertCheckpoint" c ON c.id=input.checkpoint_id WHERE c.completed""",
json.dumps(tuple({"rule_id": rule.id, "checkpoint_id": self._checkpoint(rule, day)} for rule in rules)),
)
)
return frozenset(row.id for row in rows)
async def record_evaluations(
self,
evaluations: tuple[tuple[UsageAlertRule, UsageAlertEvaluationStatus], ...],
day: date,
*,
completed: bool,
) -> None:
if not evaluations:
return
if len(evaluations) > 500:
raise ValueError("An evaluation batch must contain at most 500 rules")
await self.database.query_raw(
"""WITH input AS (
SELECT * FROM jsonb_to_recordset($1::jsonb)
AS i(rule_id text,checkpoint_id text,revision timestamptz,status text)
), checkpoints AS (
INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,completed)
SELECT checkpoint_id,true FROM input WHERE $3::boolean
ON CONFLICT(id) DO NOTHING
) UPDATE "LiteLLM_UsageAlertRule" r SET record=r.record || jsonb_build_object(
'last_evaluated_day',$2::text,'evaluation_status',input.status)
FROM input WHERE r.id=input.rule_id AND NOT r.deleted
AND (r.record->>'updated_at')::timestamptz=input.revision
AND (r.record->>'last_evaluated_day' IS DISTINCT FROM $2::text
OR r.record->>'evaluation_status' IS DISTINCT FROM input.status) RETURNING r.id""",
json.dumps(
tuple(
{
"rule_id": rule.id,
"checkpoint_id": self._checkpoint(rule, day),
"revision": rule.updated_at.isoformat(),
"status": status,
}
for rule, status in evaluations
)
),
day.isoformat(),
completed,
)
async def ensure_default_rules(self, rules: tuple[UsageAlertRuleInput, ...]) -> None:
for rule in rules:
await self._ensure_default_rule(rule)

View file

@ -0,0 +1,35 @@
from datetime import date
from typing import Final
import pytest
from integration._support.actors import ManagementDatabase
from litellm.repositories.usage_alerts import UsageAlertsRepository
from litellm.types.proxy.usage_alerts import UsageAlertRuleInput
@pytest.mark.asyncio(loop_scope="session")
async def test_batch_checkpoints_preserve_rule_revisions_and_incomplete_status(db: ManagementDatabase) -> None:
repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
definition: Final = UsageAlertRuleInput(
name="Daily evaluation", scope_type="team", scope_id="team-example", metric="spend", direction="increase"
)
rule: Final = await repository.save_rule(definition, "qa-admin")
assert rule is not None
day: Final = date(2026, 11, 4)
try:
assert not await repository.completed_rule_ids((rule,), day)
await repository.record_evaluations(((rule, "complete"),), day, completed=True)
assert await repository.completed_rule_ids((rule,), day) == frozenset({rule.id})
edited: Final = await repository.save_rule(definition.model_copy(update={"multiplier": 4}), "qa-admin", rule)
assert edited is not None
assert not await repository.completed_rule_ids((edited,), day)
await repository.record_evaluations(((rule, "insufficient_coverage"),), day, completed=False)
assert await repository.get_rule(rule.id) == edited
await repository.record_evaluations(((edited, "insufficient_coverage"),), day, completed=False)
incomplete: Final = await repository.get_rule(rule.id)
assert incomplete is not None and incomplete.evaluation_status == "insufficient_coverage"
assert incomplete.last_evaluated_day == day
assert not await repository.completed_rule_ids((edited,), day)
finally:
await repository.delete_rule(rule.id)