mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
perf: batch daily alert checkpoints and index scoped rules
This commit is contained in:
parent
008593cd91
commit
2fe5bad503
3 changed files with 148 additions and 22 deletions
|
|
@ -1,9 +1,9 @@
|
|||
"""Once-daily evaluation, polled hourly by one database-elected worker."""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import date, datetime, time, timedelta, timezone
|
||||
from itertools import groupby, product
|
||||
from itertools import chain, groupby, product
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Final,
|
||||
|
|
@ -22,6 +22,7 @@ from litellm.types.integrations.slack_alerting import AlertType
|
|||
from litellm.types.proxy.usage_alerts import (
|
||||
UsageAlert,
|
||||
UsageAlertChannel,
|
||||
UsageAlertEvaluationStatus,
|
||||
UsageAlertMetric,
|
||||
UsageAlertNotification,
|
||||
UsageAlertObservation,
|
||||
|
|
@ -179,9 +180,9 @@ class UsageAlertJob:
|
|||
async def _rules(self) -> tuple[UsageAlertRule, ...]:
|
||||
pages: Final[list[UsageAlertRule]] = [] # mutable-ok: collect configuration pages
|
||||
offset = 0 # rebind-ok: configuration pagination cursor
|
||||
while page := await self._store.list_enabled_rules(limit=100, offset=offset):
|
||||
while page := await self._store.list_enabled_rules(limit=500, offset=offset):
|
||||
pages.extend(page)
|
||||
if len(page) < 100:
|
||||
if len(page) < 500:
|
||||
return tuple(pages)
|
||||
offset += len(page)
|
||||
return tuple(pages)
|
||||
|
|
@ -192,7 +193,10 @@ class UsageAlertJob:
|
|||
self._defaults_registered = True
|
||||
target: Final = completed_day(now)
|
||||
rules: Final = await self._rules()
|
||||
due: Final = tuple([rule for rule in rules if not await self._store.is_day_complete(rule, target)])
|
||||
complete: Final[set[str]] = set() # mutable-ok: batched completion lookup, bounded by configuration size
|
||||
for offset in range(0, len(rules), 500):
|
||||
complete.update(await self._store.completed_rule_ids(rules[offset : offset + 500], target))
|
||||
due: Final = tuple(rule for rule in rules if rule.id not in complete)
|
||||
for scoped in (tuple(rule for rule in due if rule.scope_type == scope) for scope in _SCOPES):
|
||||
if scoped:
|
||||
await self._scope(scoped, target, rules)
|
||||
|
|
@ -209,37 +213,71 @@ class UsageAlertJob:
|
|||
rule for rule in rules if coverage.after_grace and (rule.direction == "increase" or coverage.hours == 24)
|
||||
)
|
||||
suppressed: Final = {rule.id: _suppressed_entities(rule, enabled) for rule in eligible}
|
||||
for rule in rules:
|
||||
if rule not in eligible:
|
||||
await self._store.record_evaluation_status(rule, target, "insufficient_coverage")
|
||||
eligible_ids: Final = frozenset(rule.id for rule in eligible)
|
||||
await self._record_evaluations(
|
||||
tuple((rule, "insufficient_coverage") for rule in rules if rule.id not in eligible_ids),
|
||||
target,
|
||||
completed=False,
|
||||
)
|
||||
if not eligible:
|
||||
return
|
||||
by_entity: Final = {
|
||||
entity: tuple(group)
|
||||
for entity, group in groupby(
|
||||
sorted(eligible, key=lambda rule: rule.scope_id or ""), lambda rule: rule.scope_id
|
||||
)
|
||||
}
|
||||
evaluated: Final[set[str]] = set() # mutable-ok: at most one entry per configured rule across streaming pages
|
||||
async for rows in self._metrics.batches(scope, target, max(rule.baseline_days for rule in eligible)):
|
||||
evaluated.update(await self._page(eligible, rows, target, enabled))
|
||||
for rule in eligible:
|
||||
await self._store.finish_day(rule, target)
|
||||
await self._store.record_evaluation_status(
|
||||
rule, target, "complete" if rule.id in evaluated or suppressed[rule.id] else "insufficient_history"
|
||||
)
|
||||
evaluated.update(await self._page(by_entity, rows, target, suppressed))
|
||||
await self._record_evaluations(
|
||||
tuple(
|
||||
(rule, "complete" if rule.id in evaluated or suppressed[rule.id] else "insufficient_history")
|
||||
for rule in eligible
|
||||
),
|
||||
target,
|
||||
completed=True,
|
||||
)
|
||||
|
||||
async def _record_evaluations(
|
||||
self,
|
||||
evaluations: tuple[tuple[UsageAlertRule, UsageAlertEvaluationStatus], ...],
|
||||
target: date,
|
||||
*,
|
||||
completed: bool,
|
||||
) -> None:
|
||||
for offset in range(0, len(evaluations), 500):
|
||||
await self._store.record_evaluations(evaluations[offset : offset + 500], target, completed=completed)
|
||||
|
||||
async def _page(
|
||||
self,
|
||||
rules: tuple[UsageAlertRule, ...],
|
||||
rules: Mapping[str | None, tuple[UsageAlertRule, ...]],
|
||||
rows: tuple[DailyAlertMetrics, ...],
|
||||
target: date,
|
||||
enabled: tuple[UsageAlertRule, ...],
|
||||
suppressed: Mapping[str, frozenset[str | None]],
|
||||
) -> tuple[str, ...]:
|
||||
entities: Final = (
|
||||
(None,) if rules[0].scope_type == "deployment" else tuple(dict.fromkeys(row.entity_id for row in rows))
|
||||
)
|
||||
active: Final = await self._store.breached_entities(tuple(rule.id for rule in rules), entities)
|
||||
by_entity: Final = {entity: tuple(group) for entity, group in groupby(rows, lambda row: row.entity_id)}
|
||||
applicable: Final = tuple(chain.from_iterable(rules.get(entity, ()) for entity in (None, *by_entity)))
|
||||
if not applicable:
|
||||
return ()
|
||||
entities: Final = (None,) if applicable[0].scope_type == "deployment" else tuple(by_entity)
|
||||
active: Final[dict[str, frozenset[str | None]]] = {} # mutable-ok: bounded active results for this entity page
|
||||
for offset in range(0, len(applicable), 500):
|
||||
active.update(
|
||||
await self._store.breached_entities(
|
||||
tuple(rule.id for rule in applicable[offset : offset + 500]), entities
|
||||
)
|
||||
)
|
||||
return tuple(
|
||||
[
|
||||
rule.id
|
||||
for rule in rules
|
||||
for rule in applicable
|
||||
if await self._rule_page(
|
||||
rule, rows, target, active.get(rule.id, frozenset()), _suppressed_entities(rule, enabled)
|
||||
rule,
|
||||
rows if rule.scope_id is None else by_entity[rule.scope_id],
|
||||
target,
|
||||
active.get(rule.id, frozenset()),
|
||||
suppressed[rule.id],
|
||||
)
|
||||
]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -558,6 +558,59 @@ class UsageAlertsRepository:
|
|||
self._checkpoint(rule, day),
|
||||
)
|
||||
|
||||
async def completed_rule_ids(self, rules: tuple[UsageAlertRule, ...], day: date) -> frozenset[str]:
|
||||
if len(rules) > 500:
|
||||
raise ValueError("A checkpoint batch must contain at most 500 rules")
|
||||
rows: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python(
|
||||
await self.database.query_raw(
|
||||
"""SELECT input.rule_id AS id FROM jsonb_to_recordset($1::jsonb)
|
||||
AS input(rule_id text, checkpoint_id text)
|
||||
JOIN "LiteLLM_UsageAlertCheckpoint" c ON c.id=input.checkpoint_id WHERE c.completed""",
|
||||
json.dumps(tuple({"rule_id": rule.id, "checkpoint_id": self._checkpoint(rule, day)} for rule in rules)),
|
||||
)
|
||||
)
|
||||
return frozenset(row.id for row in rows)
|
||||
|
||||
async def record_evaluations(
|
||||
self,
|
||||
evaluations: tuple[tuple[UsageAlertRule, UsageAlertEvaluationStatus], ...],
|
||||
day: date,
|
||||
*,
|
||||
completed: bool,
|
||||
) -> None:
|
||||
if not evaluations:
|
||||
return
|
||||
if len(evaluations) > 500:
|
||||
raise ValueError("An evaluation batch must contain at most 500 rules")
|
||||
await self.database.query_raw(
|
||||
"""WITH input AS (
|
||||
SELECT * FROM jsonb_to_recordset($1::jsonb)
|
||||
AS i(rule_id text,checkpoint_id text,revision timestamptz,status text)
|
||||
), checkpoints AS (
|
||||
INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,completed)
|
||||
SELECT checkpoint_id,true FROM input WHERE $3::boolean
|
||||
ON CONFLICT(id) DO NOTHING
|
||||
) UPDATE "LiteLLM_UsageAlertRule" r SET record=r.record || jsonb_build_object(
|
||||
'last_evaluated_day',$2::text,'evaluation_status',input.status)
|
||||
FROM input WHERE r.id=input.rule_id AND NOT r.deleted
|
||||
AND (r.record->>'updated_at')::timestamptz=input.revision
|
||||
AND (r.record->>'last_evaluated_day' IS DISTINCT FROM $2::text
|
||||
OR r.record->>'evaluation_status' IS DISTINCT FROM input.status) RETURNING r.id""",
|
||||
json.dumps(
|
||||
tuple(
|
||||
{
|
||||
"rule_id": rule.id,
|
||||
"checkpoint_id": self._checkpoint(rule, day),
|
||||
"revision": rule.updated_at.isoformat(),
|
||||
"status": status,
|
||||
}
|
||||
for rule, status in evaluations
|
||||
)
|
||||
),
|
||||
day.isoformat(),
|
||||
completed,
|
||||
)
|
||||
|
||||
async def ensure_default_rules(self, rules: tuple[UsageAlertRuleInput, ...]) -> None:
|
||||
for rule in rules:
|
||||
await self._ensure_default_rule(rule)
|
||||
|
|
|
|||
35
tests/integration/database/test_usage_alert_evaluation.py
Normal file
35
tests/integration/database/test_usage_alert_evaluation.py
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
from datetime import date
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.actors import ManagementDatabase
|
||||
|
||||
from litellm.repositories.usage_alerts import UsageAlertsRepository
|
||||
from litellm.types.proxy.usage_alerts import UsageAlertRuleInput
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_batch_checkpoints_preserve_rule_revisions_and_incomplete_status(db: ManagementDatabase) -> None:
|
||||
repository: Final = UsageAlertsRepository.from_prisma(db.prisma)
|
||||
definition: Final = UsageAlertRuleInput(
|
||||
name="Daily evaluation", scope_type="team", scope_id="team-example", metric="spend", direction="increase"
|
||||
)
|
||||
rule: Final = await repository.save_rule(definition, "qa-admin")
|
||||
assert rule is not None
|
||||
day: Final = date(2026, 11, 4)
|
||||
try:
|
||||
assert not await repository.completed_rule_ids((rule,), day)
|
||||
await repository.record_evaluations(((rule, "complete"),), day, completed=True)
|
||||
assert await repository.completed_rule_ids((rule,), day) == frozenset({rule.id})
|
||||
edited: Final = await repository.save_rule(definition.model_copy(update={"multiplier": 4}), "qa-admin", rule)
|
||||
assert edited is not None
|
||||
assert not await repository.completed_rule_ids((edited,), day)
|
||||
await repository.record_evaluations(((rule, "insufficient_coverage"),), day, completed=False)
|
||||
assert await repository.get_rule(rule.id) == edited
|
||||
await repository.record_evaluations(((edited, "insufficient_coverage"),), day, completed=False)
|
||||
incomplete: Final = await repository.get_rule(rule.id)
|
||||
assert incomplete is not None and incomplete.evaluation_status == "insufficient_coverage"
|
||||
assert incomplete.last_evaluated_day == day
|
||||
assert not await repository.completed_rule_ids((edited,), day)
|
||||
finally:
|
||||
await repository.delete_rule(rule.id)
|
||||
Loading…
Add table
Reference in a new issue