fix: preserve effective alert thresholds and current notification scope

This commit is contained in:
moe-berri 2026-11-05 11:57:35 +08:00 • committed by ishaan-j-berri
parent 2de895ffee
commit 008593cd91
4 changed files with 23 additions and 11 deletions

View file

@ -78,9 +78,11 @@ def compare_daily_usage(
if baseline == 0:
if rule.direction == "decrease":
return None
return DailyComparison(sample_days, observed, 0, rule.min_value, observed > rule.min_value)
threshold: Final = baseline * rule.multiplier if rule.direction == "increase" else baseline / rule.multiplier
crossed: Final = observed >= threshold if rule.direction == "increase" else observed <= threshold
return DailyComparison(
sample_days, observed, baseline, threshold, crossed and abs(observed - baseline) >= rule.min_value
return DailyComparison(sample_days, observed, 0, rule.min_value, observed > 0 and observed >= rule.min_value)
threshold: Final = (
max(baseline * rule.multiplier, baseline + rule.min_value)
if rule.direction == "increase"
else min(baseline / rule.multiplier, baseline - rule.min_value)
)
crossed: Final = observed >= threshold if rule.direction == "increase" else observed <= threshold
return DailyComparison(sample_days, observed, baseline, threshold, crossed)

View file

@ -69,7 +69,7 @@ def _has_activity_page(scope: UsageAlertScope, entity: str) -> bool:
if not entity.strip():
return False
if scope == "key":
return entity != "unattributed" and not entity.startswith("sk-")
return entity != "unattributed" and not entity.removeprefix("raw:").startswith("sk-")
if scope == "user":
return entity != "default_user_id"
if scope == "team":
@ -275,6 +275,9 @@ class UsageAlertJob:
await self._notify_incident(pending)
async def _notify_incident(self, pending: UsageAlertNotification) -> None:
current: Final = await self._store.get_rule(pending.rule.id)
if current is None or not current.enabled or current.updated_at != pending.rule.updated_at:
return
failures: Final = tuple(
[
await self._deliver_channel(pending.incident, pending.rule, channel)

View file

@ -680,6 +680,9 @@ class UsageAlertsRepository:
await self.database.query_raw(
"""SELECT i.record,r.record AS rule,i.notified_channels FROM "LiteLLM_UsageAlertIncident" i JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id
WHERE i.status='open' AND r.enabled AND NOT r.deleted
AND i.scope_type=r.scope_type AND i.metric=r.record->>'metric'
AND i.record->>'direction'=r.record->>'direction'
AND (r.scope_id IS NULL OR i.scope_id=r.scope_id)
AND COALESCE((i.record->>'next_notification_attempt')::timestamptz,'-infinity'::timestamptz)<=CURRENT_TIMESTAMP
AND EXISTS (
SELECT 1 FROM jsonb_array_elements_text(r.record->'notification_channels') channel WHERE NOT (channel=ANY(i.notified_channels)))
@ -742,8 +745,12 @@ class UsageAlertsRepository:
) -> Mapping[str, frozenset[str | None]]:
rows: Final = TypeAdapter(tuple[_BreachedEntity, ...]).validate_python(
await self.database.query_raw(
'SELECT DISTINCT rule_id,scope_id FROM "LiteLLM_UsageAlertIncident" '
"WHERE rule_id=ANY($1::text[]) AND scope_key=ANY($2::text[]) AND breached ORDER BY rule_id",
'SELECT DISTINCT i.rule_id,i.scope_id FROM "LiteLLM_UsageAlertIncident" i '
'JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id '
"WHERE i.rule_id=ANY($1::text[]) AND i.scope_key=ANY($2::text[]) AND i.breached "
"AND i.scope_type=r.scope_type AND i.metric=r.record->>'metric' "
"AND i.record->>'direction'=r.record->>'direction' "
"AND (r.scope_id IS NULL OR i.scope_id=r.scope_id) ORDER BY i.rule_id",
rule_ids,
tuple(entity or "" for entity in entity_ids),
)

View file

@ -48,7 +48,7 @@ def test_low_volume_is_insufficient_and_recorded_zero_baseline_uses_absolute_flo
)
floor: Final = rule.model_copy(update={"min_value": 100})
assert compare_daily_usage(floor, (*enough_requests, _row(_DAY, spend=100)), _DAY) == DailyComparison(
3, 100, 0, 100, False
3, 100, 0, 100, True
)
@ -57,8 +57,8 @@ def test_absolute_floor_suppresses_small_changes() -> None:
rule: Final = UsageAlertRuleInput(
name="spike", scope_type="key", metric="spend", direction="increase", min_value=100
)
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) == DailyComparison(3, 100, 20, 60, False)
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 60, False)
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) == DailyComparison(3, 100, 20, 120, False)
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 120, False)
def test_day_is_eligible_only_after_one_hour_utc_grace() -> None: