From 008593cd91e92d1c34e59bc0170971534cfce1bd Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 11:57:35 +0800 Subject: [PATCH] fix: preserve effective alert thresholds and current notification scope --- litellm/proxy/spend_tracking/daily_usage_alerts.py | 12 +++++++----- litellm/proxy/spend_tracking/usage_alert_job.py | 5 ++++- litellm/repositories/usage_alerts.py | 11 +++++++++-- .../proxy/spend_tracking/test_daily_usage_alerts.py | 6 +++--- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/spend_tracking/daily_usage_alerts.py b/litellm/proxy/spend_tracking/daily_usage_alerts.py index a79adaaedfa..2fe1d3bdffa 100644 --- a/litellm/proxy/spend_tracking/daily_usage_alerts.py +++ b/litellm/proxy/spend_tracking/daily_usage_alerts.py @@ -78,9 +78,11 @@ def compare_daily_usage( if baseline == 0: if rule.direction == "decrease": return None - return DailyComparison(sample_days, observed, 0, rule.min_value, observed > rule.min_value) - threshold: Final = baseline * rule.multiplier if rule.direction == "increase" else baseline / rule.multiplier - crossed: Final = observed >= threshold if rule.direction == "increase" else observed <= threshold - return DailyComparison( - sample_days, observed, baseline, threshold, crossed and abs(observed - baseline) >= rule.min_value + return DailyComparison(sample_days, observed, 0, rule.min_value, observed > 0 and observed >= rule.min_value) + threshold: Final = ( + max(baseline * rule.multiplier, baseline + rule.min_value) + if rule.direction == "increase" + else min(baseline / rule.multiplier, baseline - rule.min_value) ) + crossed: Final = observed >= threshold if rule.direction == "increase" else observed <= threshold + return DailyComparison(sample_days, observed, baseline, threshold, crossed) diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index 18bd2059adb..06ab1544fc0 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -69,7 +69,7 @@ def _has_activity_page(scope: UsageAlertScope, entity: str) -> bool: if not entity.strip(): return False if scope == "key": - return entity != "unattributed" and not entity.startswith("sk-") + return entity != "unattributed" and not entity.removeprefix("raw:").startswith("sk-") if scope == "user": return entity != "default_user_id" if scope == "team": @@ -275,6 +275,9 @@ class UsageAlertJob: await self._notify_incident(pending) async def _notify_incident(self, pending: UsageAlertNotification) -> None: + current: Final = await self._store.get_rule(pending.rule.id) + if current is None or not current.enabled or current.updated_at != pending.rule.updated_at: + return failures: Final = tuple( [ await self._deliver_channel(pending.incident, pending.rule, channel) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index dcab3eb7cac..deffb85429e 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -680,6 +680,9 @@ class UsageAlertsRepository: await self.database.query_raw( """SELECT i.record,r.record AS rule,i.notified_channels FROM "LiteLLM_UsageAlertIncident" i JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id WHERE i.status='open' AND r.enabled AND NOT r.deleted + AND i.scope_type=r.scope_type AND i.metric=r.record->>'metric' + AND i.record->>'direction'=r.record->>'direction' + AND (r.scope_id IS NULL OR i.scope_id=r.scope_id) AND COALESCE((i.record->>'next_notification_attempt')::timestamptz,'-infinity'::timestamptz)<=CURRENT_TIMESTAMP AND EXISTS ( SELECT 1 FROM jsonb_array_elements_text(r.record->'notification_channels') channel WHERE NOT (channel=ANY(i.notified_channels))) @@ -742,8 +745,12 @@ class UsageAlertsRepository: ) -> Mapping[str, frozenset[str | None]]: rows: Final = TypeAdapter(tuple[_BreachedEntity, ...]).validate_python( await self.database.query_raw( - 'SELECT DISTINCT rule_id,scope_id FROM "LiteLLM_UsageAlertIncident" ' - "WHERE rule_id=ANY($1::text[]) AND scope_key=ANY($2::text[]) AND breached ORDER BY rule_id", + 'SELECT DISTINCT i.rule_id,i.scope_id FROM "LiteLLM_UsageAlertIncident" i ' + 'JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id ' + "WHERE i.rule_id=ANY($1::text[]) AND i.scope_key=ANY($2::text[]) AND i.breached " + "AND i.scope_type=r.scope_type AND i.metric=r.record->>'metric' " + "AND i.record->>'direction'=r.record->>'direction' " + "AND (r.scope_id IS NULL OR i.scope_id=r.scope_id) ORDER BY i.rule_id", rule_ids, tuple(entity or "" for entity in entity_ids), ) diff --git a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py index 4aa657de48b..2a4c223fc35 100644 --- a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py +++ b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py @@ -48,7 +48,7 @@ def test_low_volume_is_insufficient_and_recorded_zero_baseline_uses_absolute_flo ) floor: Final = rule.model_copy(update={"min_value": 100}) assert compare_daily_usage(floor, (*enough_requests, _row(_DAY, spend=100)), _DAY) == DailyComparison( - 3, 100, 0, 100, False + 3, 100, 0, 100, True ) @@ -57,8 +57,8 @@ def test_absolute_floor_suppresses_small_changes() -> None: rule: Final = UsageAlertRuleInput( name="spike", scope_type="key", metric="spend", direction="increase", min_value=100 ) - assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) == DailyComparison(3, 100, 20, 60, False) - assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 60, False) + assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) == DailyComparison(3, 100, 20, 120, False) + assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 120, False) def test_day_is_eligible_only_after_one_hour_utc_grace() -> None: