mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix: preserve effective alert thresholds and current notification scope
This commit is contained in:
parent
2de895ffee
commit
008593cd91
4 changed files with 23 additions and 11 deletions
|
|
@ -78,9 +78,11 @@ def compare_daily_usage(
|
|||
if baseline == 0:
|
||||
if rule.direction == "decrease":
|
||||
return None
|
||||
return DailyComparison(sample_days, observed, 0, rule.min_value, observed > rule.min_value)
|
||||
threshold: Final = baseline * rule.multiplier if rule.direction == "increase" else baseline / rule.multiplier
|
||||
crossed: Final = observed >= threshold if rule.direction == "increase" else observed <= threshold
|
||||
return DailyComparison(
|
||||
sample_days, observed, baseline, threshold, crossed and abs(observed - baseline) >= rule.min_value
|
||||
return DailyComparison(sample_days, observed, 0, rule.min_value, observed > 0 and observed >= rule.min_value)
|
||||
threshold: Final = (
|
||||
max(baseline * rule.multiplier, baseline + rule.min_value)
|
||||
if rule.direction == "increase"
|
||||
else min(baseline / rule.multiplier, baseline - rule.min_value)
|
||||
)
|
||||
crossed: Final = observed >= threshold if rule.direction == "increase" else observed <= threshold
|
||||
return DailyComparison(sample_days, observed, baseline, threshold, crossed)
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ def _has_activity_page(scope: UsageAlertScope, entity: str) -> bool:
|
|||
if not entity.strip():
|
||||
return False
|
||||
if scope == "key":
|
||||
return entity != "unattributed" and not entity.startswith("sk-")
|
||||
return entity != "unattributed" and not entity.removeprefix("raw:").startswith("sk-")
|
||||
if scope == "user":
|
||||
return entity != "default_user_id"
|
||||
if scope == "team":
|
||||
|
|
@ -275,6 +275,9 @@ class UsageAlertJob:
|
|||
await self._notify_incident(pending)
|
||||
|
||||
async def _notify_incident(self, pending: UsageAlertNotification) -> None:
|
||||
current: Final = await self._store.get_rule(pending.rule.id)
|
||||
if current is None or not current.enabled or current.updated_at != pending.rule.updated_at:
|
||||
return
|
||||
failures: Final = tuple(
|
||||
[
|
||||
await self._deliver_channel(pending.incident, pending.rule, channel)
|
||||
|
|
|
|||
|
|
@ -680,6 +680,9 @@ class UsageAlertsRepository:
|
|||
await self.database.query_raw(
|
||||
"""SELECT i.record,r.record AS rule,i.notified_channels FROM "LiteLLM_UsageAlertIncident" i JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id
|
||||
WHERE i.status='open' AND r.enabled AND NOT r.deleted
|
||||
AND i.scope_type=r.scope_type AND i.metric=r.record->>'metric'
|
||||
AND i.record->>'direction'=r.record->>'direction'
|
||||
AND (r.scope_id IS NULL OR i.scope_id=r.scope_id)
|
||||
AND COALESCE((i.record->>'next_notification_attempt')::timestamptz,'-infinity'::timestamptz)<=CURRENT_TIMESTAMP
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM jsonb_array_elements_text(r.record->'notification_channels') channel WHERE NOT (channel=ANY(i.notified_channels)))
|
||||
|
|
@ -742,8 +745,12 @@ class UsageAlertsRepository:
|
|||
) -> Mapping[str, frozenset[str | None]]:
|
||||
rows: Final = TypeAdapter(tuple[_BreachedEntity, ...]).validate_python(
|
||||
await self.database.query_raw(
|
||||
'SELECT DISTINCT rule_id,scope_id FROM "LiteLLM_UsageAlertIncident" '
|
||||
"WHERE rule_id=ANY($1::text[]) AND scope_key=ANY($2::text[]) AND breached ORDER BY rule_id",
|
||||
'SELECT DISTINCT i.rule_id,i.scope_id FROM "LiteLLM_UsageAlertIncident" i '
|
||||
'JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id '
|
||||
"WHERE i.rule_id=ANY($1::text[]) AND i.scope_key=ANY($2::text[]) AND i.breached "
|
||||
"AND i.scope_type=r.scope_type AND i.metric=r.record->>'metric' "
|
||||
"AND i.record->>'direction'=r.record->>'direction' "
|
||||
"AND (r.scope_id IS NULL OR i.scope_id=r.scope_id) ORDER BY i.rule_id",
|
||||
rule_ids,
|
||||
tuple(entity or "" for entity in entity_ids),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ def test_low_volume_is_insufficient_and_recorded_zero_baseline_uses_absolute_flo
|
|||
)
|
||||
floor: Final = rule.model_copy(update={"min_value": 100})
|
||||
assert compare_daily_usage(floor, (*enough_requests, _row(_DAY, spend=100)), _DAY) == DailyComparison(
|
||||
3, 100, 0, 100, False
|
||||
3, 100, 0, 100, True
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -57,8 +57,8 @@ def test_absolute_floor_suppresses_small_changes() -> None:
|
|||
rule: Final = UsageAlertRuleInput(
|
||||
name="spike", scope_type="key", metric="spend", direction="increase", min_value=100
|
||||
)
|
||||
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) == DailyComparison(3, 100, 20, 60, False)
|
||||
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 60, False)
|
||||
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) == DailyComparison(3, 100, 20, 120, False)
|
||||
assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 120, False)
|
||||
|
||||
|
||||
def test_day_is_eligible_only_after_one_hour_utc_grace() -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue