From 418685df5576ebf708251b45a7204123b7c861b8 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 17:11:24 +0800 Subject: [PATCH] fix(alerts): require current healthy evidence before recovering resource alerts --- litellm/models/issues.py | 2 +- litellm/repositories/issues.py | 40 ++++++++++-- tests/integration/database/test_issues.py | 80 +++++++++++++++++++++++ 3 files changed, 116 insertions(+), 6 deletions(-) diff --git a/litellm/models/issues.py b/litellm/models/issues.py index 61d3f45f839..6357072453a 100644 --- a/litellm/models/issues.py +++ b/litellm/models/issues.py @@ -91,7 +91,7 @@ class IssueStore(Protocol): scope_ids: tuple[str, ...] | None = None, ) -> tuple[IssueProjectionSignal, ...]: ... - async def recover_issues(self, day: date) -> int: ... + async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int: ... async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Issue, ...]: ... diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index 0d68a957ebd..855b6962f61 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -32,7 +32,7 @@ from litellm.types.proxy.issues import ( IssueSignalInput, IssueSignalList, ) -from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily +from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS, PerformanceIssueFamily from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus if TYPE_CHECKING: @@ -366,26 +366,56 @@ class IssuesRepository: ) ) - async def recover_issues(self, day: date) -> int: + async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int: + if issue_ids == (): + return 0 + if issue_ids is not None and len(issue_ids) > 10: + raise ValueError("Recover at most 10 resource alerts") + selection: Final = "i.id=ANY($3::text[])" if issue_ids is not None else "$3::text[] IS NULL" rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python( await self.database.query_raw( - """WITH healthy AS ( - SELECT i.id FROM "LiteLLM_Issue" i JOIN "LiteLLM_IssueSignal" s ON s.issue_id=i.id + f"""WITH primary_signals AS ( + SELECT i.id,a.id AS source_id,a.breached,a.evaluation_day,a.evaluation_revision, + max(a.evaluation_day) OVER (PARTITION BY i.id) AS latest_observation_day, + r.id AS rule_id,r.record AS rule_record, + COALESCE(r.enabled AND NOT r.deleted + AND r.scope_type=a.scope_type AND r.record->>'metric'=a.metric + AND r.record->>'direction'=a.record->>'direction' + AND (r.scope_id IS NULL OR r.scope_id=a.scope_id) + AND (NOT(r.id=ANY($2::text[])) OR NOT EXISTS ( + SELECT 1 FROM "LiteLLM_UsageAlertRule" custom WHERE custom.enabled AND NOT custom.deleted + AND COALESCE((custom.record->>'overrides_defaults')::boolean,false) + AND NOT(custom.id=ANY($2::text[])) AND custom.scope_type=r.scope_type + AND custom.record->>'metric'=r.record->>'metric' + AND custom.record->>'direction'=r.record->>'direction' + AND (custom.scope_id IS NULL OR custom.scope_id=a.scope_id))),false) AS eligible + FROM "LiteLLM_Issue" i JOIN "LiteLLM_IssueSignal" s ON s.issue_id=i.id LEFT JOIN "LiteLLM_UsageAlertIncident" a ON a.id=s.source_id + LEFT JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id WHERE i.kind='performance' AND i.active AND i.scope_type<>'provider' AND s.source='usage_alert' + AND ({selection}) AND s.record->'usage_alert'->>'scope_type'=i.scope_type AND s.record->'usage_alert'->>'scope_id' IS NOT DISTINCT FROM i.scope_id AND ((i.family='spend' AND s.record->'usage_alert'->>'metric'='spend') OR (i.family='latency' AND s.record->'usage_alert'->>'metric'='latency') OR (i.family='traffic' AND s.record->'usage_alert'->>'metric'='requests') OR (i.family='reliability' AND s.record->'usage_alert'->>'metric' IN ('errors','error_rate'))) - GROUP BY i.id HAVING bool_and(a.id IS NOT NULL AND NOT a.breached AND a.evaluation_day<=$1) + ), observations AS ( + SELECT *,COALESCE(source_id IS NOT NULL AND NOT breached AND evaluation_day<=$1 + AND evaluation_day>=latest_observation_day + AND NULLIF(evaluation_revision,'')::timestamptz=(rule_record->>'updated_at')::timestamptz,false) AS healthy + FROM primary_signals + ), healthy AS ( + SELECT id FROM observations GROUP BY id + HAVING bool_or(eligible AND healthy) AND bool_and(NOT eligible OR healthy) ), changed AS ( UPDATE "LiteLLM_Issue" i SET active=false, status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP WHERE id IN(SELECT id FROM healthy) AND active RETURNING id ) SELECT count(*)::int AS count FROM changed""", day.isoformat(), + tuple(DEFAULT_USAGE_RULE_IDS), + issue_ids, ) ) return rows[0].count diff --git a/tests/integration/database/test_issues.py b/tests/integration/database/test_issues.py index 13bd87eb513..98530a6f3a9 100644 --- a/tests/integration/database/test_issues.py +++ b/tests/integration/database/test_issues.py @@ -969,3 +969,83 @@ async def test_issue_resource_metadata_bounds_labels_and_deleted_history( await _remove_issues(db, (issue.id,)) await _query(db, 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE id=$1', identity) await _query(db, 'DELETE FROM "LiteLLM_DeletedVerificationToken" WHERE token=$1', reporting_key) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_disabled_later_breach_requires_new_healthy_primary_evidence(db: ManagementDatabase) -> None: + store: Final = IssuesRepository.from_prisma(db.prisma) + alerts: Final = UsageAlertsRepository.from_prisma(db.prisma) + issue: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}")) + definition: Final = UsageAlertRuleInput( + name="Primary traffic", + scope_type="user", + scope_id=issue.cohort.scope_id, + metric="requests", + direction="increase", + ) + first: Final = await alerts.save_rule(definition, "qa-admin") + second: Final = await alerts.save_rule(definition.model_copy(update={"name": "Another traffic policy"}), "qa-admin") + assert first is not None and second is not None + breach: Final = UsageAlertObservation( + entity_id=issue.cohort.scope_id, + day=issue.observed_on, + observed=420, + baseline=120, + threshold=360, + breached=True, + ) + try: + sources: Final = tuple([(await alerts.record_observations(rule, (breach,)))[0] for rule in (first, second)]) + await store.upsert_issue(issue) + await store.add_signals( + issue.id, + tuple( + IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source) for source in sources + ), + ) + await alerts.record_observations( + first, + ( + breach.model_copy( + update={ + "day": issue.observed_on + timedelta(days=1), + "observed": 120, + "breached": False, + } + ), + ), + ) + await alerts.record_observations( + second, + ( + breach.model_copy( + update={ + "day": issue.observed_on + timedelta(days=2), + } + ), + ), + ) + assert await alerts.save_rule(definition.model_copy(update={"enabled": False}), "qa-admin", second) + await store.recover_issues(issue.observed_on + timedelta(days=3)) + unresolved: Final = await store.get_issue(issue.id) + assert unresolved is not None and unresolved.active + await alerts.record_observations( + first, + ( + breach.model_copy( + update={ + "day": issue.observed_on + timedelta(days=3), + "observed": 120, + "breached": False, + } + ), + ), + ) + await store.recover_issues(issue.observed_on + timedelta(days=3)) + recovered: Final = await store.get_issue(issue.id) + assert recovered is not None and not recovered.active + finally: + await _remove_issues(db, (issue.id,)) + for rule in (first, second): + await alerts.delete_rule(rule.id) + await _query(db, 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id)