mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-14 09:03:16 +00:00
fix(alerts): require current healthy evidence before recovering resource alerts
This commit is contained in:
moe-berri
parent
01e1198a88
commit
418685df55
3 changed files with 116 additions and 6 deletions
|
|
@ -91,7 +91,7 @@ class IssueStore(Protocol):
|
|||
scope_ids: tuple[str, ...] | None = None,
|
||||
) -> tuple[IssueProjectionSignal, ...]: ...
|
||||
|
||||
async def recover_issues(self, day: date) -> int: ...
|
||||
async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int: ...
|
||||
|
||||
async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Issue, ...]: ...
|
||||
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ from litellm.types.proxy.issues import (
|
|||
IssueSignalInput,
|
||||
IssueSignalList,
|
||||
)
|
||||
from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily
|
||||
from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS, PerformanceIssueFamily
|
||||
from litellm.types.proxy.usage_alerts import UsageAlertMetric, UsageAlertStatus
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -366,26 +366,56 @@ class IssuesRepository:
|
|||
)
|
||||
)
|
||||
|
||||
async def recover_issues(self, day: date) -> int:
|
||||
async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int:
|
||||
if issue_ids == ():
|
||||
return 0
|
||||
if issue_ids is not None and len(issue_ids) > 10:
|
||||
raise ValueError("Recover at most 10 resource alerts")
|
||||
selection: Final = "i.id=ANY($3::text[])" if issue_ids is not None else "$3::text[] IS NULL"
|
||||
rows: Final = TypeAdapter(tuple[_Count, ...]).validate_python(
|
||||
await self.database.query_raw(
|
||||
"""WITH healthy AS (
|
||||
SELECT i.id FROM "LiteLLM_Issue" i JOIN "LiteLLM_IssueSignal" s ON s.issue_id=i.id
|
||||
f"""WITH primary_signals AS (
|
||||
SELECT i.id,a.id AS source_id,a.breached,a.evaluation_day,a.evaluation_revision,
|
||||
max(a.evaluation_day) OVER (PARTITION BY i.id) AS latest_observation_day,
|
||||
r.id AS rule_id,r.record AS rule_record,
|
||||
COALESCE(r.enabled AND NOT r.deleted
|
||||
AND r.scope_type=a.scope_type AND r.record->>'metric'=a.metric
|
||||
AND r.record->>'direction'=a.record->>'direction'
|
||||
AND (r.scope_id IS NULL OR r.scope_id=a.scope_id)
|
||||
AND (NOT(r.id=ANY($2::text[])) OR NOT EXISTS (
|
||||
SELECT 1 FROM "LiteLLM_UsageAlertRule" custom WHERE custom.enabled AND NOT custom.deleted
|
||||
AND COALESCE((custom.record->>'overrides_defaults')::boolean,false)
|
||||
AND NOT(custom.id=ANY($2::text[])) AND custom.scope_type=r.scope_type
|
||||
AND custom.record->>'metric'=r.record->>'metric'
|
||||
AND custom.record->>'direction'=r.record->>'direction'
|
||||
AND (custom.scope_id IS NULL OR custom.scope_id=a.scope_id))),false) AS eligible
|
||||
FROM "LiteLLM_Issue" i JOIN "LiteLLM_IssueSignal" s ON s.issue_id=i.id
|
||||
LEFT JOIN "LiteLLM_UsageAlertIncident" a ON a.id=s.source_id
|
||||
LEFT JOIN "LiteLLM_UsageAlertRule" r ON r.id=a.rule_id
|
||||
WHERE i.kind='performance' AND i.active AND i.scope_type<>'provider' AND s.source='usage_alert'
|
||||
AND ({selection})
|
||||
AND s.record->'usage_alert'->>'scope_type'=i.scope_type
|
||||
AND s.record->'usage_alert'->>'scope_id' IS NOT DISTINCT FROM i.scope_id
|
||||
AND ((i.family='spend' AND s.record->'usage_alert'->>'metric'='spend')
|
||||
OR (i.family='latency' AND s.record->'usage_alert'->>'metric'='latency')
|
||||
OR (i.family='traffic' AND s.record->'usage_alert'->>'metric'='requests')
|
||||
OR (i.family='reliability' AND s.record->'usage_alert'->>'metric' IN ('errors','error_rate')))
|
||||
GROUP BY i.id HAVING bool_and(a.id IS NOT NULL AND NOT a.breached AND a.evaluation_day<=$1)
|
||||
), observations AS (
|
||||
SELECT *,COALESCE(source_id IS NOT NULL AND NOT breached AND evaluation_day<=$1
|
||||
AND evaluation_day>=latest_observation_day
|
||||
AND NULLIF(evaluation_revision,'')::timestamptz=(rule_record->>'updated_at')::timestamptz,false) AS healthy
|
||||
FROM primary_signals
|
||||
), healthy AS (
|
||||
SELECT id FROM observations GROUP BY id
|
||||
HAVING bool_or(eligible AND healthy) AND bool_and(NOT eligible OR healthy)
|
||||
), changed AS (
|
||||
UPDATE "LiteLLM_Issue" i SET active=false,
|
||||
status=CASE WHEN status='open' THEN 'resolved' ELSE status END,updated_at=CURRENT_TIMESTAMP
|
||||
WHERE id IN(SELECT id FROM healthy) AND active RETURNING id
|
||||
) SELECT count(*)::int AS count FROM changed""",
|
||||
day.isoformat(),
|
||||
tuple(DEFAULT_USAGE_RULE_IDS),
|
||||
issue_ids,
|
||||
)
|
||||
)
|
||||
return rows[0].count
|
||||
|
|
|
|||
|
|
@ -969,3 +969,83 @@ async def test_issue_resource_metadata_bounds_labels_and_deleted_history(
|
|||
await _remove_issues(db, (issue.id,))
|
||||
await _query(db, 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE id=$1', identity)
|
||||
await _query(db, 'DELETE FROM "LiteLLM_DeletedVerificationToken" WHERE token=$1', reporting_key)
|
||||
|
||||
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_disabled_later_breach_requires_new_healthy_primary_evidence(db: ManagementDatabase) -> None:
|
||||
store: Final = IssuesRepository.from_prisma(db.prisma)
|
||||
alerts: Final = UsageAlertsRepository.from_prisma(db.prisma)
|
||||
issue: Final = _issue(IssueCohort(scope_type="user", scope_id=f"fixture-{uuid4()}"))
|
||||
definition: Final = UsageAlertRuleInput(
|
||||
name="Primary traffic",
|
||||
scope_type="user",
|
||||
scope_id=issue.cohort.scope_id,
|
||||
metric="requests",
|
||||
direction="increase",
|
||||
)
|
||||
first: Final = await alerts.save_rule(definition, "qa-admin")
|
||||
second: Final = await alerts.save_rule(definition.model_copy(update={"name": "Another traffic policy"}), "qa-admin")
|
||||
assert first is not None and second is not None
|
||||
breach: Final = UsageAlertObservation(
|
||||
entity_id=issue.cohort.scope_id,
|
||||
day=issue.observed_on,
|
||||
observed=420,
|
||||
baseline=120,
|
||||
threshold=360,
|
||||
breached=True,
|
||||
)
|
||||
try:
|
||||
sources: Final = tuple([(await alerts.record_observations(rule, (breach,)))[0] for rule in (first, second)])
|
||||
await store.upsert_issue(issue)
|
||||
await store.add_signals(
|
||||
issue.id,
|
||||
tuple(
|
||||
IssueSignalInput(source="usage_alert", source_id=source.id, usage_alert=source) for source in sources
|
||||
),
|
||||
)
|
||||
await alerts.record_observations(
|
||||
first,
|
||||
(
|
||||
breach.model_copy(
|
||||
update={
|
||||
"day": issue.observed_on + timedelta(days=1),
|
||||
"observed": 120,
|
||||
"breached": False,
|
||||
}
|
||||
),
|
||||
),
|
||||
)
|
||||
await alerts.record_observations(
|
||||
second,
|
||||
(
|
||||
breach.model_copy(
|
||||
update={
|
||||
"day": issue.observed_on + timedelta(days=2),
|
||||
}
|
||||
),
|
||||
),
|
||||
)
|
||||
assert await alerts.save_rule(definition.model_copy(update={"enabled": False}), "qa-admin", second)
|
||||
await store.recover_issues(issue.observed_on + timedelta(days=3))
|
||||
unresolved: Final = await store.get_issue(issue.id)
|
||||
assert unresolved is not None and unresolved.active
|
||||
await alerts.record_observations(
|
||||
first,
|
||||
(
|
||||
breach.model_copy(
|
||||
update={
|
||||
"day": issue.observed_on + timedelta(days=3),
|
||||
"observed": 120,
|
||||
"breached": False,
|
||||
}
|
||||
),
|
||||
),
|
||||
)
|
||||
await store.recover_issues(issue.observed_on + timedelta(days=3))
|
||||
recovered: Final = await store.get_issue(issue.id)
|
||||
assert recovered is not None and not recovered.active
|
||||
finally:
|
||||
await _remove_issues(db, (issue.id,))
|
||||
for rule in (first, second):
|
||||
await alerts.delete_rule(rule.id)
|
||||
await _query(db, 'DELETE FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 RETURNING id', rule.id)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue