diff --git a/litellm/integrations/SlackAlerting/usage_alerts.py b/litellm/integrations/SlackAlerting/usage_alerts.py index 0d69406ca1a..dd10b5aea6d 100644 --- a/litellm/integrations/SlackAlerting/usage_alerts.py +++ b/litellm/integrations/SlackAlerting/usage_alerts.py @@ -5,15 +5,18 @@ import hashlib import json import os from html import escape +from itertools import islice from typing import TYPE_CHECKING, ClassVar, Final from urllib.parse import urlencode, urlsplit +from uuid import NAMESPACE_URL, uuid5 from httpx import Response from pydantic import TypeAdapter +from litellm.models.issues import IssueStore from litellm.types.integrations.slack_alerting import AlertType -from litellm.types.proxy.issues import IssuePendingDelivery -from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertChannel +from litellm.types.proxy.issues import IssueDeliveryBinding, IssueDeliveryInput, IssuePendingDelivery +from litellm.types.proxy.usage_alerts import UsageAlert if TYPE_CHECKING: from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting @@ -56,19 +59,43 @@ class UsageAlertDelivery: # One retained send per worker bounds SMTP threads even when a job is cancelled. _email_task: ClassVar[asyncio.Task[None] | None] = None _email_key: ClassVar[str | None] = None + _email_store: ClassVar[IssueStore | None] = None - def __init__(self, slack: "SlackAlerting") -> None: + def __init__(self, slack: "SlackAlerting", store: IssueStore | None = None) -> None: self._slack = slack + self._store = store async def send_issue(self, pending: IssuePendingDelivery) -> bool: + urls: Final = self._webhooks() if pending.delivery.channel == "slack" else () + if len(urls) > 1 and pending.delivery.slack_destination is None: + if self._store is None: + raise RuntimeError("Multiple alert destinations require a delivery store") + bindings: Final = ( + IssueDeliveryBinding( + delivery=IssueDeliveryInput( + id=str(uuid5(NAMESPACE_URL, f"{pending.delivery.id}:{destination}")), + issue_id=pending.issue.id, + channel="slack", + slack_destination=destination, + parent_delivery_id=pending.delivery.id, + ), + rule_id=pending.rule.id, + rule_revision=pending.rule.updated_at.isoformat(), + source_id=pending.alert.id, + ) + for destination in (hashlib.sha256(url.encode()).hexdigest() for url in urls) + ) + while batch := tuple(islice(bindings, 10)): + await self._store.queue_deliveries(batch) + return True path: Final = "/alerts?" + urlencode({"issue": pending.issue.id}) issue_url: Final = activity_url( pending.alert.model_copy(update={"activity_path": path}), os.getenv("PROXY_BASE_URL") ) source_url: Final = activity_url(pending.alert, os.getenv("PROXY_BASE_URL")) return await self._send( - pending.delivery.channel, - pending.delivery.email_recipients, + pending, + urls, incident_message(pending.alert, None), f"LiteLLM alert: {pending.issue.title}", (("View alert", issue_url), ("View source activity", source_url)), @@ -76,12 +103,14 @@ class UsageAlertDelivery: async def _send( self, - channel: UsageAlertChannel, - recipients: tuple[str, ...], + pending: IssuePendingDelivery, + urls: tuple[str, ...], message: str, subject: str, links: tuple[tuple[str, str], ...], ) -> bool: + channel: Final = pending.delivery.channel + recipients: Final = pending.delivery.email_recipients if channel not in configured_destinations(self._slack): return False if channel == "email": @@ -89,8 +118,19 @@ class UsageAlertDelivery: f'
' for label, url in links ) for recipient in recipients: - await self._email(recipient, subject, html) + await self._email(pending.delivery.id, recipient, subject, html) return bool(recipients) + destination: Final = pending.delivery.slack_destination + webhook: Final = next( + (url for url in urls if destination is None or hashlib.sha256(url.encode()).hexdigest() == destination), + None, + ) + if webhook is None: + return False + await self._post(webhook, message + "\n" + "\n".join(f"{label}: {url}" for label, url in links)) + return True + + def _webhooks(self) -> tuple[str, ...]: overrides: Final = self._slack.alert_to_webhook_url or {} configured_url: Final = ( overrides.get(AlertType.usage_alerts) @@ -99,38 +139,51 @@ class UsageAlertDelivery: or os.getenv("ALERTING_WEBHOOK_URL") ) if not configured_url: - return False - urls: Final = (configured_url,) if isinstance(configured_url, str) else _URLS.validate_python(configured_url) - for webhook in urls: - await self._post(webhook, message + "\n" + "\n".join(f"{label}: {url}" for label, url in links)) - return True + return () + return (configured_url,) if isinstance(configured_url, str) else _URLS.validate_python(configured_url) @staticmethod def _email_finished(task: asyncio.Task[None]) -> None: if not task.cancelled(): task.exception() # consume a late failure after the calling job was cancelled - @classmethod - async def _email(cls, recipient: str, subject: str, html: str) -> None: - key: Final = hashlib.sha256(json.dumps((recipient, subject, html)).encode()).hexdigest() + async def _email(self, key: str, recipient: str, subject: str, html: str) -> None: + cls: Final = type(self) previous: Final = cls._email_task if previous is not None: if not previous.done(): raise RuntimeError("An alert email is still in flight; retry later") - if cls._email_key == key and not previous.cancelled() and previous.exception() is None: - cls._email_task = None - cls._email_key = None - return previous.result() + if not previous.cancelled() and previous.exception() is None: + if cls._email_store is not None and cls._email_key is not None: + await cls._email_store.mark_delivery_sent(cls._email_key) + if cls._email_task is not previous: + raise RuntimeError("Another alert email is still in flight; retry later") + elif cls._email_key != key: + raise RuntimeError("A completed alert email is awaiting its receipt") + if cls._email_key == key: + cls._clear_email(previous) + return previous.result() task: Final = asyncio.create_task(cls._send_email(recipient, subject, html)) cls._email_task = task cls._email_key = key + cls._email_store = self._store task.add_done_callback(cls._email_finished) try: await asyncio.shield(task) - finally: - if task.done() and cls._email_task is task: - cls._email_task = None - cls._email_key = None + except asyncio.CancelledError: + raise + except Exception: + cls._clear_email(task) + raise + else: + cls._clear_email(task) + + @classmethod + def _clear_email(cls, task: asyncio.Task[None]) -> None: + if cls._email_task is task: + cls._email_task = None + cls._email_key = None + cls._email_store = None @staticmethod async def _send_email(recipient: str, subject: str, html: str) -> None: diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index d26b463ea70..374d9ee433a 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -145,7 +145,8 @@ class UsageAlertJob: background=True, ) metrics: Final = UsageAlertMetricsRepository(database) - delivery: Final = UsageAlertDelivery(slack) + issues: Final = IssuesRepository(database) + delivery: Final = UsageAlertDelivery(slack, issues) return cls( UsageAlertsRepository(database), metrics, @@ -158,7 +159,7 @@ class UsageAlertJob: ), deprecation_projection=deprecation_projection, performance_projection=PerformanceIssueProjector( - IssuesRepository(database), + issues, metrics, ProviderLatencyRepository(database), delivery, diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index dbf5e539f41..24d5088ea25 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -607,7 +607,7 @@ class IssuesRepository: SELECT r.record AS rule,a.record AS alert FROM "LiteLLM_IssueDeliveryRule" b JOIN "LiteLLM_UsageAlertRule" r ON r.id=b.rule_id JOIN "LiteLLM_UsageAlertIncident" a ON a.id=b.source_id AND a.rule_id=r.id - WHERE b.delivery_id=d.id AND r.enabled AND NOT r.deleted AND a.breached AND a.status='open' + WHERE b.delivery_id=COALESCE(d.record->>'parent_delivery_id',d.id) AND r.enabled AND NOT r.deleted AND a.breached AND a.status='open' AND (r.record->'notification_channels') ? (d.record->>'channel') AND (d.record->>'channel'<>'email' OR (jsonb_array_length(d.record->'email_recipients')>0 diff --git a/litellm/types/proxy/issues.py b/litellm/types/proxy/issues.py index 51b834a3630..8d185ac3930 100644 --- a/litellm/types/proxy/issues.py +++ b/litellm/types/proxy/issues.py @@ -207,6 +207,8 @@ class IssueDeliveryInput(BaseModel, frozen=True): issue_id: str = Field(min_length=1, max_length=200) channel: UsageAlertChannel email_recipients: tuple[str, ...] = Field(default=(), max_length=20) + slack_destination: str | None = Field(default=None, pattern="^[a-f0-9]{64}$") + parent_delivery_id: str | None = Field(default=None, min_length=1, max_length=200) class IssueDeliveryBinding(BaseModel, frozen=True): diff --git a/tests/integration/database/test_performance_issues.py b/tests/integration/database/test_performance_issues.py index 2e2379f8a1b..269a2a83942 100644 --- a/tests/integration/database/test_performance_issues.py +++ b/tests/integration/database/test_performance_issues.py @@ -1,4 +1,5 @@ import asyncio +import os from collections.abc import AsyncIterator from contextlib import asynccontextmanager from datetime import date, datetime, timedelta, timezone @@ -7,11 +8,14 @@ from typing import Final from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from uuid import uuid4 +import httpx import pytest from integration._support.actors import ManagementDatabase from integration._support.prisma_clients import connected_proxy_prisma from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy.spend_tracking.usage_alert_job import UsageAlertJob from litellm.proxy.utils import PrismaClient from litellm.repositories.daily_activity_buckets import api_key_bucket @@ -154,8 +158,7 @@ async def test_retirement_usage_omits_oversized_models_before_materialization(db @asynccontextmanager async def _isolated(db: ManagementDatabase) -> AsyncIterator[PrismaClient]: schema: Final = "issue_test_" + uuid4().hex - assert db.gateway.database_url is not None - parsed: Final = urlsplit(db.gateway.database_url) + parsed: Final = urlsplit(db.gateway.database_url or os.environ["DATABASE_URL"]) query: Final = {**dict(parse_qsl(parsed.query)), "schema": schema} url: Final = urlunsplit(parsed._replace(query=urlencode(query))) await db.prisma.writer_db.execute_raw(f'CREATE SCHEMA "{schema}"') @@ -871,3 +874,175 @@ async def test_projection_resume_keeps_spend_and_traffic_in_one_episode(db: Mana assert tuple(item.id for item in after.data) == tuple(item.id for item in before.data) links: Final = await IssuesRepository.from_prisma(prisma).list_signals(after.data[0].id) assert any(link.usage_alert is not None and link.usage_alert.metric == "requests" for link in links.data) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_slack_retry_only_repeats_the_failed_destination(db: ManagementDatabase) -> None: + paths: Final[list[str]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + paths.append(request.url.path) + return httpx.Response(503 if request.url.path == "/two" and paths.count("/two") == 1 else 200) + + handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) + try: + async with _isolated(db) as prisma: + await _seed(prisma) + rules: Final = UsageAlertsRepository.from_prisma(prisma) + assert ( + await rules.save_rule( + UsageAlertRuleInput( + name="Project spend", + scope_type="key", + scope_id=_KEY_A, + metric="spend", + direction="increase", + notification_channels=("slack",), + ), + "qa-admin", + ) + is not None + ) + assert ( + await rules.save_rule( + UsageAlertRuleInput( + name="Another project rule", + scope_type="key", + scope_id=_KEY_A, + metric="spend", + direction="increase", + notification_channels=("slack",), + ), + "qa-admin", + ) + is not None + ) + slack: Final = SlackAlerting( + alerting=["slack"], + alert_types=[], + async_http_handler=handler, + alert_to_webhook_url={ + "usage_alerts": [ + "https://hooks.example.com/one", + "https://hooks.example.com/two", + "https://hooks.example.com/one", + ] + }, + ) + job: Final = UsageAlertJob.from_prisma(prisma, slack) + await job._run(_NOW) + assert paths == ["/one", "/two"] or paths == ["/two", "/one"] + await prisma.writer_db.query_raw( + 'UPDATE "LiteLLM_IssueDelivery" SET next_attempt=CURRENT_TIMESTAMP WHERE NOT sent RETURNING id' + ) + retry: Final = (await IssuesRepository.from_prisma(prisma).pending_deliveries())[0] + assert ( + await rules.save_rule( + UsageAlertRuleInput.model_validate( + retry.rule.model_dump(include=set(UsageAlertRuleInput.model_fields)) + ).model_copy(update={"enabled": False}), + "qa-admin", + retry.rule, + ) + is not None + ) + await job._performance_projection.notify() + assert paths.count("/one") == 1 and paths.count("/two") == 2 + assert await IssuesRepository.from_prisma(prisma).pending_deliveries() == () + records: Final = await prisma.writer_db.query_raw( + 'SELECT record::text AS value FROM "LiteLLM_IssueDelivery"' + ) + assert all("hooks.example.com" not in row["value"] for row in records) + finally: + await handler.close() + + +@pytest.mark.asyncio(loop_scope="session") +async def test_cancelled_email_receipt_survives_a_different_delivery( + db: ManagementDatabase, monkeypatch: pytest.MonkeyPatch +) -> None: + accepted: Final = asyncio.Event() + release: Final = asyncio.Event() + messages: Final[list[bytes]] = [] + + async def smtp(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + writer.write(b"220 example.com ready\r\n") + await writer.drain() + try: + while command := await reader.readline(): + if command.upper().startswith(b"DATA"): + writer.write(b"354 End with a dot\r\n") + await writer.drain() + messages.append(await reader.readuntil(b"\r\n.\r\n")) + accepted.set() + await release.wait() + writer.write(b"250 Accepted\r\n") + elif command.upper().startswith(b"QUIT"): + writer.write(b"221 Closing\r\n") + break + else: + writer.write(b"250 OK\r\n") + await writer.drain() + finally: + writer.close() + await writer.wait_closed() + + server: Final = await asyncio.start_server(smtp, "127.0.0.1", 0) + monkeypatch.setenv("SMTP_HOST", "127.0.0.1") + monkeypatch.setenv("SMTP_PORT", str(server.sockets[0].getsockname()[1])) + monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com") + monkeypatch.setenv("SMTP_TLS", "False") + monkeypatch.setenv("SMTP_USE_SSL", "False") + for name in ("SMTP_TIMEOUT", "SMTP_USERNAME", "SMTP_PASSWORD"): + monkeypatch.delenv(name, raising=False) + try: + async with _isolated(db) as prisma: + await _seed(prisma) + rules: Final = UsageAlertsRepository.from_prisma(prisma) + assert ( + await rules.save_rule( + UsageAlertRuleInput( + name="Project spend", + scope_type="key", + scope_id=_KEY_A, + metric="spend", + direction="increase", + notification_channels=("email",), + email_recipients=("qa@example.com",), + ), + "qa-admin", + ) + is not None + ) + job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) + await job._run(_NOW) + store: Final = IssuesRepository.from_prisma(prisma) + await prisma.writer_db.query_raw( + 'UPDATE "LiteLLM_IssueDelivery" SET next_attempt=CURRENT_TIMESTAMP WHERE NOT sent RETURNING id' + ) + pending: Final = (await store.pending_deliveries())[0] + delivery: Final = UsageAlertDelivery(SlackAlerting(alerting=["email"], alert_types=[]), store) + sending: Final = asyncio.create_task(delivery.send_issue(pending)) + await asyncio.wait_for(accepted.wait(), 10) + sending.cancel() + with pytest.raises(asyncio.CancelledError): + await sending + release.set() + retained: Final = UsageAlertDelivery._email_task + assert retained is not None + await asyncio.wait_for(asyncio.shield(retained), 10) + different: Final = pending.model_copy( + update={"delivery": pending.delivery.model_copy(update={"id": "another-delivery"})} + ) + assert await delivery.send_issue(different) + assert await store.pending_delivery(pending.delivery.id) is None + assert len(messages) == 2 + finally: + release.set() + server.close() + await server.wait_closed() + if UsageAlertDelivery._email_task is not None: + await asyncio.wait_for(asyncio.shield(UsageAlertDelivery._email_task), 10) + UsageAlertDelivery._email_task = None + UsageAlertDelivery._email_key = None + UsageAlertDelivery._email_store = None diff --git a/tests/integration/database/test_usage_alert_evaluation.py b/tests/integration/database/test_usage_alert_evaluation.py index ddb3baf1b8b..799c36df03f 100644 --- a/tests/integration/database/test_usage_alert_evaluation.py +++ b/tests/integration/database/test_usage_alert_evaluation.py @@ -1,10 +1,23 @@ import asyncio +import json from datetime import date, datetime, timedelta, timezone -from typing import Final +from typing import Final, cast # noqa: TID251 # generated Prisma transaction implements the accounting protocol +from uuid import uuid4 import pytest from integration._support.actors import ManagementDatabase +from integration.database.test_performance_issues import _isolated +from litellm.accounting.application import apply_deliveries +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter +from litellm.proxy.utils import ProxyLogging +from litellm.repositories.accounting_delivery_repository import ( + AccountingDeliveryRepository, + DeliveryProjection, + LedgerDatabase, + LedgerTransaction, +) from litellm.repositories.usage_alert_health import UsageAlertFlushRecorder, UsageAlertHealthRepository from litellm.repositories.usage_alerts import UsageAlertsRepository from litellm.types.proxy.usage_alerts import UsageAlertRuleInput @@ -68,3 +81,61 @@ async def test_freshness_lock_cannot_stall_committed_spend(db: ManagementDatabas "SELECT source FROM \"LiteLLM_UsageAlertFlush\" WHERE source='user' AND hour=date_trunc('hour',CURRENT_TIMESTAMP AT TIME ZONE 'UTC')" ) assert rows == [{"source": "user"}] + + +@pytest.mark.asyncio(loop_scope="session") +async def test_freshness_waits_until_accounting_has_committed_the_daily_totals(db: ManagementDatabase) -> None: + async with _isolated(db) as prisma: + await prisma.writer_db.execute_raw( + 'CREATE TABLE "LiteLLM_DailyModelUsage" (LIKE public."LiteLLM_DailyModelUsage" INCLUDING ALL)' + ) + identity: Final = "coverage-" + uuid4().hex + now: Final = datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0) + timedelta(days=3) + row: Final = { + "request_id": identity, + "user": identity, + "api_key": "synthetic-key", + "startTime": (now - timedelta(hours=2)).isoformat(), + "model": "example-model", + "model_group": "example-model", + "custom_llm_provider": "example-provider", + "spend": 5, + "metadata": {}, + } + health: Final = UsageAlertHealthRepository.from_prisma(prisma) + writer: Final = DBSpendUpdateWriter() + logging: Final = ProxyLogging(UserApiKeyCache()) + + async def apply(transaction: LedgerTransaction, deliveries: tuple[DeliveryProjection, ...]) -> None: + await apply_deliveries( + transaction, deliveries, prisma_client=prisma, writer=writer, proxy_logging_obj=logging + ) + assert not await health.record("user", now) + + try: + await prisma.writer_db.query_raw( + 'INSERT INTO "LiteLLM_SpendLogDelivery" (request_id,accounting_payload,created_at,is_batch) VALUES ($1,$2::jsonb,$3::timestamp,false) RETURNING delivery_id', + identity, + json.dumps(row), + (now - timedelta(hours=1)).replace(tzinfo=None), + ) + assert not await health.record("user", now) + database: Final = cast(LedgerDatabase, prisma) # cast-ok: generated Prisma transaction delegate + result: Final = await AccountingDeliveryRepository(database).apply_pending(apply, batch_size=1) + assert result.applied == 1 + assert await health.record("user", now) + totals: Final = await prisma.writer_db.query_raw( + 'SELECT spend,api_requests FROM "LiteLLM_DailyUserSpend" WHERE user_id=$1', identity + ) + assert totals == [{"spend": 5, "api_requests": 1}] + finally: + await prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_SpendLogDelivery" WHERE request_id=$1 RETURNING delivery_id', identity + ) + await prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE user_id=$1 RETURNING id', identity + ) + await prisma.writer_db.query_raw( + "DELETE FROM \"LiteLLM_UsageAlertFlush\" WHERE source='user' AND hour=$1::timestamp RETURNING source", + now.replace(tzinfo=None), + )