From 2de895ffee6895e5f1b370f42ed30d932d588e34 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 11:55:27 +0800 Subject: [PATCH 01/44] feat: evaluate daily usage alerts from existing aggregates --- .../migration.sql | 6 + .../litellm_proxy_extras/schema.prisma | 9 + .../SlackAlerting/usage_alerts.py | 88 ++++++ litellm/proxy/db/db_spend_update_writer.py | 7 + litellm/proxy/proxy_server.py | 15 + litellm/proxy/schema.prisma | 9 + .../spend_tracking/daily_usage_alerts.py | 86 ++++++ .../proxy/spend_tracking/usage_alert_job.py | 286 ++++++++++++++++++ litellm/proxy/utils.py | 4 + litellm/repositories/usage_alert_health.py | 95 ++++++ litellm/repositories/usage_alert_metrics.py | 92 ++++++ litellm/repositories/usage_alerts.py | 56 +++- litellm/types/integrations/slack_alerting.py | 1 + litellm/types/proxy/usage_alerts.py | 5 + schema.prisma | 9 + .../SlackAlerting/test_usage_alerts.py | 78 +++++ .../spend_tracking/test_daily_usage_alerts.py | 155 ++++++++++ .../utils/prisma_and_spend/test_send_email.py | 14 +- 18 files changed, 1011 insertions(+), 4 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20261105110000_usage_alert_flush/migration.sql create mode 100644 litellm/integrations/SlackAlerting/usage_alerts.py create mode 100644 litellm/proxy/spend_tracking/daily_usage_alerts.py create mode 100644 litellm/proxy/spend_tracking/usage_alert_job.py create mode 100644 litellm/repositories/usage_alert_health.py create mode 100644 litellm/repositories/usage_alert_metrics.py create mode 100644 tests/unit/integrations/SlackAlerting/test_usage_alerts.py create mode 100644 tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105110000_usage_alert_flush/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105110000_usage_alert_flush/migration.sql new file mode 100644 index 00000000000..217fa4f1510 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105110000_usage_alert_flush/migration.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS "LiteLLM_UsageAlertFlush" ( + source TEXT NOT NULL, + hour TIMESTAMP(3) NOT NULL, + observed_at TIMESTAMP(3) NOT NULL, + PRIMARY KEY (source, hour) +); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 8a67d1b123a..3a08c094b3d 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -2203,3 +2203,12 @@ model LiteLLM_IssueDeliveryRule { @@id([delivery_id, rule_id, source_id]) } + +// Hourly samples of successful background daily-aggregate flushes, retained for 35 days. +model LiteLLM_UsageAlertFlush { + source String + hour DateTime + observed_at DateTime + + @@id([source, hour]) +} diff --git a/litellm/integrations/SlackAlerting/usage_alerts.py b/litellm/integrations/SlackAlerting/usage_alerts.py new file mode 100644 index 00000000000..9b09d8fb05c --- /dev/null +++ b/litellm/integrations/SlackAlerting/usage_alerts.py @@ -0,0 +1,88 @@ +"""Deliver persisted usage incidents through the existing configured transports.""" + +import json +import os +from html import escape +from typing import TYPE_CHECKING, Final +from urllib.parse import urlsplit + +from pydantic import TypeAdapter + +from litellm.types.integrations.slack_alerting import AlertType +from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertChannel, UsageAlertRule + +if TYPE_CHECKING: + from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting + +_URLS: Final = TypeAdapter(tuple[str, ...]) + + +def configured_destinations(slack: "SlackAlerting") -> tuple[str, ...]: + return _URLS.validate_python(slack.alerting or ()) # pyright: ignore[reportUnknownMemberType] # validate legacy destination config + + +def activity_url(alert: UsageAlert, base_url: str | None) -> str: + if not base_url: + return alert.activity_path + parsed: Final = urlsplit(base_url) + if parsed.scheme not in ("http", "https") or not parsed.netloc: + return alert.activity_path + return base_url.rstrip("/") + "/ui" + alert.activity_path + + +def incident_message(alert: UsageAlert, url: str) -> str: + scope: Final = "Gateway" if alert.scope_type == "deployment" else f"{alert.scope_type}: {alert.scope_id}" + unit: Final = " ms" if alert.metric == "latency" else " USD" if alert.metric == "spend" else "" + return ( + f"{alert.rule_name}\n{scope}\n{alert.day.isoformat()} UTC: " + f"{alert.observed_value:,.2f}{unit}; baseline {alert.baseline_value:,.2f}{unit}; " + f"threshold {alert.threshold:,.2f}{unit}.\nView activity: {url}" + ) + + +class UsageAlertDelivery: + def __init__(self, slack: "SlackAlerting") -> None: + self._slack = slack + + async def send(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> bool: + configured: Final = configured_destinations(self._slack) + if channel not in configured: + return False + url: Final = activity_url(alert, os.getenv("PROXY_BASE_URL")) + message: Final = incident_message(alert, url) + if channel == "email": + from litellm.proxy.utils import send_email + + for recipient in rule.email_recipients: + await send_email( + receiver_email=recipient, + subject=f"LiteLLM usage alert: {alert.rule_name}", + html=( + f"

{escape(message).replace(chr(10), '
')}

" + f'

View activity

' + ), + raise_on_error=True, + ) + return bool(rule.email_recipients) + overrides: Final = self._slack.alert_to_webhook_url or {} + configured_url: Final = ( + overrides.get(AlertType.usage_alerts) + or self._slack.default_webhook_url + or os.getenv("SLACK_WEBHOOK_URL") + or os.getenv("ALERTING_WEBHOOK_URL") + ) + if not configured_url: + return False + urls: Final = (configured_url,) if isinstance(configured_url, str) else _URLS.validate_python(configured_url) + for webhook in urls: + await self._post(webhook, message) + return True + + async def _post(self, webhook: str, message: str) -> None: + response: Final = await self._slack.async_http_handler.post( # pyright: ignore[reportUnknownMemberType] # legacy HTTP signature; supplied arguments are concrete + url=webhook, + headers={"Content-type": "application/json"}, + data=json.dumps({"text": message}), + timeout=10, + ) + response.raise_for_status() diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 80f09df6c86..53b95e6e889 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -665,6 +665,9 @@ class DBSpendUpdateWriter: self, redis_cache: RedisCache | None = None, ): + from litellm.repositories.usage_alert_health import UsageAlertFlushRecorder + + self.usage_alert_flush_recorder = UsageAlertFlushRecorder() self.redis_cache = redis_cache self.redis_update_buffer = RedisUpdateBuffer(redis_cache=self.redis_cache) self.pod_lock_manager = PodLockManager() @@ -1879,6 +1882,7 @@ class DBSpendUpdateWriter: daily_spend_transactions=daily_spend_update_transactions, ) uncommitted.pop("daily_spend_update_transactions", None) + await self.usage_alert_flush_recorder.record(prisma_client, "user") if daily_team_spend_update_transactions is not None: await DBSpendUpdateWriter.update_daily_team_spend( @@ -1888,6 +1892,7 @@ class DBSpendUpdateWriter: daily_spend_transactions=daily_team_spend_update_transactions, ) uncommitted.pop("daily_team_spend_update_transactions", None) + await self.usage_alert_flush_recorder.record(prisma_client, "team") if daily_org_spend_update_transactions is not None: await DBSpendUpdateWriter.update_daily_org_spend( @@ -1970,6 +1975,8 @@ class DBSpendUpdateWriter: ) try: await asyncio.shield(commit_task) + if entity_type in ("user", "team"): + await self.usage_alert_flush_recorder.record(prisma_client, entity_type) except asyncio.CancelledError: if commit_started.is_set(): queue.track_interrupted_commit( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 4d5f8439f37..144819876bd 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11678,6 +11678,21 @@ class ProxyStartupEvent: misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, ) + from litellm.proxy.spend_tracking.usage_alert_job import USAGE_ALERT_JOB_ID, UsageAlertJob + + usage_alert_job: Final = UsageAlertJob.from_prisma(prisma_client, proxy_logging_obj.slack_alerting_instance) + scheduler.add_job( + usage_alert_job.run, + "interval", + seconds=3600, + next_run_time=datetime.now() + timedelta(seconds=batch_writing_interval + 10 + random.randint(0, 20)), + id=USAGE_ALERT_JOB_ID, + replace_existing=True, + max_instances=1, + coalesce=True, + misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, + ) + ### TOOL DISCOVERY (own lock, same interval, never rides the spend commit) ### from litellm.proxy.utils import commit_tool_discoveries diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 8a67d1b123a..3a08c094b3d 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2203,3 +2203,12 @@ model LiteLLM_IssueDeliveryRule { @@id([delivery_id, rule_id, source_id]) } + +// Hourly samples of successful background daily-aggregate flushes, retained for 35 days. +model LiteLLM_UsageAlertFlush { + source String + hour DateTime + observed_at DateTime + + @@id([source, hour]) +} diff --git a/litellm/proxy/spend_tracking/daily_usage_alerts.py b/litellm/proxy/spend_tracking/daily_usage_alerts.py new file mode 100644 index 00000000000..a79adaaedfa --- /dev/null +++ b/litellm/proxy/spend_tracking/daily_usage_alerts.py @@ -0,0 +1,86 @@ +"""Daily comparisons over existing aggregated counters, outside the request path.""" + +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import date, timedelta +from typing import Final, Literal, Protocol + +from litellm.repositories.usage_alert_metrics import DailyAlertMetrics + + +class ComparisonRule(Protocol): + @property + def metric(self) -> Literal["spend", "requests", "latency", "errors"]: ... + + @property + def direction(self) -> Literal["increase", "decrease"]: ... + + @property + def multiplier(self) -> float: ... + + @property + def baseline_days(self) -> int: ... + + @property + def min_history_days(self) -> int: ... + + @property + def min_requests(self) -> int: ... + + @property + def min_value(self) -> float: ... + + +@dataclass(frozen=True, slots=True) +class DailyComparison: + baseline_sample_days: int + observed: float + baseline: float + threshold: float + breached: bool + + +def _value(metric: str, rows: Sequence[DailyAlertMetrics], days: int) -> float | None: + if metric == "latency": + timed: Final = sum(row.timed_requests for row in rows) + return sum(row.response_time_ms for row in rows) / timed if timed else None + if metric == "spend": + return sum(row.spend for row in rows) / days + if metric == "errors": + return sum(row.errors for row in rows) / days + return sum(row.requests for row in rows) / days + + +def compare_daily_usage( + rule: ComparisonRule, rows: Sequence[DailyAlertMetrics], target: date +) -> DailyComparison | None: + """None means insufficient evidence; it must not resolve an existing incident.""" + start: Final = target - timedelta(days=rule.baseline_days) + history: Final = tuple(row for row in rows if start <= row.day < target) + current: Final = tuple(row for row in rows if row.day == target) + sample_days: Final = len({row.day for row in history}) + if sample_days < rule.min_history_days: + return None + baseline_requests: Final = sum(row.requests for row in history) / sample_days + current_requests: Final = sum(row.requests for row in current) + if baseline_requests < rule.min_requests: + return None + if rule.direction == "increase" and current_requests < rule.min_requests: + return None + if rule.metric == "latency": + baseline_timed: Final = sum(row.timed_requests for row in history) / sample_days + if baseline_timed < rule.min_requests or sum(row.timed_requests for row in current) < rule.min_requests: + return None + baseline: Final = _value(rule.metric, history, sample_days) + observed: Final = _value(rule.metric, current, 1) + if baseline is None or baseline < 0 or observed is None: + return None + if baseline == 0: + if rule.direction == "decrease": + return None + return DailyComparison(sample_days, observed, 0, rule.min_value, observed > rule.min_value) + threshold: Final = baseline * rule.multiplier if rule.direction == "increase" else baseline / rule.multiplier + crossed: Final = observed >= threshold if rule.direction == "increase" else observed <= threshold + return DailyComparison( + sample_days, observed, baseline, threshold, crossed and abs(observed - baseline) >= rule.min_value + ) diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py new file mode 100644 index 00000000000..18bd2059adb --- /dev/null +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -0,0 +1,286 @@ +"""Once-daily evaluation, polled hourly by one database-elected worker.""" + +import asyncio +from collections.abc import Sequence +from datetime import date, datetime, time, timedelta, timezone +from itertools import groupby, product +from typing import ( + TYPE_CHECKING, + Final, + Protocol, + cast, # noqa: TID251 # generated Prisma delegate supplies the repository protocol +) +from uuid import NAMESPACE_URL, uuid4, uuid5 + +from litellm._logging import verbose_proxy_logger +from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, configured_destinations +from litellm.proxy.spend_tracking.daily_usage_alerts import compare_daily_usage +from litellm.repositories.usage_alert_health import UsageAlertHealthRepository +from litellm.repositories.usage_alert_metrics import DailyAlertMetrics, MetricsDatabase, UsageAlertMetricsRepository +from litellm.repositories.usage_alerts import UsageAlertsRepository +from litellm.types.integrations.slack_alerting import AlertType +from litellm.types.proxy.usage_alerts import ( + UsageAlert, + UsageAlertChannel, + UsageAlertMetric, + UsageAlertNotification, + UsageAlertObservation, + UsageAlertRule, + UsageAlertRuleInput, + UsageAlertScope, +) + +if TYPE_CHECKING: + from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting + from litellm.proxy.utils import PrismaClient + +USAGE_ALERT_JOB_ID: Final = "daily_usage_alerts" +_SCOPES: Final[tuple[UsageAlertScope, ...]] = ("deployment", "key", "user", "team", "model") +_METRICS: Final[tuple[UsageAlertMetric, ...]] = ("spend", "requests", "latency", "errors") +_MIN_VALUES: Final = {"spend": 10, "requests": 100, "latency": 100, "errors": 10} + + +def default_usage_rules( + channels: tuple[UsageAlertChannel, ...] = (), *, legacy_user_spend_alerts: bool = False +) -> tuple[UsageAlertRuleInput, ...]: + return tuple( + UsageAlertRuleInput( + name=f"Daily {scope} {metric} {direction}", + scope_type=scope, + metric=metric, + direction=direction, + min_value=_MIN_VALUES[metric], + notification_channels=( + () + if legacy_user_spend_alerts and (scope, metric, direction) == ("user", "spend", "increase") + else channels + ), + ) + for scope, metric, direction in product(_SCOPES, _METRICS, ("increase", "decrease")) + ) + + +def completed_day(now: datetime) -> date: + """A UTC day gets an hour for normal buffered writes before it is eligible.""" + return (now.astimezone(timezone.utc) - timedelta(hours=1)).date() - timedelta(days=1) + + +def _has_activity_page(scope: UsageAlertScope, entity: str) -> bool: + if not entity.strip(): + return False + if scope == "key": + return entity != "unattributed" and not entity.startswith("sk-") + if scope == "user": + return entity != "default_user_id" + if scope == "team": + return entity != "litellm-dashboard" + return True + + +def _observations( + rule: UsageAlertRule, + rows: Sequence[DailyAlertMetrics], + target: date, + suppressed: frozenset[str | None] = frozenset(), +) -> tuple[UsageAlertObservation, ...]: + def observation(entity: str, entity_rows: Sequence[DailyAlertMetrics]) -> UsageAlertObservation | None: + if None in suppressed or entity in suppressed: + return None + if not _has_activity_page(rule.scope_type, entity): + return None + if rule.scope_id is not None and rule.scope_id != entity: + return None + comparison: Final = compare_daily_usage(rule, entity_rows, target) + if comparison is None: + return None + identity: Final = None if rule.scope_type == "deployment" else entity + return UsageAlertObservation( + entity_id=identity, + baseline_sample_days=comparison.baseline_sample_days, + day=target, + observed=comparison.observed, + baseline=comparison.baseline, + threshold=comparison.threshold, + breached=comparison.breached, + ) + + candidates: Final = tuple( + observation(entity, tuple(group)) for entity, group in groupby(rows, lambda row: row.entity_id) + ) + return tuple(item for item in candidates if item is not None) + + +def _default_rule(rule: UsageAlertRule) -> bool: + return rule.id == str(uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.scope_type}:{rule.metric}:{rule.direction}")) + + +def _suppressed_entities(rule: UsageAlertRule, enabled: tuple[UsageAlertRule, ...]) -> frozenset[str | None]: + if not _default_rule(rule): + return frozenset() + matching: Final = tuple( + other + for other in enabled + if (other.scope_type, other.metric, other.direction) == (rule.scope_type, rule.metric, rule.direction) + ) + return frozenset(other.scope_id for other in matching if not _default_rule(other)) + + +class IncidentDelivery(Protocol): + async def send(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> bool: ... + + +class UsageAlertJob: + def __init__( + self, + store: UsageAlertsRepository, + metrics: UsageAlertMetricsRepository, + health: UsageAlertHealthRepository, + delivery: IncidentDelivery, + defaults: tuple[UsageAlertRuleInput, ...] | None = None, + ) -> None: + self._store = store + self._metrics = metrics + self._health = health + self._delivery = delivery + self._defaults = default_usage_rules() if defaults is None else defaults + self._defaults_registered = False + + @classmethod + def from_prisma(cls, prisma: "PrismaClient", slack: "SlackAlerting") -> "UsageAlertJob": + database: Final = cast(MetricsDatabase, prisma.writer_db) # cast-ok: generated Prisma delegate + return cls( + UsageAlertsRepository.from_prisma(prisma), + UsageAlertMetricsRepository(database), + UsageAlertHealthRepository(database), + UsageAlertDelivery(slack), + default_usage_rules( + ("slack",) + if "slack" in configured_destinations(slack) and AlertType.usage_alerts in slack.alert_types + else (), + legacy_user_spend_alerts=AlertType.user_spend_anomalies in slack.alert_types, + ), + ) + + async def run(self, now: datetime | None = None) -> None: + owner: Final = str(uuid4()) + if not await self._store.claim_job(owner): + return + try: + await asyncio.wait_for(self._run(now or datetime.now(timezone.utc)), timeout=840) + except asyncio.TimeoutError: + verbose_proxy_logger.warning( + "Usage alert evaluation exceeded its 14-minute lease budget; retrying next hour" + ) + except Exception: # noqa: BLE001 # the scheduler retries; completed rule/day checkpoints survive a failed pass + verbose_proxy_logger.exception("Daily usage alert evaluation failed") + finally: + await self._store.release_job(owner) + + async def _rules(self) -> tuple[UsageAlertRule, ...]: + pages: Final[list[UsageAlertRule]] = [] # mutable-ok: collect configuration pages + offset = 0 # rebind-ok: configuration pagination cursor + while page := await self._store.list_enabled_rules(limit=100, offset=offset): + pages.extend(page) + if len(page) < 100: + return tuple(pages) + offset += len(page) + return tuple(pages) + + async def _run(self, now: datetime) -> None: + if not self._defaults_registered: + await self._store.ensure_default_rules(self._defaults) + self._defaults_registered = True + target: Final = completed_day(now) + rules: Final = await self._rules() + due: Final = tuple([rule for rule in rules if not await self._store.is_day_complete(rule, target)]) + for scoped in (tuple(rule for rule in due if rule.scope_type == scope) for scope in _SCOPES): + if scoped: + await self._scope(scoped, target, rules) + await self._notify() + await self._store.prune() + + async def _scope( + self, rules: tuple[UsageAlertRule, ...], target: date, enabled: tuple[UsageAlertRule, ...] + ) -> None: + scope: Final = rules[0].scope_type + start: Final = datetime.combine(target, time(), tzinfo=timezone.utc) + coverage: Final = await self._health.coverage("team" if scope == "team" else "user", start) + eligible: Final = tuple( + rule for rule in rules if coverage.after_grace and (rule.direction == "increase" or coverage.hours == 24) + ) + suppressed: Final = {rule.id: _suppressed_entities(rule, enabled) for rule in eligible} + for rule in rules: + if rule not in eligible: + await self._store.record_evaluation_status(rule, target, "insufficient_coverage") + if not eligible: + return + evaluated: Final[set[str]] = set() # mutable-ok: at most one entry per configured rule across streaming pages + async for rows in self._metrics.batches(scope, target, max(rule.baseline_days for rule in eligible)): + evaluated.update(await self._page(eligible, rows, target, enabled)) + for rule in eligible: + await self._store.finish_day(rule, target) + await self._store.record_evaluation_status( + rule, target, "complete" if rule.id in evaluated or suppressed[rule.id] else "insufficient_history" + ) + + async def _page( + self, + rules: tuple[UsageAlertRule, ...], + rows: tuple[DailyAlertMetrics, ...], + target: date, + enabled: tuple[UsageAlertRule, ...], + ) -> tuple[str, ...]: + entities: Final = ( + (None,) if rules[0].scope_type == "deployment" else tuple(dict.fromkeys(row.entity_id for row in rows)) + ) + active: Final = await self._store.breached_entities(tuple(rule.id for rule in rules), entities) + return tuple( + [ + rule.id + for rule in rules + if await self._rule_page( + rule, rows, target, active.get(rule.id, frozenset()), _suppressed_entities(rule, enabled) + ) + ] + ) + + async def _rule_page( + self, + rule: UsageAlertRule, + rows: tuple[DailyAlertMetrics, ...], + target: date, + active: frozenset[str | None], + suppressed: frozenset[str | None], + ) -> bool: + observations: Final = _observations(rule, rows, target, suppressed) + changed: Final = tuple(item for item in observations if item.breached or item.entity_id in active) + if changed: + await self._store.record_observations(rule, changed) + return bool(observations) + + async def _deliver_channel(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> str | None: + try: + delivered: Final = await asyncio.wait_for(self._delivery.send(alert, rule, channel), timeout=30) + if not delivered: + return "transport_not_configured" + await self._store.mark_notified(alert.id, channel) + except Exception: # noqa: BLE001 # retry only failed channels; persisted acknowledgements survive restarts + verbose_proxy_logger.warning("Usage alert delivery failed for incident %s via %s", alert.id, channel) + return "delivery_failed" + return None + + async def _notify(self) -> None: + while page := await self._store.pending_notifications(limit=100): + for pending in page: + await self._notify_incident(pending) + + async def _notify_incident(self, pending: UsageAlertNotification) -> None: + failures: Final = tuple( + [ + await self._deliver_channel(pending.incident, pending.rule, channel) + for channel in pending.pending_channels + ] + ) + if any(failures): + error: Final = "delivery_failed" if "delivery_failed" in failures else "transport_not_configured" + await self._store.defer_notification(pending.incident.id, error) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index a2928bba287..732eb0b4b35 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -8282,6 +8282,8 @@ async def send_email( html: str | None = None, text: str | None = None, message_id: str | None = None, + *, + raise_on_error: bool = False, ) -> bool: """ smtp_host, @@ -8340,6 +8342,8 @@ async def send_email( ) except Exception as e: + if raise_on_error: + raise verbose_proxy_logger.exception("An error occurred while sending the email:" + str(e)) return False return True diff --git a/litellm/repositories/usage_alert_health.py b/litellm/repositories/usage_alert_health.py new file mode 100644 index 00000000000..50049eb126a --- /dev/null +++ b/litellm/repositories/usage_alert_health.py @@ -0,0 +1,95 @@ +"""Sample successful aggregate flushes without adding request-time telemetry.""" + +from datetime import datetime, timedelta, timezone +from typing import ( + TYPE_CHECKING, + Final, + Literal, + TypeAlias, + cast, # noqa: TID251 # generated Prisma delegate supplies the repository protocol +) + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from litellm.repositories.usage_alert_metrics import MetricsDatabase + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + +FlushSource: TypeAlias = Literal["user", "team"] + + +class FlushCoverage(BaseModel): + model_config = ConfigDict(frozen=True) + hours: int + after_grace: bool + + +_COVERAGE: Final = TypeAdapter(tuple[FlushCoverage, ...]) + + +class UsageAlertHealthRepository: + def __init__(self, database: MetricsDatabase) -> None: + self._database = database + + async def record(self, source: FlushSource, now: datetime) -> bool: + hour: Final = now.astimezone(timezone.utc).replace(minute=0, second=0, microsecond=0, tzinfo=None) + rows: Final = await self._database.query_raw( + """INSERT INTO "LiteLLM_UsageAlertFlush" (source,hour,observed_at) + SELECT $1,$2,$3 WHERE NOT EXISTS ( + SELECT 1 FROM "LiteLLM_SpendLogDelivery" WHERE state='pending' AND created_at<$2 LIMIT 1 + ) ON CONFLICT (source,hour) DO UPDATE SET observed_at=EXCLUDED.observed_at RETURNING source""", + source, + hour, + now.astimezone(timezone.utc).replace(tzinfo=None), + ) + await self._database.query_raw( + 'DELETE FROM "LiteLLM_UsageAlertFlush" WHERE hour<$1 RETURNING source', hour - timedelta(days=35) + ) + return bool(rows) + + async def coverage(self, source: FlushSource, start: datetime) -> FlushCoverage: + utc_start: Final = start.astimezone(timezone.utc).replace(tzinfo=None) + end: Final = utc_start + timedelta(days=1) + rows: Final = _COVERAGE.validate_python( + await self._database.query_raw( + """SELECT COUNT(*) FILTER(WHERE hour >= $2 AND hour < $3)::integer AS hours, + COALESCE(BOOL_OR(observed_at >= $3 + INTERVAL '1 hour'),FALSE) AS after_grace + FROM "LiteLLM_UsageAlertFlush" WHERE source=$1 AND hour >= $2""", + source, + utc_start, + end, + ) + ) + return rows[0] if rows else FlushCoverage(hours=0, after_grace=False) + + +class UsageAlertFlushRecorder: + """At most one heartbeat attempt per source per worker per hour after success.""" + + def __init__(self) -> None: + self._user_hour: datetime | None = None + self._team_hour: datetime | None = None + + async def record(self, prisma_client: "PrismaClient", source: FlushSource) -> None: + from litellm._logging import verbose_proxy_logger + from litellm.proxy.utils import ProxyUpdateSpend + + if ProxyUpdateSpend.disable_spend_updates(): + return + now: Final = datetime.now(timezone.utc) + hour: Final = now.replace(minute=0, second=0, microsecond=0) + prior: Final = self._user_hour if source == "user" else self._team_hour + if prior == hour: + return + database: Final = cast(MetricsDatabase, prisma_client.writer_db) # cast-ok: generated Prisma delegate + try: + if not await UsageAlertHealthRepository(database).record(source, now): + return + except Exception: # noqa: BLE001 # observability must not fail or requeue a committed spend flush + verbose_proxy_logger.warning("Could not record usage-alert aggregate freshness") + return + if source == "user": + self._user_hour = hour + else: + self._team_hour = hour diff --git a/litellm/repositories/usage_alert_metrics.py b/litellm/repositories/usage_alert_metrics.py new file mode 100644 index 00000000000..1623a698d2c --- /dev/null +++ b/litellm/repositories/usage_alert_metrics.py @@ -0,0 +1,92 @@ +"""Bounded reads of the daily rollups used by the Usage dashboard.""" + +from collections.abc import AsyncIterator, Mapping, Sequence +from datetime import date, timedelta +from typing import Final, Literal, Protocol, TypeAlias + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from litellm.repositories.daily_activity_buckets import api_key_bucket, model_bucket + +Scope: TypeAlias = Literal["deployment", "key", "user", "team", "model"] + + +class DailyAlertMetrics(BaseModel): + model_config = ConfigDict(frozen=True) + + entity_id: str + day: date + spend: float + requests: int + errors: int + response_time_ms: float + timed_requests: int + + +class MetricsDatabase(Protocol): + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ... + + +_ROWS: Final = TypeAdapter(tuple[DailyAlertMetrics, ...]) +_BATCH_SIZE: Final = 500 + + +def _source(scope: Scope) -> tuple[str, str]: + if scope == "team": + return '"LiteLLM_DailyTeamSpend"', "team_id" + if scope == "user": + return '"LiteLLM_DailyUserSpend"', "user_id" + if scope == "key": + return '"LiteLLM_DailyUserSpend"', "api_key" + if scope == "model": + return '"LiteLLM_DailyUserSpend"', "COALESCE(NULLIF(model, ''), 'unknown-model')" + return '"LiteLLM_DailyUserSpend"', "'deployment'::text" + + +def _query(scope: Scope) -> str: + table, entity = _source(scope) + # Identifiers come only from the closed scope mapping; request values are parameters. + return f""" +WITH entities AS MATERIALIZED ( + SELECT DISTINCT {entity} AS entity_id FROM {table} + WHERE date >= $1 AND date <= $2 AND {entity} >= COALESCE($3::text, '') AND ($3::text IS NULL OR {entity} <> $3) + ORDER BY entity_id LIMIT $4 +) +SELECT entities.entity_id, activity.date AS day, + SUM(activity.spend) AS spend, + SUM(activity.api_requests)::bigint AS requests, + SUM(activity.failed_requests)::bigint AS errors, + SUM(activity.total_response_time_ms) AS response_time_ms, + SUM(activity.timed_requests)::bigint AS timed_requests +FROM {table} activity JOIN entities ON {entity} = entities.entity_id +WHERE activity.date >= $1 AND activity.date <= $2 +GROUP BY entities.entity_id, activity.date +ORDER BY entities.entity_id, activity.date +""" + + +def _reporting_id(scope: Scope, row: DailyAlertMetrics) -> DailyAlertMetrics: + if scope == "key": + return row.model_copy(update={"entity_id": api_key_bucket(row.entity_id)}) + if scope == "model": + return row.model_copy(update={"entity_id": model_bucket(row.entity_id)}) + return row + + +class UsageAlertMetricsRepository: + def __init__(self, database: MetricsDatabase) -> None: + self._database = database + + async def batches( + self, scope: Scope, target: date, baseline_days: int + ) -> AsyncIterator[tuple[DailyAlertMetrics, ...]]: + start: Final = (target - timedelta(days=baseline_days)).isoformat() + query: Final = _query(scope) + cursor: str | None = None # rebind-ok: keyset pagination retains only the current page + while rows := _ROWS.validate_python( + await self._database.query_raw(query, start, target.isoformat(), cursor, _BATCH_SIZE) + ): + yield tuple(_reporting_id(scope, row) for row in rows) + cursor = rows[-1].entity_id + if scope == "deployment": + return diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index 3ecfe60d679..dcab3eb7cac 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -3,6 +3,7 @@ import json from collections.abc import Mapping from datetime import date, datetime, timedelta, timezone +from itertools import groupby from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -13,7 +14,7 @@ from typing import ( from urllib.parse import urlencode from uuid import NAMESPACE_URL, uuid4, uuid5 -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter from litellm.repositories.daily_activity_buckets import api_key_from_bucket from litellm.types.proxy.usage_alert_defaults import DEFAULT_USAGE_RULE_IDS @@ -45,6 +46,12 @@ if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient +class _BreachedEntity(BaseModel): + model_config = ConfigDict(frozen=True) + rule_id: str + scope_id: str | None + + class AlertDatabase(Protocol): async def query_raw(self, query: str, *args: object) -> object: ... @@ -672,7 +679,9 @@ class UsageAlertsRepository: rows: Final = TypeAdapter(tuple[_Pending, ...]).validate_python( await self.database.query_raw( """SELECT i.record,r.record AS rule,i.notified_channels FROM "LiteLLM_UsageAlertIncident" i JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id - WHERE i.status='open' AND r.enabled AND NOT r.deleted AND EXISTS ( + WHERE i.status='open' AND r.enabled AND NOT r.deleted + AND COALESCE((i.record->>'next_notification_attempt')::timestamptz,'-infinity'::timestamptz)<=CURRENT_TIMESTAMP + AND EXISTS ( SELECT 1 FROM jsonb_array_elements_text(r.record->'notification_channels') channel WHERE NOT (channel=ANY(i.notified_channels))) ORDER BY i.updated_at,i.id LIMIT $1""", limit, @@ -693,7 +702,16 @@ class UsageAlertsRepository: async def mark_notified(self, incident_id: str, channel: UsageAlertChannel) -> None: await self.database.query_raw( - 'UPDATE "LiteLLM_UsageAlertIncident" SET notified_channels=array_append(notified_channels,$2::text) WHERE id=$1 AND NOT ($2::text=ANY(notified_channels)) RETURNING id', + """UPDATE "LiteLLM_UsageAlertIncident" i SET + notified_channels=array_append(i.notified_channels,$2::text), + record=i.record || jsonb_build_object('notified_channels',array_append(i.notified_channels,$2::text), 'last_notification_attempt',CURRENT_TIMESTAMP) || + CASE WHEN NOT EXISTS ( + SELECT 1 FROM "LiteLLM_UsageAlertRule" r, + jsonb_array_elements_text(r.record->'notification_channels') channel + WHERE r.id=i.rule_id AND NOT (channel=ANY(array_append(i.notified_channels,$2::text))) + ) THEN jsonb_build_object('notification_error',NULL,'next_notification_attempt',NULL) + ELSE '{}'::jsonb END + WHERE i.id=$1 AND NOT ($2::text=ANY(i.notified_channels)) RETURNING i.id""", incident_id, channel, ) @@ -718,3 +736,35 @@ class UsageAlertsRepository: ) ) return rows[0]["role"] if rows else None + + async def breached_entities( + self, rule_ids: tuple[str, ...], entity_ids: tuple[str | None, ...] + ) -> Mapping[str, frozenset[str | None]]: + rows: Final = TypeAdapter(tuple[_BreachedEntity, ...]).validate_python( + await self.database.query_raw( + 'SELECT DISTINCT rule_id,scope_id FROM "LiteLLM_UsageAlertIncident" ' + "WHERE rule_id=ANY($1::text[]) AND scope_key=ANY($2::text[]) AND breached ORDER BY rule_id", + rule_ids, + tuple(entity or "" for entity in entity_ids), + ) + ) + return MappingProxyType( + { + rule_id: frozenset(row.scope_id for row in group) + for rule_id, group in groupby(rows, lambda row: row.rule_id) + } + ) + + async def defer_notification(self, incident_id: str, error: str) -> None: + await self.database.query_raw( + """UPDATE "LiteLLM_UsageAlertIncident" SET updated_at=CURRENT_TIMESTAMP, + record=record || jsonb_build_object( + 'notification_error',$2::text, + 'notification_attempts',COALESCE((record->>'notification_attempts')::int,0)+1, + 'last_notification_attempt',CURRENT_TIMESTAMP, + 'next_notification_attempt',CURRENT_TIMESTAMP + make_interval(secs => + LEAST(86400,3600*power(2,LEAST(5,COALESCE((record->>'notification_attempts')::int,0)))::int)) + ) WHERE id=$1 RETURNING id""", + incident_id, + error, + ) diff --git a/litellm/types/integrations/slack_alerting.py b/litellm/types/integrations/slack_alerting.py index 89c2ad841e8..19e6437dadf 100644 --- a/litellm/types/integrations/slack_alerting.py +++ b/litellm/types/integrations/slack_alerting.py @@ -177,6 +177,7 @@ class AlertType(str, Enum): failed_tracking_spend = "failed_tracking_spend" user_spend_thresholds = "user_spend_thresholds" user_spend_anomalies = "user_spend_anomalies" + usage_alerts = "usage_alerts" # Database alerts db_exceptions = "db_exceptions" diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py index 64336441fbe..cc967e03e66 100644 --- a/litellm/types/proxy/usage_alerts.py +++ b/litellm/types/proxy/usage_alerts.py @@ -85,6 +85,11 @@ class UsageAlert(BaseModel): first_seen: datetime last_seen: datetime activity_path: str + notified_channels: tuple[UsageAlertChannel, ...] = () + notification_attempts: int = 0 + last_notification_attempt: datetime | None = None + next_notification_attempt: datetime | None = None + notification_error: Literal["delivery_failed", "transport_not_configured"] | None = None class UsageAlertUpdate(BaseModel): diff --git a/schema.prisma b/schema.prisma index 8a67d1b123a..3a08c094b3d 100644 --- a/schema.prisma +++ b/schema.prisma @@ -2203,3 +2203,12 @@ model LiteLLM_IssueDeliveryRule { @@id([delivery_id, rule_id, source_id]) } + +// Hourly samples of successful background daily-aggregate flushes, retained for 35 days. +model LiteLLM_UsageAlertFlush { + source String + hour DateTime + observed_at DateTime + + @@id([source, hour]) +} diff --git a/tests/unit/integrations/SlackAlerting/test_usage_alerts.py b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py new file mode 100644 index 00000000000..a0f6088a807 --- /dev/null +++ b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py @@ -0,0 +1,78 @@ +from datetime import date, datetime, timezone +from typing import Final + +import httpx +import pytest + +from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, activity_url +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertRule + +_NOW: Final = datetime(2026, 10, 20, tzinfo=timezone.utc) +_RULE: Final = UsageAlertRule( + name="Request spike", + id="rule", + scope_type="key", + scope_id="reporting-key", + metric="requests", + direction="increase", + created_at=_NOW, + updated_at=_NOW, + created_by=None, + notification_channels=("slack",), +) +_ALERT: Final = UsageAlert( + id="incident", + rule_id="rule", + rule_name="Request spike", + scope_type="key", + scope_id="reporting-key", + metric="requests", + direction="increase", + status="open", + day=date(2026, 10, 20), + observed_value=1000, + baseline_value=200, + threshold=600, + first_seen=_NOW, + last_seen=_NOW, + baseline_start=date(2026, 10, 13), + baseline_end=date(2026, 10, 19), + baseline_sample_days=7, + activity_path="/api-keys?key=reporting-key&detail_tab=activity", +) + + +def test_activity_link_retains_dashboard_mount_and_entity_query() -> None: + assert activity_url(_ALERT, "https://gateway.example.com/") == ( + "https://gateway.example.com/ui/api-keys?key=reporting-key&detail_tab=activity" + ) + assert activity_url(_ALERT, None) == _ALERT.activity_path + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", [200, 503]) +async def test_slack_ack_requires_successful_http_response(status: int, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + requests: Final[list[httpx.Request]] = [] + + def transport(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(status) + + handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(transport)) + slack: Final = SlackAlerting( + alerting=["slack"], default_webhook_url="https://hooks.example.com/test", async_http_handler=handler + ) + delivery: Final = UsageAlertDelivery(slack) + try: + if status == 200: + assert await delivery.send(_ALERT, _RULE, "slack") is True + else: + with pytest.raises(httpx.HTTPStatusError): + await delivery.send(_ALERT, _RULE, "slack") + assert len(requests) == 1 + assert b"https://gateway.example.com/ui/api-keys?key=reporting-key&detail_tab=activity" in requests[0].content + finally: + await handler.close() diff --git a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py new file mode 100644 index 00000000000..4aa657de48b --- /dev/null +++ b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py @@ -0,0 +1,155 @@ +from datetime import date, datetime, timedelta, timezone +from typing import Final + +import pytest + +from litellm.proxy.spend_tracking.daily_usage_alerts import DailyComparison, compare_daily_usage +from litellm.proxy.spend_tracking.usage_alert_job import completed_day +from litellm.repositories.usage_alert_metrics import DailyAlertMetrics +from litellm.types.proxy.usage_alerts import UsageAlertRuleInput + +_DAY: Final = date(2026, 10, 20) + + +def _row(day: date, requests: int = 200, spend: float = 20, ms: float = 100_000, timed: int = 200) -> DailyAlertMetrics: + return DailyAlertMetrics( + entity_id="key-a", day=day, spend=spend, requests=requests, errors=10, response_time_ms=ms, timed_requests=timed + ) + + +@pytest.mark.parametrize( + "metric,observed,baseline", [("spend", 100, 20), ("requests", 1000, 200), ("latency", 2000, 500)] +) +def test_spikes_use_recorded_reference_days_and_weighted_latency(metric: str, observed: float, baseline: float) -> None: + rule: Final = UsageAlertRuleInput.model_validate( + {"name": "spike", "scope_type": "key", "metric": metric, "direction": "increase"} + ) + history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 3, 7)) + current: Final = _row(_DAY, requests=1000, spend=100, ms=400_000) + assert compare_daily_usage(rule, (*history, current), _DAY) == DailyComparison( + 3, observed, baseline, baseline * 3, True + ) + + +def test_missing_target_is_zero_for_counts_but_never_zero_latency() -> None: + history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3)) + rule: Final = UsageAlertRuleInput(name="drop", scope_type="key", metric="requests", direction="decrease") + assert compare_daily_usage(rule, history, _DAY) == DailyComparison(3, 0, 200, 200 / 3, True) + assert compare_daily_usage(rule.model_copy(update={"metric": "latency"}), history, _DAY) is None + + +def test_low_volume_is_insufficient_and_recorded_zero_baseline_uses_absolute_floor() -> None: + rule: Final = UsageAlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase") + history: Final = tuple(_row(_DAY - timedelta(days=days), requests=10, spend=0) for days in (1, 2, 3)) + assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) is None + enough_requests: Final = tuple(row.model_copy(update={"requests": 200}) for row in history) + assert compare_daily_usage(rule, (*enough_requests, _row(_DAY, spend=100)), _DAY) == DailyComparison( + 3, 100, 0, 0, True + ) + floor: Final = rule.model_copy(update={"min_value": 100}) + assert compare_daily_usage(floor, (*enough_requests, _row(_DAY, spend=100)), _DAY) == DailyComparison( + 3, 100, 0, 100, False + ) + + +def test_absolute_floor_suppresses_small_changes() -> None: + history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3)) + rule: Final = UsageAlertRuleInput( + name="spike", scope_type="key", metric="spend", direction="increase", min_value=100 + ) + assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) == DailyComparison(3, 100, 20, 60, False) + assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 60, False) + + +def test_day_is_eligible_only_after_one_hour_utc_grace() -> None: + assert completed_day(datetime(2026, 10, 21, 0, 59, tzinfo=timezone.utc)) == date(2026, 10, 19) + assert completed_day(datetime(2026, 10, 21, 1, 0, tzinfo=timezone.utc)) == _DAY + + +def test_latency_weights_request_samples_instead_of_averaging_daily_averages() -> None: + rule: Final = UsageAlertRuleInput( + name="latency", scope_type="key", metric="latency", direction="increase", min_history_days=2 + ) + rows: Final = ( + _row(_DAY - timedelta(days=1), requests=100, ms=100_000, timed=100), + _row(_DAY - timedelta(days=2), requests=400, ms=1_600_000, timed=400), + _row(_DAY, requests=100, ms=1_100_000, timed=100), + ) + assert compare_daily_usage(rule, rows, _DAY) == DailyComparison(2, 11000, 3400, 10200, True) + + +def test_errors_are_failed_request_count_and_require_absolute_change_floor() -> None: + rule: Final = UsageAlertRuleInput( + name="errors", scope_type="key", metric="errors", direction="increase", min_value=10 + ) + history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3)) + current: Final = _row(_DAY).model_copy(update={"errors": 40}) + assert compare_daily_usage(rule, (*history, current), _DAY) == DailyComparison(3, 40, 10, 30, True) + + +def test_a_value_at_the_configured_multiplier_alerts() -> None: + history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3)) + rule: Final = UsageAlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase") + assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 60, True) + + +def test_default_notifications_require_opt_in_and_do_not_duplicate_legacy_user_spend() -> None: + from litellm.proxy.spend_tracking.usage_alert_job import default_usage_rules + + defaults: Final = default_usage_rules() + assert len(defaults) == 40 + assert all(not rule.notification_channels for rule in defaults) + opted_in: Final = default_usage_rules(("slack",), legacy_user_spend_alerts=True) + quiet: Final = tuple(rule for rule in opted_in if not rule.notification_channels) + assert tuple((rule.scope_type, rule.metric, rule.direction) for rule in quiet) == (("user", "spend", "increase"),) + + +@pytest.mark.parametrize( + "scope,entity", + [("key", "unattributed"), ("key", "sk-example"), ("user", "default_user_id"), ("team", "litellm-dashboard")], +) +def test_sentinel_entities_cannot_create_alerts_with_unusable_activity_links(scope: str, entity: str) -> None: + from litellm.proxy.spend_tracking.usage_alert_job import _observations + from litellm.types.proxy.usage_alerts import UsageAlertRule + + rule: Final = UsageAlertRule.model_validate( + { + "id": "rule", + "name": "spike", + "scope_type": scope, + "metric": "spend", + "direction": "increase", + "created_at": "2026-10-01T00:00:00Z", + "updated_at": "2026-10-01T00:00:00Z", + "created_by": None, + } + ) + history: Final = tuple( + _row(_DAY - timedelta(days=days)).model_copy(update={"entity_id": entity}) for days in (1, 2, 3) + ) + current: Final = _row(_DAY, spend=100).model_copy(update={"entity_id": entity}) + assert _observations(rule, (*history, current), _DAY) == () + + +def test_custom_entity_rule_replaces_only_the_matching_default() -> None: + from uuid import NAMESPACE_URL, uuid5 + + from litellm.proxy.spend_tracking.usage_alert_job import _suppressed_entities + from litellm.types.proxy.usage_alerts import UsageAlertRule + + default: Final = UsageAlertRule( + id=str(uuid5(NAMESPACE_URL, "litellm:usage-alert:key:spend:increase")), + name="default", + scope_type="key", + metric="spend", + direction="increase", + created_at=datetime(2026, 10, 1, tzinfo=timezone.utc), + updated_at=datetime(2026, 10, 1, tzinfo=timezone.utc), + created_by=None, + ) + custom: Final = default.model_copy(update={"id": "custom", "scope_id": "key-a"}) + other_metric: Final = custom.model_copy(update={"id": "latency", "metric": "latency", "scope_id": "key-b"}) + assert _suppressed_entities(default, (default, custom, other_metric)) == frozenset({"key-a"}) + assert _suppressed_entities(custom, (default, custom, other_metric)) == frozenset() + wildcard: Final = custom.model_copy(update={"scope_id": None}) + assert _suppressed_entities(default, (default, wildcard)) == frozenset({None}) diff --git a/tests/unit/proxy/utils/prisma_and_spend/test_send_email.py b/tests/unit/proxy/utils/prisma_and_spend/test_send_email.py index d4965c64559..c4f36883270 100644 --- a/tests/unit/proxy/utils/prisma_and_spend/test_send_email.py +++ b/tests/unit/proxy/utils/prisma_and_spend/test_send_email.py @@ -7,13 +7,16 @@ Symbols pinned here: from __future__ import annotations import threading -from typing import Any +from typing import Any, TYPE_CHECKING from unittest.mock import Mock import pytest from litellm.proxy.utils import send_email +if TYPE_CHECKING: + from tests.unit.proxy.utils.prisma_and_spend.conftest import InMemorySMTP + @pytest.fixture(autouse=True) def _smtp_env(monkeypatch: pytest.MonkeyPatch) -> None: @@ -176,3 +179,12 @@ async def test_should_send_plain_text_and_html_as_multipart_alternative(monkeypa assert tuple(part.get_content_type() for part in parts) == ("text/plain", "text/html") assert parts[0].get_payload(decode=True).decode() == "Model retirement" assert parts[1].get_payload(decode=True).decode() == "

Model retirement

" + + +@pytest.mark.asyncio +async def test_durable_alert_delivery_can_observe_smtp_failure(in_memory_smtp: InMemorySMTP) -> None: + in_memory_smtp.raise_on_send = RuntimeError("smtp unavailable") + with pytest.raises(RuntimeError, match="smtp unavailable"): + await send_email(receiver_email="alerts@example.com", subject="Daily usage", html="

Alert

", + raise_on_error=True) + assert in_memory_smtp.sent == [] From 008593cd91e92d1c34e59bc0170971534cfce1bd Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 11:57:35 +0800 Subject: [PATCH 02/44] fix: preserve effective alert thresholds and current notification scope --- litellm/proxy/spend_tracking/daily_usage_alerts.py | 12 +++++++----- litellm/proxy/spend_tracking/usage_alert_job.py | 5 ++++- litellm/repositories/usage_alerts.py | 11 +++++++++-- .../proxy/spend_tracking/test_daily_usage_alerts.py | 6 +++--- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/spend_tracking/daily_usage_alerts.py b/litellm/proxy/spend_tracking/daily_usage_alerts.py index a79adaaedfa..2fe1d3bdffa 100644 --- a/litellm/proxy/spend_tracking/daily_usage_alerts.py +++ b/litellm/proxy/spend_tracking/daily_usage_alerts.py @@ -78,9 +78,11 @@ def compare_daily_usage( if baseline == 0: if rule.direction == "decrease": return None - return DailyComparison(sample_days, observed, 0, rule.min_value, observed > rule.min_value) - threshold: Final = baseline * rule.multiplier if rule.direction == "increase" else baseline / rule.multiplier - crossed: Final = observed >= threshold if rule.direction == "increase" else observed <= threshold - return DailyComparison( - sample_days, observed, baseline, threshold, crossed and abs(observed - baseline) >= rule.min_value + return DailyComparison(sample_days, observed, 0, rule.min_value, observed > 0 and observed >= rule.min_value) + threshold: Final = ( + max(baseline * rule.multiplier, baseline + rule.min_value) + if rule.direction == "increase" + else min(baseline / rule.multiplier, baseline - rule.min_value) ) + crossed: Final = observed >= threshold if rule.direction == "increase" else observed <= threshold + return DailyComparison(sample_days, observed, baseline, threshold, crossed) diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index 18bd2059adb..06ab1544fc0 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -69,7 +69,7 @@ def _has_activity_page(scope: UsageAlertScope, entity: str) -> bool: if not entity.strip(): return False if scope == "key": - return entity != "unattributed" and not entity.startswith("sk-") + return entity != "unattributed" and not entity.removeprefix("raw:").startswith("sk-") if scope == "user": return entity != "default_user_id" if scope == "team": @@ -275,6 +275,9 @@ class UsageAlertJob: await self._notify_incident(pending) async def _notify_incident(self, pending: UsageAlertNotification) -> None: + current: Final = await self._store.get_rule(pending.rule.id) + if current is None or not current.enabled or current.updated_at != pending.rule.updated_at: + return failures: Final = tuple( [ await self._deliver_channel(pending.incident, pending.rule, channel) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index dcab3eb7cac..deffb85429e 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -680,6 +680,9 @@ class UsageAlertsRepository: await self.database.query_raw( """SELECT i.record,r.record AS rule,i.notified_channels FROM "LiteLLM_UsageAlertIncident" i JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id WHERE i.status='open' AND r.enabled AND NOT r.deleted + AND i.scope_type=r.scope_type AND i.metric=r.record->>'metric' + AND i.record->>'direction'=r.record->>'direction' + AND (r.scope_id IS NULL OR i.scope_id=r.scope_id) AND COALESCE((i.record->>'next_notification_attempt')::timestamptz,'-infinity'::timestamptz)<=CURRENT_TIMESTAMP AND EXISTS ( SELECT 1 FROM jsonb_array_elements_text(r.record->'notification_channels') channel WHERE NOT (channel=ANY(i.notified_channels))) @@ -742,8 +745,12 @@ class UsageAlertsRepository: ) -> Mapping[str, frozenset[str | None]]: rows: Final = TypeAdapter(tuple[_BreachedEntity, ...]).validate_python( await self.database.query_raw( - 'SELECT DISTINCT rule_id,scope_id FROM "LiteLLM_UsageAlertIncident" ' - "WHERE rule_id=ANY($1::text[]) AND scope_key=ANY($2::text[]) AND breached ORDER BY rule_id", + 'SELECT DISTINCT i.rule_id,i.scope_id FROM "LiteLLM_UsageAlertIncident" i ' + 'JOIN "LiteLLM_UsageAlertRule" r ON r.id=i.rule_id ' + "WHERE i.rule_id=ANY($1::text[]) AND i.scope_key=ANY($2::text[]) AND i.breached " + "AND i.scope_type=r.scope_type AND i.metric=r.record->>'metric' " + "AND i.record->>'direction'=r.record->>'direction' " + "AND (r.scope_id IS NULL OR i.scope_id=r.scope_id) ORDER BY i.rule_id", rule_ids, tuple(entity or "" for entity in entity_ids), ) diff --git a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py index 4aa657de48b..2a4c223fc35 100644 --- a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py +++ b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py @@ -48,7 +48,7 @@ def test_low_volume_is_insufficient_and_recorded_zero_baseline_uses_absolute_flo ) floor: Final = rule.model_copy(update={"min_value": 100}) assert compare_daily_usage(floor, (*enough_requests, _row(_DAY, spend=100)), _DAY) == DailyComparison( - 3, 100, 0, 100, False + 3, 100, 0, 100, True ) @@ -57,8 +57,8 @@ def test_absolute_floor_suppresses_small_changes() -> None: rule: Final = UsageAlertRuleInput( name="spike", scope_type="key", metric="spend", direction="increase", min_value=100 ) - assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) == DailyComparison(3, 100, 20, 60, False) - assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 60, False) + assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) == DailyComparison(3, 100, 20, 120, False) + assert compare_daily_usage(rule, (*history, _row(_DAY, spend=60)), _DAY) == DailyComparison(3, 60, 20, 120, False) def test_day_is_eligible_only_after_one_hour_utc_grace() -> None: From 2fe5bad5038f6e98189aec41c59cdf130b4d53f4 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 12:02:29 +0800 Subject: [PATCH 03/44] perf: batch daily alert checkpoints and index scoped rules --- .../proxy/spend_tracking/usage_alert_job.py | 82 ++++++++++++++----- litellm/repositories/usage_alerts.py | 53 ++++++++++++ .../database/test_usage_alert_evaluation.py | 35 ++++++++ 3 files changed, 148 insertions(+), 22 deletions(-) create mode 100644 tests/integration/database/test_usage_alert_evaluation.py diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index 06ab1544fc0..c04eb88ead5 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -1,9 +1,9 @@ """Once-daily evaluation, polled hourly by one database-elected worker.""" import asyncio -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from datetime import date, datetime, time, timedelta, timezone -from itertools import groupby, product +from itertools import chain, groupby, product from typing import ( TYPE_CHECKING, Final, @@ -22,6 +22,7 @@ from litellm.types.integrations.slack_alerting import AlertType from litellm.types.proxy.usage_alerts import ( UsageAlert, UsageAlertChannel, + UsageAlertEvaluationStatus, UsageAlertMetric, UsageAlertNotification, UsageAlertObservation, @@ -179,9 +180,9 @@ class UsageAlertJob: async def _rules(self) -> tuple[UsageAlertRule, ...]: pages: Final[list[UsageAlertRule]] = [] # mutable-ok: collect configuration pages offset = 0 # rebind-ok: configuration pagination cursor - while page := await self._store.list_enabled_rules(limit=100, offset=offset): + while page := await self._store.list_enabled_rules(limit=500, offset=offset): pages.extend(page) - if len(page) < 100: + if len(page) < 500: return tuple(pages) offset += len(page) return tuple(pages) @@ -192,7 +193,10 @@ class UsageAlertJob: self._defaults_registered = True target: Final = completed_day(now) rules: Final = await self._rules() - due: Final = tuple([rule for rule in rules if not await self._store.is_day_complete(rule, target)]) + complete: Final[set[str]] = set() # mutable-ok: batched completion lookup, bounded by configuration size + for offset in range(0, len(rules), 500): + complete.update(await self._store.completed_rule_ids(rules[offset : offset + 500], target)) + due: Final = tuple(rule for rule in rules if rule.id not in complete) for scoped in (tuple(rule for rule in due if rule.scope_type == scope) for scope in _SCOPES): if scoped: await self._scope(scoped, target, rules) @@ -209,37 +213,71 @@ class UsageAlertJob: rule for rule in rules if coverage.after_grace and (rule.direction == "increase" or coverage.hours == 24) ) suppressed: Final = {rule.id: _suppressed_entities(rule, enabled) for rule in eligible} - for rule in rules: - if rule not in eligible: - await self._store.record_evaluation_status(rule, target, "insufficient_coverage") + eligible_ids: Final = frozenset(rule.id for rule in eligible) + await self._record_evaluations( + tuple((rule, "insufficient_coverage") for rule in rules if rule.id not in eligible_ids), + target, + completed=False, + ) if not eligible: return + by_entity: Final = { + entity: tuple(group) + for entity, group in groupby( + sorted(eligible, key=lambda rule: rule.scope_id or ""), lambda rule: rule.scope_id + ) + } evaluated: Final[set[str]] = set() # mutable-ok: at most one entry per configured rule across streaming pages async for rows in self._metrics.batches(scope, target, max(rule.baseline_days for rule in eligible)): - evaluated.update(await self._page(eligible, rows, target, enabled)) - for rule in eligible: - await self._store.finish_day(rule, target) - await self._store.record_evaluation_status( - rule, target, "complete" if rule.id in evaluated or suppressed[rule.id] else "insufficient_history" - ) + evaluated.update(await self._page(by_entity, rows, target, suppressed)) + await self._record_evaluations( + tuple( + (rule, "complete" if rule.id in evaluated or suppressed[rule.id] else "insufficient_history") + for rule in eligible + ), + target, + completed=True, + ) + + async def _record_evaluations( + self, + evaluations: tuple[tuple[UsageAlertRule, UsageAlertEvaluationStatus], ...], + target: date, + *, + completed: bool, + ) -> None: + for offset in range(0, len(evaluations), 500): + await self._store.record_evaluations(evaluations[offset : offset + 500], target, completed=completed) async def _page( self, - rules: tuple[UsageAlertRule, ...], + rules: Mapping[str | None, tuple[UsageAlertRule, ...]], rows: tuple[DailyAlertMetrics, ...], target: date, - enabled: tuple[UsageAlertRule, ...], + suppressed: Mapping[str, frozenset[str | None]], ) -> tuple[str, ...]: - entities: Final = ( - (None,) if rules[0].scope_type == "deployment" else tuple(dict.fromkeys(row.entity_id for row in rows)) - ) - active: Final = await self._store.breached_entities(tuple(rule.id for rule in rules), entities) + by_entity: Final = {entity: tuple(group) for entity, group in groupby(rows, lambda row: row.entity_id)} + applicable: Final = tuple(chain.from_iterable(rules.get(entity, ()) for entity in (None, *by_entity))) + if not applicable: + return () + entities: Final = (None,) if applicable[0].scope_type == "deployment" else tuple(by_entity) + active: Final[dict[str, frozenset[str | None]]] = {} # mutable-ok: bounded active results for this entity page + for offset in range(0, len(applicable), 500): + active.update( + await self._store.breached_entities( + tuple(rule.id for rule in applicable[offset : offset + 500]), entities + ) + ) return tuple( [ rule.id - for rule in rules + for rule in applicable if await self._rule_page( - rule, rows, target, active.get(rule.id, frozenset()), _suppressed_entities(rule, enabled) + rule, + rows if rule.scope_id is None else by_entity[rule.scope_id], + target, + active.get(rule.id, frozenset()), + suppressed[rule.id], ) ] ) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index deffb85429e..76205a20f55 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -558,6 +558,59 @@ class UsageAlertsRepository: self._checkpoint(rule, day), ) + async def completed_rule_ids(self, rules: tuple[UsageAlertRule, ...], day: date) -> frozenset[str]: + if len(rules) > 500: + raise ValueError("A checkpoint batch must contain at most 500 rules") + rows: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python( + await self.database.query_raw( + """SELECT input.rule_id AS id FROM jsonb_to_recordset($1::jsonb) + AS input(rule_id text, checkpoint_id text) + JOIN "LiteLLM_UsageAlertCheckpoint" c ON c.id=input.checkpoint_id WHERE c.completed""", + json.dumps(tuple({"rule_id": rule.id, "checkpoint_id": self._checkpoint(rule, day)} for rule in rules)), + ) + ) + return frozenset(row.id for row in rows) + + async def record_evaluations( + self, + evaluations: tuple[tuple[UsageAlertRule, UsageAlertEvaluationStatus], ...], + day: date, + *, + completed: bool, + ) -> None: + if not evaluations: + return + if len(evaluations) > 500: + raise ValueError("An evaluation batch must contain at most 500 rules") + await self.database.query_raw( + """WITH input AS ( + SELECT * FROM jsonb_to_recordset($1::jsonb) + AS i(rule_id text,checkpoint_id text,revision timestamptz,status text) + ), checkpoints AS ( + INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,completed) + SELECT checkpoint_id,true FROM input WHERE $3::boolean + ON CONFLICT(id) DO NOTHING + ) UPDATE "LiteLLM_UsageAlertRule" r SET record=r.record || jsonb_build_object( + 'last_evaluated_day',$2::text,'evaluation_status',input.status) + FROM input WHERE r.id=input.rule_id AND NOT r.deleted + AND (r.record->>'updated_at')::timestamptz=input.revision + AND (r.record->>'last_evaluated_day' IS DISTINCT FROM $2::text + OR r.record->>'evaluation_status' IS DISTINCT FROM input.status) RETURNING r.id""", + json.dumps( + tuple( + { + "rule_id": rule.id, + "checkpoint_id": self._checkpoint(rule, day), + "revision": rule.updated_at.isoformat(), + "status": status, + } + for rule, status in evaluations + ) + ), + day.isoformat(), + completed, + ) + async def ensure_default_rules(self, rules: tuple[UsageAlertRuleInput, ...]) -> None: for rule in rules: await self._ensure_default_rule(rule) diff --git a/tests/integration/database/test_usage_alert_evaluation.py b/tests/integration/database/test_usage_alert_evaluation.py new file mode 100644 index 00000000000..b6f5c46bdbb --- /dev/null +++ b/tests/integration/database/test_usage_alert_evaluation.py @@ -0,0 +1,35 @@ +from datetime import date +from typing import Final + +import pytest +from integration._support.actors import ManagementDatabase + +from litellm.repositories.usage_alerts import UsageAlertsRepository +from litellm.types.proxy.usage_alerts import UsageAlertRuleInput + + +@pytest.mark.asyncio(loop_scope="session") +async def test_batch_checkpoints_preserve_rule_revisions_and_incomplete_status(db: ManagementDatabase) -> None: + repository: Final = UsageAlertsRepository.from_prisma(db.prisma) + definition: Final = UsageAlertRuleInput( + name="Daily evaluation", scope_type="team", scope_id="team-example", metric="spend", direction="increase" + ) + rule: Final = await repository.save_rule(definition, "qa-admin") + assert rule is not None + day: Final = date(2026, 11, 4) + try: + assert not await repository.completed_rule_ids((rule,), day) + await repository.record_evaluations(((rule, "complete"),), day, completed=True) + assert await repository.completed_rule_ids((rule,), day) == frozenset({rule.id}) + edited: Final = await repository.save_rule(definition.model_copy(update={"multiplier": 4}), "qa-admin", rule) + assert edited is not None + assert not await repository.completed_rule_ids((edited,), day) + await repository.record_evaluations(((rule, "insufficient_coverage"),), day, completed=False) + assert await repository.get_rule(rule.id) == edited + await repository.record_evaluations(((edited, "insufficient_coverage"),), day, completed=False) + incomplete: Final = await repository.get_rule(rule.id) + assert incomplete is not None and incomplete.evaluation_status == "insufficient_coverage" + assert incomplete.last_evaluated_day == day + assert not await repository.completed_rule_ids((edited,), day) + finally: + await repository.delete_rule(rule.id) From f7588efee52610367768062355a035c5798a5be2 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 12:04:03 +0800 Subject: [PATCH 04/44] docs: explain saved incident evidence and inclusive thresholds --- docs/features/usage-alerts.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/features/usage-alerts.md b/docs/features/usage-alerts.md index c14aa564221..3111be514de 100644 --- a/docs/features/usage-alerts.md +++ b/docs/features/usage-alerts.md @@ -123,7 +123,7 @@ The response contains `updated_count`. Only incidents in that exact group that s ## Create a rule -This example alerts when a team's daily spend is more than three times its baseline and the absolute change is at least $10. Replace `team-example` with the team's ID. +This example alerts when a team's daily spend is at least three times its baseline and the absolute change is at least $10. Replace `team-example` with the team's ID. ```bash curl "$LITELLM_API_BASE/usage/alert-rules" \ @@ -169,7 +169,7 @@ Use `scope_type: "deployment"` with no `scope_id` for the gateway. For `model`, - `DELETE /usage/alert-rules/{rule_id}` removes a rule from evaluation and preserves existing incidents. Deleted defaults stay deleted. - `PATCH /usage/alerts/{alert_id}` with `{"status":"dismissed"}` dismisses an incident. `open` and `resolved` are also accepted. -Consecutive breached days update one incident. Re-evaluating the same day does not reopen a dismissed or resolved incident. After recovery, a later breach starts a new incident. Editing a rule does not rewrite historical comparisons. +Consecutive breached days keep one incident. Its original day, comparison values, and Activity link stay fixed; `last_seen` advances. Re-evaluating the same day does not reopen a dismissed or resolved incident. After recovery, a later breach starts a new incident. Editing a rule does not rewrite historical comparisons. Rules can specify `notification_channels: ["slack", "email"]` and `email_recipients: ["alerts@example.com"]`. Notification transports must already be configured on the gateway. An empty channel list keeps incidents available through the API and dashboard. Issue delivery is recorded per channel and destination. A repeated projection does not resend a successful delivery. Failed deliveries retry in the background, and the worker rechecks the issue, source signal, and current rule immediately before transport. Disabling a rule or acknowledging an issue cannot recall a send already in flight. Delivery is at least once: an ambiguous transport timeout, or partial success across several configured Slack webhooks, can cause a retry after a destination accepted a message. Configure `PROXY_BASE_URL` for clickable links in email and Slack. Default rules notify through Slack only when `usage_alerts` is explicitly enabled in the gateway's alert types when those rules are first registered. Otherwise they start with in-app/API incidents. To enable notifications after the defaults already exist, edit their channels through the API or dashboard; changing the gateway alert types does not rewrite saved rules. Explicit rule channels use the configured transport without an additional alert-type switch. If legacy `user_spend_anomalies` notifications are enabled, the default daily user-spend increase rule stays in-app to avoid overlapping notifications. Email requires recipients on the rule. From d28e78e79f30edaa545b62e3fedec723f2d75729 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 12:05:48 +0800 Subject: [PATCH 05/44] fix: cast alert freshness timestamps for Prisma queries --- litellm/repositories/usage_alert_health.py | 17 +++++++++----- .../database/test_usage_alert_evaluation.py | 22 ++++++++++++++++++- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/litellm/repositories/usage_alert_health.py b/litellm/repositories/usage_alert_health.py index 50049eb126a..178f515895f 100644 --- a/litellm/repositories/usage_alert_health.py +++ b/litellm/repositories/usage_alert_health.py @@ -32,19 +32,24 @@ class UsageAlertHealthRepository: def __init__(self, database: MetricsDatabase) -> None: self._database = database + @classmethod + def from_prisma(cls, prisma: "PrismaClient") -> "UsageAlertHealthRepository": + database: Final = cast(MetricsDatabase, prisma.writer_db) # cast-ok: generated Prisma delegate + return cls(database) + async def record(self, source: FlushSource, now: datetime) -> bool: hour: Final = now.astimezone(timezone.utc).replace(minute=0, second=0, microsecond=0, tzinfo=None) rows: Final = await self._database.query_raw( """INSERT INTO "LiteLLM_UsageAlertFlush" (source,hour,observed_at) - SELECT $1,$2,$3 WHERE NOT EXISTS ( - SELECT 1 FROM "LiteLLM_SpendLogDelivery" WHERE state='pending' AND created_at<$2 LIMIT 1 + SELECT $1,$2::timestamp,$3::timestamp WHERE NOT EXISTS ( + SELECT 1 FROM "LiteLLM_SpendLogDelivery" WHERE state='pending' AND created_at<$2::timestamp LIMIT 1 ) ON CONFLICT (source,hour) DO UPDATE SET observed_at=EXCLUDED.observed_at RETURNING source""", source, hour, now.astimezone(timezone.utc).replace(tzinfo=None), ) await self._database.query_raw( - 'DELETE FROM "LiteLLM_UsageAlertFlush" WHERE hour<$1 RETURNING source', hour - timedelta(days=35) + 'DELETE FROM "LiteLLM_UsageAlertFlush" WHERE hour<$1::timestamp RETURNING source', hour - timedelta(days=35) ) return bool(rows) @@ -53,9 +58,9 @@ class UsageAlertHealthRepository: end: Final = utc_start + timedelta(days=1) rows: Final = _COVERAGE.validate_python( await self._database.query_raw( - """SELECT COUNT(*) FILTER(WHERE hour >= $2 AND hour < $3)::integer AS hours, - COALESCE(BOOL_OR(observed_at >= $3 + INTERVAL '1 hour'),FALSE) AS after_grace - FROM "LiteLLM_UsageAlertFlush" WHERE source=$1 AND hour >= $2""", + """SELECT COUNT(*) FILTER(WHERE hour >= $2::timestamp AND hour < $3::timestamp)::integer AS hours, + COALESCE(BOOL_OR(observed_at >= $3::timestamp + INTERVAL '1 hour'),FALSE) AS after_grace + FROM "LiteLLM_UsageAlertFlush" WHERE source=$1 AND hour >= $2::timestamp""", source, utc_start, end, diff --git a/tests/integration/database/test_usage_alert_evaluation.py b/tests/integration/database/test_usage_alert_evaluation.py index b6f5c46bdbb..4b926174e14 100644 --- a/tests/integration/database/test_usage_alert_evaluation.py +++ b/tests/integration/database/test_usage_alert_evaluation.py @@ -1,9 +1,10 @@ -from datetime import date +from datetime import date, datetime, timedelta, timezone from typing import Final import pytest from integration._support.actors import ManagementDatabase +from litellm.repositories.usage_alert_health import UsageAlertHealthRepository from litellm.repositories.usage_alerts import UsageAlertsRepository from litellm.types.proxy.usage_alerts import UsageAlertRuleInput @@ -33,3 +34,22 @@ async def test_batch_checkpoints_preserve_rule_revisions_and_incomplete_status(d assert not await repository.completed_rule_ids((edited,), day) finally: await repository.delete_rule(rule.id) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_freshness_timestamps_use_the_real_prisma_parameter_types(db: ManagementDatabase) -> None: + repository: Final = UsageAlertHealthRepository.from_prisma(db.prisma) + start: Final = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0) + timedelta(days=2) + try: + assert await repository.record("user", start) + assert (await repository.coverage("user", start)).hours == 1 + assert not (await repository.coverage("user", start)).after_grace + assert await repository.record("user", start + timedelta(days=1, hours=1)) + coverage: Final = await repository.coverage("user", start) + assert coverage.hours == 1 and coverage.after_grace + finally: + await db.prisma.writer_db.query_raw( + "DELETE FROM \"LiteLLM_UsageAlertFlush\" WHERE source='user' AND hour IN ($1::timestamp,$2::timestamp) RETURNING source", + start.replace(tzinfo=None), + (start + timedelta(days=1, hours=1)).replace(tzinfo=None), + ) From 687f1ec400a20b86a5e32b38038af4946bee2c9e Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 12:19:42 +0800 Subject: [PATCH 06/44] fix(alerts): schedule initial checks with an explicit timezone --- litellm/proxy/proxy_server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 144819876bd..ad4bb00b344 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11681,11 +11681,12 @@ class ProxyStartupEvent: from litellm.proxy.spend_tracking.usage_alert_job import USAGE_ALERT_JOB_ID, UsageAlertJob usage_alert_job: Final = UsageAlertJob.from_prisma(prisma_client, proxy_logging_obj.slack_alerting_instance) - scheduler.add_job( + scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # apscheduler is untyped usage_alert_job.run, "interval", seconds=3600, - next_run_time=datetime.now() + timedelta(seconds=batch_writing_interval + 10 + random.randint(0, 20)), + next_run_time=datetime.now(timezone.utc) + + timedelta(seconds=batch_writing_interval + 10 + random.randint(0, 20)), id=USAGE_ALERT_JOB_ID, replace_existing=True, max_instances=1, From 2e043bd25981a00a34f5c18e4ec33019dfa6b3de Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 12:23:47 +0800 Subject: [PATCH 07/44] fix: show one activity link in usage alert emails --- litellm/integrations/SlackAlerting/usage_alerts.py | 6 +++--- .../unit/integrations/SlackAlerting/test_usage_alerts.py | 9 ++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/SlackAlerting/usage_alerts.py b/litellm/integrations/SlackAlerting/usage_alerts.py index 9b09d8fb05c..2eb6e5854fa 100644 --- a/litellm/integrations/SlackAlerting/usage_alerts.py +++ b/litellm/integrations/SlackAlerting/usage_alerts.py @@ -30,13 +30,13 @@ def activity_url(alert: UsageAlert, base_url: str | None) -> str: return base_url.rstrip("/") + "/ui" + alert.activity_path -def incident_message(alert: UsageAlert, url: str) -> str: +def incident_message(alert: UsageAlert, url: str | None) -> str: scope: Final = "Gateway" if alert.scope_type == "deployment" else f"{alert.scope_type}: {alert.scope_id}" unit: Final = " ms" if alert.metric == "latency" else " USD" if alert.metric == "spend" else "" return ( f"{alert.rule_name}\n{scope}\n{alert.day.isoformat()} UTC: " f"{alert.observed_value:,.2f}{unit}; baseline {alert.baseline_value:,.2f}{unit}; " - f"threshold {alert.threshold:,.2f}{unit}.\nView activity: {url}" + f"threshold {alert.threshold:,.2f}{unit}." + (f"\nView activity: {url}" if url is not None else "") ) @@ -49,7 +49,7 @@ class UsageAlertDelivery: if channel not in configured: return False url: Final = activity_url(alert, os.getenv("PROXY_BASE_URL")) - message: Final = incident_message(alert, url) + message: Final = incident_message(alert, None if channel == "email" else url) if channel == "email": from litellm.proxy.utils import send_email diff --git a/tests/unit/integrations/SlackAlerting/test_usage_alerts.py b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py index a0f6088a807..b4b9e8ea36f 100644 --- a/tests/unit/integrations/SlackAlerting/test_usage_alerts.py +++ b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py @@ -5,7 +5,7 @@ import httpx import pytest from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting -from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, activity_url +from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, activity_url, incident_message from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertRule @@ -51,6 +51,13 @@ def test_activity_link_retains_dashboard_mount_and_entity_query() -> None: assert activity_url(_ALERT, None) == _ALERT.activity_path +def test_email_summary_omits_the_plain_text_link_used_by_slack() -> None: + summary: Final = incident_message(_ALERT, None) + assert summary.endswith("1,000.00; baseline 200.00; threshold 600.00.") + assert "View activity" not in summary + assert incident_message(_ALERT, _ALERT.activity_path) == f"{summary}\nView activity: {_ALERT.activity_path}" + + @pytest.mark.asyncio @pytest.mark.parametrize("status", [200, 503]) async def test_slack_ack_requires_successful_http_response(status: int, monkeypatch: pytest.MonkeyPatch) -> None: From 811df975fb4a3b7725836f25e2e1475e29c4a0f9 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 13:33:26 +0800 Subject: [PATCH 08/44] Reduce daily alert metric row allocations --- litellm/repositories/usage_alert_metrics.py | 14 ++++--- .../spend_tracking/test_daily_usage_alerts.py | 40 ++++++++++++++++--- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/litellm/repositories/usage_alert_metrics.py b/litellm/repositories/usage_alert_metrics.py index 1623a698d2c..3cb80b711a3 100644 --- a/litellm/repositories/usage_alert_metrics.py +++ b/litellm/repositories/usage_alert_metrics.py @@ -1,19 +1,19 @@ """Bounded reads of the daily rollups used by the Usage dashboard.""" from collections.abc import AsyncIterator, Mapping, Sequence +from dataclasses import dataclass, replace from datetime import date, timedelta from typing import Final, Literal, Protocol, TypeAlias -from pydantic import BaseModel, ConfigDict, TypeAdapter +from pydantic import TypeAdapter from litellm.repositories.daily_activity_buckets import api_key_bucket, model_bucket Scope: TypeAlias = Literal["deployment", "key", "user", "team", "model"] -class DailyAlertMetrics(BaseModel): - model_config = ConfigDict(frozen=True) - +@dataclass(frozen=True, slots=True) +class DailyAlertMetrics: entity_id: str day: date spend: float @@ -67,9 +67,11 @@ ORDER BY entities.entity_id, activity.date def _reporting_id(scope: Scope, row: DailyAlertMetrics) -> DailyAlertMetrics: if scope == "key": - return row.model_copy(update={"entity_id": api_key_bucket(row.entity_id)}) + key: Final = api_key_bucket(row.entity_id) + return row if key == row.entity_id else replace(row, entity_id=key) if scope == "model": - return row.model_copy(update={"entity_id": model_bucket(row.entity_id)}) + model: Final = model_bucket(row.entity_id) + return row if model == row.entity_id else replace(row, entity_id=model) return row diff --git a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py index 2a4c223fc35..d5e1f0d2b80 100644 --- a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py +++ b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py @@ -1,3 +1,4 @@ +from dataclasses import replace from datetime import date, datetime, timedelta, timezone from typing import Final @@ -42,7 +43,7 @@ def test_low_volume_is_insufficient_and_recorded_zero_baseline_uses_absolute_flo rule: Final = UsageAlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase") history: Final = tuple(_row(_DAY - timedelta(days=days), requests=10, spend=0) for days in (1, 2, 3)) assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) is None - enough_requests: Final = tuple(row.model_copy(update={"requests": 200}) for row in history) + enough_requests: Final = tuple(replace(row, requests=200) for row in history) assert compare_daily_usage(rule, (*enough_requests, _row(_DAY, spend=100)), _DAY) == DailyComparison( 3, 100, 0, 0, True ) @@ -83,7 +84,7 @@ def test_errors_are_failed_request_count_and_require_absolute_change_floor() -> name="errors", scope_type="key", metric="errors", direction="increase", min_value=10 ) history: Final = tuple(_row(_DAY - timedelta(days=days)) for days in (1, 2, 3)) - current: Final = _row(_DAY).model_copy(update={"errors": 40}) + current: Final = replace(_row(_DAY), errors=40) assert compare_daily_usage(rule, (*history, current), _DAY) == DailyComparison(3, 40, 10, 30, True) @@ -124,10 +125,8 @@ def test_sentinel_entities_cannot_create_alerts_with_unusable_activity_links(sco "created_by": None, } ) - history: Final = tuple( - _row(_DAY - timedelta(days=days)).model_copy(update={"entity_id": entity}) for days in (1, 2, 3) - ) - current: Final = _row(_DAY, spend=100).model_copy(update={"entity_id": entity}) + history: Final = tuple(replace(_row(_DAY - timedelta(days=days)), entity_id=entity) for days in (1, 2, 3)) + current: Final = replace(_row(_DAY, spend=100), entity_id=entity) assert _observations(rule, (*history, current), _DAY) == () @@ -153,3 +152,32 @@ def test_custom_entity_rule_replaces_only_the_matching_default() -> None: assert _suppressed_entities(custom, (default, custom, other_metric)) == frozenset() wildcard: Final = custom.model_copy(update={"scope_id": None}) assert _suppressed_entities(default, (default, wildcard)) == frozenset({None}) + + +def test_database_metric_rows_preserve_numeric_and_date_validation() -> None: + from pydantic import TypeAdapter, ValidationError + + adapter: Final = TypeAdapter(tuple[DailyAlertMetrics, ...]) + record: Final = { + "entity_id": "key-a", + "day": "2026-10-20", + "spend": "20", + "requests": "200", + "errors": "10", + "response_time_ms": "100000", + "timed_requests": "200", + } + assert adapter.validate_python([record]) == (_row(_DAY),) + with pytest.raises(ValidationError): + adapter.validate_python([{**record, "requests": "invalid"}]) + + +def test_reporting_identity_preserves_canonical_rows_and_distinct_raw_buckets() -> None: + from litellm.repositories.usage_alert_metrics import _reporting_id + + canonical: Final = _row(_DAY) + assert _reporting_id("key", canonical) is canonical + assert _reporting_id("model", canonical) is canonical + assert _reporting_id("key", replace(canonical, entity_id="")).entity_id == "unattributed" + assert _reporting_id("key", replace(canonical, entity_id="unattributed")).entity_id == "raw:unattributed" + assert _reporting_id("key", replace(canonical, entity_id="raw:key")).entity_id == "raw:raw:key" From eea8ef9f4442d855b61a91a3363a88c63f832553 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 13:39:06 +0800 Subject: [PATCH 09/44] Skip healthy observation allocations during daily alert evaluation --- .../proxy/spend_tracking/usage_alert_job.py | 63 +++++++++++-------- .../spend_tracking/test_daily_usage_alerts.py | 37 ++++++++++- 2 files changed, 73 insertions(+), 27 deletions(-) diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index c04eb88ead5..af03e549bcd 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -2,6 +2,7 @@ import asyncio from collections.abc import Mapping, Sequence +from dataclasses import dataclass from datetime import date, datetime, time, timedelta, timezone from itertools import chain, groupby, product from typing import ( @@ -78,37 +79,48 @@ def _has_activity_page(scope: UsageAlertScope, entity: str) -> bool: return True +@dataclass(frozen=True, slots=True) +class _Observations: + evaluated: bool + changed: tuple[UsageAlertObservation, ...] + + def _observations( rule: UsageAlertRule, rows: Sequence[DailyAlertMetrics], target: date, suppressed: frozenset[str | None] = frozenset(), -) -> tuple[UsageAlertObservation, ...]: - def observation(entity: str, entity_rows: Sequence[DailyAlertMetrics]) -> UsageAlertObservation | None: + *, + active: frozenset[str | None] = frozenset(), +) -> _Observations: + changed: Final[list[UsageAlertObservation]] = [] # mutable-ok: collect only persisted changes for this page + evaluated = False # rebind-ok: retain whether any entity had sufficient evidence, including healthy entities + for entity, group in groupby(rows, lambda row: row.entity_id): if None in suppressed or entity in suppressed: - return None + continue if not _has_activity_page(rule.scope_type, entity): - return None + continue if rule.scope_id is not None and rule.scope_id != entity: - return None - comparison: Final = compare_daily_usage(rule, entity_rows, target) + continue + comparison = compare_daily_usage(rule, tuple(group), target) # rebind-ok: comparison for the current entity if comparison is None: - return None - identity: Final = None if rule.scope_type == "deployment" else entity - return UsageAlertObservation( - entity_id=identity, - baseline_sample_days=comparison.baseline_sample_days, - day=target, - observed=comparison.observed, - baseline=comparison.baseline, - threshold=comparison.threshold, - breached=comparison.breached, + continue + evaluated = True + identity = None if rule.scope_type == "deployment" else entity # rebind-ok: scope identity for this entity + if not comparison.breached and identity not in active: + continue + changed.append( + UsageAlertObservation( + entity_id=identity, + baseline_sample_days=comparison.baseline_sample_days, + day=target, + observed=comparison.observed, + baseline=comparison.baseline, + threshold=comparison.threshold, + breached=comparison.breached, + ) ) - - candidates: Final = tuple( - observation(entity, tuple(group)) for entity, group in groupby(rows, lambda row: row.entity_id) - ) - return tuple(item for item in candidates if item is not None) + return _Observations(evaluated, tuple(changed)) def _default_rule(rule: UsageAlertRule) -> bool: @@ -290,11 +302,10 @@ class UsageAlertJob: active: frozenset[str | None], suppressed: frozenset[str | None], ) -> bool: - observations: Final = _observations(rule, rows, target, suppressed) - changed: Final = tuple(item for item in observations if item.breached or item.entity_id in active) - if changed: - await self._store.record_observations(rule, changed) - return bool(observations) + observations: Final = _observations(rule, rows, target, suppressed, active=active) + if observations.changed: + await self._store.record_observations(rule, observations.changed) + return observations.evaluated async def _deliver_channel(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> str | None: try: diff --git a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py index d5e1f0d2b80..12627d40ea4 100644 --- a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py +++ b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py @@ -127,7 +127,9 @@ def test_sentinel_entities_cannot_create_alerts_with_unusable_activity_links(sco ) history: Final = tuple(replace(_row(_DAY - timedelta(days=days)), entity_id=entity) for days in (1, 2, 3)) current: Final = replace(_row(_DAY, spend=100), entity_id=entity) - assert _observations(rule, (*history, current), _DAY) == () + observations: Final = _observations(rule, (*history, current), _DAY) + assert not observations.evaluated + assert observations.changed == () def test_custom_entity_rule_replaces_only_the_matching_default() -> None: @@ -181,3 +183,36 @@ def test_reporting_identity_preserves_canonical_rows_and_distinct_raw_buckets() assert _reporting_id("key", replace(canonical, entity_id="")).entity_id == "unattributed" assert _reporting_id("key", replace(canonical, entity_id="unattributed")).entity_id == "raw:unattributed" assert _reporting_id("key", replace(canonical, entity_id="raw:key")).entity_id == "raw:raw:key" + + +@pytest.mark.parametrize( + "active,spend,history_days,evaluated,persisted", + [(False, 20, 3, True, False), (True, 20, 3, True, True), (False, 100, 3, True, True), (True, 100, 1, False, False)], +) +def test_healthy_evaluation_completes_and_recovers_only_with_sufficient_evidence( + active: bool, spend: float, history_days: int, evaluated: bool, persisted: bool +) -> None: + from litellm.proxy.spend_tracking.usage_alert_job import _observations + from litellm.types.proxy.usage_alerts import UsageAlertRule + + rule: Final = UsageAlertRule( + id="rule", + name="spike", + scope_type="key", + metric="spend", + direction="increase", + created_at=datetime(2026, 10, 1, tzinfo=timezone.utc), + updated_at=datetime(2026, 10, 1, tzinfo=timezone.utc), + created_by=None, + ) + history: Final = tuple(_row(_DAY - timedelta(days=day)) for day in range(1, history_days + 1)) + observations: Final = _observations( + rule, + (*history, _row(_DAY, spend=spend)), + _DAY, + active=frozenset({"key-a"}) if active else frozenset(), + ) + assert observations.evaluated is evaluated + assert bool(observations.changed) is persisted + if observations.changed: + assert observations.changed[0].breached is (spend >= 60) From 54d280719ee0b11354787d11b37f2df8580aca72 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 13:54:40 +0800 Subject: [PATCH 10/44] Align performance alert wording with the default rule controls --- litellm/integrations/SlackAlerting/usage_alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/SlackAlerting/usage_alerts.py b/litellm/integrations/SlackAlerting/usage_alerts.py index 2eb6e5854fa..2a06b3170ad 100644 --- a/litellm/integrations/SlackAlerting/usage_alerts.py +++ b/litellm/integrations/SlackAlerting/usage_alerts.py @@ -56,7 +56,7 @@ class UsageAlertDelivery: for recipient in rule.email_recipients: await send_email( receiver_email=recipient, - subject=f"LiteLLM usage alert: {alert.rule_name}", + subject=f"LiteLLM performance alert: {alert.rule_name}", html=( f"

{escape(message).replace(chr(10), '
')}

" f'

View activity

' From d39a98f41008eb6e776acc55265dcf892b2e1b09 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 14:51:45 +0800 Subject: [PATCH 11/44] feat: apply opinionated daily issue thresholds --- .../spend_tracking/daily_usage_alerts.py | 5 +- .../proxy/spend_tracking/usage_alert_job.py | 59 +++++++---------- .../spend_tracking/test_daily_usage_alerts.py | 66 ++++++++++++++++++- 3 files changed, 91 insertions(+), 39 deletions(-) diff --git a/litellm/proxy/spend_tracking/daily_usage_alerts.py b/litellm/proxy/spend_tracking/daily_usage_alerts.py index 2fe1d3bdffa..1278af5d148 100644 --- a/litellm/proxy/spend_tracking/daily_usage_alerts.py +++ b/litellm/proxy/spend_tracking/daily_usage_alerts.py @@ -10,7 +10,7 @@ from litellm.repositories.usage_alert_metrics import DailyAlertMetrics class ComparisonRule(Protocol): @property - def metric(self) -> Literal["spend", "requests", "latency", "errors"]: ... + def metric(self) -> Literal["spend", "requests", "latency", "errors", "error_rate"]: ... @property def direction(self) -> Literal["increase", "decrease"]: ... @@ -48,6 +48,9 @@ def _value(metric: str, rows: Sequence[DailyAlertMetrics], days: int) -> float | return sum(row.spend for row in rows) / days if metric == "errors": return sum(row.errors for row in rows) / days + if metric == "error_rate": + requests: Final = sum(row.requests for row in rows) + return 100 * sum(row.errors for row in rows) / requests if requests else None return sum(row.requests for row in rows) / days diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index af03e549bcd..95474d56609 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -1,17 +1,17 @@ """Once-daily evaluation, polled hourly by one database-elected worker.""" import asyncio -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass from datetime import date, datetime, time, timedelta, timezone -from itertools import chain, groupby, product +from itertools import chain, groupby from typing import ( TYPE_CHECKING, Final, Protocol, cast, # noqa: TID251 # generated Prisma delegate supplies the repository protocol ) -from uuid import NAMESPACE_URL, uuid4, uuid5 +from uuid import uuid4 from litellm._logging import verbose_proxy_logger from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, configured_destinations @@ -20,11 +20,11 @@ from litellm.repositories.usage_alert_health import UsageAlertHealthRepository from litellm.repositories.usage_alert_metrics import DailyAlertMetrics, MetricsDatabase, UsageAlertMetricsRepository from litellm.repositories.usage_alerts import UsageAlertsRepository from litellm.types.integrations.slack_alerting import AlertType +from litellm.types.proxy.usage_alert_defaults import USAGE_ALERT_SCOPES, default_usage_rules, is_default_usage_rule from litellm.types.proxy.usage_alerts import ( UsageAlert, UsageAlertChannel, UsageAlertEvaluationStatus, - UsageAlertMetric, UsageAlertNotification, UsageAlertObservation, UsageAlertRule, @@ -37,29 +37,6 @@ if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient USAGE_ALERT_JOB_ID: Final = "daily_usage_alerts" -_SCOPES: Final[tuple[UsageAlertScope, ...]] = ("deployment", "key", "user", "team", "model") -_METRICS: Final[tuple[UsageAlertMetric, ...]] = ("spend", "requests", "latency", "errors") -_MIN_VALUES: Final = {"spend": 10, "requests": 100, "latency": 100, "errors": 10} - - -def default_usage_rules( - channels: tuple[UsageAlertChannel, ...] = (), *, legacy_user_spend_alerts: bool = False -) -> tuple[UsageAlertRuleInput, ...]: - return tuple( - UsageAlertRuleInput( - name=f"Daily {scope} {metric} {direction}", - scope_type=scope, - metric=metric, - direction=direction, - min_value=_MIN_VALUES[metric], - notification_channels=( - () - if legacy_user_spend_alerts and (scope, metric, direction) == ("user", "spend", "increase") - else channels - ), - ) - for scope, metric, direction in product(_SCOPES, _METRICS, ("increase", "decrease")) - ) def completed_day(now: datetime) -> date: @@ -102,11 +79,11 @@ def _observations( continue if rule.scope_id is not None and rule.scope_id != entity: continue - comparison = compare_daily_usage(rule, tuple(group), target) # rebind-ok: comparison for the current entity + comparison: Final = compare_daily_usage(rule, tuple(group), target) if comparison is None: continue evaluated = True - identity = None if rule.scope_type == "deployment" else entity # rebind-ok: scope identity for this entity + identity: Final = None if rule.scope_type == "deployment" else entity if not comparison.breached and identity not in active: continue changed.append( @@ -123,19 +100,15 @@ def _observations( return _Observations(evaluated, tuple(changed)) -def _default_rule(rule: UsageAlertRule) -> bool: - return rule.id == str(uuid5(NAMESPACE_URL, f"litellm:usage-alert:{rule.scope_type}:{rule.metric}:{rule.direction}")) - - def _suppressed_entities(rule: UsageAlertRule, enabled: tuple[UsageAlertRule, ...]) -> frozenset[str | None]: - if not _default_rule(rule): + if not is_default_usage_rule(rule): return frozenset() matching: Final = tuple( other for other in enabled if (other.scope_type, other.metric, other.direction) == (rule.scope_type, rule.metric, rule.direction) ) - return frozenset(other.scope_id for other in matching if not _default_rule(other)) + return frozenset(other.scope_id for other in matching if not is_default_usage_rule(other)) class IncidentDelivery(Protocol): @@ -150,6 +123,8 @@ class UsageAlertJob: health: UsageAlertHealthRepository, delivery: IncidentDelivery, defaults: tuple[UsageAlertRuleInput, ...] | None = None, + *, + deprecation_projection: Callable[[datetime], Awaitable[None]] | None = None, ) -> None: self._store = store self._metrics = metrics @@ -157,9 +132,16 @@ class UsageAlertJob: self._delivery = delivery self._defaults = default_usage_rules() if defaults is None else defaults self._defaults_registered = False + self._deprecation_projection = deprecation_projection @classmethod - def from_prisma(cls, prisma: "PrismaClient", slack: "SlackAlerting") -> "UsageAlertJob": + def from_prisma( + cls, + prisma: "PrismaClient", + slack: "SlackAlerting", + *, + deprecation_projection: Callable[[datetime], Awaitable[None]] | None = None, + ) -> "UsageAlertJob": database: Final = cast(MetricsDatabase, prisma.writer_db) # cast-ok: generated Prisma delegate return cls( UsageAlertsRepository.from_prisma(prisma), @@ -172,6 +154,7 @@ class UsageAlertJob: else (), legacy_user_spend_alerts=AlertType.user_spend_anomalies in slack.alert_types, ), + deprecation_projection=deprecation_projection, ) async def run(self, now: datetime | None = None) -> None: @@ -200,6 +183,8 @@ class UsageAlertJob: return tuple(pages) async def _run(self, now: datetime) -> None: + if self._deprecation_projection is not None: + await self._deprecation_projection(now) if not self._defaults_registered: await self._store.ensure_default_rules(self._defaults) self._defaults_registered = True @@ -209,7 +194,7 @@ class UsageAlertJob: for offset in range(0, len(rules), 500): complete.update(await self._store.completed_rule_ids(rules[offset : offset + 500], target)) due: Final = tuple(rule for rule in rules if rule.id not in complete) - for scoped in (tuple(rule for rule in due if rule.scope_type == scope) for scope in _SCOPES): + for scoped in (tuple(rule for rule in due if rule.scope_type == scope) for scope in USAGE_ALERT_SCOPES): if scoped: await self._scope(scoped, target, rules) await self._notify() diff --git a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py index 12627d40ea4..c859cd8c922 100644 --- a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py +++ b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py @@ -7,6 +7,7 @@ import pytest from litellm.proxy.spend_tracking.daily_usage_alerts import DailyComparison, compare_daily_usage from litellm.proxy.spend_tracking.usage_alert_job import completed_day from litellm.repositories.usage_alert_metrics import DailyAlertMetrics +from litellm.types.proxy.usage_alert_defaults import default_usage_rules from litellm.types.proxy.usage_alerts import UsageAlertRuleInput _DAY: Final = date(2026, 10, 20) @@ -98,7 +99,7 @@ def test_default_notifications_require_opt_in_and_do_not_duplicate_legacy_user_s from litellm.proxy.spend_tracking.usage_alert_job import default_usage_rules defaults: Final = default_usage_rules() - assert len(defaults) == 40 + assert len(defaults) == 25 assert all(not rule.notification_channels for rule in defaults) opted_in: Final = default_usage_rules(("slack",), legacy_user_spend_alerts=True) quiet: Final = tuple(rule for rule in opted_in if not rule.notification_channels) @@ -216,3 +217,66 @@ def test_healthy_evaluation_completes_and_recovers_only_with_sufficient_evidence assert bool(observations.changed) is persisted if observations.changed: assert observations.changed[0].breached is (spend >= 60) + + +@pytest.mark.parametrize("multiple", [10, 50]) +def test_expensive_low_request_key_triggers_the_default_spend_issue(multiple: int) -> None: + rule: Final = next(rule for rule in default_usage_rules() if rule.scope_type == "key" and rule.metric == "spend") + history: Final = tuple(_row(_DAY - timedelta(days=day), requests=3, spend=10, timed=3) for day in (1, 2, 3)) + current: Final = _row(_DAY, requests=3, spend=10 * multiple, timed=3) + assert compare_daily_usage(rule, (*history, current), _DAY) == DailyComparison(3, 10 * multiple, 10, 30, True) + + +def test_proportional_failure_growth_does_not_become_a_reliability_issue() -> None: + rule: Final = next( + rule for rule in default_usage_rules() if rule.scope_type == "key" and rule.metric == "error_rate" + ) + history: Final = tuple(_row(_DAY - timedelta(days=day), requests=500) for day in (1, 2, 3)) + current: Final = replace(_row(_DAY, requests=5000), errors=100) + assert compare_daily_usage(rule, (*history, current), _DAY) == DailyComparison(3, 2, 2, 6, False) + + +@pytest.mark.parametrize("errors,breached", [(15, False), (25, True)]) +def test_error_rate_default_requires_two_percentage_points_as_well_as_the_multiplier( + errors: int, breached: bool +) -> None: + rule: Final = next( + rule for rule in default_usage_rules() if rule.scope_type == "key" and rule.metric == "error_rate" + ) + history: Final = tuple( + replace(_row(_DAY - timedelta(days=day), requests=1000), errors=5) for day in (1, 2, 3) + ) + current: Final = replace(_row(_DAY, requests=1000), errors=errors) + assert compare_daily_usage(rule, (*history, current), _DAY) == DailyComparison(3, errors / 10, 0.5, 2.5, breached) + + +def test_no_requests_cannot_report_zero_error_rate_or_resolve_a_reliability_issue() -> None: + rule: Final = UsageAlertRuleInput(name="rate", scope_type="key", metric="error_rate", direction="decrease") + history: Final = tuple(_row(_DAY - timedelta(days=day), requests=500) for day in (1, 2, 3)) + assert compare_daily_usage(rule, history, _DAY) is None + + +def test_traffic_drop_needs_a_full_recorded_week_and_an_eighty_percent_decline() -> None: + rule: Final = next( + rule + for rule in default_usage_rules() + if (rule.scope_type, rule.metric, rule.direction) == ("key", "requests", "decrease") + ) + history: Final = tuple(_row(_DAY - timedelta(days=day), requests=500) for day in range(1, 8)) + assert compare_daily_usage(rule, history[:-1], _DAY) is None + assert compare_daily_usage(rule, (*history, _row(_DAY, requests=101)), _DAY) == DailyComparison(7, 101, 500, 100, False) + assert compare_daily_usage(rule, (*history, _row(_DAY, requests=100)), _DAY) == DailyComparison(7, 100, 500, 100, True) + + +def test_latency_default_requires_a_material_change_and_enough_timed_requests() -> None: + rule: Final = next(rule for rule in default_usage_rules() if rule.scope_type == "key" and rule.metric == "latency") + history: Final = tuple( + _row(_DAY - timedelta(days=day), requests=500, ms=10_000, timed=100) for day in (1, 2, 3) + ) + assert compare_daily_usage(rule, (*history, _row(_DAY, ms=40_000, timed=100)), _DAY) == DailyComparison( + 3, 400, 100, 600, False + ) + assert compare_daily_usage(rule, (*history, _row(_DAY, ms=60_000, timed=100)), _DAY) == DailyComparison( + 3, 600, 100, 600, True + ) + assert compare_daily_usage(rule, (*history, _row(_DAY, ms=60_000, timed=99)), _DAY) is None From c0f77199ae370208991f12d1cb02d21e12e33753 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 14:44:01 +0800 Subject: [PATCH 12/44] feat(alerts): project used model retirements into issues --- .../spend_tracking/deprecation_issues.py | 174 ++++++++++++++++++ litellm/repositories/usage_alert_metrics.py | 37 ++++ .../test_deprecation_issue_evaluation.py | 61 ++++++ .../spend_tracking/test_deprecation_issues.py | 128 +++++++++++++ 4 files changed, 400 insertions(+) create mode 100644 litellm/proxy/spend_tracking/deprecation_issues.py create mode 100644 tests/integration/database/test_deprecation_issue_evaluation.py create mode 100644 tests/unit/proxy/spend_tracking/test_deprecation_issues.py diff --git a/litellm/proxy/spend_tracking/deprecation_issues.py b/litellm/proxy/spend_tracking/deprecation_issues.py new file mode 100644 index 00000000000..06f9a836261 --- /dev/null +++ b/litellm/proxy/spend_tracking/deprecation_issues.py @@ -0,0 +1,174 @@ +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import date, datetime, timedelta, timezone +from itertools import chain, groupby +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # Prisma delegates the query protocol dynamically + +from pydantic import BaseModel + +from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key +from litellm.repositories.config_repository import ConfigRepository +from litellm.repositories.usage_alert_metrics import MetricsDatabase, UsageAlertMetricsRepository +from litellm.types.proxy.issues import IssueCohort, IssueDeprecation, IssueImpact, IssueInput, IssueSignal +from litellm.types.proxy.model_deprecation import ( + DeprecationAlertState, + ModelDeprecationInfo, + ModelDeprecationResponse, + ModelDeprecationUsage, +) + +if TYPE_CHECKING: + from litellm.models.issues import IssueStore + + from litellm.proxy.utils import PrismaClient + +_PROJECTION_CHECKPOINT: Final = "issues:deprecation_projection" +_PAGE_SIZE: Final = 500 + + +class _ProjectionCheckpoint(BaseModel, frozen=True): + hour: datetime + + +@dataclass(frozen=True, slots=True) +class DeprecationCandidate: + usage_model: str + model: ModelDeprecationInfo + + @property + def state_key(self) -> str: + return deprecation_alert_state_key(self.usage_model, self.model.deprecation_date) + + +def _candidates(model: ModelDeprecationInfo) -> tuple[DeprecationCandidate, ...]: + base: Final = model.litellm_model if model.litellm_model is not None else model.model_name + primary: Final = DeprecationCandidate(base, model) + provider: Final = model.litellm_provider + if not provider or base.startswith(f"{provider}/"): + return (primary,) + return (primary, DeprecationCandidate(f"{provider}/{base}", model)) + + +def _identity(candidate: DeprecationCandidate) -> tuple[str, date]: + return candidate.usage_model, candidate.model.deprecation_date + + +def deprecation_candidates(snapshot: ModelDeprecationResponse) -> tuple[DeprecationCandidate, ...]: + ordered: Final = sorted( + chain.from_iterable(_candidates(model) for model in (*snapshot.deprecated, *snapshot.imminent)), + key=_identity, + ) + return tuple(next(group) for _, group in groupby(ordered, _identity)) + + +def deprecation_issue( + candidate: DeprecationCandidate, + usage: ModelDeprecationUsage, + state: DeprecationAlertState, + now: datetime, +) -> IssueInput: + today: Final = now.astimezone(timezone.utc).date() + model: Final = candidate.model + title: Final = ( + f"{usage.model} retired on {model.deprecation_date.isoformat()}" + if model.deprecation_date < today + else f"{usage.model} retires on {model.deprecation_date.isoformat()}" + ) + return IssueInput( + id=f"deprecation:{candidate.state_key.removeprefix('model_deprecation_alert:')}", + cohort_key=candidate.state_key, + kind="model_deprecation", + family="retirement", + title=title, + cohort=IssueCohort(scope_type="model", scope_id=usage.model), + observed_on=usage.last_used_date, + observed_at=now, + range_start=today - timedelta(days=29), + range_end=today, + grouping_reason="model_deprecation", + explanation="This model has recorded requests in the last 30 UTC dates and is retiring within 30 days or has retired.", + impact=IssueImpact(keys=usage.keys, users=usage.users, teams=usage.teams, models=1, requests=usage.requests), + deprecation=IssueDeprecation( + model=model, + usage_model=usage.model, + requests=usage.requests, + last_used_date=usage.last_used_date, + last_day_requests=usage.last_day_requests, + state=state, + ), + ) + + +def used_deprecation_issues( + candidates: Sequence[DeprecationCandidate], + usage: Mapping[str, ModelDeprecationUsage], + states: Mapping[str, DeprecationAlertState], + now: datetime, +) -> tuple[IssueInput, ...]: + def build(candidate: DeprecationCandidate) -> IssueInput: + state: Final = states.get(candidate.state_key) or DeprecationAlertState( + model=candidate.usage_model, deprecation_date=candidate.model.deprecation_date + ) + return deprecation_issue(candidate, usage[candidate.usage_model], state, now) + + return tuple(build(candidate) for candidate in candidates if candidate.usage_model in usage) + + +class DeprecationIssueProjector: + def __init__( + self, + config: ConfigRepository, + metrics: UsageAlertMetricsRepository, + issues: "IssueStore", + snapshot: Callable[[date], ModelDeprecationResponse], + ) -> None: + self._config = config + self._metrics = metrics + self._issues = issues + self._snapshot = snapshot + + @classmethod + def from_prisma( + cls, prisma: "PrismaClient", snapshot: Callable[[date], ModelDeprecationResponse] + ) -> "DeprecationIssueProjector": + from litellm.repositories.issues import IssuesRepository + + database: Final = cast(MetricsDatabase, prisma.writer_db) # cast-ok: generated Prisma query delegate + return cls( + ConfigRepository(prisma, use_writer=True), + UsageAlertMetricsRepository(database), + IssuesRepository.from_prisma(prisma), + snapshot, + ) + + async def _page(self, candidates: tuple[DeprecationCandidate, ...], now: datetime) -> None: + today: Final = now.astimezone(timezone.utc).date() + models: Final = tuple(sorted(frozenset(candidate.usage_model for candidate in candidates))) + usage: Final = await self._metrics.deprecation_usage(models, today - timedelta(days=29), today) + records: Final = await self._config.get_params(tuple(candidate.state_key for candidate in candidates)) + states: Final = MappingProxyType( + {record.param_name: DeprecationAlertState.model_validate(record.param_value) for record in records} + ) + by_model: Final = MappingProxyType({row.model: row for row in usage}) + issues: Final = used_deprecation_issues(candidates, by_model, states, now) + await self._issues.upsert_issues(issues) + await self._issues.add_signal_batch( + tuple( + IssueSignal( + issue_id=issue.id, source="model_deprecation", source_id=issue.id, deprecation=issue.deprecation + ) + for issue in issues + ) + ) + + async def run(self, now: datetime) -> None: + hour: Final = now.astimezone(timezone.utc).replace(minute=0, second=0, microsecond=0) + previous: Final = await self._config.get_param(_PROJECTION_CHECKPOINT) + if previous is not None and _ProjectionCheckpoint.model_validate(previous.param_value).hour >= hour: + return + candidates: Final = deprecation_candidates(self._snapshot(hour.date())) + for offset in range(0, len(candidates), _PAGE_SIZE): + await self._page(candidates[offset : offset + _PAGE_SIZE], now) + await self._issues.finish_deprecation_projection(now) + await self._config.set_param(_PROJECTION_CHECKPOINT, _ProjectionCheckpoint(hour=hour).model_dump(mode="json")) diff --git a/litellm/repositories/usage_alert_metrics.py b/litellm/repositories/usage_alert_metrics.py index 3cb80b711a3..c828a639c05 100644 --- a/litellm/repositories/usage_alert_metrics.py +++ b/litellm/repositories/usage_alert_metrics.py @@ -8,6 +8,7 @@ from typing import Final, Literal, Protocol, TypeAlias from pydantic import TypeAdapter from litellm.repositories.daily_activity_buckets import api_key_bucket, model_bucket +from litellm.types.proxy.model_deprecation import ModelDeprecationUsage Scope: TypeAlias = Literal["deployment", "key", "user", "team", "model"] @@ -29,6 +30,31 @@ class MetricsDatabase(Protocol): _ROWS: Final = TypeAdapter(tuple[DailyAlertMetrics, ...]) _BATCH_SIZE: Final = 500 +_DEPRECATION_USAGE: Final = TypeAdapter(tuple[ModelDeprecationUsage, ...]) + +_DEPRECATION_USAGE_QUERY: Final = """ +WITH activity AS MATERIALIZED ( + SELECT model, date, api_key, user_id, api_requests FROM "LiteLLM_DailyUserSpend" + WHERE date >= $1 AND date <= $2 AND model = ANY($3::text[]) AND api_requests > 0 +), daily AS ( + SELECT model, date, SUM(api_requests)::bigint AS requests FROM activity GROUP BY model, date +), latest AS ( + SELECT DISTINCT ON (model) model, date AS last_used_date, requests AS last_day_requests + FROM daily ORDER BY model, date DESC +), totals AS ( + SELECT model, SUM(api_requests)::bigint AS requests, + COUNT(DISTINCT NULLIF(api_key, ''))::int AS keys, + COUNT(DISTINCT NULLIF(NULLIF(user_id, ''), 'default_user_id'))::int AS users + FROM activity GROUP BY model +), teams AS ( + SELECT model, COUNT(DISTINCT NULLIF(NULLIF(team_id, ''), 'litellm-dashboard'))::int AS teams + FROM "LiteLLM_DailyTeamSpend" + WHERE date >= $1 AND date <= $2 AND model = ANY($3::text[]) AND api_requests > 0 + GROUP BY model +) +SELECT totals.*, latest.last_used_date, latest.last_day_requests, COALESCE(teams.teams, 0) AS teams +FROM totals JOIN latest USING (model) LEFT JOIN teams USING (model) ORDER BY model +""" def _source(scope: Scope) -> tuple[str, str]: @@ -79,6 +105,17 @@ class UsageAlertMetricsRepository: def __init__(self, database: MetricsDatabase) -> None: self._database = database + async def deprecation_usage( + self, models: tuple[str, ...], start: date, end: date + ) -> tuple[ModelDeprecationUsage, ...]: + if not models: + return () + if len(models) > _BATCH_SIZE or not 0 <= (end - start).days < 30: + raise ValueError("Deprecation usage accepts at most 500 models and 30 consecutive dates") + return _DEPRECATION_USAGE.validate_python( + await self._database.query_raw(_DEPRECATION_USAGE_QUERY, start.isoformat(), end.isoformat(), models) + ) + async def batches( self, scope: Scope, target: date, baseline_days: int ) -> AsyncIterator[tuple[DailyAlertMetrics, ...]]: diff --git a/tests/integration/database/test_deprecation_issue_evaluation.py b/tests/integration/database/test_deprecation_issue_evaluation.py new file mode 100644 index 00000000000..c280b51b54c --- /dev/null +++ b/tests/integration/database/test_deprecation_issue_evaluation.py @@ -0,0 +1,61 @@ +from datetime import date +from uuid import uuid4 + +import pytest +from integration._support.actors import ManagementDatabase + +from litellm.repositories.usage_alert_metrics import UsageAlertMetricsRepository + + +@pytest.mark.asyncio(loop_scope="session") +async def test_deprecation_usage_counts_distinct_resources_without_multiplying_requests(db: ManagementDatabase): + prefix = f"deprecation-issue-{uuid4()}" + model = f"example/{prefix}" + try: + await db.prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,endpoint,api_requests,updated_at) + SELECT $1||x.id,$1||'owner',x.day,$1||x.key, + CASE WHEN x.id='other-model' THEN $2||'-other' ELSE $2 END, + x.endpoint,x.requests,CURRENT_TIMESTAMP + FROM (VALUES + ('first','2026-11-04','key-a','/one',10), + ('second','2026-11-04','key-a','/two',20), + ('previous','2026-11-03','key-b','/one',7), + ('zero','2026-11-05','key-c','/one',0), + ('old','2026-10-01','key-d','/one',999), + ('other-model','2026-11-04','key-e','/one',999) + ) x(id,day,key,endpoint,requests) RETURNING id""", + prefix, + model, + ) + await db.prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyTeamSpend" + (id,team_id,date,api_key,model,endpoint,api_requests,updated_at) + SELECT $1||x.id,$1||x.team,x.day,$1||x.key,$2,x.endpoint,x.requests,CURRENT_TIMESTAMP + FROM (VALUES + ('first','2026-11-04','key-a','team-a','/one',15), + ('second','2026-11-04','key-a','team-a','/two',5), + ('previous','2026-11-03','key-b','team-b','/one',7), + ('zero','2026-11-05','key-c','team-c','/one',0), + ('old','2026-10-01','key-d','team-d','/one',999) + ) x(id,day,key,team,endpoint,requests) RETURNING id""", + prefix, + model, + ) + repository = UsageAlertMetricsRepository(db.prisma.writer_db) + rows = await repository.deprecation_usage((model, model + "-unused"), date(2026, 10, 7), date(2026, 11, 5)) + assert len(rows) == 1 + assert rows[0].model == model + assert rows[0].requests == 37 + assert rows[0].last_used_date == date(2026, 11, 4) + assert rows[0].last_day_requests == 30 + assert (rows[0].keys, rows[0].users, rows[0].teams) == (2, 1, 2) + assert await repository.deprecation_usage((), date(2026, 10, 7), date(2026, 11, 5)) == () + finally: + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE id LIKE $1 RETURNING id', prefix + "%" + ) + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_DailyTeamSpend" WHERE id LIKE $1 RETURNING id', prefix + "%" + ) diff --git a/tests/unit/proxy/spend_tracking/test_deprecation_issues.py b/tests/unit/proxy/spend_tracking/test_deprecation_issues.py new file mode 100644 index 00000000000..6ec00a44db1 --- /dev/null +++ b/tests/unit/proxy/spend_tracking/test_deprecation_issues.py @@ -0,0 +1,128 @@ +from datetime import date, datetime, timedelta, timezone + +from litellm.proxy.spend_tracking.deprecation_issues import deprecation_candidates, used_deprecation_issues +from litellm.types.proxy.model_deprecation import ( + DeprecationAlertState, + DeprecationAlertStatus, + ModelDeprecationInfo, + ModelDeprecationResponse, + ModelDeprecationUsage, + SuccessorModelSource, +) + +NOW = datetime(2026, 11, 5, 10, tzinfo=timezone.utc) +RETIREMENT = date(2026, 11, 20) + + +def model(**updates): + return ModelDeprecationInfo( + **{ + "model_name": "public-alias", + "litellm_model": "example/retiring-model", + "litellm_provider": "example", + "deprecation_date": RETIREMENT, + "days_until_deprecation": 15, + "status": "imminent", + **updates, + } + ) + + +def snapshot(*models, upcoming=()): + return ModelDeprecationResponse( + imminent=[entry for entry in models if entry.status == "imminent"], + deprecated=[entry for entry in models if entry.status == "deprecated"], + upcoming=upcoming, + warn_within_days=30, + checked_at=NOW, + ) + + +def usage(name="example/retiring-model", **updates): + return ModelDeprecationUsage( + **{ + "model": name, + "requests": 700, + "last_used_date": NOW.date(), + "last_day_requests": 30, + "keys": 12, + "users": 4, + "teams": 2, + **updates, + } + ) + + +def test_aliases_share_one_issue_only_when_actual_retiring_model_has_usage(): + candidates = deprecation_candidates(snapshot(model(), model(model_name="second-alias"))) + assert used_deprecation_issues(candidates, {"public-alias": usage("public-alias")}, {}, NOW) == () + issues = used_deprecation_issues(candidates, {usage().model: usage()}, {}, NOW) + assert len(issues) == 1 + issue = issues[0] + assert issue.cohort.scope_id == "example/retiring-model" + assert issue.impact.keys == 12 + assert issue.impact.requests == 700 + assert issue.range_start == date(2026, 10, 7) + assert issue.range_end == NOW.date() + + +def test_retirement_identity_survives_urgency_and_recommendation_changes(): + before = deprecation_candidates(snapshot(model())) + after = deprecation_candidates( + snapshot( + model( + status="deprecated", + days_until_deprecation=-1, + successor_model="example/replacement", + successor_model_source=SuccessorModelSource.DEPLOYMENT_CONFIG, + ) + ) + ) + original = used_deprecation_issues(before, {usage().model: usage()}, {}, NOW)[0] + later = datetime(2026, 11, 21, tzinfo=timezone.utc) + changed = used_deprecation_issues(after, {usage().model: usage()}, {}, later)[0] + assert original.id == changed.id + assert original.cohort_key == changed.cohort_key + assert changed.title.endswith("retired on 2026-11-20") + assert changed.range_end == later.date() + assert changed.deprecation.model.successor_model == "example/replacement" + assert changed.deprecation.model.successor_model_source == SuccessorModelSource.DEPLOYMENT_CONFIG + + +def test_each_retirement_date_keeps_its_existing_acknowledgment_identity(): + candidates = deprecation_candidates(snapshot(model(), model(deprecation_date=RETIREMENT + timedelta(days=1)))) + first = candidates[0] + state = DeprecationAlertState( + model=first.usage_model, + deprecation_date=first.model.deprecation_date, + status=DeprecationAlertStatus.DISMISSED, + last_seen_date=NOW.date(), + last_seen_requests=30, + ) + issues = used_deprecation_issues(candidates, {usage().model: usage()}, {first.state_key: state}, NOW) + assert len({issue.id for issue in issues}) == 2 + assert issues[0].deprecation.state == state + assert issues[1].deprecation.state.status == DeprecationAlertStatus.OPEN + + +def test_provider_prefix_candidates_match_only_exact_recorded_names(): + candidates = deprecation_candidates(snapshot(model(litellm_model="retiring-model"))) + rows = tuple(usage(name) for name in ("retiring-model", "example/retiring-model", "other/retiring-model")) + issues = used_deprecation_issues(candidates, {row.model: row for row in rows}, {}, NOW) + assert {issue.cohort.scope_id for issue in issues} == {"retiring-model", "example/retiring-model"} + + +def test_unused_and_future_retirements_do_not_create_issues(): + candidates = deprecation_candidates( + snapshot(model(litellm_model="example/unused"), upcoming=(model(status="upcoming", days_until_deprecation=60),)) + ) + assert used_deprecation_issues(candidates, {usage().model: usage()}, {}, NOW) == () + + +def test_maximum_model_identity_is_preserved_in_activity_and_evidence(): + name = "example/" + "m" * (1024 - len("example/")) + candidates = deprecation_candidates(snapshot(model(litellm_model=name))) + issue = used_deprecation_issues(candidates, {name: usage(name)}, {}, NOW)[0] + assert issue.cohort.scope_id == name + assert issue.deprecation.usage_model == name + assert issue.title.startswith(name) From 7c614030ea1f189f042553c566fab192d66dbd1d Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 15:20:11 +0800 Subject: [PATCH 13/44] test: verify retirement issue lifecycle through HTTP --- .../test_deprecation_issue_evaluation.py | 124 +++++++++++++++++- 1 file changed, 123 insertions(+), 1 deletion(-) diff --git a/tests/integration/database/test_deprecation_issue_evaluation.py b/tests/integration/database/test_deprecation_issue_evaluation.py index c280b51b54c..06c9f1f5c43 100644 --- a/tests/integration/database/test_deprecation_issue_evaluation.py +++ b/tests/integration/database/test_deprecation_issue_evaluation.py @@ -1,10 +1,16 @@ -from datetime import date +from datetime import date, datetime, timedelta, timezone from uuid import uuid4 import pytest from integration._support.actors import ManagementDatabase +from integration._support.client import Gateway +from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key +from litellm.proxy.spend_tracking.deprecation_issues import DeprecationIssueProjector +from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.usage_alert_metrics import UsageAlertMetricsRepository +from litellm.repositories.usage_alerts import UsageAlertsRepository +from litellm.types.proxy.model_deprecation import ModelDeprecationInfo, ModelDeprecationResponse @pytest.mark.asyncio(loop_scope="session") @@ -59,3 +65,119 @@ async def test_deprecation_usage_counts_distinct_resources_without_multiplying_r await db.prisma.writer_db.query_raw( 'DELETE FROM "LiteLLM_DailyTeamSpend" WHERE id LIKE $1 RETURNING id', prefix + "%" ) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_retirement_issue_http_state_survives_replay_and_reopens_only_for_new_usage( + gateway: Gateway, db: ManagementDatabase +): + prefix = f"retirement-lifecycle-{uuid4()}" + name = f"example/{prefix}" + now = datetime(2026, 11, 5, 10, tzinfo=timezone.utc) + retirement = now.date() + timedelta(days=15) + config = ConfigRepository(db.prisma, use_writer=True) + previous_checkpoint = await config.get_param("issues:deprecation_projection") + lease = UsageAlertsRepository.from_prisma(db.prisma) + assert await lease.claim_job(prefix) + + def schedule(today: date): + info = ModelDeprecationInfo( + model_name=prefix, + litellm_model=name, + deprecation_date=retirement, + days_until_deprecation=(retirement - today).days, + status="deprecated" if retirement < today else "imminent", + successor_model="example/replacement", + ) + return ModelDeprecationResponse( + deprecated=(info,) if retirement < today else (), + imminent=(info,) if retirement >= today else (), + upcoming=(), + warn_within_days=30, + checked_at=now, + ) + + async def record(day: date, requests: int): + await db.prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,api_requests,updated_at) + VALUES($1,$2,$3,$2,$4,$5,CURRENT_TIMESTAMP) + ON CONFLICT(id) DO UPDATE SET api_requests=EXCLUDED.api_requests RETURNING id""", + prefix + day.isoformat(), + prefix, + day.isoformat(), + name, + requests, + ) + + try: + await config.delete_param("issues:deprecation_projection") + await record(now.date(), 100) + projector = DeprecationIssueProjector.from_prisma(db.prisma, schedule) + await projector.run(now) + page = gateway.get("/issues", {"scope_type": "model", "scope_id": name}) + assert page["total"] == 1 + original = page["data"][0] + path = f"/issues/{original['id']}" + assert original["status"] == "open" + assert original["impact"]["requests"] == 100 + assert gateway.get(path + "/signals")["total"] == 1 + assert gateway.get(path + "/activity")["metadata"]["total_api_requests"] == 100 + state = {"model": name, "deprecation_date": retirement.isoformat()} + resolved = gateway.request( + "PUT", + "/alerting/model-deprecations/state", + {**state, "status": "resolved", "last_seen_date": now.date().isoformat(), "last_seen_requests": 100}, + ) + assert resolved.status_code == 200 + assert gateway.get(path)["status"] == "resolved" + await config.delete_param("issues:deprecation_projection") + await projector.run(now) + assert gateway.get(path)["status"] == "resolved" + stale = gateway.request("PATCH", path, {"status": "dismissed", "expected_status": "open"}) + assert stale.status_code == 409 + await db.prisma.writer_db.query_raw( + 'UPDATE "LiteLLM_Config" SET param_value=to_jsonb($2::text) WHERE param_name=$1 RETURNING param_name', + deprecation_alert_state_key(name, retirement), + resolved.text, + ) + assert gateway.get(path)["status"] == "resolved" + dismissed = gateway.request("PATCH", path, {"status": "dismissed", "expected_status": "resolved"}) + assert dismissed.status_code == 200 + assert gateway.get("/alerting/model-deprecations/state", state)["status"] == "dismissed" + await record(now.date(), 110) + await projector.run(now + timedelta(minutes=20)) + assert gateway.get(path)["status"] == "dismissed" + await projector.run(now + timedelta(hours=1)) + reopened = gateway.get(path) + assert reopened["status"] == "open" + assert reopened["impact"]["requests"] == 110 + assert reopened["first_seen"] == original["first_seen"] + assert gateway.get(path + "/signals")["total"] == 1 + expired_at = now + timedelta(days=35) + await projector.run(expired_at) + expired = gateway.get(path) + assert expired["status"] == "resolved" and expired["active"] is False + assert gateway.request("PATCH", path, {"status": "open", "expected_status": "resolved"}).status_code == 409 + await record(expired_at.date(), 5) + await projector.run(expired_at + timedelta(hours=1)) + returned = gateway.get(path) + assert returned["status"] == "open" and returned["active"] is True + assert returned["impact"]["requests"] == 5 + assert returned["first_seen"] == original["first_seen"] + finally: + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_IssueSignal" WHERE issue_id IN ' + '(SELECT id FROM "LiteLLM_Issue" WHERE scope_id=$1) RETURNING issue_id', + name, + ) + await db.prisma.writer_db.query_raw('DELETE FROM "LiteLLM_Issue" WHERE scope_id=$1 RETURNING id', name) + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE id LIKE $1 RETURNING id', prefix + "%" + ) + await config.delete_param(deprecation_alert_state_key(name, retirement)) + if previous_checkpoint is None: + await config.delete_param("issues:deprecation_projection") + else: + await config.set_param("issues:deprecation_projection", previous_checkpoint.param_value) + await lease.release_job(prefix) From e6a09e28dd970d9924c2de9200fea283c3bd6748 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 15:26:37 +0800 Subject: [PATCH 14/44] feat: confirm provider latency cohorts from established traffic --- litellm/repositories/provider_latency.py | 234 ++++++++++++++++++ .../database/test_provider_latency.py | 141 +++++++++++ 2 files changed, 375 insertions(+) create mode 100644 litellm/repositories/provider_latency.py create mode 100644 tests/integration/database/test_provider_latency.py diff --git a/litellm/repositories/provider_latency.py b/litellm/repositories/provider_latency.py new file mode 100644 index 00000000000..6a54288bb89 --- /dev/null +++ b/litellm/repositories/provider_latency.py @@ -0,0 +1,234 @@ +"""Confirm shared latency changes using existing daily key/model counters.""" + +from dataclasses import dataclass +from datetime import date, timedelta +from typing import TYPE_CHECKING, Final, Literal, cast # noqa: TID251 # generated Prisma structural interface + +from pydantic import BaseModel, TypeAdapter + +from litellm.constants import PTU_SENTINEL_API_KEY +from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket +from litellm.repositories.usage_alert_metrics import MetricsDatabase +from litellm.types.proxy.issues import IssueCohort, IssueComparison, IssueImpact, IssueModelPair +from litellm.types.proxy.usage_alerts import UsageAlertRuleInput + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + + +@dataclass(frozen=True, slots=True) +class ProviderLatencyEvidence: + cohort: IssueCohort + comparison: IssueComparison + impact: IssueImpact + comparable_timed_requests: int + total_timed_requests: int + + +class _CohortRow(BaseModel, frozen=True): + pairs: tuple[IssueModelPair, ...] + observed: float + expected: float + sample_days: int | None + comparable_timed: int + total_timed: int + keys: int + users: int + teams: int + requests: int + measured_keys: int + + +class _Member(BaseModel, frozen=True): + entity_id: str + + +_PAIRS: Final = TypeAdapter(tuple[IssueModelPair, ...]) +_COHORTS: Final = TypeAdapter(tuple[_CohortRow, ...]) +_MEMBERS: Final = TypeAdapter(tuple[_Member, ...]) +_CANDIDATES: Final = """ +SELECT DISTINCT custom_llm_provider AS provider, model FROM "LiteLLM_DailyUserSpend" +WHERE date=$1 AND timed_requests>0 AND custom_llm_provider<>'' AND model<>'' + AND ($2::text IS NULL OR (custom_llm_provider,model)>($2,$3)) +ORDER BY provider,model LIMIT $4 +""" +_LATENCY: Final = """ +WITH pairs AS ( + SELECT * FROM UNNEST($3::text[], $4::text[]) AS p(custom_llm_provider,model) +), daily AS MATERIALIZED ( + SELECT custom_llm_provider AS provider,model,api_key,date, + SUM(timed_requests)::bigint AS timed, + SUM(total_response_time_ms)::double precision AS response_ms, + SUM(api_requests)::bigint AS requests + FROM "LiteLLM_DailyUserSpend" JOIN pairs USING(custom_llm_provider,model) + WHERE date >= $1 AND date <= $2 + AND ($10::text[] IS NULL OR api_key=ANY($10)) + AND ($11::text[] IS NULL OR model=ANY($11)) + GROUP BY custom_llm_provider,model,api_key,date +), history AS ( + SELECT provider,model,api_key,COUNT(*) FILTER(WHERE timed>0)::int AS sample_days, + SUM(timed)::double precision AS timed, + SUM(response_ms)/NULLIF(SUM(timed),0) AS baseline + FROM daily WHERE date<$2 GROUP BY provider,model,api_key +), current AS ( + SELECT d.*,h.sample_days,h.baseline, + COALESCE(h.sample_days >= $5 AND h.baseline>0 + AND h.timed/NULLIF(h.sample_days,0) >= $6 AND d.timed >= $6 AND d.timed>0 + AND d.api_key<>'' AND d.api_key<>$9, FALSE) AS comparable + FROM daily d LEFT JOIN history h USING(provider,model,api_key) WHERE d.date=$2 +), scored AS MATERIALIZED ( + SELECT *, comparable AND response_ms/NULLIF(timed,0) >= GREATEST(baseline*$7,baseline+$8) AS degraded + FROM current +) +""" +_COHORT: Final = ( + _LATENCY + + """ +, model_scores AS ( + SELECT provider,model,SUM(timed)::bigint AS total_timed, + SUM(timed) FILTER(WHERE comparable)::bigint AS comparable_timed, + SUM(response_ms) FILTER(WHERE comparable) AS observed_ms, + SUM(timed*baseline) FILTER(WHERE comparable) AS expected_ms, + MIN(sample_days) FILTER(WHERE comparable) AS min_days, + MAX(sample_days) FILTER(WHERE comparable) AS max_days + FROM scored GROUP BY provider,model + HAVING COUNT(DISTINCT api_key) FILTER(WHERE degraded)>=2 + AND SUM(timed) FILTER(WHERE degraded) >= 0.8*SUM(timed) +), selected AS MATERIALIZED ( + SELECT s.* FROM scored s JOIN model_scores USING(provider,model) +) +SELECT jsonb_agg(jsonb_build_object('provider',provider,'model',model) ORDER BY provider,model) AS pairs, + SUM(observed_ms)/SUM(comparable_timed) AS observed, + SUM(expected_ms)/SUM(comparable_timed) AS expected, + CASE WHEN MIN(min_days)=MAX(max_days) THEN MIN(min_days) ELSE NULL END AS sample_days, + SUM(comparable_timed)::bigint AS comparable_timed,SUM(total_timed)::bigint AS total_timed, + (SELECT COUNT(DISTINCT api_key)::int FROM selected WHERE api_key<>'' AND api_key<>$9) AS keys, + (SELECT COUNT(DISTINCT api_key)::int FROM selected WHERE degraded) AS measured_keys, + (SELECT SUM(requests)::bigint FROM selected) AS requests, + (SELECT COUNT(DISTINCT NULLIF(NULLIF(user_id,''),'default_user_id'))::int + FROM "LiteLLM_DailyUserSpend" d JOIN model_scores s ON d.custom_llm_provider=s.provider AND d.model=s.model + WHERE d.date=$2 AND api_requests>0) AS users, + (SELECT COUNT(DISTINCT NULLIF(NULLIF(team_id,''),'litellm-dashboard'))::int + FROM "LiteLLM_DailyTeamSpend" d JOIN model_scores s ON d.custom_llm_provider=s.provider AND d.model=s.model + WHERE d.date=$2 AND api_requests>0) AS teams +FROM model_scores HAVING COUNT(*)>=2 + AND SUM(observed_ms)/SUM(comparable_timed) >= + GREATEST(SUM(expected_ms)/SUM(comparable_timed)*$7,SUM(expected_ms)/SUM(comparable_timed)+$8) +""" +) + + +def _arguments( + pairs: tuple[IssueModelPair, ...], + day: date, + rule: UsageAlertRuleInput, + keys: tuple[str, ...] | None = None, + models: tuple[str, ...] | None = None, +) -> tuple[object, ...]: + if not 1 <= len(pairs) <= 100 or len(frozenset(pairs)) != len(pairs): + raise ValueError("Choose 1 to 100 unique provider/model pairs") + if len({pair.provider for pair in pairs}) != 1: + raise ValueError("A latency cohort belongs to one provider") + if rule.metric != "latency" or rule.direction != "increase": + raise ValueError("Provider latency evidence requires an increasing latency rule") + return ( + (day - timedelta(days=rule.baseline_days)).isoformat(), + day.isoformat(), + tuple(pair.provider for pair in pairs), + tuple(pair.model for pair in pairs), + rule.min_history_days, + rule.min_requests, + rule.multiplier, + rule.min_value, + PTU_SENTINEL_API_KEY, + keys, + models, + ) + + +class ProviderLatencyRepository: + def __init__(self, database: MetricsDatabase) -> None: + self._database: Final = database + + @classmethod + def from_prisma(cls, prisma: "PrismaClient") -> "ProviderLatencyRepository": + return cls(cast(MetricsDatabase, prisma.writer_db)) # cast-ok: generated Prisma query_raw interface + + async def candidate_pairs( + self, day: date, cursor: IssueModelPair | None = None, limit: int = 100 + ) -> tuple[IssueModelPair, ...]: + if not 1 <= limit <= 100: + raise ValueError("Pair pages contain 1 to 100 candidates") + return _PAIRS.validate_python( + await self._database.query_raw( + _CANDIDATES, + day.isoformat(), + cursor.provider if cursor else None, + cursor.model if cursor else None, + limit, + ) + ) + + async def latency_cohort( + self, pairs: tuple[IssueModelPair, ...], day: date, rule: UsageAlertRuleInput + ) -> ProviderLatencyEvidence | None: + arguments: Final = _arguments(pairs, day, rule) + if len(pairs) < 2: + return None + rows: Final = _COHORTS.validate_python(await self._database.query_raw(_COHORT, *arguments)) + if not rows: + return None + row: Final = rows[0] + return ProviderLatencyEvidence( + cohort=IssueCohort(scope_type="provider", scope_id=pairs[0].provider, model_pairs=row.pairs), + comparison=IssueComparison( + metric="latency", + direction="increase", + observed_value=row.observed, + baseline_value=row.expected, + threshold=max(row.expected * rule.multiplier, row.expected + rule.min_value), + baseline_start=day - timedelta(days=rule.baseline_days), + baseline_end=day - timedelta(days=1), + baseline_sample_days=row.sample_days, + ), + impact=IssueImpact( + keys=row.keys, + users=row.users, + teams=row.teams, + models=len(row.pairs), + requests=row.requests, + measured_keys=row.measured_keys, + measured_models=len(row.pairs), + ), + comparable_timed_requests=row.comparable_timed, + total_timed_requests=row.total_timed, + ) + + async def source_members( + self, + pairs: tuple[IssueModelPair, ...], + day: date, + rule: UsageAlertRuleInput, + scope_type: Literal["key", "model"], + source_ids: tuple[str, ...], + ) -> frozenset[str]: + if not source_ids: + return frozenset() + if len(source_ids) > 500: + raise ValueError("Source membership accepts at most 500 IDs") + keys: Final = tuple(api_key_from_bucket(key) for key in source_ids) if scope_type == "key" else None + arguments: Final = _arguments(pairs, day, rule, keys, source_ids if scope_type == "model" else None) + column, parameter = ("api_key", 10) if scope_type == "key" else ("model", 11) + query: Final = ( + _LATENCY + + f""" + , totals AS ( + SELECT {column},SUM(timed_requests)::bigint AS total + FROM "LiteLLM_DailyUserSpend" WHERE date=$2 AND {column}=ANY(${parameter}) GROUP BY {column} + ) + SELECT s.{column} AS entity_id FROM scored s JOIN totals USING({column}) GROUP BY s.{column},totals.total + HAVING SUM(timed) FILTER(WHERE degraded) >= 0.8*totals.total AND totals.total>0 + """ + ) + rows: Final = _MEMBERS.validate_python(await self._database.query_raw(query, *arguments)) + return frozenset(api_key_bucket(row.entity_id) if scope_type == "key" else row.entity_id for row in rows) diff --git a/tests/integration/database/test_provider_latency.py b/tests/integration/database/test_provider_latency.py new file mode 100644 index 00000000000..fadfd2e99f1 --- /dev/null +++ b/tests/integration/database/test_provider_latency.py @@ -0,0 +1,141 @@ +from datetime import date +from uuid import uuid4 + +import pytest +from integration._support.actors import ManagementDatabase + +from litellm.repositories.provider_latency import ProviderLatencyRepository +from litellm.types.proxy.issues import IssueModelPair +from litellm.types.proxy.usage_alerts import UsageAlertRuleInput + + +@pytest.mark.asyncio(loop_scope="session") +async def test_provider_latency_requires_established_within_model_regressions(db: ManagementDatabase) -> None: + prefix = f"provider-latency-{uuid4()}" + day = date(2026, 11, 4) + pairs = tuple(IssueModelPair(provider=prefix, model=f"{prefix}-{suffix}") for suffix in ("a", "b")) + keys = (prefix + "-key-a", prefix + "-key-b") + rule = UsageAlertRuleInput(name="Latency", scope_type="key", metric="latency", direction="increase", min_value=500) + repository = ProviderLatencyRepository(db.prisma.writer_db) + try: + await db.prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,custom_llm_provider,api_requests,timed_requests,total_response_time_ms,updated_at) + SELECT $1||d||m.suffix||k.suffix,$1||'-owner',($2::date-d)::text,$1||'-key-'||k.suffix, + $1||'-'||m.suffix,$1,100,100,100*m.latency*(CASE WHEN d=0 THEN 5 ELSE 1 END),CURRENT_TIMESTAMP + FROM generate_series(0,7) d CROSS JOIN (VALUES('a',200),('b',1000)) m(suffix,latency) + CROSS JOIN (VALUES('a'),('b')) k(suffix) RETURNING id""", + prefix, + day.isoformat(), + ) + await db.prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyTeamSpend" + (id,team_id,date,api_key,model,custom_llm_provider,api_requests,updated_at) + SELECT $1||'team'||t.n||m,$1||'team'||t.n,$2,$1||'-key-a',$1||'-'||m,$1,100,CURRENT_TIMESTAMP + FROM (VALUES('a'),('b')) models(m) CROSS JOIN generate_series(1,2) t(n) RETURNING id""", + prefix, + day.isoformat(), + ) + first = await repository.candidate_pairs(day, IssueModelPair(provider=prefix, model="0"), limit=1) + assert first == pairs[:1] + assert await repository.candidate_pairs(day, first[0], limit=1) == pairs[1:] + evidence = await repository.latency_cohort(pairs, day, rule) + assert evidence is not None + assert evidence.cohort.model_pairs == pairs + assert evidence.comparison.observed_value == 3000 + assert evidence.comparison.baseline_value == 600 + assert evidence.comparison.baseline_sample_days == 7 + assert (evidence.impact.keys, evidence.impact.users, evidence.impact.teams) == (2, 1, 2) + assert (evidence.impact.measured_keys, evidence.impact.measured_models) == (2, 2) + assert evidence.impact.requests == 400 + assert await repository.source_members(pairs, day, rule, "key", keys) == frozenset(keys) + models = tuple(pair.model for pair in pairs) + assert await repository.source_members(pairs, day, rule, "model", models) == frozenset(models) + await db.prisma.writer_db.query_raw( + 'UPDATE "LiteLLM_DailyUserSpend" SET total_response_time_ms=total_response_time_ms/5 ' + "WHERE api_key=$1 AND date=$2 RETURNING id", + keys[1], + day.isoformat(), + ) + assert await repository.latency_cohort(pairs, day, rule) is None + await db.prisma.writer_db.query_raw( + 'UPDATE "LiteLLM_DailyUserSpend" SET total_response_time_ms=total_response_time_ms*5 ' + "WHERE api_key=$1 AND date=$2 RETURNING id", + keys[1], + day.isoformat(), + ) + await db.prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,custom_llm_provider,api_requests,timed_requests,total_response_time_ms,updated_at) + SELECT $1||'new'||m,$1||'-new-owner',$2,$1||'-new-key',$1||'-'||m,$1,50,50,2500000,CURRENT_TIMESTAMP + FROM (VALUES('a'),('b')) models(m) RETURNING id""", + prefix, + day.isoformat(), + ) + with_new_key = await repository.latency_cohort(pairs, day, rule) + assert with_new_key is not None + assert with_new_key.comparison == evidence.comparison + assert (with_new_key.total_timed_requests, with_new_key.comparable_timed_requests) == (500, 400) + assert (with_new_key.impact.keys, with_new_key.impact.users, with_new_key.impact.requests) == (3, 2, 500) + assert with_new_key.impact.measured_keys == 2 + await db.prisma.writer_db.query_raw( + 'UPDATE "LiteLLM_DailyUserSpend" SET timed_requests=51 WHERE api_key=$1 RETURNING id', prefix + "-new-key" + ) + assert await repository.latency_cohort(pairs, day, rule) is None + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE api_key=$1 RETURNING id', prefix + "-new-key" + ) + await db.prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,custom_llm_provider,api_requests,timed_requests,total_response_time_ms,updated_at) + SELECT id||'-other',user_id,date,api_key,model,$1||'-other',1000,1000,100000,CURRENT_TIMESTAMP + FROM "LiteLLM_DailyUserSpend" WHERE custom_llm_provider=$1 AND date=$2 RETURNING id""", + prefix, + day.isoformat(), + ) + assert await repository.latency_cohort(pairs, day, rule) == evidence + assert await repository.source_members(pairs, day, rule, "key", keys) == frozenset() + assert await repository.source_members(pairs, day, rule, "model", models) == frozenset() + await db.prisma.writer_db.query_raw( + """UPDATE "LiteLLM_DailyUserSpend" + SET api_requests=CASE WHEN model=$1||'-a' AND date<$2 THEN 10000 + WHEN model=$1||'-b' AND date=$2 THEN 5000 ELSE 100 END, + timed_requests=CASE WHEN model=$1||'-a' AND date<$2 THEN 10000 + WHEN model=$1||'-b' AND date=$2 THEN 5000 ELSE 100 END, + total_response_time_ms=(CASE WHEN model=$1||'-a' THEN 200 ELSE 1000 END)::bigint * + (CASE WHEN model=$1||'-a' AND date<$2 THEN 10000 + WHEN model=$1||'-b' AND date=$2 THEN 5000 ELSE 100 END) + WHERE custom_llm_provider=$1 RETURNING id""", + prefix, + day.isoformat(), + ) + assert await repository.latency_cohort(pairs, day, rule) is None + await db.prisma.writer_db.query_raw( + 'UPDATE "LiteLLM_DailyUserSpend" SET total_response_time_ms=total_response_time_ms*5 ' + "WHERE custom_llm_provider=$1 AND date=$2 RETURNING id", + prefix, + day.isoformat(), + ) + assert await repository.latency_cohort(pairs, day, rule) is not None + await db.prisma.writer_db.query_raw( + 'UPDATE "LiteLLM_DailyUserSpend" SET total_response_time_ms=0 WHERE custom_llm_provider=$1 AND date<$2 RETURNING id', + prefix, + day.isoformat(), + ) + assert await repository.latency_cohort(pairs, day, rule) is None + await db.prisma.writer_db.query_raw( + 'UPDATE "LiteLLM_DailyUserSpend" SET total_response_time_ms=timed_requests*' + "(CASE WHEN model=$1||'-a' THEN 200 ELSE 1000 END) " + "WHERE custom_llm_provider=$1 AND date<$2 RETURNING id", + prefix, + day.isoformat(), + ) + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE custom_llm_provider=$1 AND date<($2::date-2)::text RETURNING id', + prefix, + day.isoformat(), + ) + assert await repository.latency_cohort(pairs, day, rule) is None + finally: + for table in ('"LiteLLM_DailyUserSpend"', '"LiteLLM_DailyTeamSpend"'): + await db.prisma.writer_db.query_raw(f"DELETE FROM {table} WHERE id LIKE $1 RETURNING id", prefix + "%") From 9b69e20917b6b9c2e52eb5219a80f44e60ec5676 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 15:29:15 +0800 Subject: [PATCH 15/44] fix: require comparable traffic before recovering provider issues --- litellm/repositories/provider_latency.py | 47 ++++++++++++++++--- .../test_deprecation_issue_evaluation.py | 8 ++-- .../database/test_provider_latency.py | 6 +++ 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/litellm/repositories/provider_latency.py b/litellm/repositories/provider_latency.py index 6a54288bb89..ee8118e44ee 100644 --- a/litellm/repositories/provider_latency.py +++ b/litellm/repositories/provider_latency.py @@ -43,6 +43,10 @@ class _Member(BaseModel, frozen=True): entity_id: str +class _Recovery(BaseModel, frozen=True): + recovered: bool + + _PAIRS: Final = TypeAdapter(tuple[IssueModelPair, ...]) _COHORTS: Final = TypeAdapter(tuple[_CohortRow, ...]) _MEMBERS: Final = TypeAdapter(tuple[_Member, ...]) @@ -81,9 +85,7 @@ WITH pairs AS ( FROM current ) """ -_COHORT: Final = ( - _LATENCY - + """ +_MODEL_SCORES: Final = """ , model_scores AS ( SELECT provider,model,SUM(timed)::bigint AS total_timed, SUM(timed) FILTER(WHERE comparable)::bigint AS comparable_timed, @@ -94,7 +96,17 @@ _COHORT: Final = ( FROM scored GROUP BY provider,model HAVING COUNT(DISTINCT api_key) FILTER(WHERE degraded)>=2 AND SUM(timed) FILTER(WHERE degraded) >= 0.8*SUM(timed) -), selected AS MATERIALIZED ( +) +""" +_SHARED_THRESHOLD: Final = """ +HAVING COUNT(*)>=2 AND SUM(observed_ms)/SUM(comparable_timed) >= + GREATEST(SUM(expected_ms)/SUM(comparable_timed)*$7,SUM(expected_ms)/SUM(comparable_timed)+$8) +""" +_COHORT: Final = ( + _LATENCY + + _MODEL_SCORES + + """ +, selected AS MATERIALIZED ( SELECT s.* FROM scored s JOIN model_scores USING(provider,model) ) SELECT jsonb_agg(jsonb_build_object('provider',provider,'model',model) ORDER BY provider,model) AS pairs, @@ -111,10 +123,24 @@ SELECT jsonb_agg(jsonb_build_object('provider',provider,'model',model) ORDER BY (SELECT COUNT(DISTINCT NULLIF(NULLIF(team_id,''),'litellm-dashboard'))::int FROM "LiteLLM_DailyTeamSpend" d JOIN model_scores s ON d.custom_llm_provider=s.provider AND d.model=s.model WHERE d.date=$2 AND api_requests>0) AS teams -FROM model_scores HAVING COUNT(*)>=2 - AND SUM(observed_ms)/SUM(comparable_timed) >= - GREATEST(SUM(expected_ms)/SUM(comparable_timed)*$7,SUM(expected_ms)/SUM(comparable_timed)+$8) +FROM model_scores """ + + _SHARED_THRESHOLD +) +_RECOVERY_QUERY: Final = ( + _LATENCY + + _MODEL_SCORES + + """ +, covered AS ( + SELECT provider,model FROM scored GROUP BY provider,model + HAVING COUNT(DISTINCT api_key) FILTER(WHERE comparable)>=2 + AND SUM(timed) FILTER(WHERE comparable)>=0.8*SUM(timed) +) +SELECT (SELECT COUNT(*) FROM covered)=cardinality($4::text[]) AND NOT EXISTS ( + SELECT 1 FROM model_scores +""" + + _SHARED_THRESHOLD + + ") AS recovered" ) @@ -232,3 +258,10 @@ class ProviderLatencyRepository: ) rows: Final = _MEMBERS.validate_python(await self._database.query_raw(query, *arguments)) return frozenset(api_key_bucket(row.entity_id) if scope_type == "key" else row.entity_id for row in rows) + + async def cohort_recovered(self, pairs: tuple[IssueModelPair, ...], day: date, rule: UsageAlertRuleInput) -> bool: + arguments: Final = _arguments(pairs, day, rule) + rows: Final = TypeAdapter(tuple[_Recovery, ...]).validate_python( + await self._database.query_raw(_RECOVERY_QUERY, *arguments) + ) + return rows[0].recovered diff --git a/tests/integration/database/test_deprecation_issue_evaluation.py b/tests/integration/database/test_deprecation_issue_evaluation.py index 06c9f1f5c43..c267fe694e9 100644 --- a/tests/integration/database/test_deprecation_issue_evaluation.py +++ b/tests/integration/database/test_deprecation_issue_evaluation.py @@ -14,7 +14,7 @@ from litellm.types.proxy.model_deprecation import ModelDeprecationInfo, ModelDep @pytest.mark.asyncio(loop_scope="session") -async def test_deprecation_usage_counts_distinct_resources_without_multiplying_requests(db: ManagementDatabase): +async def test_deprecation_usage_counts_distinct_resources_without_multiplying_requests(db: ManagementDatabase) -> None: prefix = f"deprecation-issue-{uuid4()}" model = f"example/{prefix}" try: @@ -70,7 +70,7 @@ async def test_deprecation_usage_counts_distinct_resources_without_multiplying_r @pytest.mark.asyncio(loop_scope="session") async def test_retirement_issue_http_state_survives_replay_and_reopens_only_for_new_usage( gateway: Gateway, db: ManagementDatabase -): +) -> None: prefix = f"retirement-lifecycle-{uuid4()}" name = f"example/{prefix}" now = datetime(2026, 11, 5, 10, tzinfo=timezone.utc) @@ -80,7 +80,7 @@ async def test_retirement_issue_http_state_survives_replay_and_reopens_only_for_ lease = UsageAlertsRepository.from_prisma(db.prisma) assert await lease.claim_job(prefix) - def schedule(today: date): + def schedule(today: date) -> ModelDeprecationResponse: info = ModelDeprecationInfo( model_name=prefix, litellm_model=name, @@ -97,7 +97,7 @@ async def test_retirement_issue_http_state_survives_replay_and_reopens_only_for_ checked_at=now, ) - async def record(day: date, requests: int): + async def record(day: date, requests: int) -> None: await db.prisma.writer_db.query_raw( """INSERT INTO "LiteLLM_DailyUserSpend" (id,user_id,date,api_key,model,api_requests,updated_at) diff --git a/tests/integration/database/test_provider_latency.py b/tests/integration/database/test_provider_latency.py index fadfd2e99f1..c4f73fab9a9 100644 --- a/tests/integration/database/test_provider_latency.py +++ b/tests/integration/database/test_provider_latency.py @@ -48,6 +48,7 @@ async def test_provider_latency_requires_established_within_model_regressions(db assert (evidence.impact.keys, evidence.impact.users, evidence.impact.teams) == (2, 1, 2) assert (evidence.impact.measured_keys, evidence.impact.measured_models) == (2, 2) assert evidence.impact.requests == 400 + assert not await repository.cohort_recovered(pairs, day, rule) assert await repository.source_members(pairs, day, rule, "key", keys) == frozenset(keys) models = tuple(pair.model for pair in pairs) assert await repository.source_members(pairs, day, rule, "model", models) == frozenset(models) @@ -58,6 +59,7 @@ async def test_provider_latency_requires_established_within_model_regressions(db day.isoformat(), ) assert await repository.latency_cohort(pairs, day, rule) is None + assert await repository.cohort_recovered(pairs, day, rule) await db.prisma.writer_db.query_raw( 'UPDATE "LiteLLM_DailyUserSpend" SET total_response_time_ms=total_response_time_ms*5 ' "WHERE api_key=$1 AND date=$2 RETURNING id", @@ -82,6 +84,7 @@ async def test_provider_latency_requires_established_within_model_regressions(db 'UPDATE "LiteLLM_DailyUserSpend" SET timed_requests=51 WHERE api_key=$1 RETURNING id', prefix + "-new-key" ) assert await repository.latency_cohort(pairs, day, rule) is None + assert not await repository.cohort_recovered(pairs, day, rule) await db.prisma.writer_db.query_raw( 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE api_key=$1 RETURNING id', prefix + "-new-key" ) @@ -110,6 +113,7 @@ async def test_provider_latency_requires_established_within_model_regressions(db day.isoformat(), ) assert await repository.latency_cohort(pairs, day, rule) is None + assert await repository.cohort_recovered(pairs, day, rule) await db.prisma.writer_db.query_raw( 'UPDATE "LiteLLM_DailyUserSpend" SET total_response_time_ms=total_response_time_ms*5 ' "WHERE custom_llm_provider=$1 AND date=$2 RETURNING id", @@ -123,6 +127,7 @@ async def test_provider_latency_requires_established_within_model_regressions(db day.isoformat(), ) assert await repository.latency_cohort(pairs, day, rule) is None + assert not await repository.cohort_recovered(pairs, day, rule) await db.prisma.writer_db.query_raw( 'UPDATE "LiteLLM_DailyUserSpend" SET total_response_time_ms=timed_requests*' "(CASE WHEN model=$1||'-a' THEN 200 ELSE 1000 END) " @@ -136,6 +141,7 @@ async def test_provider_latency_requires_established_within_model_regressions(db day.isoformat(), ) assert await repository.latency_cohort(pairs, day, rule) is None + assert not await repository.cohort_recovered(pairs, day, rule) finally: for table in ('"LiteLLM_DailyUserSpend"', '"LiteLLM_DailyTeamSpend"'): await db.prisma.writer_db.query_raw(f"DELETE FROM {table} WHERE id LIKE $1 RETURNING id", prefix + "%") From d3e012bb2ef8589097ea2a15d23a7b5a2ee29c28 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 15:32:24 +0800 Subject: [PATCH 16/44] feat: correlate scoped latency signals with confirmed provider issues --- .../spend_tracking/deprecation_issues.py | 5 +- litellm/repositories/provider_latency.py | 62 ++++++++++++------- .../database/test_provider_latency.py | 41 ++++++++++-- .../spend_tracking/test_deprecation_issues.py | 2 +- 4 files changed, 79 insertions(+), 31 deletions(-) diff --git a/litellm/proxy/spend_tracking/deprecation_issues.py b/litellm/proxy/spend_tracking/deprecation_issues.py index 06f9a836261..e7970182dad 100644 --- a/litellm/proxy/spend_tracking/deprecation_issues.py +++ b/litellm/proxy/spend_tracking/deprecation_issues.py @@ -70,10 +70,11 @@ def deprecation_issue( ) -> IssueInput: today: Final = now.astimezone(timezone.utc).date() model: Final = candidate.model + name: Final = model.model_name or usage.model title: Final = ( - f"{usage.model} retired on {model.deprecation_date.isoformat()}" + f"{name} retired on {model.deprecation_date.isoformat()}" if model.deprecation_date < today - else f"{usage.model} retires on {model.deprecation_date.isoformat()}" + else f"{name} retires on {model.deprecation_date.isoformat()}" ) return IssueInput( id=f"deprecation:{candidate.state_key.removeprefix('model_deprecation_alert:')}", diff --git a/litellm/repositories/provider_latency.py b/litellm/repositories/provider_latency.py index ee8118e44ee..95bec0cdd29 100644 --- a/litellm/repositories/provider_latency.py +++ b/litellm/repositories/provider_latency.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from datetime import date, timedelta -from typing import TYPE_CHECKING, Final, Literal, cast # noqa: TID251 # generated Prisma structural interface +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # generated Prisma structural interface from pydantic import BaseModel, TypeAdapter @@ -10,7 +10,7 @@ from litellm.constants import PTU_SENTINEL_API_KEY from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket from litellm.repositories.usage_alert_metrics import MetricsDatabase from litellm.types.proxy.issues import IssueCohort, IssueComparison, IssueImpact, IssueModelPair -from litellm.types.proxy.usage_alerts import UsageAlertRuleInput +from litellm.types.proxy.usage_alerts import UsageAlertRuleInput, UsageAlertScope if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient @@ -56,35 +56,53 @@ WHERE date=$1 AND timed_requests>0 AND custom_llm_provider<>'' AND model<>'' AND ($2::text IS NULL OR (custom_llm_provider,model)>($2,$3)) ORDER BY provider,model LIMIT $4 """ -_LATENCY: Final = """ + + +def _source(scope: UsageAlertScope) -> tuple[str, str]: + if scope == "team": + return '"LiteLLM_DailyTeamSpend"', "team_id" + if scope == "user": + return '"LiteLLM_DailyUserSpend"', "user_id" + if scope == "key": + return '"LiteLLM_DailyUserSpend"', "api_key" + if scope == "model": + return '"LiteLLM_DailyUserSpend"', "model" + return '"LiteLLM_DailyUserSpend"', "'deployment'::text" + + +def _latency_query(scope: UsageAlertScope = "deployment") -> str: + table, entity = _source(scope) + return f""" WITH pairs AS ( SELECT * FROM UNNEST($3::text[], $4::text[]) AS p(custom_llm_provider,model) ), daily AS MATERIALIZED ( - SELECT custom_llm_provider AS provider,model,api_key,date, + SELECT {entity} AS source_id,custom_llm_provider AS provider,model,api_key,date, SUM(timed_requests)::bigint AS timed, SUM(total_response_time_ms)::double precision AS response_ms, SUM(api_requests)::bigint AS requests - FROM "LiteLLM_DailyUserSpend" JOIN pairs USING(custom_llm_provider,model) + FROM {table} JOIN pairs USING(custom_llm_provider,model) WHERE date >= $1 AND date <= $2 - AND ($10::text[] IS NULL OR api_key=ANY($10)) - AND ($11::text[] IS NULL OR model=ANY($11)) - GROUP BY custom_llm_provider,model,api_key,date + AND ($10::text[] IS NULL OR {entity}=ANY($10)) + GROUP BY source_id,custom_llm_provider,model,api_key,date ), history AS ( - SELECT provider,model,api_key,COUNT(*) FILTER(WHERE timed>0)::int AS sample_days, + SELECT source_id,provider,model,api_key,COUNT(*) FILTER(WHERE timed>0)::int AS sample_days, SUM(timed)::double precision AS timed, SUM(response_ms)/NULLIF(SUM(timed),0) AS baseline - FROM daily WHERE date<$2 GROUP BY provider,model,api_key + FROM daily WHERE date<$2 GROUP BY source_id,provider,model,api_key ), current AS ( SELECT d.*,h.sample_days,h.baseline, COALESCE(h.sample_days >= $5 AND h.baseline>0 AND h.timed/NULLIF(h.sample_days,0) >= $6 AND d.timed >= $6 AND d.timed>0 AND d.api_key<>'' AND d.api_key<>$9, FALSE) AS comparable - FROM daily d LEFT JOIN history h USING(provider,model,api_key) WHERE d.date=$2 + FROM daily d LEFT JOIN history h USING(source_id,provider,model,api_key) WHERE d.date=$2 ), scored AS MATERIALIZED ( SELECT *, comparable AND response_ms/NULLIF(timed,0) >= GREATEST(baseline*$7,baseline+$8) AS degraded FROM current ) """ + + +_LATENCY: Final = _latency_query() _MODEL_SCORES: Final = """ , model_scores AS ( SELECT provider,model,SUM(timed)::bigint AS total_timed, @@ -148,8 +166,7 @@ def _arguments( pairs: tuple[IssueModelPair, ...], day: date, rule: UsageAlertRuleInput, - keys: tuple[str, ...] | None = None, - models: tuple[str, ...] | None = None, + source_ids: tuple[str, ...] | None = None, ) -> tuple[object, ...]: if not 1 <= len(pairs) <= 100 or len(frozenset(pairs)) != len(pairs): raise ValueError("Choose 1 to 100 unique provider/model pairs") @@ -167,8 +184,7 @@ def _arguments( rule.multiplier, rule.min_value, PTU_SENTINEL_API_KEY, - keys, - models, + source_ids, ) @@ -235,24 +251,24 @@ class ProviderLatencyRepository: pairs: tuple[IssueModelPair, ...], day: date, rule: UsageAlertRuleInput, - scope_type: Literal["key", "model"], + scope_type: UsageAlertScope, source_ids: tuple[str, ...], ) -> frozenset[str]: if not source_ids: return frozenset() if len(source_ids) > 500: raise ValueError("Source membership accepts at most 500 IDs") - keys: Final = tuple(api_key_from_bucket(key) for key in source_ids) if scope_type == "key" else None - arguments: Final = _arguments(pairs, day, rule, keys, source_ids if scope_type == "model" else None) - column, parameter = ("api_key", 10) if scope_type == "key" else ("model", 11) + ids: Final = tuple(api_key_from_bucket(key) for key in source_ids) if scope_type == "key" else source_ids + arguments: Final = _arguments(pairs, day, rule, ids) + table, column = _source(scope_type) query: Final = ( - _LATENCY + _latency_query(scope_type) + f""" , totals AS ( - SELECT {column},SUM(timed_requests)::bigint AS total - FROM "LiteLLM_DailyUserSpend" WHERE date=$2 AND {column}=ANY(${parameter}) GROUP BY {column} + SELECT {column} AS source_id,SUM(timed_requests)::bigint AS total + FROM {table} WHERE date=$2 AND {column}=ANY($10) GROUP BY source_id ) - SELECT s.{column} AS entity_id FROM scored s JOIN totals USING({column}) GROUP BY s.{column},totals.total + SELECT source_id AS entity_id FROM scored s JOIN totals USING(source_id) GROUP BY source_id,totals.total HAVING SUM(timed) FILTER(WHERE degraded) >= 0.8*totals.total AND totals.total>0 """ ) diff --git a/tests/integration/database/test_provider_latency.py b/tests/integration/database/test_provider_latency.py index c4f73fab9a9..8175d8525bd 100644 --- a/tests/integration/database/test_provider_latency.py +++ b/tests/integration/database/test_provider_latency.py @@ -12,7 +12,7 @@ from litellm.types.proxy.usage_alerts import UsageAlertRuleInput @pytest.mark.asyncio(loop_scope="session") async def test_provider_latency_requires_established_within_model_regressions(db: ManagementDatabase) -> None: prefix = f"provider-latency-{uuid4()}" - day = date(2026, 11, 4) + day = date(2031, 1, 12) pairs = tuple(IssueModelPair(provider=prefix, model=f"{prefix}-{suffix}") for suffix in ("a", "b")) keys = (prefix + "-key-a", prefix + "-key-b") rule = UsageAlertRuleInput(name="Latency", scope_type="key", metric="latency", direction="increase", min_value=500) @@ -30,11 +30,11 @@ async def test_provider_latency_requires_established_within_model_regressions(db ) await db.prisma.writer_db.query_raw( """INSERT INTO "LiteLLM_DailyTeamSpend" - (id,team_id,date,api_key,model,custom_llm_provider,api_requests,updated_at) - SELECT $1||'team'||t.n||m,$1||'team'||t.n,$2,$1||'-key-a',$1||'-'||m,$1,100,CURRENT_TIMESTAMP - FROM (VALUES('a'),('b')) models(m) CROSS JOIN generate_series(1,2) t(n) RETURNING id""", + (id,team_id,date,api_key,model,custom_llm_provider,api_requests,timed_requests,total_response_time_ms,updated_at) + SELECT id||'team',$1||CASE WHEN api_key=$1||'-key-a' THEN 'team1' ELSE 'team2' END, + date,api_key,model,custom_llm_provider,api_requests,timed_requests,total_response_time_ms,CURRENT_TIMESTAMP + FROM "LiteLLM_DailyUserSpend" WHERE custom_llm_provider=$1 RETURNING id""", prefix, - day.isoformat(), ) first = await repository.candidate_pairs(day, IssueModelPair(provider=prefix, model="0"), limit=1) assert first == pairs[:1] @@ -52,6 +52,25 @@ async def test_provider_latency_requires_established_within_model_regressions(db assert await repository.source_members(pairs, day, rule, "key", keys) == frozenset(keys) models = tuple(pair.model for pair in pairs) assert await repository.source_members(pairs, day, rule, "model", models) == frozenset(models) + assert await repository.source_members(pairs, day, rule, "user", (prefix + "-owner",)) == {prefix + "-owner"} + teams = (prefix + "team1", prefix + "team2") + assert await repository.source_members(pairs, day, rule, "team", teams) == frozenset(teams) + assert await repository.source_members(pairs, day, rule, "deployment", ("deployment",)) == {"deployment"} + await db.prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,custom_llm_provider,api_requests,timed_requests,total_response_time_ms,updated_at) + SELECT id||'-new-owner',$1||'-new-owner',date,api_key,model,custom_llm_provider,api_requests, + timed_requests,total_response_time_ms,CURRENT_TIMESTAMP + FROM "LiteLLM_DailyUserSpend" WHERE custom_llm_provider=$1 AND date=$2 RETURNING id""", + prefix, + day.isoformat(), + ) + assert await repository.source_members( + pairs, day, rule, "user", (prefix + "-owner", prefix + "-new-owner") + ) == {prefix + "-owner"} + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE user_id=$1 RETURNING id', prefix + "-new-owner" + ) await db.prisma.writer_db.query_raw( 'UPDATE "LiteLLM_DailyUserSpend" SET total_response_time_ms=total_response_time_ms/5 ' "WHERE api_key=$1 AND date=$2 RETURNING id", @@ -99,6 +118,18 @@ async def test_provider_latency_requires_established_within_model_regressions(db assert await repository.latency_cohort(pairs, day, rule) == evidence assert await repository.source_members(pairs, day, rule, "key", keys) == frozenset() assert await repository.source_members(pairs, day, rule, "model", models) == frozenset() + assert await repository.source_members(pairs, day, rule, "user", (prefix + "-owner",)) == frozenset() + assert await repository.source_members(pairs, day, rule, "deployment", ("deployment",)) == frozenset() + assert await repository.source_members(pairs, day, rule, "team", teams) == frozenset(teams) + await db.prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyTeamSpend" + (id,team_id,date,api_key,model,custom_llm_provider,api_requests,timed_requests,total_response_time_ms,updated_at) + SELECT id||'-other',team_id,date,api_key,model,$1||'-other',1000,1000,100000,CURRENT_TIMESTAMP + FROM "LiteLLM_DailyTeamSpend" WHERE custom_llm_provider=$1 AND date=$2 RETURNING id""", + prefix, + day.isoformat(), + ) + assert await repository.source_members(pairs, day, rule, "team", teams) == frozenset() await db.prisma.writer_db.query_raw( """UPDATE "LiteLLM_DailyUserSpend" SET api_requests=CASE WHEN model=$1||'-a' AND date<$2 THEN 10000 diff --git a/tests/unit/proxy/spend_tracking/test_deprecation_issues.py b/tests/unit/proxy/spend_tracking/test_deprecation_issues.py index 6ec00a44db1..e8f47426d4e 100644 --- a/tests/unit/proxy/spend_tracking/test_deprecation_issues.py +++ b/tests/unit/proxy/spend_tracking/test_deprecation_issues.py @@ -125,4 +125,4 @@ def test_maximum_model_identity_is_preserved_in_activity_and_evidence(): issue = used_deprecation_issues(candidates, {name: usage(name)}, {}, NOW)[0] assert issue.cohort.scope_id == name assert issue.deprecation.usage_model == name - assert issue.title.startswith(name) + assert issue.title.startswith("public-alias") From e760834cc626848ec7460df4e6dc25d068b0bebf Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 15:36:48 +0800 Subject: [PATCH 17/44] feat: project daily performance signals into scoped issues --- .../SlackAlerting/usage_alerts.py | 64 +- litellm/proxy/proxy_server.py | 9 +- .../spend_tracking/daily_usage_alerts.py | 20 +- .../spend_tracking/performance_issues.py | 673 ++++++++++++++++++ .../proxy/spend_tracking/usage_alert_job.py | 32 +- litellm/repositories/usage_alert_metrics.py | 203 +++++- litellm/repositories/usage_alerts.py | 17 +- litellm/types/proxy/usage_alert_defaults.py | 14 +- .../database/test_performance_issues.py | 258 +++++++ .../SlackAlerting/test_usage_alerts.py | 49 ++ .../spend_tracking/test_daily_usage_alerts.py | 95 ++- 11 files changed, 1381 insertions(+), 53 deletions(-) create mode 100644 litellm/proxy/spend_tracking/performance_issues.py create mode 100644 tests/integration/database/test_performance_issues.py diff --git a/litellm/integrations/SlackAlerting/usage_alerts.py b/litellm/integrations/SlackAlerting/usage_alerts.py index 2a06b3170ad..0c642535d15 100644 --- a/litellm/integrations/SlackAlerting/usage_alerts.py +++ b/litellm/integrations/SlackAlerting/usage_alerts.py @@ -4,11 +4,13 @@ import json import os from html import escape from typing import TYPE_CHECKING, Final -from urllib.parse import urlsplit +from urllib.parse import urlencode, urlsplit +from httpx import Response from pydantic import TypeAdapter from litellm.types.integrations.slack_alerting import AlertType +from litellm.types.proxy.issues import IssuePendingDelivery from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertChannel, UsageAlertRule if TYPE_CHECKING: @@ -32,7 +34,15 @@ def activity_url(alert: UsageAlert, base_url: str | None) -> str: def incident_message(alert: UsageAlert, url: str | None) -> str: scope: Final = "Gateway" if alert.scope_type == "deployment" else f"{alert.scope_type}: {alert.scope_id}" - unit: Final = " ms" if alert.metric == "latency" else " USD" if alert.metric == "spend" else "" + unit: Final = ( + " ms" + if alert.metric == "latency" + else " USD" + if alert.metric == "spend" + else "%" + if alert.metric == "error_rate" + else "" + ) return ( f"{alert.rule_name}\n{scope}\n{alert.day.isoformat()} UTC: " f"{alert.observed_value:,.2f}{unit}; baseline {alert.baseline_value:,.2f}{unit}; " @@ -45,25 +55,55 @@ class UsageAlertDelivery: self._slack = slack async def send(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> bool: - configured: Final = configured_destinations(self._slack) - if channel not in configured: - return False url: Final = activity_url(alert, os.getenv("PROXY_BASE_URL")) - message: Final = incident_message(alert, None if channel == "email" else url) + return await self._send( + channel, + rule.email_recipients, + incident_message(alert, None), + f"LiteLLM performance alert: {alert.rule_name}", + (("View activity", url),), + ) + + async def send_issue(self, pending: IssuePendingDelivery) -> bool: + path: Final = "/alerts?" + urlencode({"issue": pending.issue.id}) + issue_url: Final = activity_url( + pending.alert.model_copy(update={"activity_path": path}), os.getenv("PROXY_BASE_URL") + ) + source_url: Final = activity_url(pending.alert, os.getenv("PROXY_BASE_URL")) + return await self._send( + pending.delivery.channel, + pending.delivery.email_recipients, + incident_message(pending.alert, None), + f"LiteLLM issue: {pending.issue.title}", + (("View issue", issue_url), ("View source activity", source_url)), + ) + + async def _send( + self, + channel: UsageAlertChannel, + recipients: tuple[str, ...], + message: str, + subject: str, + links: tuple[tuple[str, str], ...], + ) -> bool: + if channel not in configured_destinations(self._slack): + return False if channel == "email": from litellm.proxy.utils import send_email - for recipient in rule.email_recipients: + for recipient in recipients: await send_email( receiver_email=recipient, - subject=f"LiteLLM performance alert: {alert.rule_name}", + subject=subject, html=( f"

{escape(message).replace(chr(10), '
')}

" - f'

View activity

' + + "".join( + f'

{escape(label)}

' for label, url in links + ) ), raise_on_error=True, ) - return bool(rule.email_recipients) + return bool(recipients) overrides: Final = self._slack.alert_to_webhook_url or {} configured_url: Final = ( overrides.get(AlertType.usage_alerts) @@ -75,11 +115,11 @@ class UsageAlertDelivery: return False urls: Final = (configured_url,) if isinstance(configured_url, str) else _URLS.validate_python(configured_url) for webhook in urls: - await self._post(webhook, message) + await self._post(webhook, message + "\n" + "\n".join(f"{label}: {url}" for label, url in links)) return True async def _post(self, webhook: str, message: str) -> None: - response: Final = await self._slack.async_http_handler.post( # pyright: ignore[reportUnknownMemberType] # legacy HTTP signature; supplied arguments are concrete + response: Final[Response] = await self._slack.async_http_handler.post( # pyright: ignore[reportUnknownMemberType] # legacy HTTP signature; supplied arguments are concrete url=webhook, headers={"Content-type": "application/json"}, data=json.dumps({"text": message}), diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ad4bb00b344..948748267eb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11679,8 +11679,15 @@ class ProxyStartupEvent: ) from litellm.proxy.spend_tracking.usage_alert_job import USAGE_ALERT_JOB_ID, UsageAlertJob + from litellm.proxy.spend_tracking.deprecation_issues import DeprecationIssueProjector - usage_alert_job: Final = UsageAlertJob.from_prisma(prisma_client, proxy_logging_obj.slack_alerting_instance) + usage_alert_job: Final = UsageAlertJob.from_prisma( + prisma_client, + proxy_logging_obj.slack_alerting_instance, + deprecation_projection=DeprecationIssueProjector.from_prisma( + prisma_client, lambda today: collect_model_deprecations(llm_router, today=today) + ).run, + ) scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # apscheduler is untyped usage_alert_job.run, "interval", diff --git a/litellm/proxy/spend_tracking/daily_usage_alerts.py b/litellm/proxy/spend_tracking/daily_usage_alerts.py index 1278af5d148..17212d4c4d7 100644 --- a/litellm/proxy/spend_tracking/daily_usage_alerts.py +++ b/litellm/proxy/spend_tracking/daily_usage_alerts.py @@ -3,9 +3,12 @@ from collections.abc import Sequence from dataclasses import dataclass from datetime import date, timedelta +from itertools import groupby from typing import Final, Literal, Protocol from litellm.repositories.usage_alert_metrics import DailyAlertMetrics +from litellm.types.proxy.usage_alert_defaults import is_default_usage_rule +from litellm.types.proxy.usage_alerts import UsageAlertRule class ComparisonRule(Protocol): @@ -68,7 +71,18 @@ def compare_daily_usage( current_requests: Final = sum(row.requests for row in current) if baseline_requests < rule.min_requests: return None - if rule.direction == "increase" and current_requests < rule.min_requests: + if (rule.direction == "increase" or rule.metric == "error_rate") and current_requests < rule.min_requests: + return None + if ( + rule.metric == "requests" + and rule.direction == "decrease" + and isinstance(rule, UsageAlertRule) + and is_default_usage_rule(rule) + and any( + sum(row.requests for row in group) < rule.min_requests + for _, group in groupby(sorted(history, key=lambda row: row.day), lambda row: row.day) + ) + ): return None if rule.metric == "latency": baseline_timed: Final = sum(row.timed_requests for row in history) / sample_days @@ -79,7 +93,9 @@ def compare_daily_usage( if baseline is None or baseline < 0 or observed is None: return None if baseline == 0: - if rule.direction == "decrease": + if rule.metric != "error_rate" or rule.direction == "decrease": + return None + if sample_days < 3 or baseline_requests < 500 or current_requests < 500: return None return DailyComparison(sample_days, observed, 0, rule.min_value, observed > 0 and observed >= rule.min_value) threshold: Final = ( diff --git a/litellm/proxy/spend_tracking/performance_issues.py b/litellm/proxy/spend_tracking/performance_issues.py new file mode 100644 index 00000000000..ad133358254 --- /dev/null +++ b/litellm/proxy/spend_tracking/performance_issues.py @@ -0,0 +1,673 @@ +"""Project daily observations into scoped, persisted investigations.""" + +import asyncio +import hashlib +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import date, datetime, timedelta +from itertools import chain, groupby +from typing import Final, Protocol +from uuid import NAMESPACE_URL, uuid5 + +from litellm._logging import verbose_proxy_logger +from litellm.models.issues import IssueStore +from litellm.proxy.spend_tracking.daily_usage_alerts import DailyComparison, compare_daily_usage +from litellm.repositories.provider_latency import ProviderLatencyEvidence, ProviderLatencyRepository +from litellm.repositories.usage_alert_metrics import ( + DailyAlertMetrics, + IssueKeyAttribution, + IssueMetricDay, + IssueMetricWindow, + IssueSourceProvider, + UsageAlertMetricsRepository, +) +from litellm.types.proxy.issues import ( + Issue, + IssueCohort, + IssueComparison, + IssueDeliveryBinding, + IssueDeliveryInput, + IssueImpact, + IssueInput, + IssueModelPair, + IssuePendingDelivery, + IssueProjectionSignal, + IssueSignal, + IssueSignalGroupingReason, +) +from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily, is_default_usage_rule +from litellm.types.proxy.usage_alerts import UsageAlertDirection, UsageAlertMetric, UsageAlertRule, UsageAlertScope + +_SCOPES: Final[tuple[UsageAlertScope, ...]] = ("key", "user", "team", "model", "deployment") +_METRICS: Final[tuple[UsageAlertMetric, ...]] = ("spend", "latency", "error_rate", "errors", "requests") + + +class IssueDelivery(Protocol): + async def send_issue(self, pending: IssuePendingDelivery) -> bool: ... + + +@dataclass(frozen=True, slots=True) +class _Candidate: + source: IssueProjectionSignal + comparison: DailyComparison + rows: tuple[DailyAlertMetrics, ...] + impact: IssueImpact + window: IssueMetricWindow + + +@dataclass(frozen=True, slots=True) +class _Target: + issue: IssueInput + reason: IssueSignalGroupingReason = "same_resource" + explanation: str = "" + + +@dataclass(frozen=True, slots=True) +class _ProviderGroup: + evidence: ProviderLatencyEvidence + rule: UsageAlertRule + + +def _family(metric: UsageAlertMetric) -> PerformanceIssueFamily: + if metric in ("errors", "error_rate"): + return "reliability" + if metric == "requests": + return "traffic" + return "spend" if metric == "spend" else "latency" + + +def _cohort_key(family: str, scope: str, entity: str | None, direction: UsageAlertDirection) -> str: + return str(uuid5(NAMESPACE_URL, f"litellm:issue:{family}:{scope}:{entity}:{direction}")) + + +def _window(source: IssueProjectionSignal, day: date) -> IssueMetricWindow: + return IssueMetricWindow( + source.alert.id, source.alert.scope_id or "deployment", day - timedelta(days=source.rule.baseline_days), day + ) + + +def _comparison(candidate: _Candidate) -> IssueComparison: + return IssueComparison( + metric=candidate.source.rule.metric, + direction=candidate.source.rule.direction, + observed_value=candidate.comparison.observed, + baseline_value=candidate.comparison.baseline, + threshold=candidate.comparison.threshold, + baseline_start=candidate.window.start, + baseline_end=candidate.window.day - timedelta(days=1), + baseline_sample_days=candidate.comparison.baseline_sample_days, + ) + + +def _request_context(candidate: _Candidate) -> IssueComparison: + history: Final = tuple(row for row in candidate.rows if row.day < candidate.window.day) + baseline: Final = sum(row.requests for row in history) / len({row.day for row in history}) + observed: Final = sum(row.requests for row in candidate.rows if row.day == candidate.window.day) + return IssueComparison( + metric="requests", + direction="increase" if observed >= baseline else "decrease", + observed_value=observed, + baseline_value=baseline, + baseline_start=candidate.window.start, + baseline_end=candidate.window.day - timedelta(days=1), + baseline_sample_days=len({row.day for row in history}), + ) + + +def _resource_issue(candidate: _Candidate, now: datetime) -> IssueInput: + rule: Final = candidate.source.rule + family: Final = _family(rule.metric) + cohort_key: Final = _cohort_key(family, rule.scope_type, candidate.source.alert.scope_id, rule.direction) + title: Final = { + "spend": "Spend", + "latency": "Latency", + "error_rate": "Error rate", + "errors": "Failed requests", + "requests": "Traffic", + }[rule.metric] + return IssueInput( + id=str( + uuid5(NAMESPACE_URL, f"{cohort_key}:{candidate.source.alert.id}:{candidate.window.day}:{rule.updated_at}") + ), + cohort_key=cohort_key, + kind="performance", + family=family, + title=f"{title} {'increased' if rule.direction == 'increase' else 'decreased'}", + cohort=IssueCohort(scope_type=rule.scope_type, scope_id=candidate.source.alert.scope_id), + observed_on=candidate.window.day, + observed_at=now, + range_start=candidate.window.start, + range_end=candidate.window.day, + grouping_reason="same_resource", + explanation=f"Compared with {candidate.comparison.baseline_sample_days} recorded reference days.", + comparisons=(_comparison(candidate), _request_context(candidate)) + if rule.metric == "spend" + else (_comparison(candidate),), + impact=candidate.impact, + ) + + +def _volume_explains_spend(traffic: _Candidate, spend: _Candidate) -> bool: + if spend.comparison.baseline <= 0 or traffic.comparison.baseline <= 0: + return False + if (spend.window.start, spend.window.day) != (traffic.window.start, traffic.window.day): + return False + cost_ratio: Final = spend.comparison.observed / spend.comparison.baseline + request_ratio: Final = traffic.comparison.observed / traffic.comparison.baseline + return 0.8 <= cost_ratio / request_ratio <= 1.2 + + +def _bindings(candidate: _Candidate, issue_id: str) -> tuple[IssueDeliveryBinding, ...]: + rule: Final = candidate.source.rule + destinations: Final = ( + tuple(("email", address) for address in rule.email_recipients) if "email" in rule.notification_channels else () + ) + channels: Final = ( + *destinations, + *((("slack", "configured-slack-route"),) if "slack" in rule.notification_channels else ()), + ) + return tuple( + IssueDeliveryBinding( + delivery=IssueDeliveryInput( + id=str(uuid5(NAMESPACE_URL, f"{issue_id}:{channel}:{destination}")), + issue_id=issue_id, + channel="email" if channel == "email" else "slack", + email_recipients=(destination,) if channel == "email" else (), + ), + rule_id=rule.id, + rule_revision=rule.updated_at.isoformat(), + source_id=candidate.source.alert.id, + ) + for channel, destination in channels + ) + + +class PerformanceIssueProjector: + def __init__( + self, + store: IssueStore, + metrics: UsageAlertMetricsRepository, + providers: ProviderLatencyRepository, + delivery: IssueDelivery, + ) -> None: + self._store = store + self._metrics = metrics + self._providers = providers + self._delivery = delivery + + async def run(self, now: datetime, day: date, rules: tuple[UsageAlertRule, ...], *, ready: bool) -> None: + observed_at: Final = now.replace(microsecond=now.microsecond // 1000 * 1000) + version: Final = hashlib.sha256( + ("performance-v1:" + ",".join(f"{rule.id}:{rule.updated_at.isoformat()}" for rule in rules)).encode() + ).hexdigest() + if ready and not await self._store.projection_completed(day, version): + await self._store.recover_issues(day) + providers: Final = await self._provider_evidence(day, rules, observed_at) + for metric in _METRICS: + for scope in _SCOPES: + await self._scope(observed_at, day, scope, metric, providers) + await self._store.complete_projection(day, version) + await self._notify() + await self._store.prune() + + async def _candidates( + self, sources: tuple[IssueProjectionSignal, ...], scope: UsageAlertScope, day: date + ) -> tuple[_Candidate, ...]: + windows: Final = tuple(_window(source, day) for source in sources) + rows: Final = await self._metrics.issue_windows(scope, windows) + by_reference: Final = { + reference: tuple(group) + for reference, group in groupby( + sorted(rows, key=lambda row: row.reference_id), lambda row: row.reference_id + ) + } + return tuple( + candidate + for source in sources + if (candidate := self._candidate(source, by_reference.get(source.alert.id, ()), day)) is not None + ) + + @staticmethod + def _candidate(source: IssueProjectionSignal, days: tuple[IssueMetricDay, ...], day: date) -> _Candidate | None: + rows: Final = tuple(item.metrics for item in days) + comparison: Final = compare_daily_usage(source.rule, rows, day) + if comparison is None or not comparison.breached: + return None + return _Candidate( + source, + comparison, + rows, + IssueImpact( + keys=days[0].keys if days else None, + users=days[0].users if days else None, + teams=1 if source.rule.scope_type == "team" else None, + models=1 if source.rule.scope_type == "model" else None, + requests=sum(row.requests for row in rows if row.day == day), + ), + _window(source, day), + ) + + async def _scope( + self, + now: datetime, + day: date, + scope: UsageAlertScope, + metric: UsageAlertMetric, + providers: Mapping[str, _ProviderGroup], + ) -> None: + cursor: str | None = None # rebind-ok: bounded source pagination + while sources := await self._store.active_signals(day, scope_type=scope, metric=metric, cursor=cursor): + await self._project_page(sources, scope, day, providers, now) + cursor = sources[-1].alert.id + await asyncio.sleep(0) + + async def _project_page( + self, + sources: tuple[IssueProjectionSignal, ...], + scope: UsageAlertScope, + day: date, + providers: Mapping[str, _ProviderGroup], + now: datetime, + ) -> None: + candidates: Final = await self._candidates(sources, scope, day) + provider_targets: Final = await self._provider_targets(candidates, providers, now) + resource_targets: Final = await self._resource_targets(candidates, now) + targets: Final = { + item.source.alert.id: provider_targets.get(item.source.alert.id) + or resource_targets.get(item.source.alert.id) + or _Target(_resource_issue(item, now)) + for item in candidates + } + await self._save(candidates, targets, now) + + async def _spend_candidates( + self, scope: UsageAlertScope, entities: tuple[str, ...] | None, day: date + ) -> tuple[_Candidate, ...]: + found: Final[list[_Candidate]] = [] # mutable-ok: bounded entity batch may match multiple configured rules + cursor: str | None = None # rebind-ok: page matching rules without one query per entity + while sources := await self._store.active_signals( + day, scope_type=scope, metric="spend", scope_ids=entities, cursor=cursor + ): + found.extend(await self._candidates(sources, scope, day)) + cursor = sources[-1].alert.id + return tuple(found) + + async def _volume_targets(self, candidates: tuple[_Candidate, ...], now: datetime) -> Mapping[str, _Target]: + traffic: Final = tuple( + item + for item in candidates + if item.source.rule.metric == "requests" and item.source.rule.direction == "increase" + ) + if not traffic: + return {} + scope: Final = traffic[0].source.rule.scope_type + entities: Final = tuple( + dict.fromkeys(item.source.alert.scope_id for item in traffic if item.source.alert.scope_id is not None) + ) + spending: Final = await self._spend_candidates( + scope, entities if scope != "deployment" else None, traffic[0].window.day + ) + by_entity: Final = { + entity: tuple(group) + for entity, group in groupby( + sorted(spending, key=lambda item: item.source.alert.scope_id or ""), + lambda item: item.source.alert.scope_id, + ) + } + keys: Final = tuple(_cohort_key("spend", scope, item.source.alert.scope_id, "increase") for item in traffic) + issues: Final = { + issue.cohort_key: issue for issue in await self._store.find_active_issues(keys) if issue.last_seen == now + } + return { + item.source.alert.id: _Target( + issue.model_copy(update={"observed_at": now}), + "volume_explains_spend", + "Request volume and spend increased by similar proportions in the same comparison window.", + ) + for item in traffic + if (issue := issues.get(_cohort_key("spend", scope, item.source.alert.scope_id, "increase"))) is not None + and any(_volume_explains_spend(item, spend) for spend in by_entity.get(item.source.alert.scope_id, ())) + } + + async def _resource_targets(self, candidates: tuple[_Candidate, ...], now: datetime) -> Mapping[str, _Target]: + volume: Final = await self._volume_targets(candidates, now) + broader: Final = tuple( + item + for item in candidates + if is_default_usage_rule(item.source.rule) and item.source.rule.scope_type != "key" + ) + if not broader: + return volume + attributions: Final = { + row.reference_id: row + for row in await self._metrics.issue_key_attributions( + broader[0].source.rule.scope_type, + tuple(item.window for item in broader), + "requests" if broader[0].source.rule.metric == "requests" else "spend", + ) + } + eligible: Final = tuple( + item + for item in broader + if (attribution := attributions.get(item.source.alert.id)) is not None + and ( + attribution.equivalent + or ( + item.source.rule.metric in ("spend", "requests") + and item.source.rule.direction == "increase" + and attribution.positive_change_share >= 0.8 + ) + ) + ) + keys: Final = tuple( + _cohort_key( + _family(item.source.rule.metric), + "key", + attributions[item.source.alert.id].key_id, + item.source.rule.direction, + ) + for item in eligible + ) + issues: Final = { + issue.cohort_key: issue for issue in await self._store.find_active_issues(keys) if issue.last_seen == now + } + borrowed: Final = { + item.source.alert.id: _Target( + issue.model_copy(update={"observed_at": now}), + "same_key_traffic" if attributions[item.source.alert.id].equivalent else "dominant_key", + "This source contains the same key traffic." + if attributions[item.source.alert.id].equivalent + else f"This key accounts for {attributions[item.source.alert.id].positive_change_share:.0%} of positive {item.source.rule.metric} growth in this source.", + ) + for item in eligible + if ( + issue := issues.get( + _cohort_key( + _family(item.source.rule.metric), + "key", + attributions[item.source.alert.id].key_id, + item.source.rule.direction, + ) + ) + ) + is not None + } + key_volume: Final = await self._key_volume_targets(eligible, attributions, now) + return {**volume, **borrowed, **key_volume} + + async def _key_volume_targets( + self, + candidates: tuple[_Candidate, ...], + attributions: Mapping[str, IssueKeyAttribution], + now: datetime, + ) -> Mapping[str, _Target]: + traffic: Final = tuple( + item + for item in candidates + if item.source.rule.metric == "requests" and item.source.rule.direction == "increase" + ) + if not traffic: + return {} + key_ids: Final = tuple(dict.fromkeys(attributions[item.source.alert.id].key_id for item in traffic)) + spending: Final = await self._spend_candidates("key", key_ids, traffic[0].window.day) + matching: Final = tuple(item for item in spending if self._volume_matches_key_spend(item)) + by_key: Final = { + identity: tuple(group) + for identity, group in groupby( + sorted(matching, key=lambda item: item.source.alert.scope_id or ""), + lambda item: item.source.alert.scope_id, + ) + } + keys: Final = tuple(_cohort_key("spend", "key", key, "increase") for key in key_ids) + issues: Final = { + issue.cohort_key: issue for issue in await self._store.find_active_issues(keys) if issue.last_seen == now + } + return { + item.source.alert.id: _Target( + issue.model_copy(update={"observed_at": now}), + "volume_explains_spend", + "The same key explains this source's request growth and has a matching increase in spend.", + ) + for item in traffic + if (issue := issues.get(_cohort_key("spend", "key", attributions[item.source.alert.id].key_id, "increase"))) + is not None + and any( + (spend.window.start, spend.window.day) == (item.window.start, item.window.day) + for spend in by_key.get(attributions[item.source.alert.id].key_id, ()) + ) + } + + @staticmethod + def _volume_matches_key_spend(spend: _Candidate) -> bool: + requests: Final = _request_context(spend) + if ( + requests.baseline_value <= 0 + or requests.observed_value <= requests.baseline_value + or spend.comparison.baseline <= 0 + ): + return False + return ( + 0.8 + <= (spend.comparison.observed / spend.comparison.baseline) + / (requests.observed_value / requests.baseline_value) + <= 1.2 + ) + + @staticmethod + def _same_policy(left: UsageAlertRule, right: UsageAlertRule) -> bool: + return (left.baseline_days, left.min_history_days, left.min_requests, left.multiplier, left.min_value) == ( + right.baseline_days, + right.min_history_days, + right.min_requests, + right.multiplier, + right.min_value, + ) + + async def _provider_evidence( + self, day: date, rules: tuple[UsageAlertRule, ...], now: datetime + ) -> Mapping[str, _ProviderGroup]: + rule: Final = next( + ( + rule + for rule in rules + if is_default_usage_rule(rule) and rule.metric == "latency" and rule.direction == "increase" + ), + None, + ) + if rule is None: + return {} + groups: Final[dict[str, _ProviderGroup]] = {} # mutable-ok: retain one confirmed cohort per provider + reserved: Final[set[str]] = set() # mutable-ok: an active frozen cohort prevents a competing subset episode + cursor: str | None = None # rebind-ok: bounded existing-cohort pagination + while existing := await self._store.active_provider_issues(cursor=cursor): + for prior, recovered, evidence in await self._prior_provider_page(existing, day, rule, now): + if recovered or prior.cohort.scope_id is None: + continue + reserved.add(prior.cohort.scope_id) + if evidence is not None: + groups[prior.cohort.scope_id] = _ProviderGroup(evidence, rule) + cursor = existing[-1].id + source_cursor: str | None = None # rebind-ok: stop discovery when no compatible active source exists + compatible = False # rebind-ok: early-exit source scan + while sources := await self._store.active_signals(day, metric="latency", cursor=source_cursor): + if any(is_default_usage_rule(source.rule) and self._same_policy(source.rule, rule) for source in sources): + compatible = True + break + source_cursor = sources[-1].alert.id + if not compatible: + return groups + pair_cursor: IssueModelPair | None = None # rebind-ok: keyset pagination of model pairs + while pairs := await self._providers.candidate_pairs(day, pair_cursor): + for provider, selected in groupby(pairs, lambda pair: pair.provider): + if provider in reserved or provider in groups: + continue + if (candidate_evidence := await self._providers.latency_cohort(tuple(selected), day, rule)) is not None: + groups[provider] = _ProviderGroup(candidate_evidence, rule) + pair_cursor = pairs[-1] + return groups + + async def _prior_provider_page( + self, + existing: tuple[Issue, ...], + day: date, + rule: UsageAlertRule, + now: datetime, + ) -> tuple[tuple[Issue, bool, ProviderLatencyEvidence | None], ...]: + states: Final = tuple([await self._prior_provider(issue, day, rule) for issue in existing]) + await self._store.recover_issue_ids(tuple(issue.id for issue, recovered, _ in states if recovered), now) + return states + + async def _prior_provider( + self, + issue: Issue, + day: date, + rule: UsageAlertRule, + ) -> tuple[Issue, bool, ProviderLatencyEvidence | None]: + if await self._providers.cohort_recovered(issue.cohort.model_pairs, day, rule): + return issue, True, None + evidence: Final = await self._providers.latency_cohort(issue.cohort.model_pairs, day, rule) + return ( + issue, + False, + evidence if evidence is not None and evidence.cohort.model_pairs == issue.cohort.model_pairs else None, + ) + + async def _provider_targets( + self, candidates: tuple[_Candidate, ...], providers: Mapping[str, _ProviderGroup], now: datetime + ) -> Mapping[str, _Target]: + eligible: Final = tuple( + item + for item in candidates + if item.source.rule.metric == "latency" and is_default_usage_rule(item.source.rule) + ) + if not eligible or not providers: + return {} + scope: Final = eligible[0].source.rule.scope_type + by_reference: Final = {item.source.alert.id: item for item in eligible} + hints: Final = await self._metrics.issue_source_providers(scope, tuple(item.window for item in eligible)) + targets: Final[dict[str, _Target]] = {} # mutable-ok: each source belongs to at most one 80%-coverage provider + for provider, matches in groupby(sorted(hints, key=lambda hint: hint.provider), lambda hint: hint.provider): + targets.update(await self._provider_matches(provider, tuple(matches), by_reference, providers, scope, now)) + return targets + + async def _provider_matches( + self, + provider: str, + matches: tuple[IssueSourceProvider, ...], + by_reference: Mapping[str, _Candidate], + providers: Mapping[str, _ProviderGroup], + scope: UsageAlertScope, + now: datetime, + ) -> Mapping[str, _Target]: + group: Final = providers.get(provider) + if group is None: + return {} + selected: Final = tuple( + by_reference[hint.reference_id] + for hint in matches + if self._same_policy(by_reference[hint.reference_id].source.rule, group.rule) + ) + if not selected: + return {} + members: Final = await self._providers.source_members( + group.evidence.cohort.model_pairs, + selected[0].window.day, + group.rule, + scope, + tuple(item.source.alert.scope_id or "deployment" for item in selected), + ) + return { + item.source.alert.id: target + for item in selected + if (item.source.alert.scope_id or "deployment") in members + and (target := self._provider_target(item, group.evidence, now)) is not None + } + + @staticmethod + def _provider_target(item: _Candidate, evidence: ProviderLatencyEvidence, now: datetime) -> _Target | None: + cohort: Final = evidence.comparison + if ( + item.comparison.baseline <= 0 + or item.comparison.observed / item.comparison.baseline > 2 * cohort.observed_value / cohort.baseline_value + ): + return None + key: Final = _cohort_key("latency", "provider", evidence.cohort.scope_id, "increase") + return _Target( + IssueInput( + id=str( + uuid5( + NAMESPACE_URL, f"{key}:{item.source.alert.id}:{item.window.day}:{item.source.rule.updated_at}" + ) + ), + cohort_key=key, + kind="performance", + family="latency", + title=f"Higher latency across models through {evidence.cohort.scope_id}", + cohort=evidence.cohort, + observed_on=item.window.day, + observed_at=now, + range_start=cohort.baseline_start, + range_end=item.window.day, + grouping_reason="provider_cohort", + explanation="Several keys and models slowed down together. The comparison holds today's key and model mix constant and excludes keys still collecting a baseline. This is a shared pattern, not proof of a common cause.", + comparisons=(cohort,), + impact=evidence.impact, + ), + "provider_cohort", + "At least 80% of this source's timed requests belong to independently degraded key/model pairs in the saved cohort.", + ) + + async def _save(self, candidates: tuple[_Candidate, ...], targets: Mapping[str, _Target], now: datetime) -> None: + if not candidates: + return + distinct: Final = {target.issue.cohort_key: target.issue for target in targets.values()} + existing: Final = {item.cohort_key: item for item in await self._store.find_active_issues(tuple(distinct))} + inputs: Final = tuple( + item.model_copy(update={"id": existing[key].id}) if key in existing else item + for key, item in distinct.items() + ) + saved: Final = {item.cohort_key: item for item in await self._store.upsert_issues(inputs)} + await self._store.add_signal_batch( + tuple( + IssueSignal( + issue_id=saved[targets[candidate.source.alert.id].issue.cohort_key].id, + source="usage_alert", + source_id=candidate.source.alert.id, + usage_alert=candidate.source.alert, + grouping_reason=targets[candidate.source.alert.id].reason, + explanation=targets[candidate.source.alert.id].explanation, + ) + for candidate in candidates + ) + ) + bindings: Final = tuple( + chain.from_iterable( + _bindings(candidate, saved[targets[candidate.source.alert.id].issue.cohort_key].id) + for candidate in candidates + ) + ) + for offset in range(0, len(bindings), 500): + await self._store.queue_deliveries(bindings[offset : offset + 500]) + + async def _notify(self) -> None: + deadline: Final = asyncio.get_running_loop().time() + 60 + while asyncio.get_running_loop().time() < deadline and (pending := await self._store.pending_deliveries()): + for item in pending: + if asyncio.get_running_loop().time() >= deadline: + return + await self._notify_one(item) + + async def _notify_one(self, item: IssuePendingDelivery) -> None: + current: Final = await self._store.pending_delivery(item.delivery.id) + if current is None: + return + try: + delivered: Final = await asyncio.wait_for(self._delivery.send_issue(current), timeout=30) + except Exception: # noqa: BLE001 # persisted retry avoids losing a notification on transport failure + await self._store.defer_delivery(item.delivery.id, "delivery_failed") + verbose_proxy_logger.warning("Performance issue notification failed; queued for retry") + else: + if delivered: + await self._store.mark_delivery_sent(item.delivery.id) + else: + await self._store.defer_delivery(item.delivery.id, "transport_not_configured") diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index 95474d56609..872a7ae83df 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -16,6 +16,9 @@ from uuid import uuid4 from litellm._logging import verbose_proxy_logger from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, configured_destinations from litellm.proxy.spend_tracking.daily_usage_alerts import compare_daily_usage +from litellm.proxy.spend_tracking.performance_issues import PerformanceIssueProjector +from litellm.repositories.issues import IssuesRepository +from litellm.repositories.provider_latency import ProviderLatencyRepository from litellm.repositories.usage_alert_health import UsageAlertHealthRepository from litellm.repositories.usage_alert_metrics import DailyAlertMetrics, MetricsDatabase, UsageAlertMetricsRepository from litellm.repositories.usage_alerts import UsageAlertsRepository @@ -79,16 +82,14 @@ def _observations( continue if rule.scope_id is not None and rule.scope_id != entity: continue - comparison: Final = compare_daily_usage(rule, tuple(group), target) - if comparison is None: + if (comparison := compare_daily_usage(rule, tuple(group), target)) is None: continue evaluated = True - identity: Final = None if rule.scope_type == "deployment" else entity - if not comparison.breached and identity not in active: + if not comparison.breached and (None if rule.scope_type == "deployment" else entity) not in active: continue changed.append( UsageAlertObservation( - entity_id=identity, + entity_id=None if rule.scope_type == "deployment" else entity, baseline_sample_days=comparison.baseline_sample_days, day=target, observed=comparison.observed, @@ -125,6 +126,7 @@ class UsageAlertJob: defaults: tuple[UsageAlertRuleInput, ...] | None = None, *, deprecation_projection: Callable[[datetime], Awaitable[None]] | None = None, + performance_projection: PerformanceIssueProjector | None = None, ) -> None: self._store = store self._metrics = metrics @@ -133,6 +135,7 @@ class UsageAlertJob: self._defaults = default_usage_rules() if defaults is None else defaults self._defaults_registered = False self._deprecation_projection = deprecation_projection + self._performance_projection = performance_projection @classmethod def from_prisma( @@ -143,11 +146,13 @@ class UsageAlertJob: deprecation_projection: Callable[[datetime], Awaitable[None]] | None = None, ) -> "UsageAlertJob": database: Final = cast(MetricsDatabase, prisma.writer_db) # cast-ok: generated Prisma delegate + metrics: Final = UsageAlertMetricsRepository(database) + delivery: Final = UsageAlertDelivery(slack) return cls( UsageAlertsRepository.from_prisma(prisma), - UsageAlertMetricsRepository(database), + metrics, UsageAlertHealthRepository(database), - UsageAlertDelivery(slack), + delivery, default_usage_rules( ("slack",) if "slack" in configured_destinations(slack) and AlertType.usage_alerts in slack.alert_types @@ -155,6 +160,12 @@ class UsageAlertJob: legacy_user_spend_alerts=AlertType.user_spend_anomalies in slack.alert_types, ), deprecation_projection=deprecation_projection, + performance_projection=PerformanceIssueProjector( + IssuesRepository.from_prisma(prisma), + metrics, + ProviderLatencyRepository(database), + delivery, + ), ) async def run(self, now: datetime | None = None) -> None: @@ -197,7 +208,12 @@ class UsageAlertJob: for scoped in (tuple(rule for rule in due if rule.scope_type == scope) for scope in USAGE_ALERT_SCOPES): if scoped: await self._scope(scoped, target, rules) - await self._notify() + if self._performance_projection is not None: + for offset in range(0, len(due), 500): + complete.update(await self._store.completed_rule_ids(due[offset : offset + 500], target)) + await self._performance_projection.run(now, target, rules, ready=len(complete) == len(rules)) + else: + await self._notify() await self._store.prune() async def _scope( diff --git a/litellm/repositories/usage_alert_metrics.py b/litellm/repositories/usage_alert_metrics.py index c828a639c05..228c12d2795 100644 --- a/litellm/repositories/usage_alert_metrics.py +++ b/litellm/repositories/usage_alert_metrics.py @@ -1,5 +1,6 @@ """Bounded reads of the daily rollups used by the Usage dashboard.""" +import json from collections.abc import AsyncIterator, Mapping, Sequence from dataclasses import dataclass, replace from datetime import date, timedelta @@ -7,7 +8,7 @@ from typing import Final, Literal, Protocol, TypeAlias from pydantic import TypeAdapter -from litellm.repositories.daily_activity_buckets import api_key_bucket, model_bucket +from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket, model_bucket from litellm.types.proxy.model_deprecation import ModelDeprecationUsage Scope: TypeAlias = Literal["deployment", "key", "user", "team", "model"] @@ -24,6 +25,145 @@ class DailyAlertMetrics: timed_requests: int +@dataclass(frozen=True, slots=True) +class IssueMetricWindow: + reference_id: str + entity_id: str + start: date + day: date + + +@dataclass(frozen=True, slots=True) +class IssueMetricDay: + reference_id: str + metrics: DailyAlertMetrics + keys: int | None + users: int | None + + +@dataclass(frozen=True, slots=True) +class IssueKeyAttribution: + reference_id: str + key_id: str + equivalent: bool + positive_change_share: float + baseline_value: float + observed_value: float + + +@dataclass(frozen=True, slots=True) +class IssueSourceProvider: + reference_id: str + provider: str + + +def _window_payload(scope: Scope, windows: tuple[IssueMetricWindow, ...]) -> str: + if len(windows) > 500 or any(not 1 <= (window.day - window.start).days <= 30 for window in windows): + raise ValueError("Read at most 500 issue windows with 1–30 reference days per batch") + return json.dumps( + tuple( + { + "reference_id": window.reference_id, + "entity_id": api_key_from_bucket(window.entity_id) if scope == "key" else window.entity_id, + "start_day": window.start.isoformat(), + "target_day": window.day.isoformat(), + } + for window in windows + ) + ) + + +_WINDOW_INPUT: Final = """SELECT * FROM jsonb_to_recordset($1::jsonb) + AS x(reference_id text,entity_id text,start_day text,target_day text)""" + + +def _window_rows(scope: Scope) -> str: + table, entity = _source(scope) + user: Final = "NULL::text" if scope == "team" else "a.user_id" + return f"""SELECT x.reference_id,x.entity_id,x.start_day,x.target_day, + COALESCE(a.api_key,'') AS api_key,COALESCE(a.model,'') AS model, + {user} AS user_id, + a.date AS day,a.spend,a.api_requests,a.failed_requests,a.total_response_time_ms,a.timed_requests + FROM input x JOIN {table} a ON {entity}=x.entity_id + AND a.date>=x.start_day AND a.date<=x.target_day""" + + +def _key_attribution_query(scope: Scope, metric: Literal["spend", "requests"]) -> str: + measure: Final = "spend" if metric == "spend" else "api_requests" + same_identity: Final = { + "deployment": "true", + "user": "k.users=1 AND k.user_id=p.entity_id", + "model": "k.models=1 AND k.model=p.entity_id", + "team": "k.team_days IS NOT DISTINCT FROM k.user_days", + "key": "true", + }[scope] + team_days: Final = ( + """(SELECT jsonb_agg(to_jsonb(d) ORDER BY d.day) FROM ( + SELECT date AS day,SUM(spend) AS spend,SUM(api_requests)::bigint AS requests, + SUM(failed_requests)::bigint AS errors,SUM(total_response_time_ms) AS ms, + SUM(timed_requests)::bigint AS timed + FROM "LiteLLM_DailyTeamSpend" WHERE api_key=w.api_key AND team_id=w.entity_id + AND date>=w.start_day AND date<=w.target_day GROUP BY date + ) d)""" + if scope == "team" + else "NULL::jsonb" + ) + return f"""WITH input AS ({_WINDOW_INPUT}), source_rows AS MATERIALIZED (SELECT raw.*, {measure} AS score FROM ({_window_rows(scope)}) raw), + population AS ( + SELECT reference_id,MIN(entity_id) AS entity_id,MIN(start_day) AS start_day,MIN(target_day) AS target_day, + count(DISTINCT api_key) AS key_count,MIN(api_key) AS sole_key, + count(DISTINCT day) FILTER(WHERE day=3 AS comparable, + COALESCE(SUM(s.score) FILTER(WHERE s.day=s.target_day),0) AS current_value, + GREATEST(COALESCE(SUM(s.score) FILTER(WHERE s.day=s.target_day),0) - + COALESCE(SUM(s.score) FILTER(WHERE s.day'' + ), key_population AS ( + SELECT w.*, + (SELECT count(DISTINCT COALESCE(user_id,'')) FROM "LiteLLM_DailyUserSpend" + WHERE api_key=w.api_key AND date>=w.start_day AND date<=w.target_day) AS users, + (SELECT MIN(user_id) FROM "LiteLLM_DailyUserSpend" + WHERE api_key=w.api_key AND date>=w.start_day AND date<=w.target_day) AS user_id, + (SELECT count(DISTINCT COALESCE(NULLIF(model,''),'unknown-model')) FROM "LiteLLM_DailyUserSpend" + WHERE api_key=w.api_key AND date>=w.start_day AND date<=w.target_day) AS models, + (SELECT MIN(COALESCE(NULLIF(model,''),'unknown-model')) FROM "LiteLLM_DailyUserSpend" + WHERE api_key=w.api_key AND date>=w.start_day AND date<=w.target_day) AS model, + {team_days} AS team_days, + (SELECT jsonb_agg(to_jsonb(d) ORDER BY d.day) FROM ( + SELECT date AS day,SUM(spend) AS spend,SUM(api_requests)::bigint AS requests, + SUM(failed_requests)::bigint AS errors,SUM(total_response_time_ms) AS ms, + SUM(timed_requests)::bigint AS timed + FROM "LiteLLM_DailyUserSpend" WHERE api_key=w.api_key + AND date>=w.start_day AND date<=w.target_day GROUP BY date + ) d) AS user_days + FROM windows w + ) SELECT p.reference_id,p.api_key AS key_id, + p.key_count=1 AND ({same_identity}) AS equivalent,p.positive_change_share, + COALESCE(p.baseline_value,0) AS baseline_value,p.observed_value + FROM candidate p JOIN key_population k ON k.api_key=p.api_key AND k.entity_id=p.entity_id + AND k.start_day=p.start_day AND k.target_day=p.target_day""" + + class MetricsDatabase(Protocol): async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ... @@ -105,6 +245,67 @@ class UsageAlertMetricsRepository: def __init__(self, database: MetricsDatabase) -> None: self._database = database + async def issue_windows(self, scope: Scope, windows: tuple[IssueMetricWindow, ...]) -> tuple[IssueMetricDay, ...]: + if not windows: + return () + users: Final = ( + "NULL::int" + if scope == "team" + else """CASE WHEN bool_or(day=target_day AND COALESCE(user_id,'') IN ('','default_user_id')) + THEN NULL ELSE count(DISTINCT user_id) FILTER(WHERE day=target_day)::int END""" + ) + return TypeAdapter(tuple[IssueMetricDay, ...]).validate_python( + await self._database.query_raw( + f"""WITH input AS ({_WINDOW_INPUT}), source_rows AS MATERIALIZED ({_window_rows(scope)}), + impact AS ( + SELECT reference_id, + CASE WHEN bool_or(day=target_day AND api_key='') THEN NULL + ELSE count(DISTINCT api_key) FILTER(WHERE day=target_day)::int END AS keys, + {users} AS users + FROM source_rows GROUP BY reference_id + ), daily AS ( + SELECT reference_id,jsonb_build_object( + 'entity_id',MIN(entity_id),'day',day,'spend',SUM(spend), + 'requests',SUM(api_requests),'errors',SUM(failed_requests), + 'response_time_ms',SUM(total_response_time_ms),'timed_requests',SUM(timed_requests) + ) AS metrics + FROM source_rows GROUP BY reference_id,day + ) SELECT daily.*,impact.keys,impact.users FROM daily JOIN impact USING(reference_id) + ORDER BY reference_id,metrics->>'day'""", + _window_payload(scope, windows), + ) + ) + + async def issue_key_attributions( + self, scope: Scope, windows: tuple[IssueMetricWindow, ...], metric: Literal["spend", "requests"] = "spend" + ) -> tuple[IssueKeyAttribution, ...]: + if not windows: + return () + rows: Final = TypeAdapter(tuple[IssueKeyAttribution, ...]).validate_python( + await self._database.query_raw(_key_attribution_query(scope, metric), _window_payload(scope, windows)) + ) + return tuple(replace(row, key_id=api_key_bucket(row.key_id)) for row in rows) + + async def issue_source_providers( + self, scope: Scope, windows: tuple[IssueMetricWindow, ...] + ) -> tuple[IssueSourceProvider, ...]: + if not windows: + return () + table, entity = _source(scope) + return TypeAdapter(tuple[IssueSourceProvider, ...]).validate_python( + await self._database.query_raw( + f"""WITH input AS ({_WINDOW_INPUT}), grouped AS ( + SELECT x.reference_id,a.custom_llm_provider AS provider,SUM(a.timed_requests) AS timed + FROM input x JOIN {table} a ON {entity}=x.entity_id AND a.date=x.target_day + GROUP BY x.reference_id,a.custom_llm_provider + ), ranked AS ( + SELECT *,SUM(timed) OVER(PARTITION BY reference_id) AS total FROM grouped + ) SELECT reference_id,provider FROM ranked + WHERE provider<>'' AND timed>0 AND timed>=0.8*total""", + _window_payload(scope, windows), + ) + ) + async def deprecation_usage( self, models: tuple[str, ...], start: date, end: date ) -> tuple[ModelDeprecationUsage, ...]: diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index 76205a20f55..be17170a129 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -164,6 +164,7 @@ def _observation_payload(rule: UsageAlertRule, item: UsageAlertObservation, now: rule.metric, rule.direction, item.day.isoformat(), + rule.updated_at.isoformat(), ] ), ) @@ -696,7 +697,8 @@ class UsageAlertsRepository: """ WITH input AS (SELECT * FROM jsonb_to_recordset($2::jsonb) AS x(scope_key text, breached boolean, day text, record jsonb)), prior AS ( - SELECT x.*,p.id AS prior_id,p.evaluation_day,p.breached AS prior_breached,p.record AS prior_record + SELECT x.*,p.id AS prior_id,p.evaluation_day,p.evaluation_revision AS prior_revision, + p.breached AS prior_breached,p.record AS prior_record FROM input x LEFT JOIN LATERAL ( SELECT * FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1 AND scope_key=x.scope_key AND scope_type=x.record->>'scope_type' AND scope_id IS NOT DISTINCT FROM x.record->>'scope_id' @@ -706,18 +708,21 @@ class UsageAlertsRepository: WHERE EXISTS (SELECT 1 FROM "LiteLLM_UsageAlertRule" WHERE id=$1 AND enabled AND NOT deleted AND (record->>'updated_at')::timestamptz=$3::timestamptz) ), updated AS ( UPDATE "LiteLLM_UsageAlertIncident" i SET - evaluation_day=p.day, breached=p.breached, + evaluation_day=p.day, evaluation_revision=$3::text, breached=p.breached, record=i.record || jsonb_build_object( 'last_seen',CASE WHEN p.breached THEN p.record->'last_seen' ELSE i.record->'last_seen' END, 'status',CASE WHEN i.status='open' AND NOT p.breached THEN 'resolved' ELSE i.status END), status=CASE WHEN i.status='open' AND NOT p.breached THEN 'resolved' ELSE i.status END, updated_at=CURRENT_TIMESTAMP - FROM prior p WHERE i.id=p.prior_id AND p.evaluation_day>'id',$1,p.scope_key,p.record->>'scope_type',p.record->>'scope_id',p.record->>'metric','open',p.day,p.day,true,p.record - FROM prior p WHERE p.breached AND (p.prior_id IS NULL OR (NOT p.prior_breached AND p.evaluation_day>'id',$1,p.scope_key,p.record->>'scope_type',p.record->>'scope_id',p.record->>'metric','open',p.day,p.day,$3,true,p.record + FROM prior p WHERE p.breached AND (p.prior_id IS NULL OR (NOT p.prior_breached AND + (p.evaluation_day bool: - return rule.id in DEFAULT_USAGE_RULE_IDS and rule.id == default_usage_rule_id( - rule.scope_type, rule.metric, rule.direction - ) + return _DEFAULT_USAGE_IDENTITIES.get(rule.id) == (rule.scope_type, rule.metric, rule.direction) def default_usage_rules( diff --git a/tests/integration/database/test_performance_issues.py b/tests/integration/database/test_performance_issues.py new file mode 100644 index 00000000000..481db1bc89f --- /dev/null +++ b/tests/integration/database/test_performance_issues.py @@ -0,0 +1,258 @@ +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from datetime import date, datetime, timedelta, timezone +from hashlib import sha256 +from typing import Final +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit +from uuid import uuid4 + +import pytest +from integration._support.actors import ManagementDatabase +from integration._support.prisma_clients import connected_proxy_prisma + +from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.proxy.spend_tracking.usage_alert_job import UsageAlertJob +from litellm.proxy.utils import PrismaClient +from litellm.repositories.issues import IssuesRepository +from litellm.repositories.usage_alerts import UsageAlertsRepository +from litellm.types.proxy.usage_alerts import UsageAlertObservation, UsageAlertRuleInput + +_DAY: Final = date(2026, 11, 4) +_NOW: Final = datetime(2026, 11, 5, 3, microsecond=987654, tzinfo=timezone.utc) +_KEY_A: Final = sha256(b"synthetic-project-a").hexdigest() +_KEY_B: Final = sha256(b"synthetic-project-b").hexdigest() +_TABLES: Final = ( + "LiteLLM_UsageAlertRule", + "LiteLLM_UsageAlertIncident", + "LiteLLM_UsageAlertCheckpoint", + "LiteLLM_UsageAlertFlush", + "LiteLLM_Issue", + "LiteLLM_IssueSignal", + "LiteLLM_IssueDelivery", + "LiteLLM_IssueDeliveryRule", + "LiteLLM_DailyUserSpend", + "LiteLLM_DailyTeamSpend", + "LiteLLM_Config", + "LiteLLM_VerificationToken", + "LiteLLM_UserTable", + "LiteLLM_TeamTable", + "LiteLLM_SpendLogDelivery", +) + + +@asynccontextmanager +async def _isolated(db: ManagementDatabase) -> AsyncIterator[PrismaClient]: + schema: Final = "issue_test_" + uuid4().hex + assert db.gateway.database_url is not None + parsed: Final = urlsplit(db.gateway.database_url) + query: Final = {**dict(parse_qsl(parsed.query)), "schema": schema} + url: Final = urlunsplit(parsed._replace(query=urlencode(query))) + await db.prisma.writer_db.execute_raw(f'CREATE SCHEMA "{schema}"') + try: + for table in _TABLES: + await db.prisma.writer_db.execute_raw(f'CREATE TABLE "{schema}"."{table}" (LIKE "{table}" INCLUDING ALL)') + with pytest.MonkeyPatch.context() as runtime: + runtime.setenv("DATABASE_URL", url) + async with connected_proxy_prisma(url) as prisma: + assert await prisma.writer_db.query_raw("SELECT current_schema() AS name") == [{"name": schema}] + yield prisma + finally: + await db.prisma.writer_db.execute_raw(f'DROP SCHEMA "{schema}" CASCADE') + + +async def _coverage(prisma: PrismaClient, day: date) -> None: + await prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_UsageAlertFlush" (source,hour,observed_at) + SELECT s,h,h FROM unnest(ARRAY['user','team']) s + CROSS JOIN generate_series($1::timestamp,$1::timestamp+INTERVAL '25 hours',INTERVAL '1 hour') h + ON CONFLICT(source,hour) DO UPDATE SET observed_at=EXCLUDED.observed_at RETURNING source""", + day.isoformat(), + ) + + +async def _seed(prisma: PrismaClient, *, reliability: bool = False) -> None: + for table, entity in (("LiteLLM_DailyUserSpend", "user_id"), ("LiteLLM_DailyTeamSpend", "team_id")): + await prisma.writer_db.query_raw( + f"""INSERT INTO "{table}" + (id,{entity},date,api_key,model,custom_llm_provider,spend,api_requests,successful_requests, + failed_requests,total_response_time_ms,timed_requests,updated_at) + SELECT $1||d||k.n,CASE WHEN $2='user' THEN 'owner-'||k.n ELSE 'team' END, + ($3::date-d)::text,k.key,'example-model','example-provider', + CASE WHEN d=0 AND k.n=1 AND NOT $6::boolean THEN 50 ELSE 1 END, + CASE WHEN d=0 AND k.n=1 AND NOT $6::boolean THEN 50000 ELSE 1000 END, + CASE WHEN d=0 AND $6::boolean THEN 900 ELSE 1000 END, + CASE WHEN d=0 AND $6::boolean THEN 100 ELSE 0 END, + CASE WHEN d=0 AND k.n=1 AND NOT $6::boolean THEN 5000000 ELSE 100000 END, + CASE WHEN d=0 AND k.n=1 AND NOT $6::boolean THEN 50000 ELSE 1000 END,CURRENT_TIMESTAMP + FROM generate_series(0,7) d CROSS JOIN (VALUES(1,$4::text),(2,$5::text)) k(n,key) + WHERE NOT $6::boolean OR k.n=1 RETURNING id""", + table, + "user" if entity == "user_id" else "team", + _DAY.isoformat(), + _KEY_A, + _KEY_B, + reliability, + ) + await _coverage(prisma, _DAY) + + +async def _next_day(prisma: PrismaClient, day: date, first: int, second: int) -> None: + for table, entity in (("LiteLLM_DailyUserSpend", "user_id"), ("LiteLLM_DailyTeamSpend", "team_id")): + await prisma.writer_db.query_raw( + f"""INSERT INTO "{table}" + (id,{entity},date,api_key,model,custom_llm_provider,spend,api_requests,successful_requests, + failed_requests,total_response_time_ms,timed_requests,updated_at) + SELECT id||$1,{entity},$1,api_key,model,custom_llm_provider, + CASE WHEN api_key=$3 THEN $4::double precision ELSE $5::double precision END, + 1000,1000,0,100000,1000,CURRENT_TIMESTAMP FROM "{table}" WHERE date=$2 RETURNING id""", + day.isoformat(), + _DAY.isoformat(), + _KEY_A, + first, + second, + ) + await _coverage(prisma, day) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_key_spend_issue_consolidates_traffic_and_uses_fresh_days(db: ManagementDatabase) -> None: + async with _isolated(db) as prisma: + await _seed(prisma) + job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) + issues: Final = IssuesRepository.from_prisma(prisma) + await job._run(_NOW) + first: Final = await issues.list_issues() + assert first.total == 1 + original: Final = first.data[0] + assert (original.family, original.cohort.scope_type, original.cohort.scope_id) == ("spend", "key", _KEY_A) + assert original.comparisons[0].observed_value == 50 + assert original.comparisons[0].baseline_value == 1 + assert original.comparisons[1].observed_value == 50000 + assert original.comparisons[1].threshold is None + assert (await issues.list_signals(original.id)).total == 10 + await job._run(_NOW + timedelta(hours=1)) + assert (await issues.list_issues()).data == first.data + assert (await issues.list_signals(original.id)).total == 10 + + await _next_day(prisma, _DAY + timedelta(days=1), 50, 100) + await job._run(_NOW + timedelta(days=1)) + current: Final = await issues.get_issue(original.id) + assert current is not None and current.comparisons == original.comparisons + assert current.range_start == original.range_start and current.range_end == original.range_end + new_key: Final = await issues.list_issues(scope_type="key", scope_id=_KEY_B) + assert new_key.total == 1 and new_key.data[0].observed_on == _DAY + timedelta(days=1) + broader: Final = await issues.list_issues(scope_type="deployment") + assert broader.total == 1 + assert broader.data[0].comparisons[0].observed_value == 150 + assert broader.data[0].comparisons[0].baseline_value == 9 + + await _next_day(prisma, _DAY + timedelta(days=2), 1, 100) + await job._run(_NOW + timedelta(days=2)) + recovered: Final = await issues.get_issue(original.id) + assert recovered is not None and not recovered.active and recovered.status == "resolved" + assert (await issues.list_issues(scope_type="key", scope_id=_KEY_B)).data[0].active + + +@pytest.mark.asyncio(loop_scope="session") +async def test_single_key_reliability_collapses_matching_scopes_without_traffic_growth(db: ManagementDatabase) -> None: + async with _isolated(db) as prisma: + await _seed(prisma, reliability=True) + await UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[]))._run(_NOW) + issues: Final = IssuesRepository.from_prisma(prisma) + page: Final = await issues.list_issues() + assert page.total == 1, tuple((item.cohort.scope_type, item.cohort.scope_id, item.title) for item in page.data) + issue: Final = page.data[0] + assert (issue.family, issue.cohort.scope_type, issue.cohort.scope_id) == ("reliability", "key", _KEY_A) + assert issue.comparisons[0].metric == "error_rate" + assert issue.comparisons[0].observed_value == 10 and issue.comparisons[0].baseline_value == 0 + assert (await issues.list_signals(issue.id)).total == 5 + + +@pytest.mark.asyncio(loop_scope="session") +async def test_same_day_rule_revision_requires_a_fresh_observation(db: ManagementDatabase) -> None: + async with _isolated(db) as prisma: + alerts: Final = UsageAlertsRepository.from_prisma(prisma) + issues: Final = IssuesRepository.from_prisma(prisma) + definition: Final = UsageAlertRuleInput( + name="Project spend", + scope_type="key", + scope_id=_KEY_A, + metric="spend", + direction="increase", + min_requests=0, + ) + rule: Final = await alerts.save_rule(definition, None) + assert rule is not None + observation: Final = UsageAlertObservation( + entity_id=_KEY_A, day=_DAY, observed=4, baseline=1, threshold=3, breached=True, baseline_sample_days=7 + ) + await alerts.record_observations(rule, (observation,)) + original: Final = (await issues.active_signals(_DAY))[0] + await alerts.set_status(original.alert.id, "dismissed") + revised: Final = await alerts.save_rule(definition.model_copy(update={"multiplier": 5}), None, existing=rule) + assert revised is not None + await alerts.finish_day(revised, _DAY) + assert await issues.active_signals(_DAY) == () + await alerts.record_observations(revised, (observation.model_copy(update={"observed": 6, "threshold": 5}),)) + updated: Final = (await issues.active_signals(_DAY))[0] + assert updated.alert.id == original.alert.id and updated.alert.status == "dismissed" + assert updated.alert.observed_value == original.alert.observed_value + assert updated.rule.updated_at == revised.updated_at + + +@pytest.mark.asyncio(loop_scope="session") +async def test_provider_issue_keeps_an_independent_noisy_key_and_recovers_from_fresh_evidence( + db: ManagementDatabase, +) -> None: + async with _isolated(db) as prisma: + noisy: Final = sha256(b"synthetic-noisy-project").hexdigest() + for table, entity in (("LiteLLM_DailyUserSpend", "user_id"), ("LiteLLM_DailyTeamSpend", "team_id")): + await prisma.writer_db.query_raw( + f"""INSERT INTO "{table}" + (id,{entity},date,api_key,model,custom_llm_provider,spend,api_requests,successful_requests, + failed_requests,total_response_time_ms,timed_requests,updated_at) + SELECT $1||d||k.n||m,CASE WHEN $2='user' THEN 'owner-'||k.n ELSE 'team' END, + ($3::date-d)::text,k.key,'model-'||m,'example-provider',1,k.requests,k.requests,0, + k.requests*(CASE WHEN d=0 THEN k.latency ELSE 100 END),k.requests,CURRENT_TIMESTAMP + FROM generate_series(0,7) d CROSS JOIN + (VALUES(1,$4::text,500,1000),(2,$5::text,500,1000),(3,$6::text,100,10000)) k(n,key,requests,latency) + CROSS JOIN generate_series(1,2) m + WHERE k.n<>3 OR m=1 RETURNING id""", + table, + "user" if entity == "user_id" else "team", + _DAY.isoformat(), + _KEY_A, + _KEY_B, + noisy, + ) + await _coverage(prisma, _DAY) + job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) + await job._run(_NOW) + issues: Final = IssuesRepository.from_prisma(prisma) + page: Final = await issues.list_issues() + assert page.total == 2, tuple((item.cohort.scope_type, item.cohort.scope_id, item.title) for item in page.data) + provider: Final = next(item for item in page.data if item.cohort.scope_type == "provider") + independent: Final = next(item for item in page.data if item.cohort.scope_type == "key") + assert independent.cohort.scope_id == noisy + assert len(provider.cohort.model_pairs) == 2 + assert provider.impact.keys == 3 and provider.impact.models == 2 + assert (await issues.list_signals(provider.id)).total == 8 + assert (await issues.list_signals(independent.id)).total == 2 + for table, entity in (("LiteLLM_DailyUserSpend", "user_id"), ("LiteLLM_DailyTeamSpend", "team_id")): + await prisma.writer_db.query_raw( + f"""INSERT INTO "{table}" + (id,{entity},date,api_key,model,custom_llm_provider,spend,api_requests,successful_requests, + failed_requests,total_response_time_ms,timed_requests,updated_at) + SELECT id||'-next',{entity},$1,api_key,model,custom_llm_provider,spend,api_requests,successful_requests, + failed_requests,timed_requests*(CASE WHEN api_key=$3 THEN 10000 ELSE 100 END),timed_requests,CURRENT_TIMESTAMP + FROM "{table}" WHERE date=$2 RETURNING id""", + (_DAY + timedelta(days=1)).isoformat(), + _DAY.isoformat(), + noisy, + ) + await _coverage(prisma, _DAY + timedelta(days=1)) + await job._run(_NOW + timedelta(days=1)) + recovered: Final = await issues.get_issue(provider.id) + assert recovered is not None and not recovered.active + continuing: Final = await issues.get_issue(independent.id) + assert continuing is not None and continuing.active diff --git a/tests/unit/integrations/SlackAlerting/test_usage_alerts.py b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py index b4b9e8ea36f..af0f7e1da25 100644 --- a/tests/unit/integrations/SlackAlerting/test_usage_alerts.py +++ b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py @@ -7,6 +7,7 @@ import pytest from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, activity_url, incident_message from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.types.proxy.issues import Issue, IssueCohort, IssueDelivery, IssuePendingDelivery from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertRule _NOW: Final = datetime(2026, 10, 20, tzinfo=timezone.utc) @@ -83,3 +84,51 @@ async def test_slack_ack_requires_successful_http_response(status: int, monkeypa assert b"https://gateway.example.com/ui/api-keys?key=reporting-key&detail_tab=activity" in requests[0].content finally: await handler.close() + + +@pytest.mark.asyncio +async def test_issue_delivery_links_to_investigation_and_preserves_the_bound_source( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + requests: Final[list[httpx.Request]] = [] + + def transport(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200) + + issue: Final = Issue( + id="issue-example", + cohort_key="cohort-example", + kind="performance", + family="traffic", + title="Traffic increased", + cohort=IssueCohort(scope_type="key", scope_id="reporting-key"), + observed_on=_ALERT.day, + observed_at=_NOW, + range_start=_ALERT.baseline_start, + range_end=_ALERT.day, + grouping_reason="same_resource", + status="open", + first_seen=_NOW, + last_seen=_NOW, + updated_at=_NOW, + ) + handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(transport)) + slack: Final = SlackAlerting( + alerting=["slack"], default_webhook_url="https://hooks.example.com/test", async_http_handler=handler + ) + pending: Final = IssuePendingDelivery( + issue=issue, + rule=_RULE, + alert=_ALERT, + delivery=IssueDelivery(id="delivery-example", issue_id=issue.id, channel="slack"), + ) + try: + assert await UsageAlertDelivery(slack).send_issue(pending) + assert len(requests) == 1 + assert b"https://gateway.example.com/ui/alerts?issue=issue-example" in requests[0].content + assert b"https://gateway.example.com/ui/api-keys?key=reporting-key&detail_tab=activity" in requests[0].content + assert b"1,000.00; baseline 200.00" in requests[0].content + finally: + await handler.close() diff --git a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py index c859cd8c922..88954fbbafb 100644 --- a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py +++ b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py @@ -7,8 +7,8 @@ import pytest from litellm.proxy.spend_tracking.daily_usage_alerts import DailyComparison, compare_daily_usage from litellm.proxy.spend_tracking.usage_alert_job import completed_day from litellm.repositories.usage_alert_metrics import DailyAlertMetrics -from litellm.types.proxy.usage_alert_defaults import default_usage_rules -from litellm.types.proxy.usage_alerts import UsageAlertRuleInput +from litellm.types.proxy.usage_alert_defaults import default_usage_rule_id, default_usage_rules +from litellm.types.proxy.usage_alerts import UsageAlertRule, UsageAlertRuleInput _DAY: Final = date(2026, 10, 20) @@ -40,18 +40,14 @@ def test_missing_target_is_zero_for_counts_but_never_zero_latency() -> None: assert compare_daily_usage(rule.model_copy(update={"metric": "latency"}), history, _DAY) is None -def test_low_volume_is_insufficient_and_recorded_zero_baseline_uses_absolute_floor() -> None: +def test_low_volume_and_recorded_zero_spend_do_not_establish_a_multiplier_baseline() -> None: rule: Final = UsageAlertRuleInput(name="spike", scope_type="key", metric="spend", direction="increase") history: Final = tuple(_row(_DAY - timedelta(days=days), requests=10, spend=0) for days in (1, 2, 3)) assert compare_daily_usage(rule, (*history, _row(_DAY, spend=100)), _DAY) is None enough_requests: Final = tuple(replace(row, requests=200) for row in history) - assert compare_daily_usage(rule, (*enough_requests, _row(_DAY, spend=100)), _DAY) == DailyComparison( - 3, 100, 0, 0, True - ) + assert compare_daily_usage(rule, (*enough_requests, _row(_DAY, spend=100)), _DAY) is None floor: Final = rule.model_copy(update={"min_value": 100}) - assert compare_daily_usage(floor, (*enough_requests, _row(_DAY, spend=100)), _DAY) == DailyComparison( - 3, 100, 0, 100, True - ) + assert compare_daily_usage(floor, (*enough_requests, _row(_DAY, spend=100)), _DAY) is None def test_absolute_floor_suppresses_small_changes() -> None: @@ -243,9 +239,7 @@ def test_error_rate_default_requires_two_percentage_points_as_well_as_the_multip rule: Final = next( rule for rule in default_usage_rules() if rule.scope_type == "key" and rule.metric == "error_rate" ) - history: Final = tuple( - replace(_row(_DAY - timedelta(days=day), requests=1000), errors=5) for day in (1, 2, 3) - ) + history: Final = tuple(replace(_row(_DAY - timedelta(days=day), requests=1000), errors=5) for day in (1, 2, 3)) current: Final = replace(_row(_DAY, requests=1000), errors=errors) assert compare_daily_usage(rule, (*history, current), _DAY) == DailyComparison(3, errors / 10, 0.5, 2.5, breached) @@ -264,15 +258,17 @@ def test_traffic_drop_needs_a_full_recorded_week_and_an_eighty_percent_decline() ) history: Final = tuple(_row(_DAY - timedelta(days=day), requests=500) for day in range(1, 8)) assert compare_daily_usage(rule, history[:-1], _DAY) is None - assert compare_daily_usage(rule, (*history, _row(_DAY, requests=101)), _DAY) == DailyComparison(7, 101, 500, 100, False) - assert compare_daily_usage(rule, (*history, _row(_DAY, requests=100)), _DAY) == DailyComparison(7, 100, 500, 100, True) + assert compare_daily_usage(rule, (*history, _row(_DAY, requests=101)), _DAY) == DailyComparison( + 7, 101, 500, 100, False + ) + assert compare_daily_usage(rule, (*history, _row(_DAY, requests=100)), _DAY) == DailyComparison( + 7, 100, 500, 100, True + ) def test_latency_default_requires_a_material_change_and_enough_timed_requests() -> None: rule: Final = next(rule for rule in default_usage_rules() if rule.scope_type == "key" and rule.metric == "latency") - history: Final = tuple( - _row(_DAY - timedelta(days=day), requests=500, ms=10_000, timed=100) for day in (1, 2, 3) - ) + history: Final = tuple(_row(_DAY - timedelta(days=day), requests=500, ms=10_000, timed=100) for day in (1, 2, 3)) assert compare_daily_usage(rule, (*history, _row(_DAY, ms=40_000, timed=100)), _DAY) == DailyComparison( 3, 400, 100, 600, False ) @@ -280,3 +276,68 @@ def test_latency_default_requires_a_material_change_and_enough_timed_requests() 3, 600, 100, 600, True ) assert compare_daily_usage(rule, (*history, _row(_DAY, ms=60_000, timed=99)), _DAY) is None + + +@pytest.mark.parametrize("metric", ["spend", "requests", "latency", "errors"]) +def test_multiplier_alerts_need_a_positive_established_baseline(metric: str) -> None: + rule: Final = UsageAlertRuleInput.model_validate( + {"name": "baseline", "scope_type": "key", "metric": metric, "direction": "increase", "min_requests": 0} + ) + current: Final = replace(_row(_DAY, requests=2000, spend=500), errors=100) + assert compare_daily_usage(rule, (current,), _DAY) is None + empty: Final = tuple( + replace(_row(_DAY - timedelta(days=day), requests=0, spend=0, ms=0, timed=0), errors=0) for day in (1, 2, 3) + ) + assert compare_daily_usage(rule, (*empty, current), _DAY) is None + + +def test_established_healthy_traffic_can_alert_on_new_failures_but_a_new_key_cannot() -> None: + rule: Final = next( + rule for rule in default_usage_rules() if rule.scope_type == "key" and rule.metric == "error_rate" + ) + history: Final = tuple(replace(_row(_DAY - timedelta(days=day), requests=500), errors=0) for day in (1, 2, 3)) + current: Final = replace(_row(_DAY, requests=500), errors=10) + assert compare_daily_usage(rule, (current,), _DAY) is None + assert compare_daily_usage(rule, (*history[:2], current), _DAY) is None + assert compare_daily_usage(rule, (*history, current), _DAY) == DailyComparison(3, 2, 0, 2, True) + relaxed: Final = rule.model_copy(update={"min_requests": 0, "min_history_days": 1}) + assert compare_daily_usage(relaxed, (*history[:2], current), _DAY) is None + assert compare_daily_usage(relaxed, (*history, replace(current, requests=499)), _DAY) is None + + +def test_default_traffic_drop_does_not_treat_a_normal_quiet_weekend_as_an_outage() -> None: + definition: Final = next( + rule + for rule in default_usage_rules() + if (rule.scope_type, rule.metric, rule.direction) == ("key", "requests", "decrease") + ) + rule: Final = UsageAlertRule.model_validate( + { + **definition.model_dump(), + "id": default_usage_rule_id("key", "requests", "decrease"), + "created_at": "2026-10-01T00:00:00Z", + "updated_at": "2026-10-01T00:00:00Z", + "created_by": None, + } + ) + busy: Final = tuple(_row(_DAY - timedelta(days=day), requests=1000) for day in range(1, 7)) + history: Final = (*busy, _row(_DAY - timedelta(days=7), requests=1)) + current: Final = _row(_DAY, requests=1) + assert compare_daily_usage(rule, (*history, current), _DAY) is None + custom: Final = rule.model_copy(update={"id": "custom-policy"}) + custom_result: Final = compare_daily_usage(custom, (*history, current), _DAY) + assert custom_result is not None and custom_result.breached + + +def test_error_rate_drop_requires_enough_current_requests() -> None: + rule: Final = UsageAlertRuleInput( + name="Reliability", + scope_type="key", + metric="error_rate", + direction="decrease", + min_requests=500, + ) + history: Final = tuple(replace(_row(_DAY - timedelta(days=day), requests=1000), errors=100) for day in range(1, 8)) + assert compare_daily_usage(rule, (*history, replace(_row(_DAY, requests=1), errors=0)), _DAY) is None + result: Final = compare_daily_usage(rule, (*history, replace(_row(_DAY, requests=500), errors=0)), _DAY) + assert result is not None and result.breached From 3ce2231589b5f5e6104e6139e5e549220105bdbc Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 15:40:11 +0800 Subject: [PATCH 18/44] fix: integrate issue delivery with retirement email support --- docs/features/usage-alerts.md | 10 +++++++++- .../test_deprecation_issue_evaluation.py | 2 +- .../utils/prisma_and_spend/test_send_email.py | 7 ++++--- ui/litellm-dashboard/src/lib/http/schema.d.ts | 18 +++++++++++++++++- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/docs/features/usage-alerts.md b/docs/features/usage-alerts.md index 3111be514de..cfe853da626 100644 --- a/docs/features/usage-alerts.md +++ b/docs/features/usage-alerts.md @@ -16,7 +16,7 @@ Daily Performance defaults cover spend, latency, reliability, and traffic across | Latency | At least 3× average latency and a 500 ms increase | Three recorded reference days and at least 100 timed requests in both periods | | Reliability | At least 3× error rate and a two percentage-point increase | Three recorded reference days and at least 500 daily requests in both periods | | Traffic increase | At least 10× requests and an increase of 1,000 requests | Three recorded reference days with a positive request baseline | -| Traffic decrease | At most one fifth of baseline requests | Seven recorded reference days and a baseline of at least 500 daily requests | +| Traffic decrease | At most one fifth of baseline requests | Seven recorded reference days, each with at least 500 requests | New resources need history before they can produce a default spike. A zero spend, request, or latency baseline does not establish a spike. A zero error-rate baseline is usable only after sufficient historical traffic has been observed; the percentage-point floor still applies. @@ -28,6 +28,14 @@ Decrease rules need a successful background aggregate-flush sample for every hou The baseline averages recorded reference days; it does not turn missing reference days into zero. Average latency is weighted by timed request count, using the same successful-request duration totals as Activity. `baseline_sample_days` records how many reference days contributed. Cost, traffic, and latency comparisons require a positive baseline. Reliability can compare against a recorded 0% error rate when historical and current request volume meet the rule's minimum. Missing history and low-volume comparisons remain `insufficient_history`. +## How related signals form one issue + +A key with unusually high spend gets a spend issue, including when it makes only a few expensive requests. Its comparison includes request volume so you can distinguish more traffic from more expensive traffic. A traffic-increase signal joins that spend issue when both increases use the same reference window and their ratios are within 20% of each other. + +Default user, team, model, and gateway signals can join a key issue when they describe the same key traffic. For spend and request increases, they can also join when that key accounts for at least 80% of the source's positive growth. Each supporting signal retains its own comparison. A reliability or latency change remains a separate issue from increased spend. + +Provider latency grouping requires measured deterioration within established key/model traffic: at least two keys per model, two models, and 80% coverage of current timed requests. A change in the mix of fast and slow models alone is insufficient. A source joins only when at least 80% of its timed traffic independently meets the shared pattern; a source with more than twice the provider cohort's latency ratio stays separate. The issue keeps its original provider/model cohort until recovery. These relationships are evidence for a shared investigation, not proof of a provider outage or root cause. Custom rules retain their configured resource scope. + ## Read and investigate issues Set `LITELLM_API_BASE` to your gateway's base URL and `LITELLM_MANAGEMENT_KEY` to a management-capable credential. Keep the credential in your environment. diff --git a/tests/integration/database/test_deprecation_issue_evaluation.py b/tests/integration/database/test_deprecation_issue_evaluation.py index c267fe694e9..789c2358b4d 100644 --- a/tests/integration/database/test_deprecation_issue_evaluation.py +++ b/tests/integration/database/test_deprecation_issue_evaluation.py @@ -73,7 +73,7 @@ async def test_retirement_issue_http_state_survives_replay_and_reopens_only_for_ ) -> None: prefix = f"retirement-lifecycle-{uuid4()}" name = f"example/{prefix}" - now = datetime(2026, 11, 5, 10, tzinfo=timezone.utc) + now = datetime(2026, 11, 5, 10, microsecond=123456, tzinfo=timezone.utc) retirement = now.date() + timedelta(days=15) config = ConfigRepository(db.prisma, use_writer=True) previous_checkpoint = await config.get_param("issues:deprecation_projection") diff --git a/tests/unit/proxy/utils/prisma_and_spend/test_send_email.py b/tests/unit/proxy/utils/prisma_and_spend/test_send_email.py index c4f36883270..330c2994069 100644 --- a/tests/unit/proxy/utils/prisma_and_spend/test_send_email.py +++ b/tests/unit/proxy/utils/prisma_and_spend/test_send_email.py @@ -7,7 +7,7 @@ Symbols pinned here: from __future__ import annotations import threading -from typing import Any, TYPE_CHECKING +from typing import TYPE_CHECKING, Any from unittest.mock import Mock import pytest @@ -185,6 +185,7 @@ async def test_should_send_plain_text_and_html_as_multipart_alternative(monkeypa async def test_durable_alert_delivery_can_observe_smtp_failure(in_memory_smtp: InMemorySMTP) -> None: in_memory_smtp.raise_on_send = RuntimeError("smtp unavailable") with pytest.raises(RuntimeError, match="smtp unavailable"): - await send_email(receiver_email="alerts@example.com", subject="Daily usage", html="

Alert

", - raise_on_error=True) + await send_email( + receiver_email="alerts@example.com", subject="Daily usage", html="

Alert

", raise_on_error=True + ) assert in_memory_smtp.sent == [] diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 3517ddb0aef..d8c9dde9b17 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27041,7 +27041,7 @@ export interface components { * @description Enum for alert types and management event types * @enum {string} */ - AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "user_spend_thresholds" | "user_spend_anomalies" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted" | "mcp_tool_description_blocked" | "mcp_pinned_tools_changed"; + AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "user_spend_thresholds" | "user_spend_anomalies" | "usage_alerts" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted" | "mcp_tool_description_blocked" | "mcp_pinned_tools_changed"; /** AllowedVectorStoreIndexItem */ AllowedVectorStoreIndexItem: { /** Index Name */ @@ -50947,6 +50947,8 @@ export interface components { first_seen: string; /** Id */ id: string; + /** Last Notification Attempt */ + last_notification_attempt?: string | null; /** * Last Seen * Format: date-time @@ -50957,6 +50959,20 @@ export interface components { * @enum {string} */ metric: "spend" | "requests" | "latency" | "errors" | "error_rate"; + /** Next Notification Attempt */ + next_notification_attempt?: string | null; + /** + * Notification Attempts + * @default 0 + */ + notification_attempts: number; + /** Notification Error */ + notification_error?: ("delivery_failed" | "transport_not_configured") | null; + /** + * Notified Channels + * @default [] + */ + notified_channels: ("email" | "slack")[]; /** Observed Value */ observed_value: number; /** Rule Id */ From 1d2a5d624991ba3c3486f79579a60a7af0ec7f35 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 15:43:06 +0800 Subject: [PATCH 19/44] fix: apply scoped overrides before projecting daily issues --- .../spend_tracking/performance_issues.py | 21 +++++++++-- .../database/test_performance_issues.py | 36 +++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/spend_tracking/performance_issues.py b/litellm/proxy/spend_tracking/performance_issues.py index ad133358254..c7d55c450a6 100644 --- a/litellm/proxy/spend_tracking/performance_issues.py +++ b/litellm/proxy/spend_tracking/performance_issues.py @@ -194,8 +194,16 @@ class PerformanceIssueProjector: self._metrics = metrics self._providers = providers self._delivery = delivery + self._overrides: frozenset[tuple[UsageAlertScope, UsageAlertMetric, UsageAlertDirection, str | None]] = ( + frozenset() + ) async def run(self, now: datetime, day: date, rules: tuple[UsageAlertRule, ...], *, ready: bool) -> None: + self._overrides = frozenset( + (rule.scope_type, rule.metric, rule.direction, rule.scope_id) + for rule in rules + if rule.enabled and not is_default_usage_rule(rule) + ) observed_at: Final = now.replace(microsecond=now.microsecond // 1000 * 1000) version: Final = hashlib.sha256( ("performance-v1:" + ",".join(f"{rule.id}:{rule.updated_at.isoformat()}" for rule in rules)).encode() @@ -213,7 +221,16 @@ class PerformanceIssueProjector: async def _candidates( self, sources: tuple[IssueProjectionSignal, ...], scope: UsageAlertScope, day: date ) -> tuple[_Candidate, ...]: - windows: Final = tuple(_window(source, day) for source in sources) + eligible: Final = tuple( + source + for source in sources + if not is_default_usage_rule(source.rule) + or not any( + (scope, source.rule.metric, source.rule.direction, entity) in self._overrides + for entity in (None, source.alert.scope_id) + ) + ) + windows: Final = tuple(_window(source, day) for source in eligible) rows: Final = await self._metrics.issue_windows(scope, windows) by_reference: Final = { reference: tuple(group) @@ -223,7 +240,7 @@ class PerformanceIssueProjector: } return tuple( candidate - for source in sources + for source in eligible if (candidate := self._candidate(source, by_reference.get(source.alert.id, ()), day)) is not None ) diff --git a/tests/integration/database/test_performance_issues.py b/tests/integration/database/test_performance_issues.py index 481db1bc89f..ab80a8cef96 100644 --- a/tests/integration/database/test_performance_issues.py +++ b/tests/integration/database/test_performance_issues.py @@ -200,6 +200,42 @@ async def test_same_day_rule_revision_requires_a_fresh_observation(db: Managemen assert updated.rule.updated_at == revised.updated_at +@pytest.mark.asyncio(loop_scope="session") +async def test_late_custom_key_rule_only_overrides_that_keys_default_signal(db: ManagementDatabase) -> None: + async with _isolated(db) as prisma: + await _seed(prisma) + await prisma.writer_db.query_raw( + 'UPDATE "LiteLLM_DailyUserSpend" SET spend=40 WHERE date=$1 AND api_key=$2 RETURNING id', + _DAY.isoformat(), + _KEY_B, + ) + job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) + # A rule can change after the daily observations, before Issue projection. + await UsageAlertJob(job._store, job._metrics, job._health, job._delivery)._run(_NOW) + issues: Final = IssuesRepository.from_prisma(prisma) + original: Final = await issues.active_signals(_DAY, scope_type="key", metric="spend") + assert len(original) == 2 + overridden: Final = next(item for item in original if item.alert.scope_id == _KEY_A) + retained: Final = next(item for item in original if item.alert.scope_id == _KEY_B) + await UsageAlertsRepository.from_prisma(prisma).save_rule( + UsageAlertRuleInput( + name="Project A high spend threshold", + scope_type="key", + scope_id=_KEY_A, + metric="spend", + direction="increase", + multiplier=100, + min_requests=0, + ), + None, + ) + await job._run(_NOW + timedelta(minutes=1)) + attached: Final = await prisma.writer_db.query_raw('SELECT source_id FROM "LiteLLM_IssueSignal"') + assert overridden.alert.id not in {row["source_id"] for row in attached} + assert retained.alert.id in {row["source_id"] for row in attached} + assert await UsageAlertsRepository.from_prisma(prisma).get_alert(overridden.alert.id) is not None + + @pytest.mark.asyncio(loop_scope="session") async def test_provider_issue_keeps_an_independent_noisy_key_and_recovers_from_fresh_evidence( db: ManagementDatabase, From c68a5607c0dff283ab87bd6b1368cc773b1a13c1 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 15:44:53 +0800 Subject: [PATCH 20/44] perf: stop repeated shared-key population scans --- .../spend_tracking/performance_issues.py | 4 +- litellm/repositories/usage_alert_metrics.py | 30 ++++++------ .../database/test_performance_issues.py | 46 ++++++++++++++++++- 3 files changed, 65 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/spend_tracking/performance_issues.py b/litellm/proxy/spend_tracking/performance_issues.py index c7d55c450a6..53d5a460d38 100644 --- a/litellm/proxy/spend_tracking/performance_issues.py +++ b/litellm/proxy/spend_tracking/performance_issues.py @@ -288,7 +288,9 @@ class PerformanceIssueProjector: ) -> None: candidates: Final = await self._candidates(sources, scope, day) provider_targets: Final = await self._provider_targets(candidates, providers, now) - resource_targets: Final = await self._resource_targets(candidates, now) + resource_targets: Final = await self._resource_targets( + tuple(item for item in candidates if item.source.alert.id not in provider_targets), now + ) targets: Final = { item.source.alert.id: provider_targets.get(item.source.alert.id) or resource_targets.get(item.source.alert.id) diff --git a/litellm/repositories/usage_alert_metrics.py b/litellm/repositories/usage_alert_metrics.py index 228c12d2795..955564b323e 100644 --- a/litellm/repositories/usage_alert_metrics.py +++ b/litellm/repositories/usage_alert_metrics.py @@ -90,11 +90,20 @@ def _window_rows(scope: Scope) -> str: def _key_attribution_query(scope: Scope, metric: Literal["spend", "requests"]) -> str: measure: Final = "spend" if metric == "spend" else "api_requests" + # Shared keys must stop at the first different owner, rather than recounting + # every owner for every source. Compare team totals only for an exclusive key. same_identity: Final = { "deployment": "true", - "user": "k.users=1 AND k.user_id=p.entity_id", - "model": "k.models=1 AND k.model=p.entity_id", - "team": "k.team_days IS NOT DISTINCT FROM k.user_days", + "user": """NOT EXISTS (SELECT 1 FROM "LiteLLM_DailyUserSpend" + WHERE api_key=w.api_key AND date>=w.start_day AND date<=w.target_day + AND COALESCE(user_id,'')<>w.entity_id)""", + "model": """NOT EXISTS (SELECT 1 FROM "LiteLLM_DailyUserSpend" + WHERE api_key=w.api_key AND date>=w.start_day AND date<=w.target_day + AND COALESCE(NULLIF(model,''),'unknown-model')<>w.entity_id)""", + "team": """CASE WHEN EXISTS (SELECT 1 FROM "LiteLLM_DailyTeamSpend" + WHERE api_key=w.api_key AND date>=w.start_day AND date<=w.target_day + AND COALESCE(team_id,'')<>w.entity_id AND api_requests>0) + THEN false ELSE team_days IS NOT DISTINCT FROM user_days END""", "key": "true", }[scope] team_days: Final = ( @@ -140,14 +149,6 @@ def _key_attribution_query(scope: Scope, metric: Literal["spend", "requests"]) - SELECT DISTINCT api_key,entity_id,start_day,target_day FROM candidate WHERE api_key<>'' ), key_population AS ( SELECT w.*, - (SELECT count(DISTINCT COALESCE(user_id,'')) FROM "LiteLLM_DailyUserSpend" - WHERE api_key=w.api_key AND date>=w.start_day AND date<=w.target_day) AS users, - (SELECT MIN(user_id) FROM "LiteLLM_DailyUserSpend" - WHERE api_key=w.api_key AND date>=w.start_day AND date<=w.target_day) AS user_id, - (SELECT count(DISTINCT COALESCE(NULLIF(model,''),'unknown-model')) FROM "LiteLLM_DailyUserSpend" - WHERE api_key=w.api_key AND date>=w.start_day AND date<=w.target_day) AS models, - (SELECT MIN(COALESCE(NULLIF(model,''),'unknown-model')) FROM "LiteLLM_DailyUserSpend" - WHERE api_key=w.api_key AND date>=w.start_day AND date<=w.target_day) AS model, {team_days} AS team_days, (SELECT jsonb_agg(to_jsonb(d) ORDER BY d.day) FROM ( SELECT date AS day,SUM(spend) AS spend,SUM(api_requests)::bigint AS requests, @@ -157,10 +158,13 @@ def _key_attribution_query(scope: Scope, metric: Literal["spend", "requests"]) - AND date>=w.start_day AND date<=w.target_day GROUP BY date ) d) AS user_days FROM windows w + ), equivalent_keys AS ( + SELECT w.api_key,w.entity_id,w.start_day,w.target_day,({same_identity}) AS equivalent + FROM key_population w ) SELECT p.reference_id,p.api_key AS key_id, - p.key_count=1 AND ({same_identity}) AS equivalent,p.positive_change_share, + p.key_count=1 AND k.equivalent AS equivalent,p.positive_change_share, COALESCE(p.baseline_value,0) AS baseline_value,p.observed_value - FROM candidate p JOIN key_population k ON k.api_key=p.api_key AND k.entity_id=p.entity_id + FROM candidate p JOIN equivalent_keys k ON k.api_key=p.api_key AND k.entity_id=p.entity_id AND k.start_day=p.start_day AND k.target_day=p.target_day""" diff --git a/tests/integration/database/test_performance_issues.py b/tests/integration/database/test_performance_issues.py index ab80a8cef96..7cad556f1cf 100644 --- a/tests/integration/database/test_performance_issues.py +++ b/tests/integration/database/test_performance_issues.py @@ -14,8 +14,9 @@ from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.proxy.spend_tracking.usage_alert_job import UsageAlertJob from litellm.proxy.utils import PrismaClient from litellm.repositories.issues import IssuesRepository +from litellm.repositories.usage_alert_metrics import IssueMetricWindow from litellm.repositories.usage_alerts import UsageAlertsRepository -from litellm.types.proxy.usage_alerts import UsageAlertObservation, UsageAlertRuleInput +from litellm.types.proxy.usage_alerts import UsageAlertObservation, UsageAlertRuleInput, UsageAlertScope _DAY: Final = date(2026, 11, 4) _NOW: Final = datetime(2026, 11, 5, 3, microsecond=987654, tzinfo=timezone.utc) @@ -236,6 +237,49 @@ async def test_late_custom_key_rule_only_overrides_that_keys_default_signal(db: assert await UsageAlertsRepository.from_prisma(prisma).get_alert(overridden.alert.id) is not None +@pytest.mark.asyncio(loop_scope="session") +async def test_shared_key_does_not_make_distinct_resource_populations_equivalent(db: ManagementDatabase) -> None: + async with _isolated(db) as prisma: + await _seed(prisma, reliability=True) + job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) + scopes: Final[tuple[tuple[UsageAlertScope, str], ...]] = ( + ("user", "owner-1"), + ("team", "team"), + ("model", "example-model"), + ) + for scope, entity in scopes: + assert ( + await job._metrics.issue_key_attributions( + scope, + ( + IssueMetricWindow( + reference_id=entity, entity_id=entity, start=_DAY - timedelta(days=7), day=_DAY + ), + ), + ) + )[0].equivalent + for table, entity in (("LiteLLM_DailyUserSpend", "user_id"), ("LiteLLM_DailyTeamSpend", "team_id")): + await prisma.writer_db.query_raw( + f"""INSERT INTO "{table}" + (id,{entity},date,api_key,model,custom_llm_provider,spend,api_requests,successful_requests, + failed_requests,total_response_time_ms,timed_requests,updated_at) + SELECT id||'-shared','other-owner',date,api_key,'other-model',custom_llm_provider,spend, + api_requests,successful_requests,failed_requests,total_response_time_ms,timed_requests,updated_at + FROM "{table}" RETURNING id""" + ) + for scope, entity in scopes: + assert not ( + await job._metrics.issue_key_attributions( + scope, + ( + IssueMetricWindow( + reference_id=entity, entity_id=entity, start=_DAY - timedelta(days=7), day=_DAY + ), + ), + ) + )[0].equivalent + + @pytest.mark.asyncio(loop_scope="session") async def test_provider_issue_keeps_an_independent_noisy_key_and_recovers_from_fresh_evidence( db: ManagementDatabase, From ea269ad6c2d0f7626cf92997f74e24c06127d838 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 16:42:33 +0800 Subject: [PATCH 21/44] fix: bound and resume daily alert evaluation --- .../SlackAlerting/usage_alerts.py | 62 +++- litellm/models/issues.py | 3 + .../spend_tracking/performance_issues.py | 316 +++++++++--------- .../proxy/spend_tracking/usage_alert_job.py | 252 ++++++++++---- litellm/proxy/utils.py | 6 +- litellm/repositories/issues.py | 37 +- litellm/repositories/provider_latency.py | 11 + litellm/repositories/usage_alert_metrics.py | 68 +++- litellm/repositories/usage_alerts.py | 160 ++++++++- litellm/types/proxy/usage_alerts.py | 32 +- .../database/test_performance_issues.py | 227 ++++++++++++- .../SlackAlerting/test_usage_alerts.py | 89 ++++- .../spend_tracking/test_daily_usage_alerts.py | 46 ++- 13 files changed, 1034 insertions(+), 275 deletions(-) diff --git a/litellm/integrations/SlackAlerting/usage_alerts.py b/litellm/integrations/SlackAlerting/usage_alerts.py index 0c642535d15..24fbdd5a2d5 100644 --- a/litellm/integrations/SlackAlerting/usage_alerts.py +++ b/litellm/integrations/SlackAlerting/usage_alerts.py @@ -1,9 +1,11 @@ """Deliver persisted usage incidents through the existing configured transports.""" +import asyncio +import hashlib import json import os from html import escape -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, ClassVar, Final from urllib.parse import urlencode, urlsplit from httpx import Response @@ -51,6 +53,10 @@ def incident_message(alert: UsageAlert, url: str | None) -> str: class UsageAlertDelivery: + # One retained send per worker bounds SMTP threads even when a job is cancelled. + _email_task: ClassVar[asyncio.Task[None] | None] = None + _email_key: ClassVar[str | None] = None + def __init__(self, slack: "SlackAlerting") -> None: self._slack = slack @@ -74,8 +80,8 @@ class UsageAlertDelivery: pending.delivery.channel, pending.delivery.email_recipients, incident_message(pending.alert, None), - f"LiteLLM issue: {pending.issue.title}", - (("View issue", issue_url), ("View source activity", source_url)), + f"LiteLLM alert: {pending.issue.title}", + (("View alert", issue_url), ("View source activity", source_url)), ) async def _send( @@ -89,20 +95,11 @@ class UsageAlertDelivery: if channel not in configured_destinations(self._slack): return False if channel == "email": - from litellm.proxy.utils import send_email - + html: Final = f"

{escape(message).replace(chr(10), '
')}

" + "".join( + f'

{escape(label)}

' for label, url in links + ) for recipient in recipients: - await send_email( - receiver_email=recipient, - subject=subject, - html=( - f"

{escape(message).replace(chr(10), '
')}

" - + "".join( - f'

{escape(label)}

' for label, url in links - ) - ), - raise_on_error=True, - ) + await self._email(recipient, subject, html) return bool(recipients) overrides: Final = self._slack.alert_to_webhook_url or {} configured_url: Final = ( @@ -118,11 +115,42 @@ class UsageAlertDelivery: await self._post(webhook, message + "\n" + "\n".join(f"{label}: {url}" for label, url in links)) return True + @staticmethod + def _email_finished(task: asyncio.Task[None]) -> None: + if not task.cancelled(): + task.exception() # consume a late failure after the calling job was cancelled + + @classmethod + async def _email(cls, recipient: str, subject: str, html: str) -> None: + key: Final = hashlib.sha256(json.dumps((recipient, subject, html)).encode()).hexdigest() + previous: Final = cls._email_task + if previous is not None: + if not previous.done(): + raise RuntimeError("An alert email is still in flight; retry later") + if cls._email_key == key: + cls._email_task = None + cls._email_key = None + return previous.result() + task: Final = asyncio.create_task(cls._send_email(recipient, subject, html)) + cls._email_task = task + cls._email_key = key + task.add_done_callback(cls._email_finished) + await asyncio.shield(task) + if cls._email_task is task: + cls._email_task = None + cls._email_key = None + + @staticmethod + async def _send_email(recipient: str, subject: str, html: str) -> None: + from litellm.proxy.utils import send_email + + await send_email(receiver_email=recipient, subject=subject, html=html, raise_on_error=True, attempt_timeout=2) + async def _post(self, webhook: str, message: str) -> None: response: Final[Response] = await self._slack.async_http_handler.post( # pyright: ignore[reportUnknownMemberType] # legacy HTTP signature; supplied arguments are concrete url=webhook, headers={"Content-type": "application/json"}, - data=json.dumps({"text": message}), + data=json.dumps({"text": escape(message, quote=False)}), timeout=10, ) response.raise_for_status() diff --git a/litellm/models/issues.py b/litellm/models/issues.py index 6357072453a..bff56e13f0e 100644 --- a/litellm/models/issues.py +++ b/litellm/models/issues.py @@ -89,6 +89,7 @@ class IssueStore(Protocol): scope_type: IssueScope | None = None, metric: UsageAlertMetric | None = None, scope_ids: tuple[str, ...] | None = None, + completed_only: bool = False, ) -> tuple[IssueProjectionSignal, ...]: ... async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int: ... @@ -123,6 +124,8 @@ class IssueStore(Protocol): async def pending_delivery(self, delivery_id: str) -> IssuePendingDelivery | None: ... + async def reserve_delivery(self, delivery_id: str) -> bool: ... + async def mark_delivery_sent(self, delivery_id: str) -> None: ... async def defer_delivery( diff --git a/litellm/proxy/spend_tracking/performance_issues.py b/litellm/proxy/spend_tracking/performance_issues.py index 53d5a460d38..ec55339b512 100644 --- a/litellm/proxy/spend_tracking/performance_issues.py +++ b/litellm/proxy/spend_tracking/performance_issues.py @@ -1,11 +1,11 @@ """Project daily observations into scoped, persisted investigations.""" import asyncio -import hashlib -from collections.abc import Mapping +from collections import OrderedDict +from collections.abc import AsyncIterator, Mapping from dataclasses import dataclass from datetime import date, datetime, timedelta -from itertools import chain, groupby +from itertools import chain, groupby, islice from typing import Final, Protocol from uuid import NAMESPACE_URL, uuid5 @@ -21,6 +21,7 @@ from litellm.repositories.usage_alert_metrics import ( IssueSourceProvider, UsageAlertMetricsRepository, ) +from litellm.repositories.usage_alerts import UsageAlertsRepository from litellm.types.proxy.issues import ( Issue, IssueCohort, @@ -29,14 +30,24 @@ from litellm.types.proxy.issues import ( IssueDeliveryInput, IssueImpact, IssueInput, - IssueModelPair, IssuePendingDelivery, IssueProjectionSignal, IssueSignal, IssueSignalGroupingReason, ) -from litellm.types.proxy.usage_alert_defaults import PerformanceIssueFamily, is_default_usage_rule -from litellm.types.proxy.usage_alerts import UsageAlertDirection, UsageAlertMetric, UsageAlertRule, UsageAlertScope +from litellm.types.proxy.usage_alert_defaults import ( + PerformanceIssueFamily, + default_usage_rule_id, + is_default_usage_rule, +) +from litellm.types.proxy.usage_alerts import ( + UsageAlertDirection, + UsageAlertMetric, + UsageAlertProjectionCursor, + UsageAlertProjectionWork, + UsageAlertRule, + UsageAlertScope, +) _SCOPES: Final[tuple[UsageAlertScope, ...]] = ("key", "user", "team", "model", "deployment") _METRICS: Final[tuple[UsageAlertMetric, ...]] = ("spend", "latency", "error_rate", "errors", "requests") @@ -147,6 +158,10 @@ def _resource_issue(candidate: _Candidate, now: datetime) -> IssueInput: ) +def _comparison_window(candidate: _Candidate) -> tuple[str | None, date, date]: + return candidate.source.alert.scope_id, candidate.window.start, candidate.window.day + + def _volume_explains_spend(traffic: _Candidate, spend: _Candidate) -> bool: if spend.comparison.baseline <= 0 or traffic.comparison.baseline <= 0: return False @@ -194,42 +209,62 @@ class PerformanceIssueProjector: self._metrics = metrics self._providers = providers self._delivery = delivery - self._overrides: frozenset[tuple[UsageAlertScope, UsageAlertMetric, UsageAlertDirection, str | None]] = ( - frozenset() - ) + self._provider_rule: UsageAlertRule | None = None + self._provider_cache: OrderedDict[str, _ProviderGroup | None] = OrderedDict() # mutable-ok: max 32 providers - async def run(self, now: datetime, day: date, rules: tuple[UsageAlertRule, ...], *, ready: bool) -> None: - self._overrides = frozenset( - (rule.scope_type, rule.metric, rule.direction, rule.scope_id) - for rule in rules - if rule.enabled and not is_default_usage_rule(rule) - ) + async def run_pending( + self, now: datetime, work: UsageAlertProjectionWork, rules: UsageAlertsRepository, deadline: float + ) -> bool: + self._provider_cache.clear() + self._provider_rule = None + for scope in _SCOPES: + if ( + rule := await rules.get_rule(default_usage_rule_id(scope, "latency", "increase")) + ) is not None and rule.enabled: + self._provider_rule = rule + break + position = await rules.projection_cursor(work) # rebind-ok: persisted bounded source cursor observed_at: Final = now.replace(microsecond=now.microsecond // 1000 * 1000) - version: Final = hashlib.sha256( - ("performance-v1:" + ",".join(f"{rule.id}:{rule.updated_at.isoformat()}" for rule in rules)).encode() - ).hexdigest() - if ready and not await self._store.projection_completed(day, version): - await self._store.recover_issues(day) - providers: Final = await self._provider_evidence(day, rules, observed_at) - for metric in _METRICS: - for scope in _SCOPES: - await self._scope(observed_at, day, scope, metric, providers) - await self._store.complete_projection(day, version) - await self._notify() - await self._store.prune() + projection_deadline: Final = deadline - min(2, max(0, deadline - asyncio.get_running_loop().time()) / 4) + while asyncio.get_running_loop().time() < projection_deadline: + if (next_position := await self._projection_page(position, work, observed_at)) is None: + await self._store.recover_issues(work.day, completed_only=True) + await self._store.complete_projection(work.day, work.version) + await self._notify(deadline) + await self._store.prune() + return True + position = next_position + await rules.save_projection_cursor(position) + await asyncio.sleep(0.01) + await self._notify(deadline) + return False + + async def _projection_page( + self, position: UsageAlertProjectionCursor, work: UsageAlertProjectionWork, observed_at: datetime + ) -> UsageAlertProjectionCursor | None: + if position.step == 0: + prior: Final = await self._store.active_provider_issues(limit=1, cursor=position.after_id) + if prior: + if self._provider_rule is not None: + await self._prior_provider_page(prior, work.day, self._provider_rule, observed_at) + return UsageAlertProjectionCursor(version=work.version, after_id=prior[-1].id) + return UsageAlertProjectionCursor(version=work.version, step=1) + if position.step > len(_SCOPES) * len(_METRICS): + return None + metric: Final = _METRICS[(position.step - 1) // len(_SCOPES)] + scope: Final = _SCOPES[(position.step - 1) % len(_SCOPES)] + sources: Final = await self._store.active_signals( + work.day, scope_type=scope, metric=metric, cursor=position.after_id, limit=10, completed_only=True + ) + if sources: + await self._project_page(sources, scope, work.day, observed_at) + return UsageAlertProjectionCursor(version=work.version, step=position.step, after_id=sources[-1].alert.id) + return UsageAlertProjectionCursor(version=work.version, step=position.step + 1) async def _candidates( self, sources: tuple[IssueProjectionSignal, ...], scope: UsageAlertScope, day: date ) -> tuple[_Candidate, ...]: - eligible: Final = tuple( - source - for source in sources - if not is_default_usage_rule(source.rule) - or not any( - (scope, source.rule.metric, source.rule.direction, entity) in self._overrides - for entity in (None, source.alert.scope_id) - ) - ) + eligible: Final = sources windows: Final = tuple(_window(source, day) for source in eligible) rows: Final = await self._metrics.issue_windows(scope, windows) by_reference: Final = { @@ -264,30 +299,15 @@ class PerformanceIssueProjector: _window(source, day), ) - async def _scope( - self, - now: datetime, - day: date, - scope: UsageAlertScope, - metric: UsageAlertMetric, - providers: Mapping[str, _ProviderGroup], - ) -> None: - cursor: str | None = None # rebind-ok: bounded source pagination - while sources := await self._store.active_signals(day, scope_type=scope, metric=metric, cursor=cursor): - await self._project_page(sources, scope, day, providers, now) - cursor = sources[-1].alert.id - await asyncio.sleep(0) - async def _project_page( self, sources: tuple[IssueProjectionSignal, ...], scope: UsageAlertScope, day: date, - providers: Mapping[str, _ProviderGroup], now: datetime, ) -> None: candidates: Final = await self._candidates(sources, scope, day) - provider_targets: Final = await self._provider_targets(candidates, providers, now) + provider_targets: Final = await self._provider_targets(candidates, now) resource_targets: Final = await self._resource_targets( tuple(item for item in candidates if item.source.alert.id not in provider_targets), now ) @@ -299,17 +319,21 @@ class PerformanceIssueProjector: } await self._save(candidates, targets, now) - async def _spend_candidates( + async def _spend_pages( self, scope: UsageAlertScope, entities: tuple[str, ...] | None, day: date - ) -> tuple[_Candidate, ...]: - found: Final[list[_Candidate]] = [] # mutable-ok: bounded entity batch may match multiple configured rules - cursor: str | None = None # rebind-ok: page matching rules without one query per entity + ) -> AsyncIterator[tuple[_Candidate, ...]]: + cursor: str | None = None # rebind-ok: stream configured rules without retaining earlier candidate pages while sources := await self._store.active_signals( - day, scope_type=scope, metric="spend", scope_ids=entities, cursor=cursor + day, + scope_type=scope, + metric="spend", + scope_ids=entities, + cursor=cursor, + limit=10, + completed_only=True, ): - found.extend(await self._candidates(sources, scope, day)) + yield await self._candidates(sources, scope, day) cursor = sources[-1].alert.id - return tuple(found) async def _volume_targets(self, candidates: tuple[_Candidate, ...], now: datetime) -> Mapping[str, _Target]: traffic: Final = tuple( @@ -320,32 +344,37 @@ class PerformanceIssueProjector: if not traffic: return {} scope: Final = traffic[0].source.rule.scope_type - entities: Final = tuple( - dict.fromkeys(item.source.alert.scope_id for item in traffic if item.source.alert.scope_id is not None) - ) - spending: Final = await self._spend_candidates( - scope, entities if scope != "deployment" else None, traffic[0].window.day - ) - by_entity: Final = { - entity: tuple(group) - for entity, group in groupby( - sorted(spending, key=lambda item: item.source.alert.scope_id or ""), - lambda item: item.source.alert.scope_id, - ) - } keys: Final = tuple(_cohort_key("spend", scope, item.source.alert.scope_id, "increase") for item in traffic) issues: Final = { issue.cohort_key: issue for issue in await self._store.find_active_issues(keys) if issue.last_seen == now } + targets: Final = { + _comparison_window(item): item + for item in traffic + if _cohort_key("spend", scope, item.source.alert.scope_id, "increase") in issues + } + if not targets: + return {} + entities: Final = tuple(dict.fromkeys(identity for identity, _, _ in targets if identity is not None)) + matched: Final[set[tuple[str | None, date, date]]] = set() # mutable-ok: max 10 flags + async for page in self._spend_pages(scope, entities if scope != "deployment" else None, traffic[0].window.day): + for spend in page: + if (identity := _comparison_window(spend)) in targets and _volume_explains_spend( + targets[identity], spend + ): + matched.add(identity) + if len(matched) == len(targets): + break return { item.source.alert.id: _Target( - issue.model_copy(update={"observed_at": now}), + issues[_cohort_key("spend", scope, item.source.alert.scope_id, "increase")].model_copy( + update={"observed_at": now} + ), "volume_explains_spend", "Request volume and spend increased by similar proportions in the same comparison window.", ) for item in traffic - if (issue := issues.get(_cohort_key("spend", scope, item.source.alert.scope_id, "increase"))) is not None - and any(_volume_explains_spend(item, spend) for spend in by_entity.get(item.source.alert.scope_id, ())) + if _comparison_window(item) in matched } async def _resource_targets(self, candidates: tuple[_Candidate, ...], now: datetime) -> Mapping[str, _Target]: @@ -428,19 +457,24 @@ class PerformanceIssueProjector: if not traffic: return {} key_ids: Final = tuple(dict.fromkeys(attributions[item.source.alert.id].key_id for item in traffic)) - spending: Final = await self._spend_candidates("key", key_ids, traffic[0].window.day) - matching: Final = tuple(item for item in spending if self._volume_matches_key_spend(item)) - by_key: Final = { - identity: tuple(group) - for identity, group in groupby( - sorted(matching, key=lambda item: item.source.alert.scope_id or ""), - lambda item: item.source.alert.scope_id, - ) - } keys: Final = tuple(_cohort_key("spend", "key", key, "increase") for key in key_ids) issues: Final = { issue.cohort_key: issue for issue in await self._store.find_active_issues(keys) if issue.last_seen == now } + needed: Final = frozenset( + (attributions[item.source.alert.id].key_id, item.window.start, item.window.day) + for item in traffic + if _cohort_key("spend", "key", attributions[item.source.alert.id].key_id, "increase") in issues + ) + if not needed: + return {} + matched: Final[set[tuple[str | None, date, date]]] = set() # mutable-ok: max 10 flags + async for page in self._spend_pages("key", key_ids, traffic[0].window.day): + for spend in page: + if (identity := _comparison_window(spend)) in needed and self._volume_matches_key_spend(spend): + matched.add(identity) + if len(matched) == len(needed): + break return { item.source.alert.id: _Target( issue.model_copy(update={"observed_at": now}), @@ -450,10 +484,7 @@ class PerformanceIssueProjector: for item in traffic if (issue := issues.get(_cohort_key("spend", "key", attributions[item.source.alert.id].key_id, "increase"))) is not None - and any( - (spend.window.start, spend.window.day) == (item.window.start, item.window.day) - for spend in by_key.get(attributions[item.source.alert.id].key_id, ()) - ) + and (attributions[item.source.alert.id].key_id, item.window.start, item.window.day) in matched } @staticmethod @@ -482,48 +513,31 @@ class PerformanceIssueProjector: right.min_value, ) - async def _provider_evidence( - self, day: date, rules: tuple[UsageAlertRule, ...], now: datetime - ) -> Mapping[str, _ProviderGroup]: - rule: Final = next( - ( - rule - for rule in rules - if is_default_usage_rule(rule) and rule.metric == "latency" and rule.direction == "increase" - ), - None, - ) + async def _provider_group(self, provider: str, day: date, now: datetime) -> _ProviderGroup | None: + if provider in self._provider_cache: + self._provider_cache.move_to_end(provider) + return self._provider_cache[provider] + rule: Final = self._provider_rule if rule is None: - return {} - groups: Final[dict[str, _ProviderGroup]] = {} # mutable-ok: retain one confirmed cohort per provider - reserved: Final[set[str]] = set() # mutable-ok: an active frozen cohort prevents a competing subset episode - cursor: str | None = None # rebind-ok: bounded existing-cohort pagination - while existing := await self._store.active_provider_issues(cursor=cursor): - for prior, recovered, evidence in await self._prior_provider_page(existing, day, rule, now): - if recovered or prior.cohort.scope_id is None: - continue - reserved.add(prior.cohort.scope_id) - if evidence is not None: - groups[prior.cohort.scope_id] = _ProviderGroup(evidence, rule) - cursor = existing[-1].id - source_cursor: str | None = None # rebind-ok: stop discovery when no compatible active source exists - compatible = False # rebind-ok: early-exit source scan - while sources := await self._store.active_signals(day, metric="latency", cursor=source_cursor): - if any(is_default_usage_rule(source.rule) and self._same_policy(source.rule, rule) for source in sources): - compatible = True - break - source_cursor = sources[-1].alert.id - if not compatible: - return groups - pair_cursor: IssueModelPair | None = None # rebind-ok: keyset pagination of model pairs - while pairs := await self._providers.candidate_pairs(day, pair_cursor): - for provider, selected in groupby(pairs, lambda pair: pair.provider): - if provider in reserved or provider in groups: - continue - if (candidate_evidence := await self._providers.latency_cohort(tuple(selected), day, rule)) is not None: - groups[provider] = _ProviderGroup(candidate_evidence, rule) - pair_cursor = pairs[-1] - return groups + return None + existing: Final = await self._store.find_active_issues( + (_cohort_key("latency", "provider", provider, "increase"),) + ) + evidence: ProviderLatencyEvidence | None = None # rebind-ok: fresh or saved cohort evidence + if existing: + prior: Final = existing[0] + _, recovered, evidence = await self._prior_provider(prior, day, rule) # rebind-ok: saved cohort + if recovered: + await self._store.recover_issue_ids((prior.id,), now) + else: + pairs: Final = await self._providers.provider_pairs(provider, day) + if 2 <= len(pairs) <= 100: + evidence = await self._providers.latency_cohort(pairs, day, rule) # rebind-ok: new cohort + result: Final = _ProviderGroup(evidence, rule) if evidence is not None else None + self._provider_cache[provider] = result + if len(self._provider_cache) > 32: + self._provider_cache.popitem(last=False) + return result async def _prior_provider_page( self, @@ -551,22 +565,23 @@ class PerformanceIssueProjector: evidence if evidence is not None and evidence.cohort.model_pairs == issue.cohort.model_pairs else None, ) - async def _provider_targets( - self, candidates: tuple[_Candidate, ...], providers: Mapping[str, _ProviderGroup], now: datetime - ) -> Mapping[str, _Target]: + async def _provider_targets(self, candidates: tuple[_Candidate, ...], now: datetime) -> Mapping[str, _Target]: eligible: Final = tuple( item for item in candidates if item.source.rule.metric == "latency" and is_default_usage_rule(item.source.rule) ) - if not eligible or not providers: + if not eligible or self._provider_rule is None: return {} scope: Final = eligible[0].source.rule.scope_type by_reference: Final = {item.source.alert.id: item for item in eligible} hints: Final = await self._metrics.issue_source_providers(scope, tuple(item.window for item in eligible)) targets: Final[dict[str, _Target]] = {} # mutable-ok: each source belongs to at most one 80%-coverage provider for provider, matches in groupby(sorted(hints, key=lambda hint: hint.provider), lambda hint: hint.provider): - targets.update(await self._provider_matches(provider, tuple(matches), by_reference, providers, scope, now)) + if (group := await self._provider_group(provider, eligible[0].window.day, now)) is not None: + targets.update( + await self._provider_matches(provider, tuple(matches), by_reference, {provider: group}, scope, now) + ) return targets async def _provider_matches( @@ -628,7 +643,7 @@ class PerformanceIssueProjector: range_start=cohort.baseline_start, range_end=item.window.day, grouping_reason="provider_cohort", - explanation="Several keys and models slowed down together. The comparison holds today's key and model mix constant and excludes keys still collecting a baseline. This is a shared pattern, not proof of a common cause.", + explanation="Several keys and models slowed down together. The comparison holds the observed day's key and model mix constant and excludes keys still collecting a baseline. This is a shared pattern, not proof of a common cause.", comparisons=(cohort,), impact=evidence.impact, ), @@ -659,29 +674,32 @@ class PerformanceIssueProjector: for candidate in candidates ) ) - bindings: Final = tuple( - chain.from_iterable( - _bindings(candidate, saved[targets[candidate.source.alert.id].issue.cohort_key].id) - for candidate in candidates - ) + bindings: Final = chain.from_iterable( + _bindings(candidate, saved[targets[candidate.source.alert.id].issue.cohort_key].id) + for candidate in candidates ) - for offset in range(0, len(bindings), 500): - await self._store.queue_deliveries(bindings[offset : offset + 500]) + while batch := tuple(islice(bindings, 500)): + await self._store.queue_deliveries(batch) - async def _notify(self) -> None: - deadline: Final = asyncio.get_running_loop().time() + 60 - while asyncio.get_running_loop().time() < deadline and (pending := await self._store.pending_deliveries()): + async def _notify(self, stop_at: float | None = None) -> None: + deadline: Final = stop_at if stop_at is not None else asyncio.get_running_loop().time() + 60 + while asyncio.get_running_loop().time() + 2 < deadline and (pending := await self._store.pending_deliveries()): for item in pending: - if asyncio.get_running_loop().time() >= deadline: + if asyncio.get_running_loop().time() + 2 >= deadline: return - await self._notify_one(item) + await self._notify_one(item, deadline) - async def _notify_one(self, item: IssuePendingDelivery) -> None: + async def _notify_one(self, item: IssuePendingDelivery, deadline: float) -> None: current: Final = await self._store.pending_delivery(item.delivery.id) if current is None: return + remaining: Final = deadline - asyncio.get_running_loop().time() + if remaining <= 2 or not await self._store.reserve_delivery(item.delivery.id): + return try: - delivered: Final = await asyncio.wait_for(self._delivery.send_issue(current), timeout=30) + delivered: Final = await asyncio.wait_for( + self._delivery.send_issue(current), timeout=min(30, remaining - 1) + ) except Exception: # noqa: BLE001 # persisted retry avoids losing a notification on transport failure await self._store.defer_delivery(item.delivery.id, "delivery_failed") verbose_proxy_logger.warning("Performance issue notification failed; queued for retry") diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index 872a7ae83df..242603a85a3 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -4,7 +4,7 @@ import asyncio from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass from datetime import date, datetime, time, timedelta, timezone -from itertools import chain, groupby +from itertools import chain, groupby, takewhile from typing import ( TYPE_CHECKING, Final, @@ -17,22 +17,30 @@ from litellm._logging import verbose_proxy_logger from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery, configured_destinations from litellm.proxy.spend_tracking.daily_usage_alerts import compare_daily_usage from litellm.proxy.spend_tracking.performance_issues import PerformanceIssueProjector +from litellm.repositories.analytics_budget import ( + AnalyticsCapacityError, + AnalyticsDatabase, + BoundedAnalyticsDatabase, + analytics_deadline, +) from litellm.repositories.issues import IssuesRepository from litellm.repositories.provider_latency import ProviderLatencyRepository from litellm.repositories.usage_alert_health import UsageAlertHealthRepository -from litellm.repositories.usage_alert_metrics import DailyAlertMetrics, MetricsDatabase, UsageAlertMetricsRepository +from litellm.repositories.usage_alert_metrics import DailyAlertMetrics, UsageAlertMetricsRepository from litellm.repositories.usage_alerts import UsageAlertsRepository from litellm.types.integrations.slack_alerting import AlertType -from litellm.types.proxy.usage_alert_defaults import USAGE_ALERT_SCOPES, default_usage_rules, is_default_usage_rule +from litellm.types.proxy.usage_alert_defaults import USAGE_ALERT_SCOPES, default_usage_rules from litellm.types.proxy.usage_alerts import ( UsageAlert, UsageAlertChannel, UsageAlertEvaluationStatus, UsageAlertNotification, UsageAlertObservation, + UsageAlertProgress, UsageAlertRule, UsageAlertRuleInput, UsageAlertScope, + UsageAlertWorkCursor, ) if TYPE_CHECKING: @@ -76,7 +84,9 @@ def _observations( changed: Final[list[UsageAlertObservation]] = [] # mutable-ok: collect only persisted changes for this page evaluated = False # rebind-ok: retain whether any entity had sufficient evidence, including healthy entities for entity, group in groupby(rows, lambda row: row.entity_id): - if None in suppressed or entity in suppressed: + if (is_suppressed := None in suppressed or entity in suppressed) and ( + None if rule.scope_type == "deployment" else entity + ) not in active: continue if not _has_activity_page(rule.scope_type, entity): continue @@ -85,6 +95,8 @@ def _observations( if (comparison := compare_daily_usage(rule, tuple(group), target)) is None: continue evaluated = True + if is_suppressed and comparison.breached: + continue if not comparison.breached and (None if rule.scope_type == "deployment" else entity) not in active: continue changed.append( @@ -101,17 +113,6 @@ def _observations( return _Observations(evaluated, tuple(changed)) -def _suppressed_entities(rule: UsageAlertRule, enabled: tuple[UsageAlertRule, ...]) -> frozenset[str | None]: - if not is_default_usage_rule(rule): - return frozenset() - matching: Final = tuple( - other - for other in enabled - if (other.scope_type, other.metric, other.direction) == (rule.scope_type, rule.metric, rule.direction) - ) - return frozenset(other.scope_id for other in matching if not is_default_usage_rule(other)) - - class IncidentDelivery(Protocol): async def send(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> bool: ... @@ -127,6 +128,7 @@ class UsageAlertJob: *, deprecation_projection: Callable[[datetime], Awaitable[None]] | None = None, performance_projection: PerformanceIssueProjector | None = None, + work_budget_seconds: float = 60, ) -> None: self._store = store self._metrics = metrics @@ -136,6 +138,7 @@ class UsageAlertJob: self._defaults_registered = False self._deprecation_projection = deprecation_projection self._performance_projection = performance_projection + self._work_budget_seconds = work_budget_seconds @classmethod def from_prisma( @@ -145,11 +148,14 @@ class UsageAlertJob: *, deprecation_projection: Callable[[datetime], Awaitable[None]] | None = None, ) -> "UsageAlertJob": - database: Final = cast(MetricsDatabase, prisma.writer_db) # cast-ok: generated Prisma delegate + database: Final = BoundedAnalyticsDatabase( + cast(AnalyticsDatabase, prisma.writer_db), # cast-ok: generated Prisma transaction delegate + background=True, + ) metrics: Final = UsageAlertMetricsRepository(database) delivery: Final = UsageAlertDelivery(slack) return cls( - UsageAlertsRepository.from_prisma(prisma), + UsageAlertsRepository(database), metrics, UsageAlertHealthRepository(database), delivery, @@ -161,7 +167,7 @@ class UsageAlertJob: ), deprecation_projection=deprecation_projection, performance_projection=PerformanceIssueProjector( - IssuesRepository.from_prisma(prisma), + IssuesRepository(database), metrics, ProviderLatencyRepository(database), delivery, @@ -170,62 +176,123 @@ class UsageAlertJob: async def run(self, now: datetime | None = None) -> None: owner: Final = str(uuid4()) - if not await self._store.claim_job(owner): + deadline: Final = asyncio.get_running_loop().time() + self._work_budget_seconds - 1 + try: + with analytics_deadline(deadline): + claimed: Final = await self._store.claim_job(owner, lease_seconds=int(self._work_budget_seconds) + 15) + except AnalyticsCapacityError: + verbose_proxy_logger.info("Usage alert evaluation deferred while analytics capacity is busy") + return + if not claimed: return try: - await asyncio.wait_for(self._run(now or datetime.now(timezone.utc)), timeout=840) - except asyncio.TimeoutError: - verbose_proxy_logger.warning( - "Usage alert evaluation exceeded its 14-minute lease budget; retrying next hour" + await asyncio.wait_for( + self._run(now or datetime.now(timezone.utc), deadline), + timeout=max(0, deadline - asyncio.get_running_loop().time()), ) + except AnalyticsCapacityError: + verbose_proxy_logger.info("Usage alert work deferred within its database budget") + except asyncio.TimeoutError: + verbose_proxy_logger.warning("Usage alert evaluation reached its work budget; resuming next run") except Exception: # noqa: BLE001 # the scheduler retries; completed rule/day checkpoints survive a failed pass verbose_proxy_logger.exception("Daily usage alert evaluation failed") finally: - await self._store.release_job(owner) + try: + with analytics_deadline(asyncio.get_running_loop().time() + 1): + await self._store.release_job(owner) + except AnalyticsCapacityError: + verbose_proxy_logger.info("Usage alert lease will expire after bounded cleanup") - async def _rules(self) -> tuple[UsageAlertRule, ...]: - pages: Final[list[UsageAlertRule]] = [] # mutable-ok: collect configuration pages - offset = 0 # rebind-ok: configuration pagination cursor - while page := await self._store.list_enabled_rules(limit=500, offset=offset): - pages.extend(page) - if len(page) < 500: - return tuple(pages) - offset += len(page) - return tuple(pages) + async def _run(self, now: datetime, stop_at: float | None = None) -> None: + deadline: Final = ( + stop_at if stop_at is not None else asyncio.get_running_loop().time() + self._work_budget_seconds - 1 + ) + with analytics_deadline(deadline): + await self._work(now, deadline) - async def _run(self, now: datetime) -> None: + async def _work(self, now: datetime, deadline: float) -> None: if self._deprecation_projection is not None: await self._deprecation_projection(now) if not self._defaults_registered: await self._store.ensure_default_rules(self._defaults) self._defaults_registered = True target: Final = completed_day(now) - rules: Final = await self._rules() - complete: Final[set[str]] = set() # mutable-ok: batched completion lookup, bounded by configuration size - for offset in range(0, len(rules), 500): - complete.update(await self._store.completed_rule_ids(rules[offset : offset + 500], target)) - due: Final = tuple(rule for rule in rules if rule.id not in complete) - for scoped in (tuple(rule for rule in due if rule.scope_type == scope) for scope in USAGE_ALERT_SCOPES): - if scoped: - await self._scope(scoped, target, rules) + await self._store.ensure_projection_work(target) + await self._evaluate( + target, asyncio.get_running_loop().time() + max(0, deadline - asyncio.get_running_loop().time()) * 0.65 + ) if self._performance_projection is not None: - for offset in range(0, len(due), 500): - complete.update(await self._store.completed_rule_ids(due[offset : offset + 500], target)) - await self._performance_projection.run(now, target, rules, ready=len(complete) == len(rules)) + if work := await self._store.pending_projection_work(): + # Source eligibility is per completed rule/day, never a global all-rules gate. + if await self._performance_projection.run_pending(now, work, self._store, deadline): + await self._store.finish_projection_work(work) else: await self._notify() await self._store.prune() - async def _scope( - self, rules: tuple[UsageAlertRule, ...], target: date, enabled: tuple[UsageAlertRule, ...] - ) -> None: + @staticmethod + def _position(rule: UsageAlertRule, target: date) -> UsageAlertProgress: + if rule.evaluation_progress is not None and rule.evaluation_progress.revision == rule.updated_at: + return rule.evaluation_progress + return UsageAlertProgress(day=target, revision=rule.updated_at) + + async def _evaluate(self, target: date, deadline: float) -> None: + cursor = await self._store.work_cursor() # rebind-ok: durable round-robin position + idle_phases = 0 # rebind-ok: stop once every phase has no remaining usable work + phase_progress = False # rebind-ok: whether a page advanced in the current phase + while asyncio.get_running_loop().time() < deadline and idle_phases < 10: + cursor, advanced, finished = await self._evaluation_page(cursor, target) + phase_progress = phase_progress or advanced + if finished: + idle_phases = 0 if phase_progress else idle_phases + 1 + phase_progress = False + await self._store.save_work_cursor(cursor) + await asyncio.sleep(0.01) + + async def _evaluation_page( + self, cursor: UsageAlertWorkCursor, target: date + ) -> tuple[UsageAlertWorkCursor, bool, bool]: + scope: Final = USAGE_ALERT_SCOPES[cursor.phase // 2] + wildcard: Final = cursor.phase % 2 == 0 + rules: Final = await self._store.enabled_rule_page( + scope, wildcard, cursor.after_id, limit=8 if wildcard else 10 + ) + if not rules: + return UsageAlertWorkCursor(phase=(cursor.phase + 1) % 10), False, True + # Keep one date/cursor per page. A newly edited rule cannot rewind another rule. + position: Final = self._position(rules[0], target) + selected: Final = tuple( + takewhile( + lambda rule: ( + ( + self._position(rule, target).day, + self._position(rule, target).cursor if wildcard else None, + ) + == (position.day, position.cursor if wildcard else None) + ), + rules, + ) + ) + complete: Final = await self._store.completed_rule_ids(selected, position.day) + due: Final = tuple(rule for rule in selected if rule.id not in complete) + advanced: Final = bool(due) and await self._attempt_scope_page(due, position.day, wildcard) + return UsageAlertWorkCursor(phase=cursor.phase, after_id=selected[-1].id), advanced, False + + async def _attempt_scope_page(self, rules: tuple[UsageAlertRule, ...], target: date, wildcard: bool) -> bool: + try: + return await self._scope_page(rules, target, wildcard) + except AnalyticsCapacityError: + await self._store.save_progress(tuple((rule, self._position(rule, target)) for rule in rules)) + verbose_proxy_logger.info("Usage alert page deferred within its database budget") + return False + + async def _scope_page(self, rules: tuple[UsageAlertRule, ...], target: date, wildcard: bool) -> bool: scope: Final = rules[0].scope_type start: Final = datetime.combine(target, time(), tzinfo=timezone.utc) coverage: Final = await self._health.coverage("team" if scope == "team" else "user", start) eligible: Final = tuple( rule for rule in rules if coverage.after_grace and (rule.direction == "increase" or coverage.hours == 24) ) - suppressed: Final = {rule.id: _suppressed_entities(rule, enabled) for rule in eligible} eligible_ids: Final = frozenset(rule.id for rule in eligible) await self._record_evaluations( tuple((rule, "insufficient_coverage") for rule in rules if rule.id not in eligible_ids), @@ -233,24 +300,60 @@ class UsageAlertJob: completed=False, ) if not eligible: - return + return False + await self._store.save_progress( + tuple((rule, self._position(rule, target)) for rule in eligible if rule.evaluation_progress is None) + ) + page: Final = await self._metrics.page( + scope, + target, + max(rule.baseline_days for rule in eligible), + cursor=self._position(eligible[0], target).cursor if wildcard else None, + entity_ids=None + if wildcard + else tuple(dict.fromkeys(rule.scope_id for rule in eligible if rule.scope_id is not None)), + ) by_entity: Final = { entity: tuple(group) for entity, group in groupby( sorted(eligible, key=lambda rule: rule.scope_id or ""), lambda rule: rule.scope_id ) } - evaluated: Final[set[str]] = set() # mutable-ok: at most one entry per configured rule across streaming pages - async for rows in self._metrics.batches(scope, target, max(rule.baseline_days for rule in eligible)): - evaluated.update(await self._page(by_entity, rows, target, suppressed)) - await self._record_evaluations( - tuple( - (rule, "complete" if rule.id in evaluated or suppressed[rule.id] else "insufficient_history") - for rule in eligible - ), - target, - completed=True, - ) + entities: Final = (None,) if scope == "deployment" else tuple(dict.fromkeys(row.entity_id for row in page.rows)) + suppressed: Final = await self._store.suppressed_entities(eligible, entities) + evaluated: Final = frozenset(await self._page(by_entity, page.rows, target, suppressed)) + if page.complete: + await self._record_evaluations( + tuple( + ( + rule, + "complete" + if rule.id in evaluated or self._position(rule, target).evaluated or suppressed.get(rule.id) + else "insufficient_history", + ) + for rule in eligible + ), + target, + completed=True, + ) + else: + await self._store.save_progress( + tuple( + ( + rule, + UsageAlertProgress( + day=target, + revision=rule.updated_at, + cursor=page.cursor, + evaluated=rule.id in evaluated + or self._position(rule, target).evaluated + or bool(suppressed.get(rule.id)), + ), + ) + for rule in eligible + ) + ) + return True async def _record_evaluations( self, @@ -274,23 +377,36 @@ class UsageAlertJob: if not applicable: return () entities: Final = (None,) if applicable[0].scope_type == "deployment" else tuple(by_entity) - active: Final[dict[str, frozenset[str | None]]] = {} # mutable-ok: bounded active results for this entity page - for offset in range(0, len(applicable), 500): - active.update( - await self._store.breached_entities( - tuple(rule.id for rule in applicable[offset : offset + 500]), entities + evaluated: Final[list[str]] = [] # mutable-ok: at most 500 rule IDs in the current config page + chunk_size: Final = 8 if None in rules else 500 + for offset in range(0, len(applicable), chunk_size): + evaluated.extend( + await self._rule_chunk( + applicable[offset : offset + chunk_size], entities, rows, by_entity, target, suppressed ) ) + return tuple(evaluated) + + async def _rule_chunk( + self, + selected: tuple[UsageAlertRule, ...], + entities: tuple[str | None, ...], + rows: tuple[DailyAlertMetrics, ...], + by_entity: Mapping[str, tuple[DailyAlertMetrics, ...]], + target: date, + suppressed: Mapping[str, frozenset[str | None]], + ) -> tuple[str, ...]: + active: Final = await self._store.breached_entities(tuple(rule.id for rule in selected), entities) return tuple( [ rule.id - for rule in applicable + for rule in selected if await self._rule_page( rule, rows if rule.scope_id is None else by_entity[rule.scope_id], target, active.get(rule.id, frozenset()), - suppressed[rule.id], + suppressed.get(rule.id, frozenset()), ) ] ) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 732eb0b4b35..258c81dfaa9 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -8284,6 +8284,7 @@ async def send_email( message_id: str | None = None, *, raise_on_error: bool = False, + attempt_timeout: float | None = None, ) -> bool: """ smtp_host, @@ -8328,7 +8329,10 @@ async def send_email( email_message.attach(MIMEText(html, "html")) try: - smtp_timeout: Final = float(os.getenv("SMTP_TIMEOUT", "30")) + configured_timeout: Final = float(os.getenv("SMTP_TIMEOUT", "30")) + smtp_timeout: Final = ( + configured_timeout if attempt_timeout is None else min(configured_timeout, attempt_timeout) + ) await asyncio.to_thread( _send_smtp_message, email_message=email_message, diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index 855b6962f61..7faad896b8c 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -338,6 +338,7 @@ class IssuesRepository: scope_type: IssueScope | None = None, metric: UsageAlertMetric | None = None, scope_ids: tuple[str, ...] | None = None, + completed_only: bool = False, ) -> tuple[IssueProjectionSignal, ...]: if not 1 <= limit <= 500: raise ValueError("Read between one and 500 source signals") @@ -356,13 +357,39 @@ class IssuesRepository: AND (r.scope_id IS NULL OR a.scope_id=r.scope_id) AND ($2::text IS NULL OR a.id>$2) AND ($4::text IS NULL OR a.scope_type=$4) AND ($5::text IS NULL OR a.metric=$5) - AND ($6::text[] IS NULL OR a.scope_id=ANY($6::text[])) ORDER BY a.id LIMIT $3""", + AND ($6::text[] IS NULL OR a.scope_id=ANY($6::text[])) + AND (NOT $7::boolean OR EXISTS ( + SELECT 1 FROM "LiteLLM_UsageAlertCheckpoint" c WHERE c.completed + AND c.id=r.id||':'||replace(r.record->>'updated_at','Z','+00:00')||':'||$1::text)) + AND (NOT $7::boolean OR NOT(r.id=ANY($8::text[])) OR NOT EXISTS ( + SELECT 1 FROM "LiteLLM_UsageAlertRule" dependency + WHERE dependency.enabled AND NOT dependency.deleted AND dependency.id=ANY($8::text[]) + AND ((a.scope_type<>'key' AND dependency.scope_type='key' + AND dependency.record->>'metric'=r.record->>'metric' + AND dependency.record->>'direction'=r.record->>'direction') + OR (r.record->>'metric'='requests' AND r.record->>'direction'='increase' + AND dependency.record->>'metric'='spend' AND dependency.record->>'direction'='increase' + AND dependency.scope_type IN ('key',r.scope_type))) + AND NOT (dependency.record->>'evaluation_status'='insufficient_coverage' + AND dependency.record->>'last_evaluated_day'=$1::text) + AND NOT EXISTS (SELECT 1 FROM "LiteLLM_UsageAlertCheckpoint" completed WHERE completed.completed + AND completed.id=dependency.id||':'||replace(dependency.record->>'updated_at','Z','+00:00')||':'||$1::text))) + AND (NOT(r.id=ANY($8::text[])) OR NOT EXISTS ( + SELECT 1 FROM "LiteLLM_UsageAlertRule" override WHERE override.enabled AND NOT override.deleted + AND COALESCE((override.record->>'overrides_defaults')::boolean,false) + AND NOT(override.id=ANY($8::text[])) AND override.scope_type=r.scope_type + AND override.record->>'metric'=r.record->>'metric' + AND override.record->>'direction'=r.record->>'direction' + AND (override.scope_id IS NULL OR override.scope_id=a.scope_id))) + ORDER BY a.id LIMIT $3""", day.isoformat(), cursor, limit, scope_type, metric, list(scope_ids) if scope_ids is not None else None, + completed_only, + tuple(DEFAULT_USAGE_RULE_IDS), ) ) @@ -572,6 +599,14 @@ class IssuesRepository: ) ) + async def reserve_delivery(self, delivery_id: str) -> bool: + rows: Final = await self.database.query_raw( + """UPDATE "LiteLLM_IssueDelivery" SET next_attempt=CURRENT_TIMESTAMP+interval '1 hour', + updated_at=CURRENT_TIMESTAMP WHERE id=$1 AND NOT sent AND next_attempt<=CURRENT_TIMESTAMP RETURNING id""", + delivery_id, + ) + return bool(rows) + async def mark_delivery_sent(self, delivery_id: str) -> None: await self.database.query_raw( """UPDATE "LiteLLM_IssueDelivery" SET sent=true,updated_at=CURRENT_TIMESTAMP, diff --git a/litellm/repositories/provider_latency.py b/litellm/repositories/provider_latency.py index 95bec0cdd29..eef64462aab 100644 --- a/litellm/repositories/provider_latency.py +++ b/litellm/repositories/provider_latency.py @@ -196,6 +196,17 @@ class ProviderLatencyRepository: def from_prisma(cls, prisma: "PrismaClient") -> "ProviderLatencyRepository": return cls(cast(MetricsDatabase, prisma.writer_db)) # cast-ok: generated Prisma query_raw interface + async def provider_pairs(self, provider: str, day: date) -> tuple[IssueModelPair, ...]: + return TypeAdapter(tuple[IssueModelPair, ...]).validate_python( + await self._database.query_raw( + """SELECT DISTINCT custom_llm_provider AS provider,model FROM "LiteLLM_DailyUserSpend" + WHERE date=$1 AND custom_llm_provider=$2 AND model IS NOT NULL AND model<>'' AND timed_requests>0 + ORDER BY model LIMIT 101""", + day.isoformat(), + provider, + ) + ) + async def candidate_pairs( self, day: date, cursor: IssueModelPair | None = None, limit: int = 100 ) -> tuple[IssueModelPair, ...]: diff --git a/litellm/repositories/usage_alert_metrics.py b/litellm/repositories/usage_alert_metrics.py index 955564b323e..09dec617b24 100644 --- a/litellm/repositories/usage_alert_metrics.py +++ b/litellm/repositories/usage_alert_metrics.py @@ -25,6 +25,13 @@ class DailyAlertMetrics: timed_requests: int +@dataclass(frozen=True, slots=True) +class DailyMetricsPage: + rows: tuple[DailyAlertMetrics, ...] + cursor: str | None + complete: bool + + @dataclass(frozen=True, slots=True) class IssueMetricWindow: reference_id: str @@ -173,7 +180,7 @@ class MetricsDatabase(Protocol): _ROWS: Final = TypeAdapter(tuple[DailyAlertMetrics, ...]) -_BATCH_SIZE: Final = 500 +_BATCH_SIZE: Final = 10 _DEPRECATION_USAGE: Final = TypeAdapter(tuple[ModelDeprecationUsage, ...]) _DEPRECATION_USAGE_QUERY: Final = """ @@ -216,7 +223,7 @@ def _source(scope: Scope) -> tuple[str, str]: def _query(scope: Scope) -> str: table, entity = _source(scope) # Identifiers come only from the closed scope mapping; request values are parameters. - return f""" + query: Final = f""" WITH entities AS MATERIALIZED ( SELECT DISTINCT {entity} AS entity_id FROM {table} WHERE date >= $1 AND date <= $2 AND {entity} >= COALESCE($3::text, '') AND ($3::text IS NULL OR {entity} <> $3) @@ -233,6 +240,21 @@ WHERE activity.date >= $1 AND activity.date <= $2 GROUP BY entities.entity_id, activity.date ORDER BY entities.entity_id, activity.date """ + if scope != "key": + return query + return ( + query[: query.index("\nSELECT entities")] + + """ +SELECT entities.entity_id, daily.* FROM entities CROSS JOIN LATERAL ( + SELECT activity.date AS day,SUM(activity.spend) AS spend, + SUM(activity.api_requests)::bigint AS requests,SUM(activity.failed_requests)::bigint AS errors, + SUM(activity.total_response_time_ms) AS response_time_ms,SUM(activity.timed_requests)::bigint AS timed_requests + FROM "LiteLLM_DailyUserSpend" activity + WHERE activity.api_key=entities.entity_id AND activity.date>=$1 AND activity.date<=$2 + GROUP BY activity.date +) daily ORDER BY entities.entity_id,daily.day +""" + ) def _reporting_id(scope: Scope, row: DailyAlertMetrics) -> DailyAlertMetrics: @@ -321,6 +343,48 @@ class UsageAlertMetricsRepository: await self._database.query_raw(_DEPRECATION_USAGE_QUERY, start.isoformat(), end.isoformat(), models) ) + async def page( + self, + scope: Scope, + target: date, + baseline_days: int, + *, + cursor: str | None = None, + entity_ids: tuple[str, ...] | None = None, + ) -> DailyMetricsPage: + if entity_ids is not None and len(entity_ids) > _BATCH_SIZE: + raise ValueError("Read at most 10 exact entities") + if entity_ids == (): + return DailyMetricsPage((), cursor, True) + query: Final = _query(scope) + arguments: Final = ( + (target - timedelta(days=baseline_days)).isoformat(), + target.isoformat(), + cursor, + _BATCH_SIZE, + ) + exact_query: Final = ( + query[: query.index("WITH entities")] + + "WITH entities AS MATERIALIZED (SELECT unnest($3::text[]) AS entity_id)" + + query[query.index("\nSELECT entities") :] + ) + raw: Final = _ROWS.validate_python( + await self._database.query_raw( + exact_query if entity_ids is not None else query, + *( + arguments[:2] + + (tuple(api_key_from_bucket(item) if scope == "key" else item for item in entity_ids),) + if entity_ids is not None + else arguments + ), + ) + ) + return DailyMetricsPage( + tuple(_reporting_id(scope, row) for row in raw), + raw[-1].entity_id if raw else cursor, + entity_ids is not None or scope == "deployment" or len({row.entity_id for row in raw}) < _BATCH_SIZE, + ) + async def batches( self, scope: Scope, target: date, baseline_days: int ) -> AsyncIterator[tuple[DailyAlertMetrics, ...]]: diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index 775afe67aee..a17da9f8a6c 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -34,12 +34,16 @@ from litellm.types.proxy.usage_alerts import ( UsageAlertModelOption, UsageAlertNotification, UsageAlertObservation, + UsageAlertProgress, + UsageAlertProjectionCursor, + UsageAlertProjectionWork, UsageAlertRule, UsageAlertRuleInput, UsageAlertRuleKind, UsageAlertRuleList, UsageAlertScope, UsageAlertStatus, + UsageAlertWorkCursor, ) if TYPE_CHECKING: @@ -287,6 +291,7 @@ class UsageAlertsRepository: jsonb_build_object('total',count(*) FILTER (WHERE builtin), 'enabled',count(*) FILTER (WHERE builtin AND enabled), 'evaluation_status_counts',jsonb_build_object( + 'delayed',count(*) FILTER (WHERE builtin AND enabled AND record->>'evaluation_status'='delayed'), 'pending',count(*) FILTER (WHERE builtin AND enabled AND COALESCE(record->>'evaluation_status','pending')='pending'), 'complete',count(*) FILTER (WHERE builtin AND enabled AND record->>'evaluation_status'='complete'), 'insufficient_coverage',count(*) FILTER (WHERE builtin AND enabled AND record->>'evaluation_status'='insufficient_coverage'), @@ -321,6 +326,136 @@ class UsageAlertsRepository: ) return tuple(UsageAlertRule.model_validate(row.record) for row in rows) + async def enabled_rule_page( + self, scope: UsageAlertScope, wildcard: bool, cursor: str | None, limit: int = 500 + ) -> tuple[UsageAlertRule, ...]: + if not 1 <= limit <= 500: + raise ValueError("Read at most 500 rules") + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw( + 'SELECT record FROM "LiteLLM_UsageAlertRule" WHERE enabled AND NOT deleted AND scope_type=$1 AND (scope_id IS NULL)=$2 AND ($3::text IS NULL OR id>$3) ORDER BY id LIMIT $4', + scope, + wildcard, + cursor, + limit, + ) + ) + return tuple(UsageAlertRule.model_validate(row.record) for row in rows) + + async def work_cursor(self) -> UsageAlertWorkCursor: + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw( + 'SELECT param_value AS record FROM "LiteLLM_Config" WHERE param_name=$1', "usage_alert_work_cursor" + ) + ) + return UsageAlertWorkCursor.model_validate(rows[0].record) if rows else UsageAlertWorkCursor() + + async def save_work_cursor(self, cursor: UsageAlertWorkCursor) -> None: + await self.database.query_raw( + 'INSERT INTO "LiteLLM_Config" (param_name,param_value) VALUES ($1,$2::jsonb) ON CONFLICT(param_name) DO UPDATE SET param_value=EXCLUDED.param_value RETURNING param_name', + "usage_alert_work_cursor", + cursor.model_dump_json(), + ) + + async def save_progress(self, updates: tuple[tuple[UsageAlertRule, UsageAlertProgress], ...]) -> None: + if not updates: + return + if len(updates) > 500: + raise ValueError("Save at most 500 rule positions") + await self.database.query_raw( + """WITH input AS (SELECT * FROM jsonb_to_recordset($1::jsonb) + AS x(id text,revision timestamptz,progress jsonb)) + UPDATE "LiteLLM_UsageAlertRule" r SET record=r.record || jsonb_build_object( + 'evaluation_progress',x.progress,'evaluation_status','delayed','last_evaluated_day',x.progress->'day') + FROM input x WHERE r.id=x.id AND r.enabled AND NOT r.deleted + AND (r.record->>'updated_at')::timestamptz=x.revision RETURNING r.id""", + json.dumps( + tuple( + { + "id": rule.id, + "revision": rule.updated_at.isoformat(), + "progress": progress.model_dump(mode="json"), + } + for rule, progress in updates + ) + ), + ) + + async def suppressed_entities( + self, rules: tuple[UsageAlertRule, ...], entities: tuple[str | None, ...] + ) -> Mapping[str, frozenset[str | None]]: + defaults: Final = tuple(rule for rule in rules if rule.id in _DEFAULT_RULE_IDS) + if not defaults or not entities: + return {} + if len(rules) > 500 or len(entities) > 500: + raise ValueError("Check at most 500 rules and entities") + rows: Final = TypeAdapter(tuple[_BreachedEntity, ...]).validate_python( + await self.database.query_raw( + """WITH input AS (SELECT * FROM jsonb_to_recordset($1::jsonb) + AS x(id text,scope_type text,metric text,direction text)) + SELECT DISTINCT x.id AS rule_id,r.scope_id FROM input x JOIN "LiteLLM_UsageAlertRule" r + ON r.scope_type=x.scope_type AND r.record->>'metric'=x.metric AND r.record->>'direction'=x.direction + WHERE r.enabled AND NOT r.deleted AND COALESCE((r.record->>'overrides_defaults')::boolean,false) AND NOT(r.id=ANY($2::text[])) + AND (r.scope_id IS NULL OR r.scope_id=ANY($3::text[])) ORDER BY rule_id""", + json.dumps( + tuple( + {"id": r.id, "scope_type": r.scope_type, "metric": r.metric, "direction": r.direction} + for r in defaults + ) + ), + tuple(_DEFAULT_RULE_IDS), + tuple(entity for entity in entities if entity is not None), + ) + ) + return { + rule_id: frozenset(row.scope_id for row in group) + for rule_id, group in groupby(rows, lambda row: row.rule_id) + } + + async def ensure_projection_work(self, day: date) -> None: + await self.database.query_raw( + 'INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,owner,completed) VALUES ($1,$2,false) ON CONFLICT(id) DO NOTHING RETURNING id', + "performance-dirty:" + day.isoformat(), + str(uuid4()), + ) + + async def pending_projection_work(self) -> UsageAlertProjectionWork | None: + rows: Final = TypeAdapter(tuple[UsageAlertProjectionWork, ...]).validate_python( + await self.database.query_raw( + """SELECT substring(id FROM 19)::date AS day,owner AS version FROM "LiteLLM_UsageAlertCheckpoint" + WHERE id LIKE 'performance-dirty:%' AND NOT completed ORDER BY id LIMIT 1""" + ) + ) + return rows[0] if rows else None + + async def projection_cursor(self, work: UsageAlertProjectionWork) -> UsageAlertProjectionCursor: + rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( + await self.database.query_raw( + 'SELECT param_value AS record FROM "LiteLLM_Config" WHERE param_name=$1', + "usage_issue_projection_cursor", + ) + ) + current: Final = UsageAlertProjectionCursor.model_validate(rows[0].record) if rows else None + return ( + current + if current is not None and current.version == work.version + else UsageAlertProjectionCursor(version=work.version) + ) + + async def save_projection_cursor(self, cursor: UsageAlertProjectionCursor) -> None: + await self.database.query_raw( + 'INSERT INTO "LiteLLM_Config" (param_name,param_value) VALUES ($1,$2::jsonb) ON CONFLICT(param_name) DO UPDATE SET param_value=EXCLUDED.param_value RETURNING param_name', + "usage_issue_projection_cursor", + cursor.model_dump_json(), + ) + + async def finish_projection_work(self, work: UsageAlertProjectionWork) -> None: + await self.database.query_raw( + 'UPDATE "LiteLLM_UsageAlertCheckpoint" SET completed=true WHERE id=$1 AND owner=$2 RETURNING id', + "performance-dirty:" + work.day.isoformat(), + work.version, + ) + async def get_alert(self, alert_id: str) -> UsageAlert | None: rows: Final = TypeAdapter(tuple[_Record, ...]).validate_python( await self.database.query_raw('SELECT record FROM "LiteLLM_UsageAlertIncident" WHERE id=$1', alert_id) @@ -609,16 +744,24 @@ class UsageAlertsRepository: """WITH input AS ( SELECT * FROM jsonb_to_recordset($1::jsonb) AS i(rule_id text,checkpoint_id text,revision timestamptz,status text) + ), valid AS ( + SELECT input.* FROM input JOIN "LiteLLM_UsageAlertRule" r ON r.id=input.rule_id + WHERE r.enabled AND NOT r.deleted AND (r.record->>'updated_at')::timestamptz=input.revision ), checkpoints AS ( INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,completed) - SELECT checkpoint_id,true FROM input WHERE $3::boolean - ON CONFLICT(id) DO NOTHING - ) UPDATE "LiteLLM_UsageAlertRule" r SET record=r.record || jsonb_build_object( - 'last_evaluated_day',$2::text,'evaluation_status',input.status) - FROM input WHERE r.id=input.rule_id AND NOT r.deleted - AND (r.record->>'updated_at')::timestamptz=input.revision - AND (r.record->>'last_evaluated_day' IS DISTINCT FROM $2::text - OR r.record->>'evaluation_status' IS DISTINCT FROM input.status) RETURNING r.id""", + SELECT checkpoint_id,true FROM valid WHERE $3::boolean + ON CONFLICT(id) DO NOTHING RETURNING id + ), dirty AS ( + INSERT INTO "LiteLLM_UsageAlertCheckpoint" (id,owner,completed) + SELECT 'performance-dirty:'||$2::text,$4,false WHERE EXISTS(SELECT 1 FROM checkpoints) + ON CONFLICT(id) DO UPDATE SET owner=EXCLUDED.owner,completed=false + ) UPDATE "LiteLLM_UsageAlertRule" r SET record= + (CASE WHEN $3::boolean THEN r.record-'evaluation_progress' ELSE r.record END) || jsonb_build_object( + 'last_evaluated_day',$2::text,'evaluation_status',input.status) + FROM valid input WHERE r.id=input.rule_id AND ( + r.record->>'last_evaluated_day' IS DISTINCT FROM $2::text + OR r.record->>'evaluation_status' IS DISTINCT FROM input.status + OR ($3::boolean AND r.record ? 'evaluation_progress')) RETURNING r.id""", json.dumps( tuple( { @@ -632,6 +775,7 @@ class UsageAlertsRepository: ), day.isoformat(), completed, + str(uuid4()), ) async def ensure_default_rules(self, rules: tuple[UsageAlertRuleInput, ...]) -> None: diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py index d6f7e599683..73da957b0c9 100644 --- a/litellm/types/proxy/usage_alerts.py +++ b/litellm/types/proxy/usage_alerts.py @@ -10,7 +10,9 @@ UsageAlertMetric: TypeAlias = Literal["spend", "requests", "latency", "errors", UsageAlertDirection: TypeAlias = Literal["increase", "decrease"] UsageAlertStatus: TypeAlias = Literal["open", "resolved", "dismissed"] UsageAlertChannel: TypeAlias = Literal["email", "slack"] -UsageAlertEvaluationStatus: TypeAlias = Literal["pending", "complete", "insufficient_coverage", "insufficient_history"] +UsageAlertEvaluationStatus: TypeAlias = Literal[ + "pending", "complete", "insufficient_coverage", "insufficient_history", "delayed" +] UsageAlertRuleKind: TypeAlias = Literal["all", "default", "custom"] @@ -54,6 +56,31 @@ class UsageAlertRuleInput(BaseModel): return self +class UsageAlertProgress(BaseModel, frozen=True): + """Internal durable position; never part of rule input or public output.""" + + day: date + revision: datetime + cursor: str | None = None + evaluated: bool = False + + +class UsageAlertWorkCursor(BaseModel, frozen=True): + phase: int = Field(default=0, ge=0, lt=10) + after_id: str | None = None + + +class UsageAlertProjectionWork(BaseModel, frozen=True): + day: date + version: str + + +class UsageAlertProjectionCursor(BaseModel, frozen=True): + version: str + step: int = Field(default=0, ge=0, le=26) + after_id: str | None = None + + class UsageAlertRule(UsageAlertRuleInput): overrides_defaults: bool = Field(default=False, exclude=True) id: str @@ -63,6 +90,8 @@ class UsageAlertRule(UsageAlertRuleInput): last_evaluated_day: date | None = None evaluation_status: UsageAlertEvaluationStatus = "pending" is_default: bool = False + evaluation_progress: UsageAlertProgress | None = Field(default=None, exclude=True) + overrides_defaults: bool = Field(default=False, exclude=True) class UsageAlert(BaseModel): @@ -99,6 +128,7 @@ class UsageAlertUpdate(BaseModel): class UsageAlertEvaluationCounts(BaseModel, frozen=True): + delayed: int = 0 pending: int = 0 complete: int = 0 insufficient_coverage: int = 0 diff --git a/tests/integration/database/test_performance_issues.py b/tests/integration/database/test_performance_issues.py index 7cad556f1cf..fabb741118b 100644 --- a/tests/integration/database/test_performance_issues.py +++ b/tests/integration/database/test_performance_issues.py @@ -16,7 +16,7 @@ from litellm.proxy.utils import PrismaClient from litellm.repositories.issues import IssuesRepository from litellm.repositories.usage_alert_metrics import IssueMetricWindow from litellm.repositories.usage_alerts import UsageAlertsRepository -from litellm.types.proxy.usage_alerts import UsageAlertObservation, UsageAlertRuleInput, UsageAlertScope +from litellm.types.proxy.usage_alerts import UsageAlertObservation, UsageAlertRule, UsageAlertRuleInput, UsageAlertScope _DAY: Final = date(2026, 11, 4) _NOW: Final = datetime(2026, 11, 5, 3, microsecond=987654, tzinfo=timezone.utc) @@ -218,7 +218,7 @@ async def test_late_custom_key_rule_only_overrides_that_keys_default_signal(db: assert len(original) == 2 overridden: Final = next(item for item in original if item.alert.scope_id == _KEY_A) retained: Final = next(item for item in original if item.alert.scope_id == _KEY_B) - await UsageAlertsRepository.from_prisma(prisma).save_rule( + custom: Final = await UsageAlertsRepository.from_prisma(prisma).save_rule( UsageAlertRuleInput( name="Project A high spend threshold", scope_type="key", @@ -230,6 +230,13 @@ async def test_late_custom_key_rule_only_overrides_that_keys_default_signal(db: ), None, ) + assert custom is not None + unprivileged: Final = await issues.active_signals(_DAY, scope_type="key", metric="spend") + assert overridden.alert.id in {item.alert.id for item in unprivileged} + await prisma.writer_db.query_raw( + 'UPDATE "LiteLLM_UsageAlertRule" SET record=record || \'{"overrides_defaults":true}\'::jsonb WHERE id=$1 RETURNING id', + custom.id, + ) await job._run(_NOW + timedelta(minutes=1)) attached: Final = await prisma.writer_db.query_raw('SELECT source_id FROM "LiteLLM_IssueSignal"') assert overridden.alert.id not in {row["source_id"] for row in attached} @@ -305,6 +312,16 @@ async def test_provider_issue_keeps_an_independent_noisy_key_and_recovers_from_f _KEY_B, noisy, ) + # Put the two correlated models on opposite sides of the former 100-pair page boundary. + await prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,custom_llm_provider,spend,api_requests, + successful_requests,failed_requests,total_response_time_ms,timed_requests,updated_at) + SELECT 'boundary-'||n,'boundary-owner',$1,md5(n::text)||md5(n::text), + 'unrelated-'||n,'aaa-unrelated',0,1,1,0,100,1,CURRENT_TIMESTAMP + FROM generate_series(1,99) n RETURNING id""", + _DAY.isoformat(), + ) await _coverage(prisma, _DAY) job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) await job._run(_NOW) @@ -336,3 +353,209 @@ async def test_provider_issue_keeps_an_independent_noisy_key_and_recovers_from_f assert recovered is not None and not recovered.active continuing: Final = await issues.get_issue(independent.id) assert continuing is not None and continuing.active + + +@pytest.mark.asyncio(loop_scope="session") +async def test_missing_drop_coverage_does_not_block_established_spend_issue(db: ManagementDatabase) -> None: + async with _isolated(db) as prisma: + await _seed(prisma) + await prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_UsageAlertFlush" WHERE hour<$1::timestamp RETURNING source', + (_DAY + timedelta(days=1)).isoformat(), + ) + job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) + await job._run(_NOW) + rules: Final = await UsageAlertsRepository.from_prisma(prisma).list_enabled_rules(limit=100) + assert any(rule.direction == "decrease" and rule.evaluation_status == "insufficient_coverage" for rule in rules) + issues: Final = await IssuesRepository.from_prisma(prisma).list_issues() + assert any(issue.family == "spend" and issue.cohort.scope_id == _KEY_A for issue in issues.data) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_entity_page_progress_resumes_old_day_without_early_completion(db: ManagementDatabase) -> None: + async with _isolated(db) as prisma: + await prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,custom_llm_provider,spend,api_requests, + successful_requests,failed_requests,total_response_time_ms,timed_requests,updated_at) + SELECT 'resume-'||k||'-'||d,'owner',($1::date-d)::text,md5(k::text)||md5(k::text), + 'model','example-provider',CASE WHEN d=0 THEN 50 ELSE 1 END, + 1000,1000,0,100000,1000,CURRENT_TIMESTAMP + FROM generate_series(1,31) k CROSS JOIN generate_series(0,7) d RETURNING id""", + _DAY.isoformat(), + ) + await _coverage(prisma, _DAY) + job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) + rule: Final = await job._store.save_rule( + UsageAlertRuleInput( + name="Resume each key", + scope_type="key", + metric="spend", + direction="increase", + min_requests=0, + ), + None, + ) + assert rule is not None + await job._scope_page((rule,), _DAY, True) + partial: Final = await job._store.get_rule(rule.id) + assert partial is not None and partial.evaluation_status == "delayed" + assert partial.evaluation_progress is not None and partial.evaluation_progress.cursor is not None + assert not await job._store.is_day_complete(rule, _DAY) + assert "evaluation_progress" not in partial.model_dump() + assert "evaluation_progress" not in UsageAlertRule.model_json_schema(mode="serialization")["properties"] + first: Final = await job._store.list_alerts(limit=500) + assert first.total == 10 + await job._run(_NOW + timedelta(days=1)) + assert await job._store.is_day_complete(rule, _DAY) + final: Final = await job._store.list_alerts(limit=500) + assert sum(1 for item in final.data if item.rule_id == rule.id) == 31 + count: Final = await prisma.writer_db.query_raw( + 'SELECT count(*)::int AS count FROM "LiteLLM_UsageAlertIncident" WHERE rule_id=$1', + rule.id, + ) + assert count == [{"count": 31}] + resumed: Final = await job._store.get_rule(rule.id) + assert resumed is not None and resumed.evaluation_progress is None + edited: Final = await job._store.save_rule( + UsageAlertRuleInput.model_validate(rule.model_dump(include=set(UsageAlertRuleInput.model_fields))), + None, + resumed, + ) + assert edited is not None and edited.evaluation_progress is None + + +@pytest.mark.asyncio(loop_scope="session") +async def test_disabled_primary_requires_another_eligible_healthy_source_to_recover(db: ManagementDatabase) -> None: + async with _isolated(db) as prisma: + await _seed(prisma) + job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) + definition: Final = UsageAlertRuleInput( + name="Extra key policy", + scope_type="key", + scope_id=_KEY_A, + metric="spend", + direction="increase", + min_requests=0, + ) + custom: Final = await job._store.save_rule(definition, None) + assert custom is not None + await job._run(_NOW) + issues: Final = IssuesRepository.from_prisma(prisma) + issue: Final = next( + item + for item in (await issues.list_issues()).data + if item.family == "spend" and item.cohort.scope_id == _KEY_A + ) + assert await job._store.save_rule(definition.model_copy(update={"enabled": False}), None, custom) + await issues.recover_issues(_DAY) + unchanged: Final = await issues.get_issue(issue.id) + assert unchanged is not None and unchanged.active + await _next_day(prisma, _DAY + timedelta(days=1), 1, 1) + await job._run(_NOW + timedelta(days=1)) + recovered: Final = await issues.get_issue(issue.id) + assert recovered is not None and not recovered.active + history: Final = await job._store.list_alerts(scope_type="key", scope_id=_KEY_A) + assert any(item.rule_id == custom.id and item.status == "open" for item in history.data) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_lone_disabled_source_does_not_claim_recovery(db: ManagementDatabase) -> None: + async with _isolated(db) as prisma: + await _seed(prisma) + await prisma.writer_db.query_raw( + 'UPDATE "LiteLLM_DailyUserSpend" SET spend=5 WHERE date=$1 AND api_key=$2 RETURNING id', + _DAY.isoformat(), + _KEY_A, + ) + job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) + definition: Final = UsageAlertRuleInput( + name="Sensitive key policy", + scope_type="key", + scope_id=_KEY_A, + metric="spend", + direction="increase", + min_requests=0, + ) + custom: Final = await job._store.save_rule(definition, None) + assert custom is not None + await job._run(_NOW) + issues: Final = IssuesRepository.from_prisma(prisma) + issue: Final = next( + item + for item in (await issues.list_issues()).data + if item.family == "spend" and item.cohort.scope_id == _KEY_A + ) + assert await job._store.save_rule(definition.model_copy(update={"enabled": False}), None, custom) + await _next_day(prisma, _DAY + timedelta(days=1), 1, 1) + await job._run(_NOW + timedelta(days=1)) + preserved: Final = await issues.get_issue(issue.id) + assert preserved is not None and preserved.active + + +@pytest.mark.asyncio(loop_scope="session") +async def test_rule_edit_discards_old_cursor_and_rejects_stale_progress(db: ManagementDatabase) -> None: + from litellm.types.proxy.usage_alerts import UsageAlertProgress + + async with _isolated(db) as prisma: + store: Final = UsageAlertsRepository.from_prisma(prisma) + definition: Final = UsageAlertRuleInput( + name="Resumable", scope_type="key", metric="spend", direction="increase" + ) + rule: Final = await store.save_rule(definition, None) + assert rule is not None + position: Final = UsageAlertProgress(day=_DAY, revision=rule.updated_at, cursor="a" * 64, evaluated=True) + await store.save_progress(((rule, position),)) + partial: Final = await store.get_rule(rule.id) + assert partial is not None and partial.evaluation_progress == position + edited: Final = await store.save_rule(definition.model_copy(update={"multiplier": 10}), None, partial) + assert edited is not None and edited.evaluation_progress is None + await store.save_progress(((rule, position),)) + current: Final = await store.get_rule(rule.id) + assert current is not None and current.evaluation_progress is None + assert current.updated_at == edited.updated_at + await store.record_evaluations(((rule, "complete"),), _DAY, completed=True) + assert not await store.is_day_complete(edited, _DAY) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_busy_background_capacity_defers_and_releases_the_worker_lease(db: ManagementDatabase) -> None: + async with _isolated(db) as prisma: + await _seed(prisma) + job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) + async with prisma.writer_db.tx() as transaction: + await transaction.query_raw("SELECT pg_advisory_xact_lock(1876539401,2)::text") + await job.run(_NOW) + assert not (await IssuesRepository.from_prisma(prisma).list_issues()).data + assert await prisma.writer_db.query_raw( + "SELECT count(*)::int AS count FROM \"LiteLLM_UsageAlertCheckpoint\" WHERE id='worker'" + ) == [{"count": 0}] + await job.run(_NOW) + assert (await IssuesRepository.from_prisma(prisma).list_issues()).total == 1 + assert await prisma.writer_db.query_raw( + "SELECT count(*)::int AS count FROM \"LiteLLM_UsageAlertCheckpoint\" WHERE id='worker'" + ) == [{"count": 0}] + + +@pytest.mark.asyncio(loop_scope="session") +async def test_partial_broad_default_waits_for_its_key_family_only(db: ManagementDatabase) -> None: + from litellm.types.proxy.usage_alert_defaults import default_usage_rules + + async with _isolated(db) as prisma: + await _seed(prisma) + job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) + await job._store.ensure_default_rules(default_usage_rules()) + broad: Final = await job._store.enabled_rule_page("deployment", True, None) + await job._scope_page(broad, _DAY, True) + sources: Final = IssuesRepository.from_prisma(prisma) + assert await sources.active_signals(_DAY, scope_type="deployment", metric="spend") + assert not await sources.active_signals(_DAY, scope_type="deployment", metric="spend", completed_only=True) + key_rules: Final = await job._store.enabled_rule_page("key", True, None) + spend: Final = tuple(rule for rule in key_rules if rule.metric == "spend") + await job._scope_page(spend, _DAY, True) + assert await sources.active_signals(_DAY, scope_type="deployment", metric="spend", completed_only=True) + unfinished: Final = tuple(rule for rule in key_rules if rule.metric != "spend") + assert not await job._store.completed_rule_ids(unfinished, _DAY) + await job._run(_NOW) + result: Final = await sources.list_issues() + assert result.total == 1 and result.data[0].cohort.scope_id == _KEY_A diff --git a/tests/unit/integrations/SlackAlerting/test_usage_alerts.py b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py index af0f7e1da25..278eb1bcb65 100644 --- a/tests/unit/integrations/SlackAlerting/test_usage_alerts.py +++ b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py @@ -81,7 +81,9 @@ async def test_slack_ack_requires_successful_http_response(status: int, monkeypa with pytest.raises(httpx.HTTPStatusError): await delivery.send(_ALERT, _RULE, "slack") assert len(requests) == 1 - assert b"https://gateway.example.com/ui/api-keys?key=reporting-key&detail_tab=activity" in requests[0].content + assert ( + b"https://gateway.example.com/ui/api-keys?key=reporting-key&detail_tab=activity" in requests[0].content + ) finally: await handler.close() @@ -128,7 +130,90 @@ async def test_issue_delivery_links_to_investigation_and_preserves_the_bound_sou assert await UsageAlertDelivery(slack).send_issue(pending) assert len(requests) == 1 assert b"https://gateway.example.com/ui/alerts?issue=issue-example" in requests[0].content - assert b"https://gateway.example.com/ui/api-keys?key=reporting-key&detail_tab=activity" in requests[0].content + assert ( + b"https://gateway.example.com/ui/api-keys?key=reporting-key&detail_tab=activity" in requests[0].content + ) assert b"1,000.00; baseline 200.00" in requests[0].content finally: await handler.close() + + +@pytest.mark.asyncio +async def test_slack_source_labels_cannot_inject_mentions_or_links() -> None: + requests: Final[list[httpx.Request]] = [] + + def transport(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200) + + handler: Final = AsyncHTTPHandler(transport=httpx.MockTransport(transport)) + slack: Final = SlackAlerting( + alerting=["slack"], default_webhook_url="https://hooks.example.com/test", async_http_handler=handler + ) + try: + assert await UsageAlertDelivery(slack).send( + _ALERT.model_copy( + update={"rule_name": " & ", "scope_id": "<@someone>"} + ), + _RULE, + "slack", + ) + assert b"" not in requests[0].content + assert b"<!channel> & <https://example.com|fake>" in requests[0].content + assert b"<@someone>" in requests[0].content + finally: + await handler.close() + + +@pytest.mark.asyncio +async def test_cancelled_smtp_attempt_retains_only_one_send(monkeypatch: pytest.MonkeyPatch) -> None: + import asyncio + import threading + + connected: Final = asyncio.Event() + release: Final = asyncio.Event() + accepted: Final[list[asyncio.StreamWriter]] = [] + + async def stalled(_: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + accepted.append(writer) + connected.set() + await release.wait() + writer.close() + await writer.wait_closed() + + server: Final = await asyncio.start_server(stalled, "127.0.0.1", 0) + monkeypatch.setenv("SMTP_HOST", "127.0.0.1") + monkeypatch.setenv("SMTP_PORT", str(server.sockets[0].getsockname()[1])) + monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com") + monkeypatch.setenv("SMTP_TLS", "False") + monkeypatch.setenv("SMTP_USE_SSL", "False") + monkeypatch.delenv("SMTP_USERNAME", raising=False) + monkeypatch.delenv("SMTP_PASSWORD", raising=False) + slack: Final = SlackAlerting(alerting=["email"], alert_types=[]) + delivery: Final = UsageAlertDelivery(slack) + rule: Final = _RULE.model_copy( + update={"notification_channels": ("email",), "email_recipients": ("qa@example.com",)} + ) + try: + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(delivery.send(_ALERT, rule, "email"), timeout=0.1) + await asyncio.wait_for(connected.wait(), timeout=1) + threads: Final = threading.active_count() + pending: Final = UsageAlertDelivery._email_task + assert pending is not None and not pending.done() + for _ in range(100): + with pytest.raises(RuntimeError, match="still in flight"): + await delivery.send(_ALERT, rule, "email") + assert UsageAlertDelivery._email_task is pending + assert len(accepted) == 1 and threading.active_count() == threads + with pytest.raises(Exception): + await asyncio.wait_for(asyncio.shield(pending), timeout=3) + assert pending.done() + finally: + release.set() + server.close() + await server.wait_closed() + for writer in accepted: + writer.close() + UsageAlertDelivery._email_task = None + UsageAlertDelivery._email_key = None diff --git a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py index 88954fbbafb..a6d083c41e1 100644 --- a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py +++ b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py @@ -129,30 +129,6 @@ def test_sentinel_entities_cannot_create_alerts_with_unusable_activity_links(sco assert observations.changed == () -def test_custom_entity_rule_replaces_only_the_matching_default() -> None: - from uuid import NAMESPACE_URL, uuid5 - - from litellm.proxy.spend_tracking.usage_alert_job import _suppressed_entities - from litellm.types.proxy.usage_alerts import UsageAlertRule - - default: Final = UsageAlertRule( - id=str(uuid5(NAMESPACE_URL, "litellm:usage-alert:key:spend:increase")), - name="default", - scope_type="key", - metric="spend", - direction="increase", - created_at=datetime(2026, 10, 1, tzinfo=timezone.utc), - updated_at=datetime(2026, 10, 1, tzinfo=timezone.utc), - created_by=None, - ) - custom: Final = default.model_copy(update={"id": "custom", "scope_id": "key-a"}) - other_metric: Final = custom.model_copy(update={"id": "latency", "metric": "latency", "scope_id": "key-b"}) - assert _suppressed_entities(default, (default, custom, other_metric)) == frozenset({"key-a"}) - assert _suppressed_entities(custom, (default, custom, other_metric)) == frozenset() - wildcard: Final = custom.model_copy(update={"scope_id": None}) - assert _suppressed_entities(default, (default, wildcard)) == frozenset({None}) - - def test_database_metric_rows_preserve_numeric_and_date_validation() -> None: from pydantic import TypeAdapter, ValidationError @@ -341,3 +317,25 @@ def test_error_rate_drop_requires_enough_current_requests() -> None: assert compare_daily_usage(rule, (*history, replace(_row(_DAY, requests=1), errors=0)), _DAY) is None result: Final = compare_daily_usage(rule, (*history, replace(_row(_DAY, requests=500), errors=0)), _DAY) assert result is not None and result.breached + + +def test_suppressed_default_can_recover_existing_episode_without_creating_a_new_one() -> None: + from litellm.proxy.spend_tracking.usage_alert_job import _observations + + rule: Final = UsageAlertRule( + id="rule", + name="Default", + scope_type="key", + metric="spend", + direction="increase", + created_at=datetime(2026, 10, 1, tzinfo=timezone.utc), + updated_at=datetime(2026, 10, 1, tzinfo=timezone.utc), + created_by=None, + ) + history: Final = tuple(_row(_DAY - timedelta(days=day)) for day in range(1, 4)) + suppressed: Final = frozenset({"key-a"}) + active: Final = frozenset({"key-a"}) + recovered: Final = _observations(rule, (*history, _row(_DAY)), _DAY, suppressed, active=active) + assert len(recovered.changed) == 1 and not recovered.changed[0].breached + assert not _observations(rule, (*history, _row(_DAY, spend=100)), _DAY, suppressed, active=active).changed + assert not _observations(rule, (*history, _row(_DAY, spend=100)), _DAY, suppressed).changed From f0b3926240bc0db19e59097780f55bc67e19325f Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 16:52:55 +0800 Subject: [PATCH 22/44] fix: page alert recovery within the work budget --- litellm/models/issues.py | 2 ++ litellm/proxy/spend_tracking/performance_issues.py | 7 ++++++- litellm/repositories/issues.py | 10 ++++++++++ tests/integration/database/test_performance_issues.py | 7 +++++++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/litellm/models/issues.py b/litellm/models/issues.py index bff56e13f0e..348de00abbb 100644 --- a/litellm/models/issues.py +++ b/litellm/models/issues.py @@ -92,6 +92,8 @@ class IssueStore(Protocol): completed_only: bool = False, ) -> tuple[IssueProjectionSignal, ...]: ... + async def active_resource_issue_ids(self, cursor: str | None = None) -> tuple[str, ...]: ... + async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int: ... async def active_provider_issues(self, limit: int = 500, cursor: str | None = None) -> tuple[Issue, ...]: ... diff --git a/litellm/proxy/spend_tracking/performance_issues.py b/litellm/proxy/spend_tracking/performance_issues.py index ec55339b512..f209122560e 100644 --- a/litellm/proxy/spend_tracking/performance_issues.py +++ b/litellm/proxy/spend_tracking/performance_issues.py @@ -228,7 +228,6 @@ class PerformanceIssueProjector: projection_deadline: Final = deadline - min(2, max(0, deadline - asyncio.get_running_loop().time()) / 4) while asyncio.get_running_loop().time() < projection_deadline: if (next_position := await self._projection_page(position, work, observed_at)) is None: - await self._store.recover_issues(work.day, completed_only=True) await self._store.complete_projection(work.day, work.version) await self._notify(deadline) await self._store.prune() @@ -250,6 +249,12 @@ class PerformanceIssueProjector: return UsageAlertProjectionCursor(version=work.version, after_id=prior[-1].id) return UsageAlertProjectionCursor(version=work.version, step=1) if position.step > len(_SCOPES) * len(_METRICS): + identities: Final = await self._store.active_resource_issue_ids(position.after_id) + if identities: + await self._store.recover_issues(work.day, completed_only=True, issue_ids=identities) + return UsageAlertProjectionCursor( + version=work.version, step=position.step, after_id=identities[-1] + ) return None metric: Final = _METRICS[(position.step - 1) // len(_SCOPES)] scope: Final = _SCOPES[(position.step - 1) % len(_SCOPES)] diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index 7faad896b8c..98495034ce9 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -393,6 +393,16 @@ class IssuesRepository: ) ) + async def active_resource_issue_ids(self, cursor: str | None = None) -> tuple[str, ...]: + rows: Final = TypeAdapter(tuple[_Identifier, ...]).validate_python( + await self.database.query_raw( + """SELECT id FROM "LiteLLM_Issue" WHERE kind='performance' AND active AND scope_type<>'provider' + AND ($1::text IS NULL OR id>$1) ORDER BY id LIMIT 10""", + cursor, + ) + ) + return tuple(row.id for row in rows) + async def recover_issues(self, day: date, issue_ids: tuple[str, ...] | None = None) -> int: if issue_ids == (): return 0 diff --git a/tests/integration/database/test_performance_issues.py b/tests/integration/database/test_performance_issues.py index fabb741118b..fd7896ba4d9 100644 --- a/tests/integration/database/test_performance_issues.py +++ b/tests/integration/database/test_performance_issues.py @@ -415,6 +415,13 @@ async def test_entity_page_progress_resumes_old_day_without_early_completion(db: rule.id, ) assert count == [{"count": 31}] + issues: Final = IssuesRepository.from_prisma(prisma) + first_page: Final = await issues.active_resource_issue_ids() + second_page: Final = await issues.active_resource_issue_ids(first_page[-1]) + assert len(first_page) == len(second_page) == 10 + assert set(first_page).isdisjoint(second_page) + with pytest.raises(ValueError, match="at most 10"): + await issues.recover_issues(_DAY, issue_ids=first_page + second_page[:1]) resumed: Final = await job._store.get_rule(rule.id) assert resumed is not None and resumed.evaluation_progress is None edited: Final = await job._store.save_rule( From 81e97a9359242b62475a7d85acad6adf196a39bc Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 17:04:18 +0800 Subject: [PATCH 23/44] fix: bound retirement projection pages and resume interrupted scans without resolving active model alerts prematurely --- .../proxy/common_utils/model_deprecation.py | 112 ++++++++++- litellm/proxy/proxy_server.py | 6 +- .../spend_tracking/deprecation_issues.py | 108 ++++++++--- litellm/repositories/config_repository.py | 38 ++++ litellm/repositories/issues.py | 5 +- .../test_deprecation_issue_evaluation.py | 179 +++++++++++++++--- .../common_utils/test_model_deprecation.py | 71 +++++++ .../spend_tracking/test_deprecation_issues.py | 18 +- 8 files changed, 472 insertions(+), 65 deletions(-) diff --git a/litellm/proxy/common_utils/model_deprecation.py b/litellm/proxy/common_utils/model_deprecation.py index a210ba6eb69..12310b95f87 100644 --- a/litellm/proxy/common_utils/model_deprecation.py +++ b/litellm/proxy/common_utils/model_deprecation.py @@ -8,7 +8,15 @@ from dataclasses import dataclass from datetime import date, datetime, timezone from itertools import groupby from types import MappingProxyType -from typing import TYPE_CHECKING, Final, TypeAlias +from typing import ( # noqa: TID251 # router exposes its live validated deployment list + TYPE_CHECKING, + Final, + TypeAlias, + cast, +) +from uuid import uuid4 + +from pydantic import BaseModel, Field import litellm from litellm._logging import verbose_logger @@ -206,6 +214,8 @@ def _build_info( warn_within_days: int, catalog: Mapping[str, object], successor_log_level: int, + *, + check_successor: bool = True, ) -> _DeploymentInfo | None: model_name: Final = deployment.get("model_name") if not isinstance(model_name, str) or not model_name: @@ -217,7 +227,11 @@ def _build_info( days_until: Final = (resolved.deprecation_date - today).days model_info: Final = _mapping_field(deployment, "model_info") - configured: Final = _checked_successor(model_name, model_info, llm_router, successor_log_level) + configured: Final = ( + _checked_successor(model_name, model_info, llm_router, successor_log_level) + if check_successor + else _non_empty_string(model_info.get("successor_model")) + ) team_public_model_name_value: Final = model_info.get("team_public_model_name") team_public_model_name: Final = ( team_public_model_name_value @@ -355,6 +369,100 @@ def collect_model_deprecations( ) +class DeprecationScanCursor(BaseModel, frozen=True): + deployment: int = Field(default=0, ge=0) + group_position: int = Field(default=0, ge=0) + model: ModelDeprecationInfo | None = None + configured_successor: bool = False + + +@dataclass(frozen=True, slots=True) +class DeprecationModelPage: + models: tuple[ModelDeprecationInfo, ...] + cursor: DeprecationScanCursor + complete: bool + + +def _scan_deprecation_model( + router: Router, + deployments: Sequence[Mapping[str, object]], + cursor: DeprecationScanCursor, + today: date, + catalog: Mapping[str, object], +) -> tuple[DeprecationScanCursor, ModelDeprecationInfo | None]: + def info(position: int) -> _DeploymentInfo | None: + return _build_info( + deployments[position], + router, + today, + DEFAULT_DEPRECATION_WARN_DAYS, + catalog, + logging.DEBUG, + check_successor=False, + ) + + if cursor.model is None: + current: Final = info(cursor.deployment) + if current is None or current.info.status == "upcoming": + return DeprecationScanCursor(deployment=cursor.deployment + 1), None + return cursor.model_copy( + update={"model": current.info, "configured_successor": current.configured_successor} + ), None + positions: Final = router.registry().index("model_name").get(cursor.model.model_name, ()) + if cursor.group_position >= len(positions): + return DeprecationScanCursor(deployment=cursor.deployment + 1), cursor.model + position: Final = positions[cursor.group_position] + candidate: Final = info(position) + advanced: Final = cursor.model_copy(update={"group_position": cursor.group_position + 1}) + if candidate is None or candidate.info.deprecation_date != cursor.model.deprecation_date: + return advanced, None + if position < cursor.deployment: + return DeprecationScanCursor(deployment=cursor.deployment + 1), None + merged: Final = _merge_group((_DeploymentInfo(cursor.model, cursor.configured_successor), candidate), logging.DEBUG) + return advanced.model_copy( + update={ + "model": merged, + "configured_successor": cursor.configured_successor or candidate.configured_successor, + } + ), None + + +class RouterDeprecationSource: + """Read pages from router-owned views; never retain a second deployment catalog.""" + + def __init__( + self, router: Callable[[], Router | None], loaded_catalog: LoadedCatalog = GetModelCostMap.loaded_model_cost_map + ) -> None: + self._router = router + self._loaded_catalog = loaded_catalog + self._instance = str(uuid4()) + + @property + def revision(self) -> str: + router: Final = self._router() + version: Final = router.registry().version if router is not None else 0 + model_cost_identity: Final = id(litellm.model_cost) # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType] # identity only, from the legacy untyped catalog + return f"{self._instance}:{id(router)}:{version}:{model_cost_identity}:{id(self._loaded_catalog())}" + + def page(self, today: date, cursor: DeprecationScanCursor) -> DeprecationModelPage: + router: Final = self._router() + if router is None: + return DeprecationModelPage((), cursor, False) + deployments: Final = cast( # cast-ok: router validated each deployment + Sequence[Mapping[str, object]], router.registry().all() + ) + catalog: Final = self._loaded_catalog() + position = cursor # rebind-ok: bounded scan continuation + models: tuple[ModelDeprecationInfo, ...] = () # rebind-ok: bounded result page + for _ in range(100): + if position.deployment >= len(deployments) or len(models) == 10: + break + position, model = _scan_deprecation_model(router, deployments, position, today, catalog) + if model is not None: + models += (model,) + return DeprecationModelPage(models, position, position.deployment >= len(deployments)) + + def _escape_slack_mrkdwn(value: str) -> str: """Neutralize Slack control characters so a model name cannot forge a mention or link""" return value.replace("&", "&").replace("<", "<").replace(">", ">") diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 948748267eb..d3384483d94 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -557,7 +557,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( resolve_realtime_route_model as _resolve_realtime_route_model, ) from litellm.proxy.common_utils.load_config_utils import get_config_from_bucket -from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations +from litellm.proxy.common_utils.model_deprecation import RouterDeprecationSource, collect_model_deprecations from litellm.proxy.common_utils.model_listing_utils import ( ClaudeCodeRoutingNames, DeployedModelGroups, @@ -11678,14 +11678,14 @@ class ProxyStartupEvent: misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME, ) - from litellm.proxy.spend_tracking.usage_alert_job import USAGE_ALERT_JOB_ID, UsageAlertJob from litellm.proxy.spend_tracking.deprecation_issues import DeprecationIssueProjector + from litellm.proxy.spend_tracking.usage_alert_job import USAGE_ALERT_JOB_ID, UsageAlertJob usage_alert_job: Final = UsageAlertJob.from_prisma( prisma_client, proxy_logging_obj.slack_alerting_instance, deprecation_projection=DeprecationIssueProjector.from_prisma( - prisma_client, lambda today: collect_model_deprecations(llm_router, today=today) + prisma_client, RouterDeprecationSource(lambda: llm_router) ).run, ) scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # apscheduler is untyped diff --git a/litellm/proxy/spend_tracking/deprecation_issues.py b/litellm/proxy/spend_tracking/deprecation_issues.py index e7970182dad..31b66058a8a 100644 --- a/litellm/proxy/spend_tracking/deprecation_issues.py +++ b/litellm/proxy/spend_tracking/deprecation_issues.py @@ -1,34 +1,49 @@ -from collections.abc import Callable, Mapping, Sequence +import asyncio +from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import date, datetime, timedelta, timezone from itertools import chain, groupby from types import MappingProxyType from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # Prisma delegates the query protocol dynamically -from pydantic import BaseModel +from pydantic import BaseModel, Field -from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key -from litellm.repositories.config_repository import ConfigRepository -from litellm.repositories.usage_alert_metrics import MetricsDatabase, UsageAlertMetricsRepository +from litellm._logging import verbose_proxy_logger +from litellm.proxy.common_utils.model_deprecation import ( + DeprecationScanCursor, + RouterDeprecationSource, + deprecation_alert_state_key, +) +from litellm.repositories.analytics_budget import ( + AnalyticsCapacityError, + AnalyticsDatabase, + BoundedAnalyticsDatabase, + analytics_deadline, +) +from litellm.repositories.config_repository import BoundedConfigRepository +from litellm.repositories.usage_alert_metrics import UsageAlertMetricsRepository from litellm.types.proxy.issues import IssueCohort, IssueDeprecation, IssueImpact, IssueInput, IssueSignal from litellm.types.proxy.model_deprecation import ( DeprecationAlertState, ModelDeprecationInfo, - ModelDeprecationResponse, ModelDeprecationUsage, ) if TYPE_CHECKING: from litellm.models.issues import IssueStore - from litellm.proxy.utils import PrismaClient _PROJECTION_CHECKPOINT: Final = "issues:deprecation_projection" -_PAGE_SIZE: Final = 500 +_PAGE_SIZE: Final = 10 +_WORK_BUDGET_SECONDS: Final = 5 class _ProjectionCheckpoint(BaseModel, frozen=True): hour: datetime + observed_at: datetime | None = None + source_revision: str = "" + cursor: DeprecationScanCursor = Field(default_factory=DeprecationScanCursor) + complete: bool = True @dataclass(frozen=True, slots=True) @@ -54,10 +69,12 @@ def _identity(candidate: DeprecationCandidate) -> tuple[str, date]: return candidate.usage_model, candidate.model.deprecation_date -def deprecation_candidates(snapshot: ModelDeprecationResponse) -> tuple[DeprecationCandidate, ...]: +def deprecation_candidates(models: Sequence[ModelDeprecationInfo]) -> tuple[DeprecationCandidate, ...]: + if len(models) > _PAGE_SIZE: + raise ValueError("Read at most 10 retirement models per page") ordered: Final = sorted( - chain.from_iterable(_candidates(model) for model in (*snapshot.deprecated, *snapshot.imminent)), - key=_identity, + chain.from_iterable(_candidates(model) for model in models if model.status != "upcoming"), + key=lambda candidate: (*_identity(candidate), candidate.model.model_name), ) return tuple(next(group) for _, group in groupby(ordered, _identity)) @@ -119,28 +136,29 @@ def used_deprecation_issues( class DeprecationIssueProjector: def __init__( self, - config: ConfigRepository, + config: BoundedConfigRepository, metrics: UsageAlertMetricsRepository, issues: "IssueStore", - snapshot: Callable[[date], ModelDeprecationResponse], + source: RouterDeprecationSource, ) -> None: self._config = config self._metrics = metrics self._issues = issues - self._snapshot = snapshot + self._source = source @classmethod - def from_prisma( - cls, prisma: "PrismaClient", snapshot: Callable[[date], ModelDeprecationResponse] - ) -> "DeprecationIssueProjector": + def from_prisma(cls, prisma: "PrismaClient", source: RouterDeprecationSource) -> "DeprecationIssueProjector": from litellm.repositories.issues import IssuesRepository - database: Final = cast(MetricsDatabase, prisma.writer_db) # cast-ok: generated Prisma query delegate + database: Final = BoundedAnalyticsDatabase( + cast(AnalyticsDatabase, prisma.writer_db), # cast-ok: generated Prisma transaction delegate + background=True, + ) return cls( - ConfigRepository(prisma, use_writer=True), + BoundedConfigRepository(database), UsageAlertMetricsRepository(database), - IssuesRepository.from_prisma(prisma), - snapshot, + IssuesRepository(database), + source, ) async def _page(self, candidates: tuple[DeprecationCandidate, ...], now: datetime) -> None: @@ -153,23 +171,55 @@ class DeprecationIssueProjector: ) by_model: Final = MappingProxyType({row.model: row for row in usage}) issues: Final = used_deprecation_issues(candidates, by_model, states, now) - await self._issues.upsert_issues(issues) + saved: Final = await self._issues.upsert_issues(issues) await self._issues.add_signal_batch( tuple( IssueSignal( issue_id=issue.id, source="model_deprecation", source_id=issue.id, deprecation=issue.deprecation ) - for issue in issues + for issue in saved ) ) async def run(self, now: datetime) -> None: + deadline: Final = asyncio.get_running_loop().time() + _WORK_BUDGET_SECONDS + try: + with analytics_deadline(deadline): + await asyncio.wait_for(self._run(now), timeout=_WORK_BUDGET_SECONDS) + except (AnalyticsCapacityError, asyncio.TimeoutError): + verbose_proxy_logger.debug("Model retirement scan deferred within its work budget") + + async def _run(self, now: datetime) -> None: hour: Final = now.astimezone(timezone.utc).replace(minute=0, second=0, microsecond=0) previous: Final = await self._config.get_param(_PROJECTION_CHECKPOINT) - if previous is not None and _ProjectionCheckpoint.model_validate(previous.param_value).hour >= hour: + stored: Final = _ProjectionCheckpoint.model_validate(previous.param_value) if previous is not None else None + if stored is not None and stored.complete and stored.hour >= hour: return - candidates: Final = deprecation_candidates(self._snapshot(hour.date())) - for offset in range(0, len(candidates), _PAGE_SIZE): - await self._page(candidates[offset : offset + _PAGE_SIZE], now) - await self._issues.finish_deprecation_projection(now) - await self._config.set_param(_PROJECTION_CHECKPOINT, _ProjectionCheckpoint(hour=hour).model_dump(mode="json")) + state = ( # rebind-ok: durable continuation after each completed page + stored + if stored is not None + and not stored.complete + and stored.source_revision == self._source.revision + and stored.hour.date() == hour.date() + else _ProjectionCheckpoint( + hour=hour, observed_at=now, source_revision=self._source.revision, complete=False + ) + ) + while self._source.revision == state.source_revision: + page = self._source.page(state.hour.date(), state.cursor) + if not page.complete and not page.models and page.cursor == state.cursor: + return + observed_at = state.observed_at or now + candidates = deprecation_candidates(page.models) + for offset in range(0, len(candidates), _PAGE_SIZE): + await self._page(candidates[offset : offset + _PAGE_SIZE], observed_at) + state = state.model_copy(update={"cursor": page.cursor}) + await self._config.set_param(_PROJECTION_CHECKPOINT, state.model_dump(mode="json")) + if self._source.revision != state.source_revision: + return + if page.complete: + await self._issues.finish_deprecation_projection(observed_at) + finished = state.model_copy(update={"hour": hour, "complete": True}) + await self._config.set_param(_PROJECTION_CHECKPOINT, finished.model_dump(mode="json")) + return + await asyncio.sleep(0) diff --git a/litellm/repositories/config_repository.py b/litellm/repositories/config_repository.py index 5cf5dafcce5..653be00ddf9 100644 --- a/litellm/repositories/config_repository.py +++ b/litellm/repositories/config_repository.py @@ -7,6 +7,10 @@ from collections.abc import Mapping, Sequence from datetime import datetime from typing import TYPE_CHECKING, Final, Protocol, cast +from pydantic import BaseModel, TypeAdapter + +from litellm.repositories.analytics_budget import BoundedAnalyticsDatabase + if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient @@ -151,3 +155,37 @@ class ConfigRepository: return await self._config_table.delete_many( where={"param_name": {"startswith": prefix}, "last_run_at": {"lt": before}} ) + + +class _ConfigValue(BaseModel, frozen=True): + param_name: str + param_value: object + + +class BoundedConfigRepository: + def __init__(self, database: BoundedAnalyticsDatabase) -> None: + self._database = database + + async def get_params(self, names: Sequence[str]) -> tuple[_ConfigValue, ...]: + if len(names) > 10: + raise ValueError("Read at most 10 retirement states per page") + rows: Final = await self._database.query_raw( + """SELECT param_name, CASE WHEN jsonb_typeof(param_value)='string' + THEN (param_value #>> '{}')::jsonb ELSE param_value END AS param_value + FROM "LiteLLM_Config" WHERE param_name=ANY($1::text[])""", + tuple(names), + ) + return TypeAdapter(tuple[_ConfigValue, ...]).validate_python(rows) + + async def get_param(self, name: str) -> _ConfigValue | None: + rows: Final = await self.get_params((name,)) + return rows[0] if rows else None + + async def set_param(self, name: str, value: object) -> None: + await self._database.query_raw( + """INSERT INTO "LiteLLM_Config" (param_name,param_value) VALUES($1,$2::jsonb) + ON CONFLICT(param_name) DO UPDATE SET param_value=EXCLUDED.param_value + RETURNING param_name""", + name, + json.dumps(value), + ) diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index 98495034ce9..930ff95dcb4 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -140,7 +140,10 @@ class IssuesRepository: (record->>'observed_at')::timestamptz AT TIME ZONE 'UTC', (record->>'observed_at')::timestamptz AT TIME ZONE 'UTC' FROM incoming ON CONFLICT(id) DO UPDATE SET - record=CASE WHEN old.kind='model_deprecation' AND EXCLUDED.last_seen >= old.last_seen + record=CASE WHEN old.kind='model_deprecation' AND ( + EXCLUDED.last_seen > old.last_seen OR (EXCLUDED.last_seen = old.last_seen AND + EXCLUDED.record->'deprecation'->'model'->>'model_name' < + old.record->'deprecation'->'model'->>'model_name')) THEN EXCLUDED.record ELSE old.record END, last_seen=GREATEST(old.last_seen,EXCLUDED.last_seen), active=CASE WHEN old.kind='model_deprecation' AND EXCLUDED.last_seen >= old.last_seen diff --git a/tests/integration/database/test_deprecation_issue_evaluation.py b/tests/integration/database/test_deprecation_issue_evaluation.py index 789c2358b4d..e7de0784f32 100644 --- a/tests/integration/database/test_deprecation_issue_evaluation.py +++ b/tests/integration/database/test_deprecation_issue_evaluation.py @@ -1,3 +1,5 @@ +import asyncio +from contextlib import suppress from datetime import date, datetime, timedelta, timezone from uuid import uuid4 @@ -5,12 +7,19 @@ import pytest from integration._support.actors import ManagementDatabase from integration._support.client import Gateway -from litellm.proxy.common_utils.model_deprecation import deprecation_alert_state_key -from litellm.proxy.spend_tracking.deprecation_issues import DeprecationIssueProjector +from litellm import Router +from litellm.proxy.common_utils.model_deprecation import RouterDeprecationSource, deprecation_alert_state_key +from litellm.proxy.spend_tracking.deprecation_issues import ( + DeprecationCandidate, + DeprecationIssueProjector, + _ProjectionCheckpoint, + deprecation_issue, +) from litellm.repositories.config_repository import ConfigRepository +from litellm.repositories.issues import IssuesRepository from litellm.repositories.usage_alert_metrics import UsageAlertMetricsRepository from litellm.repositories.usage_alerts import UsageAlertsRepository -from litellm.types.proxy.model_deprecation import ModelDeprecationInfo, ModelDeprecationResponse +from litellm.types.proxy.model_deprecation import DeprecationAlertState, ModelDeprecationInfo, ModelDeprecationUsage @pytest.mark.asyncio(loop_scope="session") @@ -72,30 +81,24 @@ async def test_retirement_issue_http_state_survives_replay_and_reopens_only_for_ gateway: Gateway, db: ManagementDatabase ) -> None: prefix = f"retirement-lifecycle-{uuid4()}" - name = f"example/{prefix}" - now = datetime(2026, 11, 5, 10, microsecond=123456, tzinfo=timezone.utc) + name = f"openai/{prefix}" + now = datetime(2026, 11, 5, 10, tzinfo=timezone.utc) retirement = now.date() + timedelta(days=15) config = ConfigRepository(db.prisma, use_writer=True) previous_checkpoint = await config.get_param("issues:deprecation_projection") lease = UsageAlertsRepository.from_prisma(db.prisma) assert await lease.claim_job(prefix) - def schedule(today: date) -> ModelDeprecationResponse: - info = ModelDeprecationInfo( - model_name=prefix, - litellm_model=name, - deprecation_date=retirement, - days_until_deprecation=(retirement - today).days, - status="deprecated" if retirement < today else "imminent", - successor_model="example/replacement", - ) - return ModelDeprecationResponse( - deprecated=(info,) if retirement < today else (), - imminent=(info,) if retirement >= today else (), - upcoming=(), - warn_within_days=30, - checked_at=now, - ) + router = Router( + model_list=[ + { + "model_name": prefix, + "litellm_params": {"model": name}, + "model_info": {"deprecation_date": retirement.isoformat(), "successor_model": "replacement"}, + } + ] + ) + source = RouterDeprecationSource(lambda: router) async def record(day: date, requests: int) -> None: await db.prisma.writer_db.query_raw( @@ -113,7 +116,7 @@ async def test_retirement_issue_http_state_survives_replay_and_reopens_only_for_ try: await config.delete_param("issues:deprecation_projection") await record(now.date(), 100) - projector = DeprecationIssueProjector.from_prisma(db.prisma, schedule) + projector = DeprecationIssueProjector.from_prisma(db.prisma, source) await projector.run(now) page = gateway.get("/issues", {"scope_type": "model", "scope_id": name}) assert page["total"] == 1 @@ -181,3 +184,135 @@ async def test_retirement_issue_http_state_survives_replay_and_reopens_only_for_ else: await config.set_param("issues:deprecation_projection", previous_checkpoint.param_value) await lease.release_job(prefix) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_retirement_scan_resumes_bounded_pages_without_premature_recovery(db: ManagementDatabase) -> None: + prefix = f"retirement-pages-{uuid4()}" + now = datetime(2026, 11, 5, 10, tzinfo=timezone.utc) + retirement = date(2026, 11, 20) + config = ConfigRepository(db.prisma, use_writer=True) + checkpoint_name = "issues:deprecation_projection" + previous = await config.get_param(checkpoint_name) + lease = UsageAlertsRepository.from_prisma(db.prisma) + assert await lease.claim_job(prefix) + repository = IssuesRepository(db.prisma.writer_db) + old_model = f"openai/{prefix}-old" + old = deprecation_issue( + DeprecationCandidate( + old_model, + ModelDeprecationInfo( + model_name=prefix + "-old", + litellm_model=old_model, + deprecation_date=retirement, + days_until_deprecation=15, + status="imminent", + ), + ), + ModelDeprecationUsage( + model=old_model, requests=1, last_used_date=now.date(), last_day_requests=1, keys=1, users=1, teams=0 + ), + DeprecationAlertState(model=old_model, deprecation_date=retirement), + now - timedelta(hours=1), + ) + router = Router( + model_list=[ + { + "model_name": f"{prefix}-{index}", + "litellm_params": {"model": f"openai/{prefix}-{index}"}, + "model_info": {"id": f"{prefix}-{index}", "deprecation_date": retirement.isoformat()}, + } + for index in range(600) + ] + ) + try: + await config.delete_param(checkpoint_name) + await repository.upsert_issues((old,)) + earlier_alias = old.model_copy( + update={ + "title": "alphabetically first configured alias", + "deprecation": old.deprecation.model_copy( + update={ + "model": old.deprecation.model.model_copy(update={"model_name": "a-" + prefix}), + } + ), + } + ) + await repository.upsert_issues((earlier_alias,)) + await repository.upsert_issues((old,)) + assert (await repository.get_issue(old.id)).title == earlier_alias.title + await db.prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,api_requests,updated_at) + SELECT $1||n,$1,$2,$1,'openai/'||$1||'-'||n,1,CURRENT_TIMESTAMP + FROM generate_series(0,599) n RETURNING id""", + prefix, + now.date().isoformat(), + ) + source = RouterDeprecationSource(lambda: router) + projector = DeprecationIssueProjector.from_prisma(db.prisma, source) + + async def interrupt_after_page( + projector: DeprecationIssueProjector, observed_at: datetime, revision: str + ) -> _ProjectionCheckpoint: + task = asyncio.create_task(projector.run(observed_at)) + try: + + async def checkpoint() -> _ProjectionCheckpoint: + while not task.done(): + value = await config.get_param(checkpoint_name) + if value is not None: + state = _ProjectionCheckpoint.model_validate(value.param_value) + if state.source_revision == revision and 0 < state.cursor.deployment < 600: + return state + await asyncio.sleep(0) + pytest.fail("projector finished before observing a partial scan") + + return await asyncio.wait_for(checkpoint(), timeout=10) + finally: + task.cancel() + with suppress(asyncio.CancelledError): + await task + + partial = await interrupt_after_page(projector, now, source.revision) + assert partial.complete is False + assert (await repository.get_issue(old.id)).active is True + # A different process/router generation cannot trust the same numeric registry version. + next_source = RouterDeprecationSource(lambda: router) + replacement = DeprecationIssueProjector.from_prisma(db.prisma, next_source) + restarted = await interrupt_after_page(replacement, now + timedelta(seconds=10), next_source.revision) + assert restarted.source_revision != partial.source_revision + assert restarted.complete is False + assert (await repository.get_issue(old.id)).active is True + for iteration in range(10): + await replacement.run(now + timedelta(seconds=20 + iteration)) + complete = await config.get_param(checkpoint_name) + if _ProjectionCheckpoint.model_validate(complete.param_value).complete: + break + else: + pytest.fail("retirement scan did not resume to completion") + assert (await repository.get_issue(old.id)).active is False + rows = await db.prisma.writer_db.query_raw( + 'SELECT count(*)::int AS count FROM "LiteLLM_Issue" WHERE scope_id LIKE $1 AND active', + f"openai/{prefix}-%", + ) + assert rows[0]["count"] == 600 + finally: + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_IssueSignal" WHERE issue_id IN (SELECT id FROM "LiteLLM_Issue" ' + "WHERE scope_id LIKE $1) RETURNING issue_id", + f"openai/{prefix}-%", + ) + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_Issue" WHERE scope_id LIKE $1 RETURNING id', + f"openai/{prefix}-%", + ) + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE id LIKE $1 RETURNING id', + prefix + "%", + ) + if previous is None: + await config.delete_param(checkpoint_name) + else: + await config.set_param(checkpoint_name, previous.param_value) + await lease.release_job(prefix) diff --git a/tests/unit/proxy/common_utils/test_model_deprecation.py b/tests/unit/proxy/common_utils/test_model_deprecation.py index a0fe560e6fb..082e5731114 100644 --- a/tests/unit/proxy/common_utils/test_model_deprecation.py +++ b/tests/unit/proxy/common_utils/test_model_deprecation.py @@ -17,6 +17,8 @@ from litellm import Router from litellm._logging import verbose_logger from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry from litellm.proxy.common_utils.model_deprecation import ( + DeprecationScanCursor, + RouterDeprecationSource, _classify, _parse_deprecation_date, collect_model_deprecations, @@ -753,3 +755,72 @@ class TestRecommendationSources: assert info.successor_model == "b-next" assert info.successor_model_source == "model_cost_map" assert info.deprecation_source == "https://provider.example/b" + + +def test_retirement_pages_resume_inside_large_alias_without_losing_configured_successor() -> None: + router = Router( + model_list=[ + { + "model_name": "alias", + "litellm_params": {"model": f"openai/paged-model-{index}"}, + "model_info": { + "id": f"paged-{index}", + "deprecation_date": "2026-11-20", + **({"successor_model": "replacement"} if index == 149 else {}), + }, + } + for index in range(150) + ] + ) + source = RouterDeprecationSource(lambda: router) + today = date(2026, 11, 5) + first = source.page(today, DeprecationScanCursor()) + assert not first.complete and first.models == () + cursor = DeprecationScanCursor.model_validate_json(first.cursor.model_dump_json()) + models = () + for _ in range(20): + page = source.page(today, cursor) + assert len(page.models) <= 10 + models += page.models + cursor = DeprecationScanCursor.model_validate_json(page.cursor.model_dump_json()) + if page.complete: + break + else: + pytest.fail("bounded alias scan did not finish") + expected = collect_model_deprecations(router, today=today).imminent + assert models == tuple(expected) + assert len(models) == 1 + assert models[0].litellm_model == "openai/paged-model-0" + assert models[0].successor_model == "replacement" + + +def test_retirement_source_bounds_pages_and_restarts_identity_after_router_replacement() -> None: + router = Router( + model_list=[ + { + "model_name": f"paged-{index}", + "litellm_params": {"model": f"openai/paged-model-{index}"}, + "model_info": {"id": f"page-{index}", "deprecation_date": "2026-11-20"}, + } + for index in range(25) + ] + ) + source = RouterDeprecationSource(lambda: router) + first = source.page(date(2026, 11, 5), DeprecationScanCursor()) + assert len(first.models) == 10 and not first.complete + second = source.page(date(2026, 11, 5), first.cursor) + assert len(second.models) == 10 and not second.complete + third = source.page(date(2026, 11, 5), second.cursor) + assert len(third.models) == 5 and third.complete + assert len({model.model_name for model in (*first.models, *second.models, *third.models)}) == 25 + revision = source.revision + router = Router(model_list=[]) + assert source.revision != revision + assert source.revision != RouterDeprecationSource(lambda: router).revision + + +def test_unavailable_retirement_router_is_not_a_completed_empty_catalog() -> None: + source = RouterDeprecationSource(lambda: None) + cursor = DeprecationScanCursor() + page = source.page(date(2026, 11, 5), cursor) + assert page.models == () and page.cursor == cursor and not page.complete diff --git a/tests/unit/proxy/spend_tracking/test_deprecation_issues.py b/tests/unit/proxy/spend_tracking/test_deprecation_issues.py index e8f47426d4e..9da14479fee 100644 --- a/tests/unit/proxy/spend_tracking/test_deprecation_issues.py +++ b/tests/unit/proxy/spend_tracking/test_deprecation_issues.py @@ -1,11 +1,12 @@ from datetime import date, datetime, timedelta, timezone +import pytest + from litellm.proxy.spend_tracking.deprecation_issues import deprecation_candidates, used_deprecation_issues from litellm.types.proxy.model_deprecation import ( DeprecationAlertState, DeprecationAlertStatus, ModelDeprecationInfo, - ModelDeprecationResponse, ModelDeprecationUsage, SuccessorModelSource, ) @@ -29,13 +30,7 @@ def model(**updates): def snapshot(*models, upcoming=()): - return ModelDeprecationResponse( - imminent=[entry for entry in models if entry.status == "imminent"], - deprecated=[entry for entry in models if entry.status == "deprecated"], - upcoming=upcoming, - warn_within_days=30, - checked_at=NOW, - ) + return (*models, *upcoming) def usage(name="example/retiring-model", **updates): @@ -126,3 +121,10 @@ def test_maximum_model_identity_is_preserved_in_activity_and_evidence(): assert issue.cohort.scope_id == name assert issue.deprecation.usage_model == name assert issue.title.startswith("public-alias") + + +def test_retirement_candidates_are_bounded_and_preserve_alias_choice() -> None: + candidates = deprecation_candidates((model(model_name="z-alias"), model(model_name="a-alias"))) + assert len(candidates) == 1 and candidates[0].model.model_name == "a-alias" + with pytest.raises(ValueError, match="at most 10"): + deprecation_candidates(tuple(model(model_name=f"alias-{index}") for index in range(11))) From ca81207f51ffc04d560405bb88b1af7055328ac6 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 17:08:50 +0800 Subject: [PATCH 24/44] fix: publish completed entity alerts during partial scans --- .../spend_tracking/performance_issues.py | 22 +++--- .../proxy/spend_tracking/usage_alert_job.py | 2 +- litellm/repositories/issues.py | 10 +-- litellm/repositories/usage_alerts.py | 11 ++- litellm/types/proxy/usage_alerts.py | 1 + .../database/test_performance_issues.py | 70 +++++++++++++++++++ 6 files changed, 99 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/spend_tracking/performance_issues.py b/litellm/proxy/spend_tracking/performance_issues.py index f209122560e..e20b3a2b6ae 100644 --- a/litellm/proxy/spend_tracking/performance_issues.py +++ b/litellm/proxy/spend_tracking/performance_issues.py @@ -224,7 +224,7 @@ class PerformanceIssueProjector: self._provider_rule = rule break position = await rules.projection_cursor(work) # rebind-ok: persisted bounded source cursor - observed_at: Final = now.replace(microsecond=now.microsecond // 1000 * 1000) + observed_at: Final = position.observed_at or now.replace(microsecond=now.microsecond // 1000 * 1000) projection_deadline: Final = deadline - min(2, max(0, deadline - asyncio.get_running_loop().time()) / 4) while asyncio.get_running_loop().time() < projection_deadline: if (next_position := await self._projection_page(position, work, observed_at)) is None: @@ -246,14 +246,14 @@ class PerformanceIssueProjector: if prior: if self._provider_rule is not None: await self._prior_provider_page(prior, work.day, self._provider_rule, observed_at) - return UsageAlertProjectionCursor(version=work.version, after_id=prior[-1].id) - return UsageAlertProjectionCursor(version=work.version, step=1) + return UsageAlertProjectionCursor(version=work.version, after_id=prior[-1].id, observed_at=observed_at) + return UsageAlertProjectionCursor(version=work.version, step=1, observed_at=observed_at) if position.step > len(_SCOPES) * len(_METRICS): identities: Final = await self._store.active_resource_issue_ids(position.after_id) if identities: - await self._store.recover_issues(work.day, completed_only=True, issue_ids=identities) + await self._store.recover_issues(work.day, issue_ids=identities) return UsageAlertProjectionCursor( - version=work.version, step=position.step, after_id=identities[-1] + version=work.version, step=position.step, after_id=identities[-1], observed_at=observed_at ) return None metric: Final = _METRICS[(position.step - 1) // len(_SCOPES)] @@ -263,8 +263,10 @@ class PerformanceIssueProjector: ) if sources: await self._project_page(sources, scope, work.day, observed_at) - return UsageAlertProjectionCursor(version=work.version, step=position.step, after_id=sources[-1].alert.id) - return UsageAlertProjectionCursor(version=work.version, step=position.step + 1) + return UsageAlertProjectionCursor( + version=work.version, step=position.step, after_id=sources[-1].alert.id, observed_at=observed_at + ) + return UsageAlertProjectionCursor(version=work.version, step=position.step + 1, observed_at=observed_at) async def _candidates( self, sources: tuple[IssueProjectionSignal, ...], scope: UsageAlertScope, day: date @@ -683,12 +685,14 @@ class PerformanceIssueProjector: _bindings(candidate, saved[targets[candidate.source.alert.id].issue.cohort_key].id) for candidate in candidates ) - while batch := tuple(islice(bindings, 500)): + while batch := tuple(islice(bindings, 10)): await self._store.queue_deliveries(batch) async def _notify(self, stop_at: float | None = None) -> None: deadline: Final = stop_at if stop_at is not None else asyncio.get_running_loop().time() + 60 - while asyncio.get_running_loop().time() + 2 < deadline and (pending := await self._store.pending_deliveries()): + while asyncio.get_running_loop().time() + 2 < deadline and ( + pending := await self._store.pending_deliveries(limit=10) + ): for item in pending: if asyncio.get_running_loop().time() + 2 >= deadline: return diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index 242603a85a3..a12026237f9 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -436,7 +436,7 @@ class UsageAlertJob: return None async def _notify(self) -> None: - while page := await self._store.pending_notifications(limit=100): + while page := await self._store.pending_notifications(limit=10): for pending in page: await self._notify_incident(pending) diff --git a/litellm/repositories/issues.py b/litellm/repositories/issues.py index 930ff95dcb4..11753fda3f0 100644 --- a/litellm/repositories/issues.py +++ b/litellm/repositories/issues.py @@ -361,9 +361,6 @@ class IssuesRepository: AND ($2::text IS NULL OR a.id>$2) AND ($4::text IS NULL OR a.scope_type=$4) AND ($5::text IS NULL OR a.metric=$5) AND ($6::text[] IS NULL OR a.scope_id=ANY($6::text[])) - AND (NOT $7::boolean OR EXISTS ( - SELECT 1 FROM "LiteLLM_UsageAlertCheckpoint" c WHERE c.completed - AND c.id=r.id||':'||replace(r.record->>'updated_at','Z','+00:00')||':'||$1::text)) AND (NOT $7::boolean OR NOT(r.id=ANY($8::text[])) OR NOT EXISTS ( SELECT 1 FROM "LiteLLM_UsageAlertRule" dependency WHERE dependency.enabled AND NOT dependency.deleted AND dependency.id=ANY($8::text[]) @@ -372,7 +369,12 @@ class IssuesRepository: AND dependency.record->>'direction'=r.record->>'direction') OR (r.record->>'metric'='requests' AND r.record->>'direction'='increase' AND dependency.record->>'metric'='spend' AND dependency.record->>'direction'='increase' - AND dependency.scope_type IN ('key',r.scope_type))) + AND dependency.scope_type IN ('key',r.scope_type) + AND NOT COALESCE(a.scope_type='key' AND dependency.scope_type='key' + AND dependency.record->'evaluation_progress'->>'day'=$1::text + AND (dependency.record->'evaluation_progress'->>'revision')::timestamptz= + (dependency.record->>'updated_at')::timestamptz + AND dependency.record->'evaluation_progress'->>'cursor'>=a.scope_id,false))) AND NOT (dependency.record->>'evaluation_status'='insufficient_coverage' AND dependency.record->>'last_evaluated_day'=$1::text) AND NOT EXISTS (SELECT 1 FROM "LiteLLM_UsageAlertCheckpoint" completed WHERE completed.completed diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index a17da9f8a6c..8ed9f0ab547 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -883,18 +883,23 @@ class UsageAlertsRepository: FROM prior p WHERE i.id=p.prior_id AND (p.evaluation_day>'id',$1,p.scope_key,p.record->>'scope_type',p.record->>'scope_id',p.record->>'metric','open',p.day,p.day,$3,true,p.record FROM prior p WHERE p.breached AND (p.prior_id IS NULL OR (NOT p.prior_breached AND (p.evaluation_day None: + from litellm.types.proxy.usage_alerts import UsageAlertProjectionCursor + + async with _isolated(db) as prisma: + await _seed(prisma) + job: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) + await job._store.ensure_default_rules(job._defaults) + await job._evaluate(_DAY, asyncio.get_running_loop().time() + 10) + work: Final = await job._store.pending_projection_work() + projector: Final = job._performance_projection + assert work is not None and projector is not None + observed_at: Final = _NOW.replace(microsecond=987000) + position = UsageAlertProjectionCursor(version=work.version, observed_at=observed_at) # rebind-ok: cursor + while position.step < 21: + next_position: Final = await projector._projection_page(position, work, observed_at) + assert next_position is not None + position = next_position + await job._store.save_projection_cursor(position) + before: Final = await IssuesRepository.from_prisma(prisma).list_issues() + assert len(before.data) == 1 and before.data[0].family == "spend" + restarted: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) + assert restarted._performance_projection is not None + assert await restarted._performance_projection.run_pending( + _NOW + timedelta(hours=1), work, restarted._store, asyncio.get_running_loop().time() + 10 + ) + after: Final = await IssuesRepository.from_prisma(prisma).list_issues() + assert tuple(item.id for item in after.data) == tuple(item.id for item in before.data) + links: Final = await IssuesRepository.from_prisma(prisma).list_signals(after.data[0].id) + assert any(link.usage_alert is not None and link.usage_alert.metric == "requests" for link in links.data) From 7c8f47e243dc4b4fec56fd17b157bcd903ee679a Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 17:14:49 +0800 Subject: [PATCH 25/44] fix: retain projection progress as new observations arrive --- .../spend_tracking/performance_issues.py | 29 +++++++++++++++---- .../proxy/spend_tracking/usage_alert_job.py | 25 +++++++++++----- litellm/repositories/usage_alerts.py | 9 ++++-- litellm/types/proxy/usage_alerts.py | 1 + .../database/test_performance_issues.py | 17 ++++++++++- 5 files changed, 63 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/spend_tracking/performance_issues.py b/litellm/proxy/spend_tracking/performance_issues.py index e20b3a2b6ae..e3c5d1fb723 100644 --- a/litellm/proxy/spend_tracking/performance_issues.py +++ b/litellm/proxy/spend_tracking/performance_issues.py @@ -228,9 +228,12 @@ class PerformanceIssueProjector: projection_deadline: Final = deadline - min(2, max(0, deadline - asyncio.get_running_loop().time()) / 4) while asyncio.get_running_loop().time() < projection_deadline: if (next_position := await self._projection_page(position, work, observed_at)) is None: - await self._store.complete_projection(work.day, work.version) + await self._store.complete_projection(work.day, position.version) await self._notify(deadline) await self._store.prune() + if position.version != work.version: + await rules.save_projection_cursor(UsageAlertProjectionCursor(version=work.version, day=work.day)) + return False return True position = next_position await rules.save_projection_cursor(position) @@ -246,14 +249,22 @@ class PerformanceIssueProjector: if prior: if self._provider_rule is not None: await self._prior_provider_page(prior, work.day, self._provider_rule, observed_at) - return UsageAlertProjectionCursor(version=work.version, after_id=prior[-1].id, observed_at=observed_at) - return UsageAlertProjectionCursor(version=work.version, step=1, observed_at=observed_at) + return UsageAlertProjectionCursor( + version=position.version, day=work.day, after_id=prior[-1].id, observed_at=observed_at + ) + return UsageAlertProjectionCursor(version=position.version, day=work.day, step=1, observed_at=observed_at) if position.step > len(_SCOPES) * len(_METRICS): + if position.version != work.version: + return None identities: Final = await self._store.active_resource_issue_ids(position.after_id) if identities: await self._store.recover_issues(work.day, issue_ids=identities) return UsageAlertProjectionCursor( - version=work.version, step=position.step, after_id=identities[-1], observed_at=observed_at + version=position.version, + day=work.day, + step=position.step, + after_id=identities[-1], + observed_at=observed_at, ) return None metric: Final = _METRICS[(position.step - 1) // len(_SCOPES)] @@ -264,9 +275,15 @@ class PerformanceIssueProjector: if sources: await self._project_page(sources, scope, work.day, observed_at) return UsageAlertProjectionCursor( - version=work.version, step=position.step, after_id=sources[-1].alert.id, observed_at=observed_at + version=position.version, + day=work.day, + step=position.step, + after_id=sources[-1].alert.id, + observed_at=observed_at, ) - return UsageAlertProjectionCursor(version=work.version, step=position.step + 1, observed_at=observed_at) + return UsageAlertProjectionCursor( + version=position.version, day=work.day, step=position.step + 1, observed_at=observed_at + ) async def _candidates( self, sources: tuple[IssueProjectionSignal, ...], scope: UsageAlertScope, day: date diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index a12026237f9..0baac454f25 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -227,7 +227,7 @@ class UsageAlertJob: if await self._performance_projection.run_pending(now, work, self._store, deadline): await self._store.finish_projection_work(work) else: - await self._notify() + await self._notify(deadline) await self._store.prune() @staticmethod @@ -424,9 +424,14 @@ class UsageAlertJob: await self._store.record_observations(rule, observations.changed) return observations.evaluated - async def _deliver_channel(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> str | None: + async def _deliver_channel( + self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel, deadline: float + ) -> str | None: try: - delivered: Final = await asyncio.wait_for(self._delivery.send(alert, rule, channel), timeout=30) + delivered: Final = await asyncio.wait_for( + self._delivery.send(alert, rule, channel), + timeout=min(30, max(0, deadline - asyncio.get_running_loop().time() - 1)), + ) if not delivered: return "transport_not_configured" await self._store.mark_notified(alert.id, channel) @@ -435,18 +440,22 @@ class UsageAlertJob: return "delivery_failed" return None - async def _notify(self) -> None: - while page := await self._store.pending_notifications(limit=10): + async def _notify(self, deadline: float) -> None: + while asyncio.get_running_loop().time() + 2 < deadline and ( + page := await self._store.pending_notifications(limit=10) + ): for pending in page: - await self._notify_incident(pending) + if asyncio.get_running_loop().time() + 2 >= deadline: + return + await self._notify_incident(pending, deadline) - async def _notify_incident(self, pending: UsageAlertNotification) -> None: + async def _notify_incident(self, pending: UsageAlertNotification, deadline: float) -> None: current: Final = await self._store.get_rule(pending.rule.id) if current is None or not current.enabled or current.updated_at != pending.rule.updated_at: return failures: Final = tuple( [ - await self._deliver_channel(pending.incident, pending.rule, channel) + await self._deliver_channel(pending.incident, pending.rule, channel, deadline) for channel in pending.pending_channels ] ) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index 8ed9f0ab547..5842f4ac3d8 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -438,8 +438,8 @@ class UsageAlertsRepository: current: Final = UsageAlertProjectionCursor.model_validate(rows[0].record) if rows else None return ( current - if current is not None and current.version == work.version - else UsageAlertProjectionCursor(version=work.version) + if current is not None and (current.day == work.day or current.version == work.version) + else UsageAlertProjectionCursor(version=work.version, day=work.day) ) async def save_projection_cursor(self, cursor: UsageAlertProjectionCursor) -> None: @@ -451,7 +451,10 @@ class UsageAlertsRepository: async def finish_projection_work(self, work: UsageAlertProjectionWork) -> None: await self.database.query_raw( - 'UPDATE "LiteLLM_UsageAlertCheckpoint" SET completed=true WHERE id=$1 AND owner=$2 RETURNING id', + """WITH finished AS (UPDATE "LiteLLM_UsageAlertCheckpoint" SET completed=true + WHERE id=$1 AND owner=$2 RETURNING id) + DELETE FROM "LiteLLM_Config" WHERE param_name='usage_issue_projection_cursor' + AND param_value->>'version'=$2 AND EXISTS(SELECT 1 FROM finished) RETURNING param_name""", "performance-dirty:" + work.day.isoformat(), work.version, ) diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py index 6cdfb834b98..4ed69609e1e 100644 --- a/litellm/types/proxy/usage_alerts.py +++ b/litellm/types/proxy/usage_alerts.py @@ -77,6 +77,7 @@ class UsageAlertProjectionWork(BaseModel, frozen=True): class UsageAlertProjectionCursor(BaseModel, frozen=True): version: str + day: date | None = None step: int = Field(default=0, ge=0, le=26) after_id: str | None = None observed_at: datetime | None = None diff --git a/tests/integration/database/test_performance_issues.py b/tests/integration/database/test_performance_issues.py index 00188d2dc6a..47c2a4386a5 100644 --- a/tests/integration/database/test_performance_issues.py +++ b/tests/integration/database/test_performance_issues.py @@ -627,10 +627,25 @@ async def test_projection_resume_keeps_spend_and_traffic_in_one_episode(db: Mana await job._store.save_projection_cursor(position) before: Final = await IssuesRepository.from_prisma(prisma).list_issues() assert len(before.data) == 1 and before.data[0].family == "spend" + added: Final = await job._store.save_rule( + UsageAlertRuleInput( + name="Later policy", scope_type="key", scope_id=_KEY_B, metric="latency", direction="increase" + ), + None, + ) + assert added is not None + await job._scope_page((added,), _DAY, False) + latest: Final = await job._store.pending_projection_work() + assert latest is not None and latest.version != work.version + retained: Final = await job._store.projection_cursor(latest) + assert retained.step == 21 and retained.version == work.version restarted: Final = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) assert restarted._performance_projection is not None + assert not await restarted._performance_projection.run_pending( + _NOW + timedelta(hours=1), latest, restarted._store, asyncio.get_running_loop().time() + 10 + ) assert await restarted._performance_projection.run_pending( - _NOW + timedelta(hours=1), work, restarted._store, asyncio.get_running_loop().time() + 10 + _NOW + timedelta(hours=2), latest, restarted._store, asyncio.get_running_loop().time() + 10 ) after: Final = await IssuesRepository.from_prisma(prisma).list_issues() assert tuple(item.id for item in after.data) == tuple(item.id for item in before.data) From 553a7a934e634a9db60520ab4537fde962cbed4f Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 17:24:08 +0800 Subject: [PATCH 26/44] fix: date alert checkpoint retention from completion --- litellm/repositories/usage_alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index 5842f4ac3d8..dcffa6b3f8d 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -451,7 +451,7 @@ class UsageAlertsRepository: async def finish_projection_work(self, work: UsageAlertProjectionWork) -> None: await self.database.query_raw( - """WITH finished AS (UPDATE "LiteLLM_UsageAlertCheckpoint" SET completed=true + """WITH finished AS (UPDATE "LiteLLM_UsageAlertCheckpoint" SET completed=true,expires_at=CURRENT_TIMESTAMP WHERE id=$1 AND owner=$2 RETURNING id) DELETE FROM "LiteLLM_Config" WHERE param_name='usage_issue_projection_cursor' AND param_value->>'version'=$2 AND EXISTS(SELECT 1 FROM finished) RETURNING param_name""", From 016d4e89de296a65474f8f5dbd8705b8858ba713 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 17:36:45 +0800 Subject: [PATCH 27/44] fix: preserve typed checkpoint reads and retirement precision during integration --- litellm/repositories/usage_alerts.py | 4 ++++ .../integration/database/test_deprecation_issue_evaluation.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/litellm/repositories/usage_alerts.py b/litellm/repositories/usage_alerts.py index dcffa6b3f8d..0cb67c117da 100644 --- a/litellm/repositories/usage_alerts.py +++ b/litellm/repositories/usage_alerts.py @@ -103,6 +103,10 @@ class _ModelIdentity(BaseModel, frozen=True): model: str +class _Identifier(BaseModel, frozen=True): + id: str + + class _Pending(_Record, frozen=True): rule: Mapping[str, object] notified_channels: tuple[UsageAlertChannel, ...] diff --git a/tests/integration/database/test_deprecation_issue_evaluation.py b/tests/integration/database/test_deprecation_issue_evaluation.py index e7de0784f32..6e049a336d3 100644 --- a/tests/integration/database/test_deprecation_issue_evaluation.py +++ b/tests/integration/database/test_deprecation_issue_evaluation.py @@ -82,7 +82,7 @@ async def test_retirement_issue_http_state_survives_replay_and_reopens_only_for_ ) -> None: prefix = f"retirement-lifecycle-{uuid4()}" name = f"openai/{prefix}" - now = datetime(2026, 11, 5, 10, tzinfo=timezone.utc) + now = datetime(2026, 11, 5, 10, microsecond=123456, tzinfo=timezone.utc) retirement = now.date() + timedelta(days=15) config = ConfigRepository(db.prisma, use_writer=True) previous_checkpoint = await config.get_param("issues:deprecation_projection") From 2ee9cfed25bbaeb59007abbb17444198425df4dd Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 17:37:30 +0800 Subject: [PATCH 28/44] docs(alerts): explain bounded checks and catch-up --- docs/features/usage-alerts.md | 46 ++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/docs/features/usage-alerts.md b/docs/features/usage-alerts.md index b7ca3ed4844..2c780b6592e 100644 --- a/docs/features/usage-alerts.md +++ b/docs/features/usage-alerts.md @@ -1,6 +1,6 @@ -# Alerts and daily Performance alerts +# Alerts -The Alerts page combines model retirements with daily Performance alerts. Open an issue to inspect its saved comparison, supporting signals, and Activity for the affected resource or provider/model cohort. Related signals can share one investigation without discarding their individual records. Grouping describes observed relationships; it does not establish a root cause. +The Alerts page combines model retirements with daily Performance alerts. Open an alert to inspect its saved comparison, supporting signals, and Activity for the affected resource or provider/model cohort. Related signals can share one investigation without discarding their individual records. Grouping describes observed relationships; it does not establish a root cause. Performance comparisons cover completed UTC days. Retirement issues show recent model usage and the provider's retirement schedule, including today's usage when present. @@ -20,15 +20,23 @@ Daily Performance defaults cover spend, latency, reliability, and traffic across New resources need history before they can produce a default spike. A zero spend, request, or latency baseline does not establish a spike. A zero error-rate baseline is usable only after sufficient historical traffic has been observed; the percentage-point floor still applies. -Use the family settings to enable or disable a family or adjust its thresholds across the five scopes. Existing custom rules remain independent. An enabled custom rule replaces the matching default for its scope, metric, and direction. A custom wildcard rule replaces that default for every entity. Deleted defaults stay deleted. +Open **Alerts → Settings** to enable or disable a family or adjust its thresholds across the five scopes. Only enabled custom rules managed by a gateway administrator replace matching defaults for their scope, metric, and direction. An administrator-managed wildcard rule replaces that default for every entity. Rules created by scoped non-admins add alerts without suppressing defaults. Deleted defaults stay deleted. -The scheduler wakes hourly and evaluates the latest completed UTC day after a one-hour grace period. A successful aggregate flush after the grace period is required before either direction is finalized. Completed rule versions are skipped on later ticks. The evaluator reads existing daily aggregates in pages of 500 entities; it adds no request-time counters or per-request database writes. +## When checks run + +An initial check is scheduled shortly after startup, followed by hourly wakeups. The evaluator checks the latest completed UTC day after a one-hour grace period. A successful aggregate flush after the grace period is required before either direction is finalized. Completed rule versions are skipped on later wakeups. These are daily comparisons, not rolling hourly alerts. + +Each run has a shared 60-second work budget, including up to five seconds for model-retirement checks. Performance evaluation reads existing daily aggregates in pages of 10 entities, with a short pause between pages. Proxy replicas share a database lease for this job. It adds no request-time counters or per-request database writes. + +Completed pages save a checkpoint. When time or database capacity runs out, unfinished work resumes from its saved position on a later wakeup. A page interrupted before its checkpoint can be repeated safely. This is best-effort progress: a large or busy deployment can need several hourly wakeups, and repeated query timeouts can prevent a page from advancing. The hourly schedule does not guarantee that every resource or notification finishes within an hour. + +Completed key alerts can appear while other resources are still being checked or related signals are being grouped. **Monitoring** shows **Catching up** for unfinished work and explains that it resumes automatically. Rule status `delayed` means work remains; `insufficient_history` means a completed comparison lacked usable baseline data, while `insufficient_coverage` means the required usage-data coverage is missing. A partial pass is not reported as a completed scan. Decrease rules need a successful background aggregate-flush sample for every hour of the target day and another sample after the grace period. Until that coverage exists, their status is `insufficient_coverage`. A key with no requests can then produce a request or spend drop; missing latency samples never become zero latency. Coverage shows that an aggregate writer was healthy each hour, not that every replica delivered every request. The baseline averages recorded reference days; it does not turn missing reference days into zero. Average latency is weighted by timed request count, using the same successful-request duration totals as Activity. `baseline_sample_days` records how many reference days contributed. Cost, traffic, and latency comparisons require a positive baseline. Reliability can compare against a recorded 0% error rate when historical and current request volume meet the rule's minimum. Missing history and low-volume comparisons remain `insufficient_history`. -## How related signals form one issue +## How related signals form one alert A key with unusually high spend gets a spend issue, including when it makes only a few expensive requests. Its comparison includes request volume so you can distinguish more traffic from more expensive traffic. A traffic-increase signal joins that spend issue when both increases use the same reference window and their ratios are within 20% of each other. @@ -36,7 +44,7 @@ Default user, team, model, and gateway signals can join a key issue when they de Provider latency grouping requires measured deterioration within established key/model traffic: at least two keys per model, two models, and 80% coverage of current timed requests. A change in the mix of fast and slow models alone is insufficient. A source joins only when at least 80% of its timed traffic independently meets the shared pattern; a source with more than twice the provider cohort's latency ratio stays separate. The issue keeps its original provider/model cohort until recovery. These relationships are evidence for a shared investigation, not proof of a provider outage or root cause. Custom rules retain their configured resource scope. -## Read and investigate issues +## Read and investigate alerts Set `LITELLM_API_BASE` to your gateway's base URL and `LITELLM_MANAGEMENT_KEY` to a management-capable credential. Keep the credential in your environment. @@ -86,7 +94,7 @@ The issue-detail response includes `can_update_status` and `can_view_activity` f ## Configure default families -`GET /issues/families` returns the four Performance families, their enabled counts, and common policy values. A null common value means the underlying scopes have different settings. This is separate from custom rules and the model-retirement warning window. +`GET /issues/families` returns the four Performance families, their enabled counts, common policy values, and `evaluation_status_counts` for enabled rules. A null common value means the underlying scopes have different settings. The top-level `projection_pending` flag indicates that alert grouping has unfinished work, even if no rule is currently `delayed`. This is separate from custom rules and the model-retirement warning window. This example sets the spend family to a 5× threshold and a $25 minimum absolute increase: @@ -156,7 +164,7 @@ curl "$LITELLM_API_BASE/usage/alert-rules" \ }' ``` -The baseline is the mean across recorded days in the preceding window. Missing reference days are unknown. At least `min_history_days` recorded days are required. A decrease uses `baseline / multiplier`; a multiplier of 2 detects a value at or below half the baseline. `min_value` is the minimum absolute difference in the metric's own units. +The baseline is the mean across recorded days in the preceding window. Missing reference days are unknown. At least `min_history_days` recorded days are required. A decrease uses `baseline / multiplier`; a multiplier of 2 detects a value at most half the baseline. `min_value` is the minimum absolute difference in the metric's own units. | Metric | Unit | | --- | --- | @@ -168,7 +176,7 @@ The baseline is the mean across recorded days in the preceding window. Missing r Counts come from the existing Activity/spend aggregates, rather than every gateway HTTP outcome. Latency averages successful timed, non-internal requests; it is not time to first token and does not include failed requests. Issue Activity and gateway source incidents use the same aggregates. An error-rate `min_value` of 2 means two percentage points, such as 1% to 3%, not a 2% relative increase. -The evaluator runs against daily aggregates. An hourly scheduler does not make these rolling hourly alerts. A complete UTC day and sufficient ingestion coverage are required before detecting drops; a missing flush must not become a false traffic-drop incident. Rule responses expose `last_evaluated_day` and `evaluation_status` so you can distinguish a completed evaluation from insufficient history or coverage. +The evaluator runs against daily aggregates. An hourly scheduler does not make these rolling hourly alerts. A complete UTC day and sufficient ingestion coverage are required before detecting drops; a missing flush must not become a false traffic-drop incident. Rule responses expose `last_evaluated_day` and `evaluation_status` so you can distinguish a completed evaluation from delayed work, insufficient history, or missing coverage. Use `scope_type: "deployment"` with no `scope_id` for the gateway. For `model`, use the underlying model identity recorded in Activity. For `key`, use its reporting ID, never the secret API key. Omitting `scope_id` for key, user, team, or model evaluates all entities of that type and requires a gateway administrator. @@ -183,12 +191,26 @@ Use `scope_type: "deployment"` with no `scope_id` for the gateway. For `model`, Consecutive breached days keep one incident. Its original day, comparison values, and Activity link stay fixed; `last_seen` advances. Re-evaluating the same day does not reopen a dismissed or resolved incident. After recovery, a later breach starts a new incident. Editing a rule does not rewrite historical comparisons. -Rules can specify `notification_channels: ["slack", "email"]` and `email_recipients: ["alerts@example.com"]`. Notification transports must already be configured on the gateway. An empty channel list keeps incidents available through the API and dashboard. Issue delivery is recorded per channel and destination. A repeated projection does not resend a successful delivery. Failed deliveries retry in the background, and the worker rechecks the issue, source signal, and current rule immediately before transport. Disabling a rule or acknowledging an issue cannot recall a send already in flight. Delivery is at least once: an ambiguous transport timeout, or partial success across several configured Slack webhooks, can cause a retry after a destination accepted a message. Configure `PROXY_BASE_URL` for clickable links in email and Slack. Default rules notify through Slack only when `usage_alerts` is explicitly enabled in the gateway's alert types when those rules are first registered. Otherwise they start with in-app/API incidents. To enable notifications after the defaults already exist, edit their channels through the API or dashboard; changing the gateway alert types does not rewrite saved rules. Explicit rule channels use the configured transport without an additional alert-type switch. If legacy `user_spend_anomalies` notifications are enabled, the default daily user-spend increase rule stays in-app to avoid overlapping notifications. Email requires recipients on the rule. +Rules can specify `notification_channels: ["slack", "email"]` and `email_recipients: ["alerts@example.com"]`. Notification transports must already be configured on the gateway. An empty channel list keeps incidents available through the API and dashboard. + +Issue delivery is recorded per channel and destination. A repeated projection does not resend a successful delivery. Failed or unconfigured deliveries become eligible for retry after one hour and are retried on a later background pass with available capacity. Delivery pages contain at most 10 entries, and each transport attempt is limited by the remaining run budget, up to 30 seconds. The worker rechecks the issue, source signal, and current rule immediately before transport. Disabling a rule or acknowledging an issue cannot recall a send already in flight. Delivery is at least once: an ambiguous transport timeout, or partial success across several configured Slack webhooks, can cause a retry after a destination accepted a message. + +Configure `PROXY_BASE_URL` for clickable links in email and Slack. Default rules notify through Slack only when `usage_alerts` is explicitly enabled in the gateway's alert types when those rules are first registered. Otherwise they start with in-app/API incidents. To enable notifications after the defaults already exist, edit their channels through the API or dashboard; changing the gateway alert types does not rewrite saved rules. Explicit rule channels use the configured transport without an additional alert-type switch. + +If legacy `user_spend_anomalies` notifications are enabled, the default daily user-spend increase rule stays in-app to avoid overlapping notifications. Email requires recipients on the rule. + +## Resource limits and delayed work + +Alert investigations and resource-page Activity tabs limit returned rows before loading results into the application. For a key, these views show exact totals and a bounded model breakdown: 20 named models by default, at most 100, plus a separate **Other** aggregate for the remaining models. Activity ranges span at most 366 days. A small result page can still require reading many aggregate rows; page size is not a database scan limit. + +Guarded queries allow at most two foreground queries and one background query at a time, coordinated across workers sharing the database. They do not build an unbounded waiting queue. Each query has a statement timeout of at most five seconds, a lock wait of at most 100 ms, and a 1 MiB PostgreSQL `work_mem` setting, with stricter existing limits preserved. Background query deadlines also shrink to the remaining run budget. PostgreSQL `work_mem` applies per operation, not to total application or database memory. + +When an Activity query is busy or exceeds its budget, the API returns `503` with `Retry-After: 5`. Background work keeps its checkpoint and tries again on a later wakeup. These limits bound work per attempt; database size, query plans, existing gateway memory, and concurrent serving traffic still affect overhead and completion time. Persistent **Catching up** means monitoring has not finished, not that the remaining resources are healthy. ## Access -Gateway administrators can manage every scope. Admin viewers can read every scope. Internal users can manage their own user and key scopes; team administrators can manage their team's scope and keys. Team members can read their team's incidents. Read-only users cannot change rules or incident state, and inference-only keys cannot access these endpoints. +Gateway administrators can manage every scope. Admin viewers can read every scope. Internal users can manage their own user and key scopes; team administrators can manage their team's scope and keys. Scoped non-admins can create additive rules without notification delivery. Only gateway administrators can configure notification destinations or change operator-managed rules. Team members can read their team's incidents. Team-wide Activity requires team-administrator access or the existing team daily-activity permission. Read-only users cannot change rules or incident state, and inference-only keys cannot access these endpoints. Read-only `info_routes` grants do not authorize mutations. -Non-admin list requests must specify a scope they can access, for example `?scope_type=user&scope_id=user-example`. A `403` indicates that the credential or scope lacks access. A `503` can indicate an unavailable database or exhausted analytics capacity; a `Retry-After` header indicates that the client should retry later. +Non-admin list requests must specify a scope they can access, for example `?scope_type=user&scope_id=user-example`. A `403` indicates that the credential or scope lacks access. A `503` can indicate a missing database connection or unavailable analytics capacity; capacity responses include `Retry-After`. The dashboard's Alerts page is available to administrators and admin viewers. Other authorized roles can use the scoped API. From 461204b564c700ac846c7adcca6405ac34a06943 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 17:43:35 +0800 Subject: [PATCH 29/44] fix: tighten timeout assertions and refresh alert status types --- .../proxy/common_utils/model_deprecation.py | 4 +-- .../proxy/spend_tracking/usage_alert_job.py | 11 ++++--- .../SlackAlerting/test_usage_alerts.py | 3 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 30 ++++++++++++++++++- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/common_utils/model_deprecation.py b/litellm/proxy/common_utils/model_deprecation.py index 12310b95f87..72ff564dbd4 100644 --- a/litellm/proxy/common_utils/model_deprecation.py +++ b/litellm/proxy/common_utils/model_deprecation.py @@ -8,11 +8,11 @@ from dataclasses import dataclass from datetime import date, datetime, timezone from itertools import groupby from types import MappingProxyType -from typing import ( # noqa: TID251 # router exposes its live validated deployment list +from typing import ( TYPE_CHECKING, Final, TypeAlias, - cast, + cast, # noqa: TID251 # router exposes its live validated deployment list ) from uuid import uuid4 diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index 0baac454f25..47c601f63ed 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -221,13 +221,12 @@ class UsageAlertJob: await self._evaluate( target, asyncio.get_running_loop().time() + max(0, deadline - asyncio.get_running_loop().time()) * 0.65 ) - if self._performance_projection is not None: - if work := await self._store.pending_projection_work(): - # Source eligibility is per completed rule/day, never a global all-rules gate. - if await self._performance_projection.run_pending(now, work, self._store, deadline): - await self._store.finish_projection_work(work) - else: + if self._performance_projection is None: await self._notify(deadline) + elif work := await self._store.pending_projection_work(): + # Source eligibility is per completed rule/day, never a global all-rules gate. + if await self._performance_projection.run_pending(now, work, self._store, deadline): + await self._store.finish_projection_work(work) await self._store.prune() @staticmethod diff --git a/tests/unit/integrations/SlackAlerting/test_usage_alerts.py b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py index 278eb1bcb65..540fd7aa5a9 100644 --- a/tests/unit/integrations/SlackAlerting/test_usage_alerts.py +++ b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py @@ -1,3 +1,4 @@ +import smtplib from datetime import date, datetime, timezone from typing import Final @@ -206,7 +207,7 @@ async def test_cancelled_smtp_attempt_retains_only_one_send(monkeypatch: pytest. await delivery.send(_ALERT, rule, "email") assert UsageAlertDelivery._email_task is pending assert len(accepted) == 1 and threading.active_count() == threads - with pytest.raises(Exception): + with pytest.raises(smtplib.SMTPServerDisconnected, match="timed out"): await asyncio.wait_for(asyncio.shield(pending), timeout=3) assert pending.done() finally: diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 13df0783936..d7e629b3a84 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -51061,6 +51061,11 @@ export interface components { * @default 0 */ complete: number; + /** + * Delayed + * @default 0 + */ + delayed: number; /** * Insufficient Coverage * @default 0 @@ -51191,6 +51196,29 @@ export interface components { /** Scope Id */ scope_id: string; }; + /** + * UsageAlertProgress + * @description Internal durable position; never part of rule input or public output. + */ + UsageAlertProgress: { + /** Cursor */ + cursor?: string | null; + /** + * Day + * Format: date + */ + day: string; + /** + * Evaluated + * @default false + */ + evaluated: boolean; + /** + * Revision + * Format: date-time + */ + revision: string; + }; /** UsageAlertRule */ UsageAlertRule: { /** @@ -51225,7 +51253,7 @@ export interface components { * @default pending * @enum {string} */ - evaluation_status: "pending" | "complete" | "insufficient_coverage" | "insufficient_history"; + evaluation_status: "pending" | "complete" | "insufficient_coverage" | "insufficient_history" | "delayed"; /** Id */ id: string; /** From c98264f459f41424daccba1bdb71daa980ef5546 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 17:54:58 +0800 Subject: [PATCH 30/44] fix: simplify optional alert projection branch --- litellm/proxy/spend_tracking/usage_alert_job.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index 47c601f63ed..6d3e9eaf035 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -223,10 +223,10 @@ class UsageAlertJob: ) if self._performance_projection is None: await self._notify(deadline) - elif work := await self._store.pending_projection_work(): - # Source eligibility is per completed rule/day, never a global all-rules gate. - if await self._performance_projection.run_pending(now, work, self._store, deadline): - await self._store.finish_projection_work(work) + elif ( + work := await self._store.pending_projection_work() + ) is not None and await self._performance_projection.run_pending(now, work, self._store, deadline): + await self._store.finish_projection_work(work) await self._store.prune() @staticmethod From f135edba8b911a6ba5f744ef6bc2672b0c3f8c02 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 5 Nov 2026 17:46:42 +0800 Subject: [PATCH 31/44] perf: skip exhausted alert evaluation phases --- .../proxy/spend_tracking/usage_alert_job.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index 6d3e9eaf035..7e28a893275 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -237,13 +237,18 @@ class UsageAlertJob: async def _evaluate(self, target: date, deadline: float) -> None: cursor = await self._store.work_cursor() # rebind-ok: durable round-robin position - idle_phases = 0 # rebind-ok: stop once every phase has no remaining usable work + exhausted: Final[set[int]] = set() # mutable-ok: at most the 10 scope/config phases in this invocation phase_progress = False # rebind-ok: whether a page advanced in the current phase - while asyncio.get_running_loop().time() < deadline and idle_phases < 10: + while asyncio.get_running_loop().time() < deadline and len(exhausted) < 10: + if cursor.phase in exhausted: + cursor = UsageAlertWorkCursor(phase=(cursor.phase + 1) % 10) + continue + phase: Final = cursor.phase cursor, advanced, finished = await self._evaluation_page(cursor, target) phase_progress = phase_progress or advanced if finished: - idle_phases = 0 if phase_progress else idle_phases + 1 + if not phase_progress: + exhausted.add(phase) phase_progress = False await self._store.save_work_cursor(cursor) await asyncio.sleep(0.01) @@ -253,9 +258,8 @@ class UsageAlertJob: ) -> tuple[UsageAlertWorkCursor, bool, bool]: scope: Final = USAGE_ALERT_SCOPES[cursor.phase // 2] wildcard: Final = cursor.phase % 2 == 0 - rules: Final = await self._store.enabled_rule_page( - scope, wildcard, cursor.after_id, limit=8 if wildcard else 10 - ) + limit: Final = 8 if wildcard else 10 + rules: Final = await self._store.enabled_rule_page(scope, wildcard, cursor.after_id, limit=limit) if not rules: return UsageAlertWorkCursor(phase=(cursor.phase + 1) % 10), False, True # Keep one date/cursor per page. A newly edited rule cannot rewind another rule. @@ -275,6 +279,8 @@ class UsageAlertJob: complete: Final = await self._store.completed_rule_ids(selected, position.day) due: Final = tuple(rule for rule in selected if rule.id not in complete) advanced: Final = bool(due) and await self._attempt_scope_page(due, position.day, wildcard) + if len(rules) < limit and len(selected) == len(rules): + return UsageAlertWorkCursor(phase=(cursor.phase + 1) % 10), advanced, True return UsageAlertWorkCursor(phase=cursor.phase, after_id=selected[-1].id), advanced, False async def _attempt_scope_page(self, rules: tuple[UsageAlertRule, ...], target: date, wildcard: bool) -> bool: From 415ec98d4cb8019b04d5f6265d4372036809181e Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 09:13:01 +0800 Subject: [PATCH 32/44] fix: keep alert rule and scheduler types consistent --- litellm/proxy/spend_tracking/usage_alert_job.py | 2 +- litellm/types/proxy/usage_alerts.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index 7e28a893275..7c1e40bad82 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -243,7 +243,7 @@ class UsageAlertJob: if cursor.phase in exhausted: cursor = UsageAlertWorkCursor(phase=(cursor.phase + 1) % 10) continue - phase: Final = cursor.phase + phase = cursor.phase # rebind-ok: current round-robin phase cursor, advanced, finished = await self._evaluation_page(cursor, target) phase_progress = phase_progress or advanced if finished: diff --git a/litellm/types/proxy/usage_alerts.py b/litellm/types/proxy/usage_alerts.py index 4ed69609e1e..9bdeb5c2984 100644 --- a/litellm/types/proxy/usage_alerts.py +++ b/litellm/types/proxy/usage_alerts.py @@ -93,7 +93,6 @@ class UsageAlertRule(UsageAlertRuleInput): evaluation_status: UsageAlertEvaluationStatus = "pending" is_default: bool = False evaluation_progress: UsageAlertProgress | None = Field(default=None, exclude=True) - overrides_defaults: bool = Field(default=False, exclude=True) class UsageAlert(BaseModel): From df942240ddcc5f10e20c3f425d08450fb69e6402 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 09:16:53 +0800 Subject: [PATCH 33/44] fix(alerts): name batched analytics database spans --- litellm/integrations/otel/model/spans.py | 1 + litellm/repositories/usage_alert_metrics.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index 0f98a8fce0e..001a18a2043 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -324,6 +324,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset( "LiteLLM_DailyGatewayRequests", "LiteLLM_DailyGatewayFailedRequests", "LiteLLM_DailyRequestErrors", + "LiteLLM_UsageAlertFlush", "LiteLLM_PromptTable", "LiteLLM_HealthCheckTable", "LiteLLM_SearchToolsTable", diff --git a/litellm/repositories/usage_alert_metrics.py b/litellm/repositories/usage_alert_metrics.py index 09dec617b24..847c8fe0170 100644 --- a/litellm/repositories/usage_alert_metrics.py +++ b/litellm/repositories/usage_alert_metrics.py @@ -388,12 +388,19 @@ class UsageAlertMetricsRepository: async def batches( self, scope: Scope, target: date, baseline_days: int ) -> AsyncIterator[tuple[DailyAlertMetrics, ...]]: + from litellm.proxy.db.db_span import db_span + from litellm.proxy.db.prisma_query_span import sql_relation + start: Final = (target - timedelta(days=baseline_days)).isoformat() query: Final = _query(scope) cursor: str | None = None # rebind-ok: keyset pagination retains only the current page - while rows := _ROWS.validate_python( - await self._database.query_raw(query, start, target.isoformat(), cursor, _BATCH_SIZE) - ): + while True: + async with db_span("daily_activity_query", sql_relation(query)): + rows = _ROWS.validate_python( # rebind-ok: only the current bounded page is retained + await self._database.query_raw(query, start, target.isoformat(), cursor, _BATCH_SIZE) + ) + if not rows: + return yield tuple(_reporting_id(scope, row) for row in rows) cursor = rows[-1].entity_id if scope == "deployment": From b2230fb13b47261d24fa66bd0f3a81ce38c7d653 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 10:43:03 +0800 Subject: [PATCH 34/44] fix(alerts): trace bounded metric page reads --- litellm/repositories/usage_alert_metrics.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/litellm/repositories/usage_alert_metrics.py b/litellm/repositories/usage_alert_metrics.py index 847c8fe0170..2b9e0be7d66 100644 --- a/litellm/repositories/usage_alert_metrics.py +++ b/litellm/repositories/usage_alert_metrics.py @@ -352,6 +352,9 @@ class UsageAlertMetricsRepository: cursor: str | None = None, entity_ids: tuple[str, ...] | None = None, ) -> DailyMetricsPage: + from litellm.proxy.db.db_span import db_span + from litellm.proxy.db.prisma_query_span import sql_relation + if entity_ids is not None and len(entity_ids) > _BATCH_SIZE: raise ValueError("Read at most 10 exact entities") if entity_ids == (): @@ -368,8 +371,8 @@ class UsageAlertMetricsRepository: + "WITH entities AS MATERIALIZED (SELECT unnest($3::text[]) AS entity_id)" + query[query.index("\nSELECT entities") :] ) - raw: Final = _ROWS.validate_python( - await self._database.query_raw( + async with db_span("daily_activity_query", sql_relation(query)): + result: Final = await self._database.query_raw( exact_query if entity_ids is not None else query, *( arguments[:2] @@ -378,7 +381,7 @@ class UsageAlertMetricsRepository: else arguments ), ) - ) + raw: Final = _ROWS.validate_python(result) return DailyMetricsPage( tuple(_reporting_id(scope, row) for row in raw), raw[-1].entity_id if raw else cursor, From 6f28dc4c6af6867810c131d2f41e831c5efa7ce9 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 10:44:32 +0800 Subject: [PATCH 35/44] fix(alerts): report the retirement batch limit accurately --- litellm/repositories/usage_alert_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/repositories/usage_alert_metrics.py b/litellm/repositories/usage_alert_metrics.py index 2b9e0be7d66..a33692b5b15 100644 --- a/litellm/repositories/usage_alert_metrics.py +++ b/litellm/repositories/usage_alert_metrics.py @@ -338,7 +338,7 @@ class UsageAlertMetricsRepository: if not models: return () if len(models) > _BATCH_SIZE or not 0 <= (end - start).days < 30: - raise ValueError("Deprecation usage accepts at most 500 models and 30 consecutive dates") + raise ValueError("Deprecation usage accepts at most 10 models and 30 consecutive dates") return _DEPRECATION_USAGE.validate_python( await self._database.query_raw(_DEPRECATION_USAGE_QUERY, start.isoformat(), end.isoformat(), models) ) From dd31c70908cf38b6fe96b344c47043f3b241c761 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 14:22:03 +0800 Subject: [PATCH 36/44] fix(alerts): annotate bounded loop values without stale suppressions --- litellm/proxy/spend_tracking/usage_alert_job.py | 2 +- litellm/repositories/usage_alert_metrics.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index 7c1e40bad82..f46603caa6e 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -243,7 +243,7 @@ class UsageAlertJob: if cursor.phase in exhausted: cursor = UsageAlertWorkCursor(phase=(cursor.phase + 1) % 10) continue - phase = cursor.phase # rebind-ok: current round-robin phase + phase: int = cursor.phase cursor, advanced, finished = await self._evaluation_page(cursor, target) phase_progress = phase_progress or advanced if finished: diff --git a/litellm/repositories/usage_alert_metrics.py b/litellm/repositories/usage_alert_metrics.py index a33692b5b15..d2a9af41913 100644 --- a/litellm/repositories/usage_alert_metrics.py +++ b/litellm/repositories/usage_alert_metrics.py @@ -399,7 +399,7 @@ class UsageAlertMetricsRepository: cursor: str | None = None # rebind-ok: keyset pagination retains only the current page while True: async with db_span("daily_activity_query", sql_relation(query)): - rows = _ROWS.validate_python( # rebind-ok: only the current bounded page is retained + rows: tuple[DailyAlertMetrics, ...] = _ROWS.validate_python( await self._database.query_raw(query, start, target.isoformat(), cursor, _BATCH_SIZE) ) if not rows: From 860e8fb22828cc3b4791a7e26d62b02d47846b5b Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 15:50:02 +0800 Subject: [PATCH 37/44] fix(alerts): bound analytics client waits and transaction cleanup --- litellm/repositories/analytics_budget.py | 33 ++++-- .../repositories/test_analytics_budget.py | 101 ++++++++++++++++++ 2 files changed, 128 insertions(+), 6 deletions(-) create mode 100644 tests/unit/repositories/test_analytics_budget.py diff --git a/litellm/repositories/analytics_budget.py b/litellm/repositories/analytics_budget.py index 042bff3fe42..d5e02e655a4 100644 --- a/litellm/repositories/analytics_budget.py +++ b/litellm/repositories/analytics_budget.py @@ -1,6 +1,6 @@ import asyncio -from collections.abc import Generator, Mapping, Sequence -from contextlib import AbstractAsyncContextManager, contextmanager +from collections.abc import AsyncGenerator, Generator, Mapping, Sequence +from contextlib import AbstractAsyncContextManager, asynccontextmanager, contextmanager from contextvars import ContextVar from datetime import timedelta from threading import BoundedSemaphore @@ -75,6 +75,23 @@ def _milliseconds(limit: int) -> int: return limit if remaining is None else min(limit, max(1, int(remaining * 1000))) +@asynccontextmanager +async def _bounded_transaction( + manager: AbstractAsyncContextManager[AnalyticsTransaction], +) -> AsyncGenerator[AnalyticsTransaction, None]: + transaction: Final = await manager.__aenter__() + try: + yield transaction + except BaseException as error: + suppressed: Final = await asyncio.wait_for( + manager.__aexit__(type(error), error, error.__traceback__), timeout=1 + ) + if not suppressed: + raise + else: + await asyncio.wait_for(manager.__aexit__(None, None, None), timeout=1) + + class BoundedAnalyticsDatabase: def __init__(self, database: AnalyticsDatabase, *, background: bool = False) -> None: self._database = database @@ -86,7 +103,9 @@ class BoundedAnalyticsDatabase: raise AnalyticsCapacityError("Analytics capacity is busy. Try again shortly") try: remaining: Final = _remaining_seconds() - return await asyncio.wait_for(self._query(query, args), timeout=remaining) + return await asyncio.wait_for( + self._query(query, args), timeout=6.25 if remaining is None else min(6.25, remaining) + ) except RawQueryError as error: if _PostgresError.model_validate(error.meta or {}).code not in ("57014", "55P03"): raise @@ -101,9 +120,11 @@ class BoundedAnalyticsDatabase: self._admission.release() async def _query(self, query: str, args: tuple[object, ...]) -> Sequence[Mapping[str, object]]: - async with self._database.tx( - max_wait=timedelta(milliseconds=_milliseconds(250)), - timeout=timedelta(milliseconds=_milliseconds(6000)), + async with _bounded_transaction( + self._database.tx( + max_wait=timedelta(milliseconds=_milliseconds(250)), + timeout=timedelta(milliseconds=_milliseconds(6000)), + ) ) as transaction: configuration: Final = await transaction.query_raw(_CONFIGURE, self._background, _milliseconds(5000)) if not configuration or not _Admission.model_validate(configuration[0]).acquired: diff --git a/tests/unit/repositories/test_analytics_budget.py b/tests/unit/repositories/test_analytics_budget.py new file mode 100644 index 00000000000..befa645c4b6 --- /dev/null +++ b/tests/unit/repositories/test_analytics_budget.py @@ -0,0 +1,101 @@ +import asyncio +from collections import deque +from collections.abc import AsyncGenerator, Mapping, Sequence +from contextlib import asynccontextmanager, nullcontext +from datetime import timedelta +from typing import Final, Literal + +import pytest + +from litellm.repositories.analytics_budget import ( + AnalyticsCapacityError, + AnalyticsTransaction, + BoundedAnalyticsDatabase, + analytics_deadline, +) + + +class _ClockedLoop(asyncio.SelectorEventLoop): + _ready: deque[asyncio.Handle] + _scheduled: list[asyncio.TimerHandle] + + def __init__(self) -> None: + super().__init__() + self.now = 10000.0 + + def time(self) -> float: + return self.now + + def _run_once(self) -> None: + if not self._ready and self._scheduled: + self.now = max(self.now, self._scheduled[0].when()) + super()._run_once() # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue] # CPython loop hook lacks stubs + + +class _Driver: + def __init__(self, stage: Literal["begin", "query", "commit", "rollback", "healthy"]) -> None: + self.stage = stage + self.cancelled = asyncio.Event() + + async def _stall(self) -> None: + try: + await asyncio.Event().wait() + finally: + self.cancelled.set() + + @asynccontextmanager + async def tx(self, *, max_wait: timedelta, timeout: timedelta) -> AsyncGenerator[AnalyticsTransaction]: + if self.stage == "begin": + await self._stall() + try: + yield self + finally: + if self.stage in ("commit", "rollback"): + await self._stall() + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + if query == "SELECT 1 AS result": + if self.stage in ("query", "rollback"): + await self._stall() + return ({"result": 1},) + return ({"acquired": True},) + + +@pytest.mark.parametrize("stage", ("begin", "query", "commit", "rollback")) +@pytest.mark.parametrize("deadline_seconds", (None, 0.25, 10.0)) +def test_stalled_driver_is_cancelled_without_retaining_admission_or_tasks( + stage: Literal["begin", "query", "commit", "rollback"], deadline_seconds: float | None +) -> None: + async def scenario() -> None: + loop: Final = asyncio.get_running_loop() + existing: Final = asyncio.all_tasks() + driver: Final = _Driver(stage) + start: Final = loop.time() + context: Final = nullcontext() if deadline_seconds is None else analytics_deadline(start + deadline_seconds) + with context: + task: Final = asyncio.create_task(BoundedAnalyticsDatabase(driver).query_raw("SELECT 1 AS result")) + done, _ = await asyncio.wait((task,), timeout=10) + if not done: + task.cancel() + watchdog: Final = loop.call_later(1, task.cancel) + try: + with pytest.raises(asyncio.CancelledError): + await task + finally: + watchdog.cancel() + assert done, "Analytics retained its task after the query and cleanup budget" + with pytest.raises(AnalyticsCapacityError): + await task + assert driver.cancelled.is_set() + assert loop.time() - start <= (7.25 if deadline_seconds != 0.25 else 1.25) + healthy: Final = BoundedAnalyticsDatabase(_Driver("healthy")) + assert await asyncio.gather( + healthy.query_raw("SELECT 1 AS result"), healthy.query_raw("SELECT 1 AS result") + ) == [({"result": 1},), ({"result": 1},)] + assert asyncio.all_tasks() == existing + + loop: Final = _ClockedLoop() + try: + loop.run_until_complete(scenario()) + finally: + loop.close() From 8f997824dbb06cbd95f43de472b91db5105a0100 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 15:51:08 +0800 Subject: [PATCH 38/44] fix(alerts): bound daily metric identifiers before materialization --- litellm/repositories/provider_latency.py | 4 + litellm/repositories/usage_alert_metrics.py | 23 +++- .../database/test_performance_issues.py | 107 +++++++++++++++++- .../database/test_provider_latency.py | 45 ++++++++ 4 files changed, 175 insertions(+), 4 deletions(-) diff --git a/litellm/repositories/provider_latency.py b/litellm/repositories/provider_latency.py index eef64462aab..ebfc52e9668 100644 --- a/litellm/repositories/provider_latency.py +++ b/litellm/repositories/provider_latency.py @@ -53,6 +53,8 @@ _MEMBERS: Final = TypeAdapter(tuple[_Member, ...]) _CANDIDATES: Final = """ SELECT DISTINCT custom_llm_provider AS provider, model FROM "LiteLLM_DailyUserSpend" WHERE date=$1 AND timed_requests>0 AND custom_llm_provider<>'' AND model<>'' + AND CASE WHEN octet_length(custom_llm_provider)<=1024 THEN length(custom_llm_provider)<=256 ELSE false END + AND CASE WHEN octet_length(model)<=4096 THEN length(model)<=1024 ELSE false END AND ($2::text IS NULL OR (custom_llm_provider,model)>($2,$3)) ORDER BY provider,model LIMIT $4 """ @@ -201,6 +203,8 @@ class ProviderLatencyRepository: await self._database.query_raw( """SELECT DISTINCT custom_llm_provider AS provider,model FROM "LiteLLM_DailyUserSpend" WHERE date=$1 AND custom_llm_provider=$2 AND model IS NOT NULL AND model<>'' AND timed_requests>0 + AND CASE WHEN octet_length(custom_llm_provider)<=1024 THEN length(custom_llm_provider)<=256 ELSE false END + AND CASE WHEN octet_length(model)<=4096 THEN length(model)<=1024 ELSE false END ORDER BY model LIMIT 101""", day.isoformat(), provider, diff --git a/litellm/repositories/usage_alert_metrics.py b/litellm/repositories/usage_alert_metrics.py index d2a9af41913..2f766df702d 100644 --- a/litellm/repositories/usage_alert_metrics.py +++ b/litellm/repositories/usage_alert_metrics.py @@ -64,6 +64,16 @@ class IssueSourceProvider: provider: str +def _bounded_identity(scope: Scope, entity: str) -> str: + # Check the stored byte length before decoding text; escaped key IDs add four characters. + maximum: Final = ( + f"CASE WHEN {entity}='unattributed' OR {entity} LIKE 'raw:%' THEN 1020 ELSE 1024 END" + if scope == "key" + else "1024" + ) + return f"CASE WHEN octet_length({entity})<=4096 THEN length({entity})<={maximum} ELSE false END" + + def _window_payload(scope: Scope, windows: tuple[IssueMetricWindow, ...]) -> str: if len(windows) > 500 or any(not 1 <= (window.day - window.start).days <= 30 for window in windows): raise ValueError("Read at most 500 issue windows with 1–30 reference days per batch") @@ -154,6 +164,7 @@ def _key_attribution_query(scope: Scope, metric: Literal["spend", "requests"]) - FROM population p JOIN ranked r USING(reference_id) WHERE r.rank=1 ), windows AS MATERIALIZED ( SELECT DISTINCT api_key,entity_id,start_day,target_day FROM candidate WHERE api_key<>'' + AND {_bounded_identity("key", "api_key")} ), key_population AS ( SELECT w.*, {team_days} AS team_days, @@ -183,10 +194,11 @@ _ROWS: Final = TypeAdapter(tuple[DailyAlertMetrics, ...]) _BATCH_SIZE: Final = 10 _DEPRECATION_USAGE: Final = TypeAdapter(tuple[ModelDeprecationUsage, ...]) -_DEPRECATION_USAGE_QUERY: Final = """ +_DEPRECATION_USAGE_QUERY: Final = f""" WITH activity AS MATERIALIZED ( SELECT model, date, api_key, user_id, api_requests FROM "LiteLLM_DailyUserSpend" WHERE date >= $1 AND date <= $2 AND model = ANY($3::text[]) AND api_requests > 0 + AND {_bounded_identity("model", "model")} ), daily AS ( SELECT model, date, SUM(api_requests)::bigint AS requests FROM activity GROUP BY model, date ), latest AS ( @@ -201,6 +213,7 @@ WITH activity AS MATERIALIZED ( SELECT model, COUNT(DISTINCT NULLIF(NULLIF(team_id, ''), 'litellm-dashboard'))::int AS teams FROM "LiteLLM_DailyTeamSpend" WHERE date >= $1 AND date <= $2 AND model = ANY($3::text[]) AND api_requests > 0 + AND {_bounded_identity("model", "model")} GROUP BY model ) SELECT totals.*, latest.last_used_date, latest.last_day_requests, COALESCE(teams.teams, 0) AS teams @@ -227,6 +240,7 @@ def _query(scope: Scope) -> str: WITH entities AS MATERIALIZED ( SELECT DISTINCT {entity} AS entity_id FROM {table} WHERE date >= $1 AND date <= $2 AND {entity} >= COALESCE($3::text, '') AND ($3::text IS NULL OR {entity} <> $3) + AND {_bounded_identity(scope, entity)} ORDER BY entity_id LIMIT $4 ) SELECT entities.entity_id, activity.date AS day, @@ -327,7 +341,8 @@ class UsageAlertMetricsRepository: ), ranked AS ( SELECT *,SUM(timed) OVER(PARTITION BY reference_id) AS total FROM grouped ) SELECT reference_id,provider FROM ranked - WHERE provider<>'' AND timed>0 AND timed>=0.8*total""", + WHERE provider<>'' AND timed>0 AND timed>=0.8*total + AND CASE WHEN octet_length(provider)<=1024 THEN length(provider)<=256 ELSE false END""", _window_payload(scope, windows), ) ) @@ -368,7 +383,9 @@ class UsageAlertMetricsRepository: ) exact_query: Final = ( query[: query.index("WITH entities")] - + "WITH entities AS MATERIALIZED (SELECT unnest($3::text[]) AS entity_id)" + + f"""WITH entities AS MATERIALIZED ( + SELECT entity_id FROM unnest($3::text[]) AS ids(entity_id) + WHERE {_bounded_identity(scope, "entity_id")})""" + query[query.index("\nSELECT entities") :] ) async with db_span("daily_activity_query", sql_relation(query)): diff --git a/tests/integration/database/test_performance_issues.py b/tests/integration/database/test_performance_issues.py index 47c2a4386a5..3c2539ac071 100644 --- a/tests/integration/database/test_performance_issues.py +++ b/tests/integration/database/test_performance_issues.py @@ -14,8 +14,9 @@ from integration._support.prisma_clients import connected_proxy_prisma from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.proxy.spend_tracking.usage_alert_job import UsageAlertJob from litellm.proxy.utils import PrismaClient +from litellm.repositories.daily_activity_buckets import api_key_bucket from litellm.repositories.issues import IssuesRepository -from litellm.repositories.usage_alert_metrics import IssueMetricWindow +from litellm.repositories.usage_alert_metrics import IssueMetricWindow, UsageAlertMetricsRepository from litellm.repositories.usage_alerts import UsageAlertsRepository from litellm.types.proxy.usage_alerts import UsageAlertObservation, UsageAlertRule, UsageAlertRuleInput, UsageAlertScope @@ -42,6 +43,110 @@ _TABLES: Final = ( ) +@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.parametrize("scope", ("user", "team", "key", "model")) +async def test_metric_pages_skip_oversized_ids_without_losing_valid_later_entities( + db: ManagementDatabase, scope: UsageAlertScope +) -> None: + identities = ( + "a" * 1025, + "\U0001f600" * 1025, + "raw:" + "x" * 1016, + "raw:" + "x" * 1017, + "oversized-" + "x" * 65536, + "oversized-" + "\U0001f600" * 8192, + *(f"valid-{index:02}" for index in range(10)), + "\U0001f600" * 1024, + ) + expected = tuple( + sorted( + api_key_bucket(value) if scope == "key" else value + for value in identities + if len(api_key_bucket(value) if scope == "key" else value) <= 1024 + ) + ) + async with _isolated(db) as prisma: + for table, entity in (("LiteLLM_DailyUserSpend", "user_id"), ("LiteLLM_DailyTeamSpend", "team_id")): + await prisma.writer_db.query_raw( + f"""INSERT INTO "{table}" (id,{entity},date,api_key,model,spend,api_requests,updated_at) + SELECT $1||n,identity,$2,identity,identity,2,3,CURRENT_TIMESTAMP + FROM unnest($3::text[]) WITH ORDINALITY AS items(identity,n) RETURNING id""", + table, + _DAY.isoformat(), + identities, + ) + metrics = UsageAlertMetricsRepository(prisma.writer_db) + rows = [] + cursor = None + for _ in range(3): + page = await metrics.page(scope, _DAY, 1, cursor=cursor) + rows.extend(page.rows) + if page.complete: + break + cursor = page.cursor + else: + pytest.fail("bounded metric pagination did not finish") + assert tuple(sorted(row.entity_id for row in rows)) == expected + assert all((row.spend, row.requests) == (2, 3) for row in rows) + assert "\U0001f600" * 1024 in expected + exact = tuple(api_key_bucket(value) if scope == "key" else value for value in identities[:6]) + ("valid-00",) + chosen = await metrics.page(scope, _DAY, 1, entity_ids=exact) + assert {row.entity_id for row in chosen.rows} == set(expected).intersection(exact) + deployment = await metrics.page("deployment", _DAY, 1) + assert len(deployment.rows) == 1 + assert deployment.rows[0].requests == 3 * len(identities) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_attribution_bounds_keep_invalid_identity_traffic_in_population_totals(db: ManagementDatabase) -> None: + async with _isolated(db) as prisma: + await prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,custom_llm_provider,spend,api_requests,timed_requests,updated_at) + SELECT owner||d||k,owner,($1::date-d)::text, + CASE WHEN k='bad' THEN owner||repeat('x',1025) ELSE owner||'-valid' END, + 'example-model',CASE WHEN k='bad' THEN repeat('p',257) ELSE 'valid-provider' END, + CASE WHEN d>0 THEN 10 WHEN (owner='invalid-dominant')=(k='bad') THEN 100 ELSE 20 END, + 1,CASE WHEN (owner='invalid-dominant')=(k='bad') THEN 90 ELSE 10 END,CURRENT_TIMESTAMP + FROM generate_series(0,3) d CROSS JOIN (VALUES('invalid-dominant'),('valid-dominant')) owners(owner) + CROSS JOIN (VALUES('bad'),('good')) keys(k) RETURNING id""", + _DAY.isoformat(), + ) + windows = tuple( + IssueMetricWindow(owner, owner, _DAY - timedelta(days=3), _DAY) + for owner in ("invalid-dominant", "valid-dominant") + ) + metrics = UsageAlertMetricsRepository(prisma.writer_db) + attributions = await metrics.issue_key_attributions("user", windows) + assert len(attributions) == 1 + attribution = attributions[0] + assert (attribution.reference_id, attribution.key_id, attribution.equivalent) == ( + "valid-dominant", + "valid-dominant-valid", + False, + ) + assert attribution.positive_change_share == pytest.approx(0.9) + assert (attribution.baseline_value, attribution.observed_value) == (20, 120) + providers = await metrics.issue_source_providers("user", windows) + assert tuple((row.reference_id, row.provider) for row in providers) == (("valid-dominant", "valid-provider"),) + + +@pytest.mark.asyncio(loop_scope="session") +async def test_retirement_usage_omits_oversized_models_before_materialization(db: ManagementDatabase) -> None: + models = ("a" * 65536, "valid-model", "\U0001f600" * 1024) + async with _isolated(db) as prisma: + await prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyUserSpend" (id,user_id,date,api_key,model,api_requests,updated_at) + SELECT 'retirement-'||n,'owner',$1,'key',model,7,CURRENT_TIMESTAMP + FROM unnest($2::text[]) WITH ORDINALITY AS items(model,n) RETURNING id""", + _DAY.isoformat(), + models, + ) + rows = await UsageAlertMetricsRepository(prisma.writer_db).deprecation_usage(models, _DAY, _DAY) + assert frozenset(row.model for row in rows) == frozenset(models[1:]) + assert tuple(row.requests for row in rows) == (7, 7) + + @asynccontextmanager async def _isolated(db: ManagementDatabase) -> AsyncIterator[PrismaClient]: schema: Final = "issue_test_" + uuid4().hex diff --git a/tests/integration/database/test_provider_latency.py b/tests/integration/database/test_provider_latency.py index 8175d8525bd..ec4986b7c29 100644 --- a/tests/integration/database/test_provider_latency.py +++ b/tests/integration/database/test_provider_latency.py @@ -9,6 +9,51 @@ from litellm.types.proxy.issues import IssueModelPair from litellm.types.proxy.usage_alerts import UsageAlertRuleInput +@pytest.mark.asyncio(loop_scope="session") +async def test_provider_pair_pages_skip_oversized_names_and_continue(db: ManagementDatabase) -> None: + prefix = f"provider-width-{uuid4()}" + day = date(2038, 1, 12) + provider = prefix + "-valid" + wide_provider = "\U0001f600" * 256 + wide_model = "\U0001f600" * 1024 + pairs = ( + ("a" * 257, "first"), + (provider, "a" * 1025), + (provider, "valid-model"), + (provider, wide_model), + (wide_provider, wide_model), + ("\U0001f600" * 257, "last"), + ("\U0001f600" * 8192, "last-large"), + (provider, "\U0001f600" * 8192), + ) + repository = ProviderLatencyRepository(db.prisma.writer_db) + try: + await db.prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,custom_llm_provider,model,api_requests,timed_requests,updated_at) + SELECT $1||n,$1,$2,$1,provider,model,1,1,CURRENT_TIMESTAMP + FROM unnest($3::text[],$4::text[]) WITH ORDINALITY AS items(provider,model,n) RETURNING id""", + prefix, + day.isoformat(), + tuple(pair[0] for pair in pairs), + tuple(pair[1] for pair in pairs), + ) + expected = tuple(IssueModelPair(provider=p, model=m) for p, m in pairs[2:5]) + first = await repository.candidate_pairs(day, limit=1) + second = await repository.candidate_pairs(day, first[0], limit=1) + third = await repository.candidate_pairs(day, second[0], limit=1) + assert len(first) == len(second) == len(third) == 1 + assert frozenset((*first, *second, *third)) == frozenset(expected) + assert await repository.candidate_pairs(day, third[0], limit=1) == () + assert frozenset(await repository.provider_pairs(provider, day)) == frozenset(expected[:2]) + assert await repository.provider_pairs(wide_provider, day) == expected[2:] + assert await repository.provider_pairs("a" * 257, day) == () + finally: + await db.prisma.writer_db.query_raw( + 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE id LIKE $1 RETURNING id', prefix + "%" + ) + + @pytest.mark.asyncio(loop_scope="session") async def test_provider_latency_requires_established_within_model_regressions(db: ManagementDatabase) -> None: prefix = f"provider-latency-{uuid4()}" From cf385e1f8159db604fce90741c76a23c519785a7 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 16:01:41 +0800 Subject: [PATCH 39/44] fix(alerts): retain admission while cancelled database work settles --- .../litellm_core_utils/pending_operations.py | 57 ++++++++ litellm/proxy/db/prisma_client.py | 25 +++- litellm/repositories/analytics_budget.py | 35 ++--- .../test_pending_operations.py | 32 +++++ .../repositories/test_analytics_budget.py | 126 ++++++++++++++++++ 5 files changed, 254 insertions(+), 21 deletions(-) create mode 100644 litellm/litellm_core_utils/pending_operations.py create mode 100644 tests/unit/litellm_core_utils/test_pending_operations.py diff --git a/litellm/litellm_core_utils/pending_operations.py b/litellm/litellm_core_utils/pending_operations.py new file mode 100644 index 00000000000..5c8a242c849 --- /dev/null +++ b/litellm/litellm_core_utils/pending_operations.py @@ -0,0 +1,57 @@ +from collections.abc import Callable, Generator +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Final + + +class PendingOperations: + def __init__(self, completed: Callable[[], None]) -> None: + self._completed = completed + self._pending = 0 + self._closed = False + self._released = False + + def started(self) -> bool: + if self._released: + return False + self._pending += 1 + return True + + def finished(self) -> None: + self._pending -= 1 + if self._closed and self._pending == 0: + self._release() + + def close(self) -> None: + self._closed = True + if self._pending == 0: + self._release() + + def _release(self) -> None: + if not self._released: + self._released = True + self._completed() + + +_PENDING: Final[ContextVar[PendingOperations | None]] = ContextVar("litellm_pending_operations", default=None) + + +@contextmanager +def defer_until_operations_finish(completed: Callable[[], None]) -> Generator[None, None, None]: + operations: Final = PendingOperations(completed) + token: Final = _PENDING.set(operations) + try: + yield + finally: + _PENDING.reset(token) + operations.close() + + +def register_pending_operation() -> PendingOperations | None: + operations: Final = _PENDING.get() + return operations if operations is not None and operations.started() else None + + +def finish_pending_operation(operations: PendingOperations | None) -> None: + if operations is not None: + operations.finished() diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index be5de040fdf..6bfb5c7680f 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -19,6 +19,11 @@ from typing import TYPE_CHECKING, Final, Protocol, TypeVar from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.pending_operations import ( + PendingOperations, + finish_pending_operation, + register_pending_operation, +) from litellm.proxy.db.db_span import db_span from litellm.proxy.db.db_url_settings import add_missing_query_params, token_refresh_params_from_url from litellm.proxy.db.log_db_metrics import db_io_claimed, record_db_io @@ -148,33 +153,45 @@ class _TrackedPrismaEngine: ) -> _EngineResponseT: self.tracker.begin_operation() task: Final = asyncio.ensure_future(request) + pending: Final = register_pending_operation() try: - return await asyncio.shield(task) + response: Final = await asyncio.shield(task) except asyncio.CancelledError: self._detached = self._detached.union((task,)) - task.add_done_callback(lambda done: self._detached_request_finished(done, abandoned)) + task.add_done_callback(lambda done: self._detached_request_finished(done, abandoned, pending)) raise + except BaseException: + finish_pending_operation(pending) + raise + else: + finish_pending_operation(pending) + return response def _detached_request_finished( self, done: "asyncio.Task[_EngineResponseT]", abandoned: Callable[[_EngineResponseT], Awaitable[None]] | None, + pending: PendingOperations | None, ) -> None: self._detached = self._detached.difference((done,)) if done.cancelled(): + finish_pending_operation(pending) return failure: Final = done.exception() if failure is not None: verbose_proxy_logger.debug("Prisma request abandoned by a cancelled caller failed: %s", failure) + finish_pending_operation(pending) return if abandoned is None: + finish_pending_operation(pending) return settle: Final = asyncio.ensure_future(abandoned(done.result())) self._detached = self._detached.union((settle,)) - settle.add_done_callback(self._abandoned_transaction_settled) + settle.add_done_callback(lambda done: self._abandoned_transaction_settled(done, pending)) - def _abandoned_transaction_settled(self, settle: "asyncio.Task[object]") -> None: + def _abandoned_transaction_settled(self, settle: "asyncio.Task[object]", pending: PendingOperations | None) -> None: self._detached = self._detached.difference((settle,)) + finish_pending_operation(pending) if settle.cancelled() or settle.exception() is None: return verbose_proxy_logger.debug( diff --git a/litellm/repositories/analytics_budget.py b/litellm/repositories/analytics_budget.py index d5e02e655a4..11ac67d970a 100644 --- a/litellm/repositories/analytics_budget.py +++ b/litellm/repositories/analytics_budget.py @@ -9,6 +9,8 @@ from typing import Final, Protocol from prisma.errors import RawQueryError, TransactionExpiredError, TransactionNotStartedError from pydantic import BaseModel +from litellm.litellm_core_utils.pending_operations import defer_until_operations_finish + class AnalyticsTransaction(Protocol): async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ... @@ -101,23 +103,22 @@ class BoundedAnalyticsDatabase: async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: if not self._admission.acquire(blocking=False): raise AnalyticsCapacityError("Analytics capacity is busy. Try again shortly") - try: - remaining: Final = _remaining_seconds() - return await asyncio.wait_for( - self._query(query, args), timeout=6.25 if remaining is None else min(6.25, remaining) - ) - except RawQueryError as error: - if _PostgresError.model_validate(error.meta or {}).code not in ("57014", "55P03"): - raise - raise AnalyticsCapacityError("Analytics exceeded its query budget. Try again shortly") from error - except ( - TransactionExpiredError, - TransactionNotStartedError, - asyncio.TimeoutError, - ) as error: - raise AnalyticsCapacityError("Analytics exceeded its connection budget. Try again shortly") from error - finally: - self._admission.release() + with defer_until_operations_finish(self._admission.release): + try: + remaining: Final = _remaining_seconds() + return await asyncio.wait_for( + self._query(query, args), timeout=6.25 if remaining is None else min(6.25, remaining) + ) + except RawQueryError as error: + if _PostgresError.model_validate(error.meta or {}).code not in ("57014", "55P03"): + raise + raise AnalyticsCapacityError("Analytics exceeded its query budget. Try again shortly") from error + except ( + TransactionExpiredError, + TransactionNotStartedError, + asyncio.TimeoutError, + ) as error: + raise AnalyticsCapacityError("Analytics exceeded its connection budget. Try again shortly") from error async def _query(self, query: str, args: tuple[object, ...]) -> Sequence[Mapping[str, object]]: async with _bounded_transaction( diff --git a/tests/unit/litellm_core_utils/test_pending_operations.py b/tests/unit/litellm_core_utils/test_pending_operations.py new file mode 100644 index 00000000000..965334bd397 --- /dev/null +++ b/tests/unit/litellm_core_utils/test_pending_operations.py @@ -0,0 +1,32 @@ +import asyncio +from threading import BoundedSemaphore +from typing import Final + +import pytest + +from litellm.litellm_core_utils.pending_operations import ( + defer_until_operations_finish, + finish_pending_operation, + register_pending_operation, +) + + +@pytest.mark.asyncio +async def test_late_child_cannot_release_a_drained_parent_scope_twice() -> None: + admission: Final = BoundedSemaphore(1) + assert admission.acquire(blocking=False) + proceed: Final = asyncio.Event() + + async def late_child() -> None: + await proceed.wait() + operation: Final = register_pending_operation() + assert operation is None + finish_pending_operation(operation) + + with defer_until_operations_finish(admission.release): + child: Final = asyncio.create_task(late_child()) + assert admission.acquire(blocking=False) + proceed.set() + await child + assert not admission.acquire(blocking=False) + admission.release() diff --git a/tests/unit/repositories/test_analytics_budget.py b/tests/unit/repositories/test_analytics_budget.py index befa645c4b6..90f8ab83f3b 100644 --- a/tests/unit/repositories/test_analytics_budget.py +++ b/tests/unit/repositories/test_analytics_budget.py @@ -1,4 +1,5 @@ import asyncio +import heapq from collections import deque from collections.abc import AsyncGenerator, Mapping, Sequence from contextlib import asynccontextmanager, nullcontext @@ -27,6 +28,8 @@ class _ClockedLoop(asyncio.SelectorEventLoop): return self.now def _run_once(self) -> None: + while self._scheduled and self._scheduled[0].cancelled(): + heapq.heappop(self._scheduled) if not self._ready and self._scheduled: self.now = max(self.now, self._scheduled[0].when()) super()._run_once() # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue] # CPython loop hook lacks stubs @@ -99,3 +102,126 @@ def test_stalled_driver_is_cancelled_without_retaining_admission_or_tasks( loop.run_until_complete(scenario()) finally: loop.close() + + +class _Process: + pid: object = None + + +class _HeldEngine: + def __init__(self, stage: Literal["begin", "query", "commit", "rollback"], *, failure: bool = False) -> None: + self.stage = stage + self.failure = failure + self.entered = asyncio.Event() + self.released = asyncio.Event() + self.rollback_entered = asyncio.Event() + self.rollback_released = asyncio.Event() + self.process = _Process() + self.calls = 0 + self.starts = 0 + + async def _request(self, stage: str) -> None: + self.calls += 1 + if stage == self.stage: + self.entered.set() + await self.released.wait() + if self.failure: + raise RuntimeError("synthetic engine error") + + async def query(self, content: str, *, tx_id: str | None) -> object: + if content == "SELECT 1 AS result": + await self._request("query") + return {} + + async def start_transaction(self, *, content: str) -> str: + self.starts += 1 + identifier: Final = f"synthetic-transaction-{self.starts}" + await self._request("begin") + return identifier + + async def commit_transaction(self, tx_id: str) -> None: + await self._request("commit") + + async def rollback_transaction(self, tx_id: str) -> None: + await self._request("rollback") + if self.stage == "begin": + self.rollback_entered.set() + await self.rollback_released.wait() + + +class _TrackedDriver: + def __init__(self, engine: _HeldEngine) -> None: + from litellm.proxy.db.prisma_client import ( + _PrismaDrainTracker, # pyright: ignore[reportPrivateUsage] # exercise real cancellation ownership + _TrackedPrismaEngine, # pyright: ignore[reportPrivateUsage] # exercise real cancellation ownership + ) + + self.stage = engine.stage + self.tracker = _PrismaDrainTracker() + self.engine = _TrackedPrismaEngine(engine, self.tracker) + + @asynccontextmanager + async def tx(self, *, max_wait: timedelta, timeout: timedelta) -> AsyncGenerator[AnalyticsTransaction]: + identifier: Final = await self.engine.start_transaction(content="{}") + try: + yield self + if self.stage == "rollback": + raise RuntimeError("synthetic query failure") + except BaseException: + await self.engine.rollback_transaction(identifier) + raise + else: + await self.engine.commit_transaction(identifier) + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + from litellm.proxy.db.log_db_metrics import claim_db_io + + with claim_db_io(): + await self.engine.query(query, tx_id="synthetic-transaction") + return ({"result": 1},) if query == "SELECT 1 AS result" else ({"acquired": True},) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stage", ("begin", "query", "commit", "rollback")) +@pytest.mark.parametrize("failure", (False, True)) +async def test_cancelled_analytics_keeps_admission_until_detached_engine_work_and_rollback_finish( + stage: Literal["begin", "query", "commit", "rollback"], + failure: bool, +) -> None: + engines: Final = (_HeldEngine(stage, failure=failure), _HeldEngine(stage, failure=failure)) + drivers: Final = tuple(_TrackedDriver(engine) for engine in engines) + tasks: Final = tuple( + asyncio.create_task(BoundedAnalyticsDatabase(driver).query_raw("SELECT 1 AS result")) for driver in drivers + ) + try: + await asyncio.gather(*(engine.entered.wait() for engine in engines)) + for task in tasks: + task.cancel() + for task in tasks: + with pytest.raises(asyncio.CancelledError): + await task + calls: Final = tuple(engine.calls for engine in engines) + for _ in range(64): + with pytest.raises(AnalyticsCapacityError): + await BoundedAnalyticsDatabase(drivers[0]).query_raw("SELECT 1 AS result") + assert tuple(engine.calls for engine in engines) == calls + assert tuple(len(driver.engine._detached) for driver in drivers) == (1, 1) # pyright: ignore[reportPrivateUsage] # bounded retained engine tasks are the behavior under test + for engine in engines: + engine.released.set() + if stage == "begin" and not failure: + await asyncio.gather(*(engine.rollback_entered.wait() for engine in engines)) + with pytest.raises(AnalyticsCapacityError): + await BoundedAnalyticsDatabase(drivers[0]).query_raw("SELECT 1 AS result") + finally: + for engine in engines: + engine.released.set() + engine.rollback_released.set() + await asyncio.gather(*(driver.tracker.wait_until_drained() for driver in drivers)) + await asyncio.sleep(0) + await asyncio.sleep(0) + assert not any(driver.engine._detached for driver in drivers) # pyright: ignore[reportPrivateUsage] # verify detached engine tasks drained + healthy: Final = BoundedAnalyticsDatabase(_Driver("healthy")) + assert await asyncio.gather(healthy.query_raw("SELECT 1 AS result"), healthy.query_raw("SELECT 1 AS result")) == [ + ({"result": 1},), + ({"result": 1},), + ] From e3b8fcb0532dba5fc27e87750be452a63fb06ad8 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 16:07:08 +0800 Subject: [PATCH 40/44] test(alerts): keep async driver annotations compatible with Python 3.10 --- tests/unit/repositories/test_analytics_budget.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/repositories/test_analytics_budget.py b/tests/unit/repositories/test_analytics_budget.py index 90f8ab83f3b..aaa2790d650 100644 --- a/tests/unit/repositories/test_analytics_budget.py +++ b/tests/unit/repositories/test_analytics_budget.py @@ -47,7 +47,7 @@ class _Driver: self.cancelled.set() @asynccontextmanager - async def tx(self, *, max_wait: timedelta, timeout: timedelta) -> AsyncGenerator[AnalyticsTransaction]: + async def tx(self, *, max_wait: timedelta, timeout: timedelta) -> AsyncGenerator[AnalyticsTransaction, None]: if self.stage == "begin": await self._stall() try: @@ -161,7 +161,7 @@ class _TrackedDriver: self.engine = _TrackedPrismaEngine(engine, self.tracker) @asynccontextmanager - async def tx(self, *, max_wait: timedelta, timeout: timedelta) -> AsyncGenerator[AnalyticsTransaction]: + async def tx(self, *, max_wait: timedelta, timeout: timedelta) -> AsyncGenerator[AnalyticsTransaction, None]: identifier: Final = await self.engine.start_transaction(content="{}") try: yield self From d36fb0f506ba6a053fe0bba4a441ac78b8f351ad Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 16:08:19 +0800 Subject: [PATCH 41/44] fix(alerts): include cancellation cleanup in the worker deadline --- .../proxy/spend_tracking/usage_alert_job.py | 7 +- .../spend_tracking/test_daily_usage_alerts.py | 123 +++++++++++++++++- 2 files changed, 124 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index f46603caa6e..63dc96eaed7 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -176,7 +176,8 @@ class UsageAlertJob: async def run(self, now: datetime | None = None) -> None: owner: Final = str(uuid4()) - deadline: Final = asyncio.get_running_loop().time() + self._work_budget_seconds - 1 + finish_at: Final = asyncio.get_running_loop().time() + self._work_budget_seconds + deadline: Final = finish_at - 2 try: with analytics_deadline(deadline): claimed: Final = await self._store.claim_job(owner, lease_seconds=int(self._work_budget_seconds) + 15) @@ -198,14 +199,14 @@ class UsageAlertJob: verbose_proxy_logger.exception("Daily usage alert evaluation failed") finally: try: - with analytics_deadline(asyncio.get_running_loop().time() + 1): + with analytics_deadline(finish_at - 1): await self._store.release_job(owner) except AnalyticsCapacityError: verbose_proxy_logger.info("Usage alert lease will expire after bounded cleanup") async def _run(self, now: datetime, stop_at: float | None = None) -> None: deadline: Final = ( - stop_at if stop_at is not None else asyncio.get_running_loop().time() + self._work_budget_seconds - 1 + stop_at if stop_at is not None else asyncio.get_running_loop().time() + self._work_budget_seconds - 2 ) with analytics_deadline(deadline): await self._work(now, deadline) diff --git a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py index a6d083c41e1..d63a8dc6e1b 100644 --- a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py +++ b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py @@ -1,3 +1,8 @@ +import asyncio +import heapq +from collections import deque +from collections.abc import AsyncGenerator, Mapping, Sequence +from contextlib import asynccontextmanager from dataclasses import replace from datetime import date, datetime, timedelta, timezone from typing import Final @@ -5,14 +10,126 @@ from typing import Final import pytest from litellm.proxy.spend_tracking.daily_usage_alerts import DailyComparison, compare_daily_usage -from litellm.proxy.spend_tracking.usage_alert_job import completed_day -from litellm.repositories.usage_alert_metrics import DailyAlertMetrics +from litellm.proxy.spend_tracking.usage_alert_job import UsageAlertJob, completed_day +from litellm.repositories.analytics_budget import AnalyticsTransaction, BoundedAnalyticsDatabase +from litellm.repositories.usage_alert_health import UsageAlertHealthRepository +from litellm.repositories.usage_alert_metrics import DailyAlertMetrics, UsageAlertMetricsRepository +from litellm.repositories.usage_alerts import UsageAlertsRepository from litellm.types.proxy.usage_alert_defaults import default_usage_rule_id, default_usage_rules -from litellm.types.proxy.usage_alerts import UsageAlertRule, UsageAlertRuleInput +from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertChannel, UsageAlertRule, UsageAlertRuleInput _DAY: Final = date(2026, 10, 20) +class _JobBudgetClock(asyncio.SelectorEventLoop): + _ready: deque[asyncio.Handle] + _scheduled: list[asyncio.TimerHandle] + + def __init__(self) -> None: + super().__init__() + self.now = 1000.0 + + def time(self) -> float: + return self.now + + def _run_once(self) -> None: + while self._scheduled and self._scheduled[0].cancelled(): + heapq.heappop(self._scheduled) + if not self._ready and self._scheduled: + self.now = max(self.now, self._scheduled[0].when()) + super()._run_once() # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue] # CPython loop hook lacks stubs + + +class _BudgetTransaction: + def __init__(self, database: "_BudgetDatabase", index: int) -> None: + self.database = database + self.index = index + self.configured = False + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + if not self.configured: + self.configured = True + return ({"acquired": True},) + if self.index == 0: + return ({"count": 1},) + try: + await asyncio.Event().wait() + finally: + self.database.cancelled_queries += (self.index,) + raise AssertionError("stalled query unexpectedly completed") + + +class _BudgetDatabase: + def __init__(self) -> None: + self.transactions = 0 + self.active = 0 + self.cancelled_queries: tuple[int, ...] = () + self.query_rollback_finished = False + self.lease_rollback_cancelled = False + + @asynccontextmanager + async def tx(self, *, max_wait: timedelta, timeout: timedelta) -> AsyncGenerator[AnalyticsTransaction]: + transaction: Final = _BudgetTransaction(self, self.transactions) + self.transactions += 1 + self.active += 1 + try: + yield transaction + finally: + try: + if transaction.index == 1: + await asyncio.sleep(0.5) + self.query_rollback_finished = True + elif transaction.index == 2: + try: + await asyncio.Event().wait() + finally: + self.lease_rollback_cancelled = True + finally: + self.active -= 1 + + +class _UnexpectedDelivery: + async def send(self, alert: UsageAlert, rule: UsageAlertRule, channel: UsageAlertChannel) -> bool: + raise AssertionError("stalled evaluation must not deliver alerts") + + +def test_job_budget_includes_query_cancellation_and_lease_cleanup() -> None: + async def scenario() -> None: + existing: Final = asyncio.all_tasks() + loop: Final = asyncio.get_running_loop() + driver: Final = _BudgetDatabase() + database: Final = BoundedAnalyticsDatabase(driver, background=True) + + async def projection(now: datetime) -> None: + await asyncio.sleep(57) + await database.query_raw("SELECT 1") + + job: Final = UsageAlertJob( + UsageAlertsRepository(database), + UsageAlertMetricsRepository(database), + UsageAlertHealthRepository(database), + _UnexpectedDelivery(), + defaults=(), + deprecation_projection=projection, + work_budget_seconds=60, + ) + started: Final = loop.time() + await asyncio.wait_for(job.run(datetime(2026, 10, 21, 2, tzinfo=timezone.utc)), timeout=65) + elapsed: Final = loop.time() - started + assert elapsed <= 60, f"Job exceeded its 60-second budget including cleanup: {elapsed:g}s" + assert driver.cancelled_queries == (1, 2) + assert driver.query_rollback_finished and driver.lease_rollback_cancelled + assert driver.active == 0 and asyncio.all_tasks() == existing + healthy: Final = BoundedAnalyticsDatabase(_BudgetDatabase(), background=True) + assert await healthy.query_raw("SELECT 1") == ({"count": 1},) + + loop: Final = _JobBudgetClock() + try: + loop.run_until_complete(scenario()) + finally: + loop.close() + + def _row(day: date, requests: int = 200, spend: float = 20, ms: float = 100_000, timed: int = 200) -> DailyAlertMetrics: return DailyAlertMetrics( entity_id="key-a", day=day, spend=spend, requests=requests, errors=10, response_time_ms=ms, timed_requests=timed From da2a44bf1fd008a3fcf21a85ae34f4306078c0b1 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 16:09:04 +0800 Subject: [PATCH 42/44] docs(alerts): describe bounded queries and cancellation recovery --- docs/features/usage-alerts.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/features/usage-alerts.md b/docs/features/usage-alerts.md index b4be79e3b0f..a3d633b3d89 100644 --- a/docs/features/usage-alerts.md +++ b/docs/features/usage-alerts.md @@ -203,7 +203,11 @@ If legacy `user_spend_anomalies` notifications are enabled, the default daily us Alert investigations and resource-page Activity tabs limit returned rows before loading results into the application. For a key, these views show exact totals and a bounded model breakdown: 20 named models by default, at most 100, plus a separate **Other** aggregate for the remaining models. Activity ranges span at most 366 days. A small result page can still require reading many aggregate rows; page size is not a database scan limit. -Guarded queries allow at most two foreground queries and one background query at a time, coordinated across workers sharing the database. They do not build an unbounded waiting queue. Each query has a statement timeout of at most five seconds, a lock wait of at most 100 ms, and a 1 MiB PostgreSQL `work_mem` setting, with stricter existing limits preserved. Background query deadlines also shrink to the remaining run budget. PostgreSQL `work_mem` applies per operation, not to total application or database memory. +Detection excludes resource and model IDs above 1,024 characters or 4,096 bytes, and provider names above 256 characters or 1,024 bytes. Escaped key IDs reserve space for their `raw:` prefix. Unsupported identifiers cannot produce individual resource or provider alerts. Their traffic still contributes to Activity totals and attribution denominators, so excluding an identifier does not increase another resource's attributed share. + +Each process admits at most two foreground analytics queries and one background analytics query. PostgreSQL advisory locks also coordinate those slots across workers sharing the database. Busy callers are rejected without an unbounded waiting queue. A query has a client wait limit of 6.25 seconds and up to one additional second for transaction cleanup. Background deadlines shrink to the remaining run budget. If cancellation leaves driver work running, the process retains its slot until that work finishes. + +Within a transaction, SQL has a statement timeout of at most five seconds, a lock wait of at most 100 ms, and PostgreSQL `work_mem` of at most 1 MiB, preserving stricter existing limits. Hash memory uses a multiplier of one, and JIT compilation and parallel query workers are disabled for that transaction. These are concurrency and work limits, not a hard limit on total application or database memory. PostgreSQL `work_mem` applies per operation. When an Activity query is busy or exceeds its budget, the API returns `503` with `Retry-After: 5`. Background work keeps its checkpoint and tries again on a later wakeup. These limits bound work per attempt; database size, query plans, existing gateway memory, and concurrent serving traffic still affect overhead and completion time. Persistent **Catching up** means monitoring has not finished, not that the remaining resources are healthy. From 2cc807cec4b23040fec80d1aec07b2034adf8a39 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 6 Nov 2026 16:09:36 +0800 Subject: [PATCH 43/44] test(alerts): support budget regression on Python 3.12 --- tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py index d63a8dc6e1b..41fab29ef44 100644 --- a/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py +++ b/tests/unit/proxy/spend_tracking/test_daily_usage_alerts.py @@ -68,7 +68,7 @@ class _BudgetDatabase: self.lease_rollback_cancelled = False @asynccontextmanager - async def tx(self, *, max_wait: timedelta, timeout: timedelta) -> AsyncGenerator[AnalyticsTransaction]: + async def tx(self, *, max_wait: timedelta, timeout: timedelta) -> AsyncGenerator[AnalyticsTransaction, None]: transaction: Final = _BudgetTransaction(self, self.transactions) self.transactions += 1 self.active += 1 From 4c4f2bfc19b89a2acc08f1ea2f3d63c0e7fd1881 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 7 Nov 2026 11:24:29 +0800 Subject: [PATCH 44/44] fix(alerts): retry delivery and retain distinct retirement models --- .../proxy/common_utils/model_deprecation.py | 6 +- .../spend_tracking/performance_issues.py | 6 +- .../proxy/spend_tracking/usage_alert_job.py | 10 +- .../database/test_performance_issues.py | 99 +++++++++++++++++ .../observability/test_usage_alert_email.py | 103 ++++++++++++++++++ .../SlackAlerting/test_usage_alerts.py | 55 ---------- .../common_utils/test_model_deprecation.py | 23 +++- 7 files changed, 237 insertions(+), 65 deletions(-) create mode 100644 tests/integration/observability/test_usage_alert_email.py diff --git a/litellm/proxy/common_utils/model_deprecation.py b/litellm/proxy/common_utils/model_deprecation.py index 72ff564dbd4..f84aa05977a 100644 --- a/litellm/proxy/common_utils/model_deprecation.py +++ b/litellm/proxy/common_utils/model_deprecation.py @@ -414,7 +414,11 @@ def _scan_deprecation_model( position: Final = positions[cursor.group_position] candidate: Final = info(position) advanced: Final = cursor.model_copy(update={"group_position": cursor.group_position + 1}) - if candidate is None or candidate.info.deprecation_date != cursor.model.deprecation_date: + if candidate is None or ( + candidate.info.deprecation_date, + candidate.info.litellm_model, + candidate.info.litellm_provider, + ) != (cursor.model.deprecation_date, cursor.model.litellm_model, cursor.model.litellm_provider): return advanced, None if position < cursor.deployment: return DeprecationScanCursor(deployment=cursor.deployment + 1), None diff --git a/litellm/proxy/spend_tracking/performance_issues.py b/litellm/proxy/spend_tracking/performance_issues.py index e3c5d1fb723..c4b148ef6e6 100644 --- a/litellm/proxy/spend_tracking/performance_issues.py +++ b/litellm/proxy/spend_tracking/performance_issues.py @@ -229,7 +229,7 @@ class PerformanceIssueProjector: while asyncio.get_running_loop().time() < projection_deadline: if (next_position := await self._projection_page(position, work, observed_at)) is None: await self._store.complete_projection(work.day, position.version) - await self._notify(deadline) + await self.notify(deadline) await self._store.prune() if position.version != work.version: await rules.save_projection_cursor(UsageAlertProjectionCursor(version=work.version, day=work.day)) @@ -238,7 +238,7 @@ class PerformanceIssueProjector: position = next_position await rules.save_projection_cursor(position) await asyncio.sleep(0.01) - await self._notify(deadline) + await self.notify(deadline) return False async def _projection_page( @@ -705,7 +705,7 @@ class PerformanceIssueProjector: while batch := tuple(islice(bindings, 10)): await self._store.queue_deliveries(batch) - async def _notify(self, stop_at: float | None = None) -> None: + async def notify(self, stop_at: float | None = None) -> None: deadline: Final = stop_at if stop_at is not None else asyncio.get_running_loop().time() + 60 while asyncio.get_running_loop().time() + 2 < deadline and ( pending := await self._store.pending_deliveries(limit=10) diff --git a/litellm/proxy/spend_tracking/usage_alert_job.py b/litellm/proxy/spend_tracking/usage_alert_job.py index 63dc96eaed7..d0b403c4643 100644 --- a/litellm/proxy/spend_tracking/usage_alert_job.py +++ b/litellm/proxy/spend_tracking/usage_alert_job.py @@ -1,6 +1,7 @@ """Once-daily evaluation, polled hourly by one database-elected worker.""" import asyncio +import re from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass from datetime import date, datetime, time, timedelta, timezone @@ -59,7 +60,8 @@ def _has_activity_page(scope: UsageAlertScope, entity: str) -> bool: if not entity.strip(): return False if scope == "key": - return entity != "unattributed" and not entity.removeprefix("raw:").startswith("sk-") + key: Final = re.sub(r"^(?:raw:)+", "", entity) + return key not in ("", "unattributed") and not key.startswith("sk-") if scope == "user": return entity != "default_user_id" if scope == "team": @@ -224,9 +226,9 @@ class UsageAlertJob: ) if self._performance_projection is None: await self._notify(deadline) - elif ( - work := await self._store.pending_projection_work() - ) is not None and await self._performance_projection.run_pending(now, work, self._store, deadline): + elif (work := await self._store.pending_projection_work()) is None: + await self._performance_projection.notify(deadline) + elif await self._performance_projection.run_pending(now, work, self._store, deadline): await self._store.finish_projection_work(work) await self._store.prune() diff --git a/tests/integration/database/test_performance_issues.py b/tests/integration/database/test_performance_issues.py index 3c2539ac071..ed810840f4d 100644 --- a/tests/integration/database/test_performance_issues.py +++ b/tests/integration/database/test_performance_issues.py @@ -260,6 +260,105 @@ async def test_key_spend_issue_consolidates_traffic_and_uses_fresh_days(db: Mana assert (await issues.list_issues(scope_type="key", scope_id=_KEY_B)).data[0].active +@pytest.mark.asyncio(loop_scope="session") +async def test_due_issue_delivery_retries_after_daily_projection_is_complete(db: ManagementDatabase) -> None: + async with _isolated(db) as prisma: + await _seed(prisma) + rules = UsageAlertsRepository.from_prisma(prisma) + rule = await rules.save_rule( + UsageAlertRuleInput( + name="Key spend notification", + scope_type="key", + scope_id=_KEY_A, + metric="spend", + direction="increase", + notification_channels=("slack",), + ), + "qa-admin", + ) + assert rule is not None + job = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) + await job._run(_NOW) + assert await rules.pending_projection_work() is None + before = await prisma.writer_db.query_raw( + "SELECT id,(record->>'attempts')::int AS attempts,record->>'last_error' AS error " + 'FROM "LiteLLM_IssueDelivery" WHERE NOT sent' + ) + assert len(before) == 1 + assert (before[0]["attempts"], before[0]["error"]) == (1, "transport_not_configured") + await prisma.writer_db.query_raw( + "UPDATE \"LiteLLM_IssueDelivery\" SET next_attempt=CURRENT_TIMESTAMP-interval '1 second' " + "WHERE id=$1 RETURNING id", + before[0]["id"], + ) + await job._run(_NOW + timedelta(hours=1)) + after = await prisma.writer_db.query_raw( + "SELECT (record->>'attempts')::int AS attempts FROM \"LiteLLM_IssueDelivery\" WHERE id=$1", + before[0]["id"], + ) + assert after == [{"attempts": 2}], "due delivery was not retried after projection completed" + assert await rules.pending_projection_work() is None + + +@pytest.mark.asyncio(loop_scope="session") +async def test_key_evaluation_rejects_normalized_sentinels_and_secret_shaped_ids(db: ManagementDatabase) -> None: + keys = ( + "unattributed", + "raw:unattributed", + "sk-synthetic-only", + "raw:sk-synthetic-only", + "raw:raw:sk-synthetic-only", + _KEY_A, + ) + async with _isolated(db) as prisma: + await prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyUserSpend" (id,user_id,date,api_key,model,spend,api_requests,updated_at) + SELECT 'key-identity-'||n||'-'||d,'owner',($1::date-d)::text,key,'example-model', + CASE WHEN d=0 THEN 100 ELSE 1 END,1000,CURRENT_TIMESTAMP + FROM unnest($2::text[]) WITH ORDINALITY AS keys(key,n) CROSS JOIN generate_series(0,7) d RETURNING id""", + _DAY.isoformat(), + keys, + ) + await _coverage(prisma, _DAY) + job = UsageAlertJob.from_prisma(prisma, SlackAlerting(alerting=[], alert_types=[])) + await job._run(_NOW) + alerts = await UsageAlertsRepository.from_prisma(prisma).list_alerts(scope_type="key") + assert alerts.total == 1, "normalized invalid keys created incidents" + assert alerts.data[0].scope_id == _KEY_A + + +@pytest.mark.asyncio(loop_scope="session") +async def test_retirement_detects_used_model_after_unused_model_in_the_same_alias(db: ManagementDatabase) -> None: + from litellm import Router + from litellm.proxy.common_utils.model_deprecation import RouterDeprecationSource + from litellm.proxy.spend_tracking.deprecation_issues import DeprecationIssueProjector + + models = ("openai/unused-retirement-example", "openai/used-retirement-example") + async with _isolated(db) as prisma: + router = Router( + model_list=[ + { + "model_name": "shared-retirement-alias", + "litellm_params": {"model": model}, + "model_info": {"id": model, "deprecation_date": "2026-11-20"}, + } + for model in models + ] + ) + await prisma.writer_db.query_raw( + """INSERT INTO "LiteLLM_DailyUserSpend" (id,user_id,date,api_key,model,api_requests,updated_at) + VALUES('used-retirement','owner',$1,$2,$3,7,CURRENT_TIMESTAMP) RETURNING id""", + _DAY.isoformat(), + _KEY_A, + models[1], + ) + await DeprecationIssueProjector.from_prisma(prisma, RouterDeprecationSource(lambda: router)).run(_NOW) + issues = await IssuesRepository.from_prisma(prisma).list_issues(kind="model_deprecation") + assert issues.total == 1, "retired model with actual usage was lost" + assert issues.data[0].cohort.scope_id == models[1] + assert issues.data[0].impact.requests == 7 + + @pytest.mark.asyncio(loop_scope="session") async def test_single_key_reliability_collapses_matching_scopes_without_traffic_growth(db: ManagementDatabase) -> None: async with _isolated(db) as prisma: diff --git a/tests/integration/observability/test_usage_alert_email.py b/tests/integration/observability/test_usage_alert_email.py new file mode 100644 index 00000000000..bcec3a022e3 --- /dev/null +++ b/tests/integration/observability/test_usage_alert_email.py @@ -0,0 +1,103 @@ +import asyncio +import smtplib +import threading +from contextlib import suppress +from datetime import date, datetime, timezone +from typing import Final + +import pytest + +from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.integrations.SlackAlerting.usage_alerts import UsageAlertDelivery +from litellm.types.proxy.usage_alerts import UsageAlert, UsageAlertRule + +_NOW: Final = datetime(2026, 10, 20, tzinfo=timezone.utc) +_RULE: Final = UsageAlertRule( + name="Request spike", + id="rule", + scope_type="key", + scope_id="reporting-key", + metric="requests", + direction="increase", + created_at=_NOW, + updated_at=_NOW, + created_by=None, + notification_channels=("slack",), +) +_ALERT: Final = UsageAlert( + id="incident", + rule_id="rule", + rule_name="Request spike", + scope_type="key", + scope_id="reporting-key", + metric="requests", + direction="increase", + status="open", + day=date(2026, 10, 20), + observed_value=1000, + baseline_value=200, + threshold=600, + first_seen=_NOW, + last_seen=_NOW, + baseline_start=date(2026, 10, 13), + baseline_end=date(2026, 10, 19), + baseline_sample_days=7, + activity_path="/api-keys?key=reporting-key&detail_tab=activity", +) + + +@pytest.mark.asyncio +async def test_cancelled_smtp_attempt_retains_only_one_send(monkeypatch: pytest.MonkeyPatch) -> None: + connected: Final = asyncio.Event() + release: Final = asyncio.Event() + accepted: Final[list[asyncio.StreamWriter]] = [] + + async def stalled(_: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + accepted.append(writer) + connected.set() + await release.wait() + writer.close() + await writer.wait_closed() + + server: Final = await asyncio.start_server(stalled, "127.0.0.1", 0) + monkeypatch.setenv("SMTP_HOST", "127.0.0.1") + monkeypatch.setenv("SMTP_PORT", str(server.sockets[0].getsockname()[1])) + monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com") + monkeypatch.setenv("SMTP_TLS", "False") + monkeypatch.setenv("SMTP_USE_SSL", "False") + monkeypatch.delenv("SMTP_USERNAME", raising=False) + monkeypatch.delenv("SMTP_PASSWORD", raising=False) + slack: Final = SlackAlerting(alerting=["email"], alert_types=[]) + delivery: Final = UsageAlertDelivery(slack) + rule: Final = _RULE.model_copy( + update={"notification_channels": ("email",), "email_recipients": ("qa@example.com",)} + ) + try: + sending = asyncio.create_task(delivery.send(_ALERT, rule, "email")) + await asyncio.wait_for(connected.wait(), timeout=10) + sending.cancel() + with pytest.raises(asyncio.CancelledError): + await sending + threads: Final = threading.active_count() + pending: Final = UsageAlertDelivery._email_task + assert pending is not None and not pending.done() + for _ in range(100): + with pytest.raises(RuntimeError, match="still in flight"): + await delivery.send(_ALERT, rule, "email") + assert UsageAlertDelivery._email_task is pending + assert len(accepted) == 1 and threading.active_count() == threads + with pytest.raises(smtplib.SMTPServerDisconnected, match="timed out"): + await asyncio.wait_for(asyncio.shield(pending), timeout=10) + assert pending.done() + finally: + release.set() + server.close() + await server.wait_closed() + for writer in accepted: + writer.close() + pending = UsageAlertDelivery._email_task + if pending is not None: + with suppress(smtplib.SMTPServerDisconnected): + await asyncio.wait_for(asyncio.shield(pending), timeout=10) + UsageAlertDelivery._email_task = None + UsageAlertDelivery._email_key = None diff --git a/tests/unit/integrations/SlackAlerting/test_usage_alerts.py b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py index 540fd7aa5a9..cc8f608bfad 100644 --- a/tests/unit/integrations/SlackAlerting/test_usage_alerts.py +++ b/tests/unit/integrations/SlackAlerting/test_usage_alerts.py @@ -1,4 +1,3 @@ -import smtplib from datetime import date, datetime, timezone from typing import Final @@ -164,57 +163,3 @@ async def test_slack_source_labels_cannot_inject_mentions_or_links() -> None: assert b"<@someone>" in requests[0].content finally: await handler.close() - - -@pytest.mark.asyncio -async def test_cancelled_smtp_attempt_retains_only_one_send(monkeypatch: pytest.MonkeyPatch) -> None: - import asyncio - import threading - - connected: Final = asyncio.Event() - release: Final = asyncio.Event() - accepted: Final[list[asyncio.StreamWriter]] = [] - - async def stalled(_: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - accepted.append(writer) - connected.set() - await release.wait() - writer.close() - await writer.wait_closed() - - server: Final = await asyncio.start_server(stalled, "127.0.0.1", 0) - monkeypatch.setenv("SMTP_HOST", "127.0.0.1") - monkeypatch.setenv("SMTP_PORT", str(server.sockets[0].getsockname()[1])) - monkeypatch.setenv("SMTP_SENDER_EMAIL", "alerts@example.com") - monkeypatch.setenv("SMTP_TLS", "False") - monkeypatch.setenv("SMTP_USE_SSL", "False") - monkeypatch.delenv("SMTP_USERNAME", raising=False) - monkeypatch.delenv("SMTP_PASSWORD", raising=False) - slack: Final = SlackAlerting(alerting=["email"], alert_types=[]) - delivery: Final = UsageAlertDelivery(slack) - rule: Final = _RULE.model_copy( - update={"notification_channels": ("email",), "email_recipients": ("qa@example.com",)} - ) - try: - with pytest.raises(asyncio.TimeoutError): - await asyncio.wait_for(delivery.send(_ALERT, rule, "email"), timeout=0.1) - await asyncio.wait_for(connected.wait(), timeout=1) - threads: Final = threading.active_count() - pending: Final = UsageAlertDelivery._email_task - assert pending is not None and not pending.done() - for _ in range(100): - with pytest.raises(RuntimeError, match="still in flight"): - await delivery.send(_ALERT, rule, "email") - assert UsageAlertDelivery._email_task is pending - assert len(accepted) == 1 and threading.active_count() == threads - with pytest.raises(smtplib.SMTPServerDisconnected, match="timed out"): - await asyncio.wait_for(asyncio.shield(pending), timeout=3) - assert pending.done() - finally: - release.set() - server.close() - await server.wait_closed() - for writer in accepted: - writer.close() - UsageAlertDelivery._email_task = None - UsageAlertDelivery._email_key = None diff --git a/tests/unit/proxy/common_utils/test_model_deprecation.py b/tests/unit/proxy/common_utils/test_model_deprecation.py index 082e5731114..9d526d37bdd 100644 --- a/tests/unit/proxy/common_utils/test_model_deprecation.py +++ b/tests/unit/proxy/common_utils/test_model_deprecation.py @@ -762,7 +762,7 @@ def test_retirement_pages_resume_inside_large_alias_without_losing_configured_su model_list=[ { "model_name": "alias", - "litellm_params": {"model": f"openai/paged-model-{index}"}, + "litellm_params": {"model": "openai/paged-model"}, "model_info": { "id": f"paged-{index}", "deprecation_date": "2026-11-20", @@ -790,10 +790,29 @@ def test_retirement_pages_resume_inside_large_alias_without_losing_configured_su expected = collect_model_deprecations(router, today=today).imminent assert models == tuple(expected) assert len(models) == 1 - assert models[0].litellm_model == "openai/paged-model-0" + assert models[0].litellm_model == "openai/paged-model" assert models[0].successor_model == "replacement" +def test_retirement_pages_preserve_distinct_models_sharing_an_alias_and_date() -> None: + router = Router( + model_list=[ + { + "model_name": "shared-alias", + "litellm_params": {"model": model}, + "model_info": {"id": model, "deprecation_date": "2026-11-20"}, + } + for model in ("openai/unused-retired-model", "openai/used-retired-model") + ] + ) + page = RouterDeprecationSource(lambda: router).page(date(2026, 11, 5), DeprecationScanCursor()) + assert page.complete + assert tuple(model.litellm_model for model in page.models) == ( + "openai/unused-retired-model", + "openai/used-retired-model", + ) + + def test_retirement_source_bounds_pages_and_restarts_identity_after_router_replacement() -> None: router = Router( model_list=[