From 6ff7b05b7e9c130f263005f909e7e0e7be53bd34 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 13:55:33 +0530 Subject: [PATCH 01/37] feat(alerting): add model deprecation email thresholds to alerting args --- litellm/constants.py | 1 + litellm/types/integrations/slack_alerting.py | 25 ++++++++++- .../test_model_deprecation_email_args.py | 42 +++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_email_args.py diff --git a/litellm/constants.py b/litellm/constants.py index 5751e6e46af..080d1853a88 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1666,6 +1666,7 @@ USER_SPEND_ALERTS_JOB_ID: Final = "user_spend_alerts_job" PROMETHEUS_FALLBACK_STATS_JOB_ID: Final = "prometheus_fallback_stats_job" SLACK_DAILY_REPORT_LOCK_ID: Final = "slack_daily_report" SLACK_MODEL_DEPRECATION_LOCK_ID: Final = "slack_model_deprecation_warning" +EMAIL_MODEL_DEPRECATION_LOCK_ID: Final = "email_model_deprecation_warning" SPEND_LOG_RUN_LOOPS: Final = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500)) SPEND_LOG_CLEANUP_BATCH_SIZE: Final = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000)) SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = int(os.getenv("SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3)) diff --git a/litellm/types/integrations/slack_alerting.py b/litellm/types/integrations/slack_alerting.py index 64c0c530e9b..7f9a59f88ad 100644 --- a/litellm/types/integrations/slack_alerting.py +++ b/litellm/types/integrations/slack_alerting.py @@ -4,7 +4,7 @@ from datetime import datetime as dt from enum import Enum from typing import Any, Final, Literal, Optional, Union -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator from typing_extensions import TypedDict from litellm.types.utils import LiteLLMPydanticObjectBase @@ -125,6 +125,28 @@ class SlackAlertingArgs(LiteLLMPydanticObjectBase): ge=60, description="How often (in seconds) to check per-user spend thresholds and anomalies. Default is hourly.", ) + model_deprecation_email_thresholds: list[int] = Field( + default_factory=lambda: [30, 7, 0], + description=( + "Days before a model's deprecation_date at which team admins are emailed. " + "One email per team per threshold. An empty list disables deprecation emails." + ), + ) + model_deprecation_email_ttl: int = Field( + default=90 * 24 * 60 * 60, + ge=1, + description=( + "Seconds a sent (team, model, threshold) deprecation email is remembered to prevent re-sending. " + "Default is 90 days." + ), + ) + + @field_validator("model_deprecation_email_thresholds") + @classmethod + def _normalize_deprecation_email_thresholds(cls, value: list[int]) -> list[int]: + if any(threshold < 0 for threshold in value): + raise ValueError("model_deprecation_email_thresholds must be non-negative") + return sorted(set(value), reverse=True) class DeploymentMetrics(LiteLLMPydanticObjectBase): @@ -156,6 +178,7 @@ class SlackAlertingCacheKeys(Enum): latency_key = "latency_daily_metrics" report_sent_key = "daily_metrics_report_sent" deprecation_alert_sent_key = "model_deprecation_alert_sent" + deprecation_email_pass_key = "model_deprecation_email_pass_at" class AlertType(str, Enum): diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_email_args.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_email_args.py new file mode 100644 index 00000000000..40365bd9925 --- /dev/null +++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_email_args.py @@ -0,0 +1,42 @@ +"""Tests for the model deprecation email settings on SlackAlertingArgs.""" + +import pytest +from pydantic import ValidationError + +from litellm.constants import EMAIL_MODEL_DEPRECATION_LOCK_ID +from litellm.types.integrations.slack_alerting import ( + SlackAlertingArgs, + SlackAlertingCacheKeys, +) + + +def test_should_default_thresholds_to_30_7_0(): + assert SlackAlertingArgs().model_deprecation_email_thresholds == [30, 7, 0] + + +def test_should_default_ttl_to_90_days(): + assert SlackAlertingArgs().model_deprecation_email_ttl == 90 * 24 * 60 * 60 + + +def test_should_dedupe_and_sort_thresholds_descending(): + args = SlackAlertingArgs(model_deprecation_email_thresholds=[0, 7, 7, 30]) + assert args.model_deprecation_email_thresholds == [30, 7, 0] + + +def test_should_allow_empty_thresholds_to_disable_emails(): + assert SlackAlertingArgs(model_deprecation_email_thresholds=[]).model_deprecation_email_thresholds == [] + + +def test_should_reject_negative_thresholds(): + with pytest.raises(ValidationError, match="non-negative"): + SlackAlertingArgs(model_deprecation_email_thresholds=[30, -1]) + + +def test_should_reject_non_positive_ttl(): + with pytest.raises(ValidationError, match="greater than or equal to 1"): + SlackAlertingArgs(model_deprecation_email_ttl=0) + + +def test_should_expose_pass_cache_key_and_lock_id(): + assert SlackAlertingCacheKeys.deprecation_email_pass_key.value == "model_deprecation_email_pass_at" + assert EMAIL_MODEL_DEPRECATION_LOCK_ID == "email_model_deprecation_warning" From 5a8b2d5e9a1b3380a708acf7af3d778d0edb4769 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 14:06:36 +0530 Subject: [PATCH 02/37] refactor(alerting): make deprecation email thresholds an immutable tuple --- litellm/types/integrations/slack_alerting.py | 12 ++++++------ .../test_model_deprecation_email_args.py | 10 ++++++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/litellm/types/integrations/slack_alerting.py b/litellm/types/integrations/slack_alerting.py index 7f9a59f88ad..91881e83758 100644 --- a/litellm/types/integrations/slack_alerting.py +++ b/litellm/types/integrations/slack_alerting.py @@ -125,8 +125,8 @@ class SlackAlertingArgs(LiteLLMPydanticObjectBase): ge=60, description="How often (in seconds) to check per-user spend thresholds and anomalies. Default is hourly.", ) - model_deprecation_email_thresholds: list[int] = Field( - default_factory=lambda: [30, 7, 0], + model_deprecation_email_thresholds: tuple[int, ...] = Field( + default=(30, 7, 0), description=( "Days before a model's deprecation_date at which team admins are emailed. " "One email per team per threshold. An empty list disables deprecation emails." @@ -136,17 +136,17 @@ class SlackAlertingArgs(LiteLLMPydanticObjectBase): default=90 * 24 * 60 * 60, ge=1, description=( - "Seconds a sent (team, model, threshold) deprecation email is remembered to prevent re-sending. " - "Default is 90 days." + "Seconds each (team, model, threshold) deprecation email is remembered after sending so the same " + "milestone is not emailed twice. Default is 90 days." ), ) @field_validator("model_deprecation_email_thresholds") @classmethod - def _normalize_deprecation_email_thresholds(cls, value: list[int]) -> list[int]: + def _normalize_deprecation_email_thresholds(cls, value: tuple[int, ...]) -> tuple[int, ...]: if any(threshold < 0 for threshold in value): raise ValueError("model_deprecation_email_thresholds must be non-negative") - return sorted(set(value), reverse=True) + return tuple(sorted(frozenset(value), reverse=True)) class DeploymentMetrics(LiteLLMPydanticObjectBase): diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_email_args.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_email_args.py index 40365bd9925..22ac2f1494c 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_email_args.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_email_args.py @@ -1,5 +1,7 @@ """Tests for the model deprecation email settings on SlackAlertingArgs.""" +from typing import Final + import pytest from pydantic import ValidationError @@ -11,7 +13,7 @@ from litellm.types.integrations.slack_alerting import ( def test_should_default_thresholds_to_30_7_0(): - assert SlackAlertingArgs().model_deprecation_email_thresholds == [30, 7, 0] + assert SlackAlertingArgs().model_deprecation_email_thresholds == (30, 7, 0) def test_should_default_ttl_to_90_days(): @@ -19,12 +21,12 @@ def test_should_default_ttl_to_90_days(): def test_should_dedupe_and_sort_thresholds_descending(): - args = SlackAlertingArgs(model_deprecation_email_thresholds=[0, 7, 7, 30]) - assert args.model_deprecation_email_thresholds == [30, 7, 0] + args: Final = SlackAlertingArgs(model_deprecation_email_thresholds=[0, 7, 7, 30]) + assert args.model_deprecation_email_thresholds == (30, 7, 0) def test_should_allow_empty_thresholds_to_disable_emails(): - assert SlackAlertingArgs(model_deprecation_email_thresholds=[]).model_deprecation_email_thresholds == [] + assert SlackAlertingArgs(model_deprecation_email_thresholds=[]).model_deprecation_email_thresholds == () def test_should_reject_negative_thresholds(): From eb1b5469ed77fc5f7184dfe812e388971e5f8ffe Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 14:12:32 +0530 Subject: [PATCH 03/37] test(alerting): move deprecation email args tests to the mirrored types path --- .../integrations/test_slack_alerting.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/test_litellm/{integrations/SlackAlerting/test_model_deprecation_email_args.py => types/integrations/test_slack_alerting.py} (100%) diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_email_args.py b/tests/test_litellm/types/integrations/test_slack_alerting.py similarity index 100% rename from tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_email_args.py rename to tests/test_litellm/types/integrations/test_slack_alerting.py From fa62054fc5b30590364a577c3b6de724b21257ac Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 14:18:20 +0530 Subject: [PATCH 04/37] feat(proxy): add milestone selection for model deprecation emails --- .../model_deprecation_notifications.py | 12 +++++++++ .../test_model_deprecation_notifications.py | 27 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 litellm/proxy/common_utils/model_deprecation_notifications.py create mode 100644 tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py new file mode 100644 index 00000000000..eaa76b0442e --- /dev/null +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -0,0 +1,12 @@ +"""Email team admins about the deprecating models their team can reach""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Final + + +def select_milestone(days_until: int, thresholds: Sequence[int]) -> int | None: + """The most urgent threshold already reached, None while the first one is still ahead""" + reached: Final = tuple(threshold for threshold in thresholds if days_until <= threshold) + return min(reached) if reached else None diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py new file mode 100644 index 00000000000..9cd3ae67ce4 --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py @@ -0,0 +1,27 @@ +"""Tests for the model deprecation email notifications module. + +Pure functions are exercised directly; the router, DB, cache and deliverer are +hand-rolled fakes injected through the module's own interfaces, so nothing here +patches litellm internals or needs a proxy, Postgres or Redis. +""" + +from __future__ import annotations + +import pytest + +from litellm.proxy.common_utils.model_deprecation_notifications import select_milestone + + +class TestSelectMilestone: + @pytest.mark.parametrize( + ("days_until", "expected"), + [(45, None), (30, 30), (25, 30), (7, 7), (3, 7), (0, 0), (-12, 0)], + ) + def test_should_pick_the_most_urgent_threshold_reached(self, days_until, expected): + assert select_milestone(days_until, (30, 7, 0)) == expected + + def test_should_return_none_when_no_thresholds_configured(self): + assert select_milestone(-5, ()) is None + + def test_should_not_depend_on_threshold_order(self): + assert select_milestone(5, (0, 30, 7)) == 7 From 87b8b5e52cc1ec6fcc057a10977778913ccccb0b Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 14:23:37 +0530 Subject: [PATCH 05/37] refactor(proxy): simplify milestone selection and tighten its order test --- .../common_utils/model_deprecation_notifications.py | 4 +--- .../common_utils/test_model_deprecation_notifications.py | 9 ++------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index eaa76b0442e..1c5b569ae4a 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -3,10 +3,8 @@ from __future__ import annotations from collections.abc import Sequence -from typing import Final def select_milestone(days_until: int, thresholds: Sequence[int]) -> int | None: """The most urgent threshold already reached, None while the first one is still ahead""" - reached: Final = tuple(threshold for threshold in thresholds if days_until <= threshold) - return min(reached) if reached else None + return min((threshold for threshold in thresholds if days_until <= threshold), default=None) diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py index 9cd3ae67ce4..cc7cdd9b434 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py @@ -1,9 +1,4 @@ -"""Tests for the model deprecation email notifications module. - -Pure functions are exercised directly; the router, DB, cache and deliverer are -hand-rolled fakes injected through the module's own interfaces, so nothing here -patches litellm internals or needs a proxy, Postgres or Redis. -""" +"""Tests for the model deprecation email notifications module.""" from __future__ import annotations @@ -24,4 +19,4 @@ class TestSelectMilestone: assert select_milestone(-5, ()) is None def test_should_not_depend_on_threshold_order(self): - assert select_milestone(5, (0, 30, 7)) == 7 + assert select_milestone(5, (0, 7, 30)) == 7 From 0d62ad7b2270b147c3150b3ed6aec0e2ea7d0019 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 14:26:41 +0530 Subject: [PATCH 06/37] feat(proxy): resolve team admin emails for deprecation notices --- litellm/integrations/email_alerting.py | 22 +++++- .../integrations/test_email_alerting.py | 69 +++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 tests/test_litellm/integrations/test_email_alerting.py diff --git a/litellm/integrations/email_alerting.py b/litellm/integrations/email_alerting.py index 351896425bb..e381dc282f1 100644 --- a/litellm/integrations/email_alerting.py +++ b/litellm/integrations/email_alerting.py @@ -3,12 +3,16 @@ Functions for sending Email Alerts """ import os -from typing import Final +from typing import TYPE_CHECKING, Final from litellm._logging import verbose_logger, verbose_proxy_logger +from litellm.models.team import LiteLLM_TeamTable from litellm.proxy._types import WebhookEvent from litellm.repositories.team_repository import TeamRepository +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + # we use this for the email header, please send a test email if you change this. verify it looks good on email LITELLM_LOGO_URL: Final = "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" LITELLM_SUPPORT_CONTACT: Final = "support@berri.ai" @@ -65,6 +69,22 @@ async def get_all_team_member_emails(team_id: str | None = None) -> list: return emails +async def get_team_admin_emails(team: LiteLLM_TeamTable, prisma_client: "PrismaClient") -> tuple[str, ...]: + """Emails of the team's admins: members_with_roles admins plus the legacy admins list, deduped""" + admin_ids: Final = frozenset( + ( + *(member.user_id for member in team.members_with_roles if member.role == "admin" and member.user_id), + *(user_id for user_id in team.admins if user_id), + ) + ) + if not admin_ids: + return () + rows: Final = await prisma_client.db.litellm_usertable.find_many( + where={"user_id": {"in": sorted(admin_ids)}} # mutable-ok: prisma filter payloads are plain dict/list + ) + return tuple(dict.fromkeys(row.user_email for row in rows if row.user_email)) + + async def send_team_budget_alert(webhook_event: WebhookEvent) -> bool: """ Send an Email Alert to All Team Members when the Team Budget is crossed diff --git a/tests/test_litellm/integrations/test_email_alerting.py b/tests/test_litellm/integrations/test_email_alerting.py new file mode 100644 index 00000000000..6b8c91c1b47 --- /dev/null +++ b/tests/test_litellm/integrations/test_email_alerting.py @@ -0,0 +1,69 @@ +"""Tests for litellm/integrations/email_alerting.py.""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Final + +import pytest + +from litellm.integrations.email_alerting import get_team_admin_emails +from litellm.models.team import LiteLLM_TeamTable, Member + + +class _FakeUserTable: + def __init__(self, rows): + self.rows = rows + self.queries = [] + + async def find_many(self, where): + self.queries.append(where) + wanted: Final = where["user_id"]["in"] + return [row for row in self.rows if row.user_id in wanted] + + +def _prisma(rows): + return SimpleNamespace(db=SimpleNamespace(litellm_usertable=_FakeUserTable(rows))) + + +USERS: Final = ( + SimpleNamespace(user_id="u-admin", user_email="admin@example.com"), + SimpleNamespace(user_id="u-member", user_email="member@example.com"), + SimpleNamespace(user_id="u-legacy", user_email="legacy@example.com"), + SimpleNamespace(user_id="u-no-email", user_email=None), +) + + +@pytest.mark.asyncio +async def test_should_return_only_admin_emails(): + team: Final = LiteLLM_TeamTable( + team_id="t1", + members_with_roles=[Member(user_id="u-admin", role="admin"), Member(user_id="u-member", role="user")], + ) + assert await get_team_admin_emails(team, _prisma(USERS)) == ("admin@example.com",) + + +@pytest.mark.asyncio +async def test_should_include_legacy_admins_list(): + team: Final = LiteLLM_TeamTable( + team_id="t1", admins=["u-legacy"], members_with_roles=[Member(user_id="u-admin", role="admin")] + ) + assert sorted(await get_team_admin_emails(team, _prisma(USERS))) == ["admin@example.com", "legacy@example.com"] + + +@pytest.mark.asyncio +async def test_should_drop_admins_without_an_email_and_dedupe(): + team: Final = LiteLLM_TeamTable( + team_id="t1", + admins=["u-admin"], + members_with_roles=[Member(user_id="u-admin", role="admin"), Member(user_id="u-no-email", role="admin")], + ) + assert await get_team_admin_emails(team, _prisma(USERS)) == ("admin@example.com",) + + +@pytest.mark.asyncio +async def test_should_not_query_when_team_has_no_admins(): + prisma: Final = _prisma(USERS) + team: Final = LiteLLM_TeamTable(team_id="t1", members_with_roles=[Member(user_id="u-member", role="user")]) + assert await get_team_admin_emails(team, prisma) == () + assert prisma.db.litellm_usertable.queries == [] From 9e47254bc50141afc1ade688edfbdc90764111bd Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 14:36:11 +0530 Subject: [PATCH 07/37] feat(proxy): map deprecating models to the teams that can reach them --- .../model_deprecation_notifications.py | 113 +++++++++++++++++- .../test_model_deprecation_notifications.py | 101 +++++++++++++++- 2 files changed, 212 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index 1c5b569ae4a..d34e0243df4 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -2,9 +2,120 @@ from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from litellm.models.team import LiteLLM_TeamTable +from litellm.proxy._types import ProxyException +from litellm.proxy.auth.auth_checks import can_team_access_model +from litellm.types.proxy.model_deprecation import ModelDeprecationInfo + +if TYPE_CHECKING: + from litellm.router import Router + + +@dataclass(frozen=True, slots=True) +class AffectedModel: + info: ModelDeprecationInfo + display_name: str + milestone: int + + +@dataclass(frozen=True, slots=True) +class _DeploymentOwner: + model_name: str + team_id: str + public_name: str def select_milestone(days_until: int, thresholds: Sequence[int]) -> int | None: """The most urgent threshold already reached, None while the first one is still ahead""" return min((threshold for threshold in thresholds if days_until <= threshold), default=None) + + +def _reached(info: ModelDeprecationInfo, thresholds: Sequence[int]) -> tuple[ModelDeprecationInfo, int] | None: + milestone: Final = select_milestone(info.days_until_deprecation, thresholds) + return None if milestone is None else (info, milestone) + + +def _owner_of(deployment: Mapping[str, object]) -> _DeploymentOwner | None: + model_name: Final = deployment.get("model_name") + model_info: Final = deployment.get("model_info") + if not isinstance(model_name, str) or not isinstance(model_info, Mapping): + return None + team_id: Final = model_info.get("team_id") + if not isinstance(team_id, str) or not team_id: + return None + public_name: Final = model_info.get("team_public_model_name") + return _DeploymentOwner( + model_name=model_name, + team_id=team_id, + public_name=public_name if isinstance(public_name, str) and public_name else model_name, + ) + + +def _deployment_owners(llm_router: Router) -> Mapping[str, _DeploymentOwner]: + """Team-scoped deployments keyed by their internal model name, with the name the team knows""" + owners: Final = (_owner_of(deployment) for deployment in llm_router.get_model_list() or ()) + return MappingProxyType({owner.model_name: owner for owner in owners if owner is not None}) + + +async def _team_can_access(model_name: str, team: LiteLLM_TeamTable, llm_router: Router) -> bool: + """Reuse the auth check so wildcards, access groups and the empty-list rule match real requests""" + try: + await can_team_access_model(model=model_name, team_object=team, llm_router=llm_router) + except ProxyException: + return False + return True + + +async def _display_name_for( + info: ModelDeprecationInfo, team: LiteLLM_TeamTable, owner: _DeploymentOwner | None, llm_router: Router +) -> str | None: + if owner is not None: + return owner.public_name if owner.team_id == team.team_id else None + if await _team_can_access(info.model_name, team, llm_router): + return info.model_name + return None + + +async def _affected_model( + reached: tuple[ModelDeprecationInfo, int], + team: LiteLLM_TeamTable, + owners: Mapping[str, _DeploymentOwner], + llm_router: Router, +) -> AffectedModel | None: + info, milestone = reached + display_name: Final = await _display_name_for(info, team, owners.get(info.model_name), llm_router) + return None if display_name is None else AffectedModel(info=info, display_name=display_name, milestone=milestone) + + +async def _affected_for_team( + team: LiteLLM_TeamTable, + reached: Sequence[tuple[ModelDeprecationInfo, int]], + owners: Mapping[str, _DeploymentOwner], + llm_router: Router, +) -> tuple[AffectedModel, ...]: + candidates: Final = tuple([await _affected_model(pair, team, owners, llm_router) for pair in reached]) + return tuple(model for model in candidates if model is not None) + + +async def resolve_affected_teams( + infos: Sequence[ModelDeprecationInfo], + llm_router: Router, + teams: Sequence[LiteLLM_TeamTable], + thresholds: Sequence[int], +) -> Mapping[str, tuple[AffectedModel, ...]]: + """Per team id, the deprecating models it can reach that have crossed a threshold""" + reached: Final = tuple(pair for pair in (_reached(info, thresholds) for info in infos) if pair is not None) + owners: Final = _deployment_owners(llm_router) + per_team: Final = MappingProxyType( + { + team.team_id: await _affected_for_team(team, reached, owners, llm_router) + for team in teams + if not team.blocked + } + ) + return MappingProxyType({team_id: models for team_id, models in per_team.items() if models}) diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py index cc7cdd9b434..7ef63ceb946 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py @@ -2,9 +2,19 @@ from __future__ import annotations +from datetime import date +from typing import Final +from unittest.mock import MagicMock + import pytest -from litellm.proxy.common_utils.model_deprecation_notifications import select_milestone +from litellm.models.team import LiteLLM_TeamTable +from litellm.proxy.common_utils.model_deprecation_notifications import ( + AffectedModel, + resolve_affected_teams, + select_milestone, +) +from litellm.types.proxy.model_deprecation import ModelDeprecationInfo class TestSelectMilestone: @@ -20,3 +30,92 @@ class TestSelectMilestone: def test_should_not_depend_on_threshold_order(self): assert select_milestone(5, (0, 7, 30)) == 7 + + +def _info(model_name: str, days_until: int, provider: str = "openai") -> ModelDeprecationInfo: + return ModelDeprecationInfo( + model_name=model_name, + litellm_model=model_name, + deprecation_date=date(2026, 10, 1), + days_until_deprecation=days_until, + status="deprecated" if days_until < 0 else ("imminent" if days_until <= 30 else "upcoming"), + litellm_provider=provider, + ) + + +def _router(deployments): + router: Final = MagicMock() + router.get_model_list.return_value = deployments + router.get_model_access_groups.return_value = {} + router.model_group_alias = {} + return router + + +def _team(team_id: str, models=(), blocked: bool = False) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable(team_id=team_id, models=list(models), blocked=blocked) + + +THRESHOLDS: Final = (30, 7, 0) + + +class TestResolveAffectedTeams: + @pytest.mark.asyncio + async def test_should_include_team_listing_the_model_explicitly(self): + affected: Final = await resolve_affected_teams( + (_info("gpt-old", 5),), _router([]), (_team("t1", ("gpt-old",)), _team("t2", ("other",))), THRESHOLDS + ) + assert set(affected) == {"t1"} + assert affected["t1"] == (AffectedModel(info=_info("gpt-old", 5), display_name="gpt-old", milestone=7),) + + @pytest.mark.asyncio + async def test_should_include_unrestricted_team(self): + affected: Final = await resolve_affected_teams((_info("gpt-old", 5),), _router([]), (_team("t1"),), THRESHOLDS) + assert set(affected) == {"t1"} + + @pytest.mark.asyncio + async def test_should_match_wildcard_allowlists(self): + affected: Final = await resolve_affected_teams( + (_info("openai/gpt-old", 5),), + _router([]), + (_team("t1", ("openai/*",)), _team("t2", ("azure/*",))), + THRESHOLDS, + ) + assert set(affected) == {"t1"} + + @pytest.mark.asyncio + async def test_should_skip_blocked_teams(self): + affected: Final = await resolve_affected_teams( + (_info("gpt-old", 5),), _router([]), (_team("t1", blocked=True),), THRESHOLDS + ) + assert dict(affected) == {} + + @pytest.mark.asyncio + async def test_should_skip_models_that_have_not_reached_a_threshold(self): + affected: Final = await resolve_affected_teams((_info("gpt-old", 45),), _router([]), (_team("t1"),), THRESHOLDS) + assert dict(affected) == {} + + @pytest.mark.asyncio + async def test_should_attribute_team_scoped_deployment_to_its_owner_only(self): + deployments: Final = [ + { + "model_name": "model_name_t2_abc", + "litellm_params": {"model": "openai/gpt-old"}, + "model_info": {"id": "1", "team_id": "t2", "team_public_model_name": "gpt-old"}, + } + ] + affected: Final = await resolve_affected_teams( + (_info("model_name_t2_abc", -3),), _router(deployments), (_team("t1"), _team("t2")), THRESHOLDS + ) + assert set(affected) == {"t2"} + assert affected["t2"][0].display_name == "gpt-old" + assert affected["t2"][0].milestone == 0 + + @pytest.mark.asyncio + async def test_should_collect_every_reachable_model_per_team(self): + affected: Final = await resolve_affected_teams( + (_info("gpt-old", 5), _info("gpt-older", -1)), + _router([]), + (_team("t1", ("gpt-old", "gpt-older")),), + THRESHOLDS, + ) + assert [m.info.model_name for m in affected["t1"]] == ["gpt-old", "gpt-older"] From 9d50497af30979b1aaa9c2a98f87dc889ca8b88b Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 14:38:37 +0530 Subject: [PATCH 08/37] refactor(proxy): validate team-scoped deployments with pydantic at the boundary --- .../model_deprecation_notifications.py | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index d34e0243df4..3717f3b22a5 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -7,6 +7,8 @@ from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Final +from pydantic import BaseModel, Field, ValidationError + from litellm.models.team import LiteLLM_TeamTable from litellm.proxy._types import ProxyException from litellm.proxy.auth.auth_checks import can_team_access_model @@ -40,19 +42,27 @@ def _reached(info: ModelDeprecationInfo, thresholds: Sequence[int]) -> tuple[Mod return None if milestone is None else (info, milestone) +class _TeamScope(BaseModel): + team_id: str | None = None + team_public_model_name: str | None = None + + +class _OwnedDeployment(BaseModel): + model_name: str + model_info: _TeamScope = Field(default_factory=_TeamScope) + + def _owner_of(deployment: Mapping[str, object]) -> _DeploymentOwner | None: - model_name: Final = deployment.get("model_name") - model_info: Final = deployment.get("model_info") - if not isinstance(model_name, str) or not isinstance(model_info, Mapping): + try: + parsed: Final = _OwnedDeployment.model_validate(deployment) + except ValidationError: return None - team_id: Final = model_info.get("team_id") - if not isinstance(team_id, str) or not team_id: + if not parsed.model_info.team_id: return None - public_name: Final = model_info.get("team_public_model_name") return _DeploymentOwner( - model_name=model_name, - team_id=team_id, - public_name=public_name if isinstance(public_name, str) and public_name else model_name, + model_name=parsed.model_name, + team_id=parsed.model_info.team_id, + public_name=parsed.model_info.team_public_model_name or parsed.model_name, ) From 0bd076ece27074b7697d8f3e5da8e8ae737f68b3 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 15:01:10 +0530 Subject: [PATCH 09/37] fix(proxy): treat a malformed team allowlist as denied in deprecation resolution --- .../model_deprecation_notifications.py | 4 ++ .../test_model_deprecation_notifications.py | 40 ++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index 3717f3b22a5..7ba0b10e903 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Final from pydantic import BaseModel, Field, ValidationError +from litellm._logging import verbose_proxy_logger from litellm.models.team import LiteLLM_TeamTable from litellm.proxy._types import ProxyException from litellm.proxy.auth.auth_checks import can_team_access_model @@ -78,6 +79,9 @@ async def _team_can_access(model_name: str, team: LiteLLM_TeamTable, llm_router: await can_team_access_model(model=model_name, team_object=team, llm_router=llm_router) except ProxyException: return False + except Exception as e: # noqa: BLE001 # a malformed allowlist (e.g. a wildcard with regex metacharacters) must not abort the pass + verbose_proxy_logger.debug("model_deprecation: access check failed for team %s: %s", team.team_id, e) + return False return True diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py index 7ef63ceb946..e3df5c8d9bc 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Mapping, Sequence from datetime import date from typing import Final from unittest.mock import MagicMock @@ -43,7 +44,7 @@ def _info(model_name: str, days_until: int, provider: str = "openai") -> ModelDe ) -def _router(deployments): +def _router(deployments: Sequence[Mapping[str, object]]) -> MagicMock: router: Final = MagicMock() router.get_model_list.return_value = deployments router.get_model_access_groups.return_value = {} @@ -51,7 +52,7 @@ def _router(deployments): return router -def _team(team_id: str, models=(), blocked: bool = False) -> LiteLLM_TeamTable: +def _team(team_id: str, models: Sequence[str] = (), blocked: bool = False) -> LiteLLM_TeamTable: return LiteLLM_TeamTable(team_id=team_id, models=list(models), blocked=blocked) @@ -119,3 +120,38 @@ class TestResolveAffectedTeams: THRESHOLDS, ) assert [m.info.model_name for m in affected["t1"]] == ["gpt-old", "gpt-older"] + + @pytest.mark.asyncio + async def test_should_treat_an_ordinary_deployment_as_shared(self): + deployments: Final = [ + {"model_name": "gpt-old", "litellm_params": {"model": "openai/gpt-old"}, "model_info": {"id": "1"}} + ] + affected: Final = await resolve_affected_teams( + (_info("gpt-old", 5),), _router(deployments), (_team("t1", ("gpt-old",)),), THRESHOLDS + ) + assert set(affected) == {"t1"} + assert affected["t1"][0].display_name == "gpt-old" + + @pytest.mark.asyncio + async def test_should_fall_back_to_the_internal_name_when_a_scoped_deployment_has_no_public_name(self): + deployments: Final = [ + { + "model_name": "model_name_t2_abc", + "litellm_params": {"model": "openai/gpt-old"}, + "model_info": {"id": "1", "team_id": "t2"}, + } + ] + affected: Final = await resolve_affected_teams( + (_info("model_name_t2_abc", -3),), _router(deployments), (_team("t2"),), THRESHOLDS + ) + assert affected["t2"][0].display_name == "model_name_t2_abc" + + @pytest.mark.asyncio + async def test_should_deny_a_team_whose_allowlist_is_malformed_and_keep_going(self): + affected: Final = await resolve_affected_teams( + (_info("openai/gpt-old", 5),), + _router([]), + (_team("t1", ("openai/[*",)), _team("t2", ("openai/gpt-old",))), + THRESHOLDS, + ) + assert set(affected) == {"t2"} From 802e82cddf4fca1bc49ea683cfa0197a471c6f9e Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 15:02:39 +0530 Subject: [PATCH 10/37] refactor(proxy): load team admin emails through the user repository --- litellm/integrations/email_alerting.py | 3 +- .../integrations/test_email_alerting.py | 32 +++++++++++++++---- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/litellm/integrations/email_alerting.py b/litellm/integrations/email_alerting.py index e381dc282f1..dff8d980430 100644 --- a/litellm/integrations/email_alerting.py +++ b/litellm/integrations/email_alerting.py @@ -9,6 +9,7 @@ from litellm._logging import verbose_logger, verbose_proxy_logger from litellm.models.team import LiteLLM_TeamTable from litellm.proxy._types import WebhookEvent from litellm.repositories.team_repository import TeamRepository +from litellm.repositories.user_repository import UserRepository if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient @@ -79,7 +80,7 @@ async def get_team_admin_emails(team: LiteLLM_TeamTable, prisma_client: "PrismaC ) if not admin_ids: return () - rows: Final = await prisma_client.db.litellm_usertable.find_many( + rows: Final = await UserRepository(prisma_client).find_many( where={"user_id": {"in": sorted(admin_ids)}} # mutable-ok: prisma filter payloads are plain dict/list ) return tuple(dict.fromkeys(row.user_email for row in rows if row.user_email)) diff --git a/tests/test_litellm/integrations/test_email_alerting.py b/tests/test_litellm/integrations/test_email_alerting.py index 6b8c91c1b47..de72cce9bac 100644 --- a/tests/test_litellm/integrations/test_email_alerting.py +++ b/tests/test_litellm/integrations/test_email_alerting.py @@ -16,10 +16,10 @@ class _FakeUserTable: self.rows = rows self.queries = [] - async def find_many(self, where): + async def find_many(self, where=None, take=None, skip=None, order=None): self.queries.append(where) wanted: Final = where["user_id"]["in"] - return [row for row in self.rows if row.user_id in wanted] + return [row for row in self.rows if row["user_id"] in wanted] def _prisma(rows): @@ -27,10 +27,10 @@ def _prisma(rows): USERS: Final = ( - SimpleNamespace(user_id="u-admin", user_email="admin@example.com"), - SimpleNamespace(user_id="u-member", user_email="member@example.com"), - SimpleNamespace(user_id="u-legacy", user_email="legacy@example.com"), - SimpleNamespace(user_id="u-no-email", user_email=None), + {"user_id": "u-admin", "user_email": "admin@example.com"}, + {"user_id": "u-member", "user_email": "member@example.com"}, + {"user_id": "u-legacy", "user_email": "legacy@example.com"}, + {"user_id": "u-no-email", "user_email": None}, ) @@ -67,3 +67,23 @@ async def test_should_not_query_when_team_has_no_admins(): team: Final = LiteLLM_TeamTable(team_id="t1", members_with_roles=[Member(user_id="u-member", role="user")]) assert await get_team_admin_emails(team, prisma) == () assert prisma.db.litellm_usertable.queries == [] + + +@pytest.mark.asyncio +async def test_should_skip_email_only_members_and_blank_legacy_ids_and_dedupe(): + prisma: Final = _prisma( + ( + {"user_id": "u-admin", "user_email": "admin@example.com"}, + {"user_id": "u-legacy", "user_email": "admin@example.com"}, + ) + ) + team: Final = LiteLLM_TeamTable( + team_id="t1", + admins=["", "u-legacy", "u-admin"], + members_with_roles=[ + Member(user_id="u-admin", role="admin"), + Member(user_email="mail-only@example.com", role="admin"), + ], + ) + assert await get_team_admin_emails(team, prisma) == ("admin@example.com",) + assert prisma.db.litellm_usertable.queries == [{"user_id": {"in": ["u-admin", "u-legacy"]}}] From d4b22323595574cb4ecabee4d39d2449fdafa1bb Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 15:07:04 +0530 Subject: [PATCH 11/37] feat(proxy): render the model deprecation digest email --- .../model_deprecation_email.py | 42 +++++++++++++ .../model_deprecation_notifications.py | 44 +++++++++++++ .../test_model_deprecation_notifications.py | 61 +++++++++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 litellm/integrations/email_templates/model_deprecation_email.py diff --git a/litellm/integrations/email_templates/model_deprecation_email.py b/litellm/integrations/email_templates/model_deprecation_email.py new file mode 100644 index 00000000000..a8e779e4713 --- /dev/null +++ b/litellm/integrations/email_templates/model_deprecation_email.py @@ -0,0 +1,42 @@ +"""Template for the model deprecation digest sent to team admins""" + +from typing import Final + +MODEL_DEPRECATION_EMAIL_TEMPLATE: Final = """ +LiteLLM Logo

+ +

Hi team admin,

+ +

The following models available to team {team_name} are being deprecated by their provider. +Requests to a deprecated model may start failing on its deprecation date.

+ + + + + + + + + + + + +{model_rows} + +
ModelProviderDeprecation dateDays leftStatus
+ +

Plan a migration to a supported model. See +the LiteLLM model management docs for guidance.

+ +

If you have any questions, please send an email to {email_support_contact}

+ +

Best,
The LiteLLM team

+""" + +MODEL_DEPRECATION_EMAIL_ROW_TEMPLATE: Final = """ + {model_name} + {provider} + {deprecation_date} + {days_left} + {status} + """ diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index 7ba0b10e903..8e1ed18beae 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -2,6 +2,7 @@ from __future__ import annotations +import html from collections.abc import Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType @@ -10,6 +11,11 @@ from typing import TYPE_CHECKING, Final from pydantic import BaseModel, Field, ValidationError from litellm._logging import verbose_proxy_logger +from litellm.integrations.email_templates.email_footer import EMAIL_FOOTER +from litellm.integrations.email_templates.model_deprecation_email import ( + MODEL_DEPRECATION_EMAIL_ROW_TEMPLATE, + MODEL_DEPRECATION_EMAIL_TEMPLATE, +) from litellm.models.team import LiteLLM_TeamTable from litellm.proxy._types import ProxyException from litellm.proxy.auth.auth_checks import can_team_access_model @@ -33,6 +39,14 @@ class _DeploymentOwner: public_name: str +@dataclass(frozen=True, slots=True) +class TeamNotification: + team_id: str + team_alias: str | None + recipients: tuple[str, ...] + models: tuple[AffectedModel, ...] + + def select_milestone(days_until: int, thresholds: Sequence[int]) -> int | None: """The most urgent threshold already reached, None while the first one is still ahead""" return min((threshold for threshold in thresholds if days_until <= threshold), default=None) @@ -133,3 +147,33 @@ async def resolve_affected_teams( } ) return MappingProxyType({team_id: models for team_id, models in per_team.items() if models}) + + +def _days_left_label(days_until: int) -> str: + return f"deprecated {abs(days_until)}d ago" if days_until < 0 else f"{days_until}d" + + +def _render_row(model: AffectedModel) -> str: + return MODEL_DEPRECATION_EMAIL_ROW_TEMPLATE.format( + model_name=html.escape(model.display_name), + provider=html.escape(model.info.litellm_provider or "unknown"), + deprecation_date=model.info.deprecation_date.isoformat(), + days_left=_days_left_label(model.info.days_until_deprecation), + status=model.info.status, + ) + + +def render_model_deprecation_email( + notification: TeamNotification, email_logo_url: str, email_support_contact: str +) -> tuple[str, str]: + """(subject, html) for one team's digest, every model-sourced string escaped""" + team_name: Final = notification.team_alias or notification.team_id + verb: Final = "deprecated" if any(model.milestone == 0 for model in notification.models) else "deprecating" + subject: Final = f"[LiteLLM] {len(notification.models)} model(s) {verb} for team {team_name}" + body: Final = MODEL_DEPRECATION_EMAIL_TEMPLATE.format( + email_logo_url=email_logo_url, + team_name=html.escape(team_name), + model_rows="\n".join(_render_row(model) for model in notification.models), + email_support_contact=html.escape(email_support_contact), + ) + return subject, body + EMAIL_FOOTER diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py index e3df5c8d9bc..15afa366801 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py @@ -12,6 +12,8 @@ import pytest from litellm.models.team import LiteLLM_TeamTable from litellm.proxy.common_utils.model_deprecation_notifications import ( AffectedModel, + TeamNotification, + render_model_deprecation_email, resolve_affected_teams, select_milestone, ) @@ -155,3 +157,62 @@ class TestResolveAffectedTeams: THRESHOLDS, ) assert set(affected) == {"t2"} + + +class TestRenderModelDeprecationEmail: + def _notification(self, models: tuple[AffectedModel, ...]) -> TeamNotification: + return TeamNotification(team_id="t1", team_alias="Data Team", recipients=("a@example.com",), models=models) + + def test_should_render_one_row_per_model_with_escaped_names(self): + hostile: Final = _info("", 5, provider="op&en") + subject, html = render_model_deprecation_email( + self._notification((AffectedModel(info=hostile, display_name=hostile.model_name, milestone=7),)), + email_logo_url="https://logo", + email_support_contact="help@example.com", + ) + assert "5d" in html + assert "https://logo" in html + assert "help@example.com" in html + assert "Data Team" in html + assert subject == "[LiteLLM] 1 model(s) deprecating for team Data Team" + + def test_should_say_deprecated_in_subject_once_a_model_reaches_day_zero(self): + models: Final = ( + AffectedModel(info=_info("gpt-old", 5), display_name="gpt-old", milestone=7), + AffectedModel(info=_info("gpt-older", -4), display_name="gpt-older", milestone=0), + ) + subject, html = render_model_deprecation_email( + self._notification(models), email_logo_url="https://logo", email_support_contact="help@example.com" + ) + assert subject == "[LiteLLM] 2 model(s) deprecated for team Data Team" + assert "deprecated 4d ago" in html + assert html.index("gpt-old") < html.index("gpt-older") + + def test_should_fall_back_to_team_id_when_alias_missing(self): + notification: Final = TeamNotification( + team_id="t1", + team_alias=None, + recipients=("a@example.com",), + models=(AffectedModel(info=_info("gpt-old", 5), display_name="gpt-old", milestone=7),), + ) + subject, _ = render_model_deprecation_email( + notification, email_logo_url="https://logo", email_support_contact="help@example.com" + ) + assert subject.endswith("for team t1") + + def test_should_escape_the_team_alias(self): + notification: Final = TeamNotification( + team_id="t1", + team_alias="Ops", + recipients=("a@example.com",), + models=(AffectedModel(info=_info("gpt-old", 5), display_name="gpt-old", milestone=7),), + ) + _, html = render_model_deprecation_email( + notification, email_logo_url="https://logo", email_support_contact="help@example.com" + ) + assert "Ops" not in html + assert "<b>Ops</b>" in html From 4696b0558ca4bd0e8a0ba34c46363d4319c4a219 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 15:16:30 +0530 Subject: [PATCH 12/37] fix(proxy): make day-zero deprecation emails consistent and harden escaping --- .../model_deprecation_notifications.py | 13 +++-- .../test_model_deprecation_notifications.py | 48 ++++++++++++++++++- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index 8e1ed18beae..4fa6771603b 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -150,7 +150,9 @@ async def resolve_affected_teams( def _days_left_label(days_until: int) -> str: - return f"deprecated {abs(days_until)}d ago" if days_until < 0 else f"{days_until}d" + if days_until < 0: + return f"deprecated {abs(days_until)}d ago" + return "today" if days_until == 0 else f"{days_until}d" def _render_row(model: AffectedModel) -> str: @@ -167,11 +169,14 @@ def render_model_deprecation_email( notification: TeamNotification, email_logo_url: str, email_support_contact: str ) -> tuple[str, str]: """(subject, html) for one team's digest, every model-sourced string escaped""" - team_name: Final = notification.team_alias or notification.team_id - verb: Final = "deprecated" if any(model.milestone == 0 for model in notification.models) else "deprecating" + alias: Final = " ".join((notification.team_alias or "").split()) + team_name: Final = alias or notification.team_id + verb: Final = ( + "deprecated" if any(model.info.days_until_deprecation < 0 for model in notification.models) else "deprecating" + ) subject: Final = f"[LiteLLM] {len(notification.models)} model(s) {verb} for team {team_name}" body: Final = MODEL_DEPRECATION_EMAIL_TEMPLATE.format( - email_logo_url=email_logo_url, + email_logo_url=html.escape(email_logo_url), team_name=html.escape(team_name), model_rows="\n".join(_render_row(model) for model in notification.models), email_support_contact=html.escape(email_support_contact), diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py index 15afa366801..89ce0a1443b 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py @@ -35,7 +35,7 @@ class TestSelectMilestone: assert select_milestone(5, (0, 7, 30)) == 7 -def _info(model_name: str, days_until: int, provider: str = "openai") -> ModelDeprecationInfo: +def _info(model_name: str, days_until: int, provider: str | None = "openai") -> ModelDeprecationInfo: return ModelDeprecationInfo( model_name=model_name, litellm_model=model_name, @@ -190,6 +190,7 @@ class TestRenderModelDeprecationEmail: ) assert subject == "[LiteLLM] 2 model(s) deprecated for team Data Team" assert "deprecated 4d ago" in html + assert "deprecated" in html assert html.index("gpt-old") < html.index("gpt-older") def test_should_fall_back_to_team_id_when_alias_missing(self): @@ -216,3 +217,48 @@ class TestRenderModelDeprecationEmail: ) assert "Ops" not in html assert "<b>Ops</b>" in html + + def test_should_render_unknown_when_the_provider_is_missing(self): + models: Final = (AffectedModel(info=_info("gpt-old", 5, provider=None), display_name="gpt-old", milestone=7),) + _, html = render_model_deprecation_email( + self._notification(models), email_logo_url="https://logo", email_support_contact="help@example.com" + ) + assert "unknown" in html + + def test_should_describe_day_zero_as_today_and_still_deprecating(self): + models: Final = (AffectedModel(info=_info("gpt-old", 0), display_name="gpt-old", milestone=0),) + subject, html = render_model_deprecation_email( + self._notification(models), email_logo_url="https://logo", email_support_contact="help@example.com" + ) + assert subject == "[LiteLLM] 1 model(s) deprecating for team Data Team" + assert "today" in html + assert "imminent" in html + + def test_should_append_the_shared_footer_and_escape_the_support_contact(self): + models: Final = (AffectedModel(info=_info("gpt-old", 5), display_name="gpt-old", milestone=7),) + _, html = render_model_deprecation_email( + self._notification(models), email_logo_url="https://logo", email_support_contact="@example.com" + ) + assert "github.com/BerriAI/litellm" in html + assert "<help>@example.com" in html + assert "@example.com" not in html + + def test_should_escape_the_logo_url(self): + models: Final = (AffectedModel(info=_info("gpt-old", 5), display_name="gpt-old", milestone=7),) + _, html = render_model_deprecation_email( + self._notification(models), email_logo_url='https://logo" onerror="x', email_support_contact="h@x.io" + ) + assert '" onerror="' not in html + assert "https://logo" onerror="x" in html + + def test_should_collapse_newlines_in_the_subject_team_name(self): + notification: Final = TeamNotification( + team_id="t1", + team_alias="Data\r\n Team", + recipients=("a@example.com",), + models=(AffectedModel(info=_info("gpt-old", 5), display_name="gpt-old", milestone=7),), + ) + subject, _ = render_model_deprecation_email( + notification, email_logo_url="https://logo", email_support_contact="h@x.io" + ) + assert subject == "[LiteLLM] 1 model(s) deprecating for team Data Team" From afcb4d0f6514dfb2d220860a57b9dbaddf83b1b6 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 15:24:22 +0530 Subject: [PATCH 13/37] feat(proxy): build per-team deprecation digests deduped by milestone --- .../model_deprecation_notifications.py | 57 +++++++++++++++- .../test_model_deprecation_notifications.py | 67 +++++++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index 4fa6771603b..968cf38fdc7 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -6,11 +6,12 @@ import html from collections.abc import Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, Protocol from pydantic import BaseModel, Field, ValidationError from litellm._logging import verbose_proxy_logger +from litellm.integrations.email_alerting import get_team_admin_emails from litellm.integrations.email_templates.email_footer import EMAIL_FOOTER from litellm.integrations.email_templates.model_deprecation_email import ( MODEL_DEPRECATION_EMAIL_ROW_TEMPLATE, @@ -22,6 +23,7 @@ from litellm.proxy.auth.auth_checks import can_team_access_model from litellm.types.proxy.model_deprecation import ModelDeprecationInfo if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient from litellm.router import Router @@ -47,6 +49,12 @@ class TeamNotification: models: tuple[AffectedModel, ...] +class DeprecationEmailCache(Protocol): + async def async_get_cache(self, key: str) -> object | None: ... + + async def async_set_cache(self, key: str, value: float, ttl: int) -> None: ... + + def select_milestone(days_until: int, thresholds: Sequence[int]) -> int | None: """The most urgent threshold already reached, None while the first one is still ahead""" return min((threshold for threshold in thresholds if days_until <= threshold), default=None) @@ -149,6 +157,53 @@ async def resolve_affected_teams( return MappingProxyType({team_id: models for team_id, models in per_team.items() if models}) +def email_sent_key(team_id: str, model_name: str, milestone: int) -> str: + return f"model_deprecation_email:{team_id}:{model_name}:{milestone}" + + +async def _unsent( + team_id: str, models: Sequence[AffectedModel], cache: DeprecationEmailCache +) -> tuple[AffectedModel, ...]: + flags: Final = tuple( + [ + await cache.async_get_cache(key=email_sent_key(team_id, model.info.model_name, model.milestone)) is None + for model in models + ] + ) + return tuple(model for model, unsent in zip(models, flags, strict=True) if unsent) + + +async def _notification_for( + team: LiteLLM_TeamTable, models: Sequence[AffectedModel], cache: DeprecationEmailCache, prisma_client: PrismaClient +) -> TeamNotification | None: + unsent: Final = await _unsent(team.team_id, models, cache) + if not unsent: + return None + recipients: Final = await get_team_admin_emails(team, prisma_client) + if not recipients: + verbose_proxy_logger.debug("model_deprecation: team %s has no admin emails, skipping", team.team_id) + return None + return TeamNotification(team_id=team.team_id, team_alias=team.team_alias, recipients=recipients, models=unsent) + + +async def build_team_notifications( + affected: Mapping[str, Sequence[AffectedModel]], + teams: Sequence[LiteLLM_TeamTable], + cache: DeprecationEmailCache, + prisma_client: PrismaClient, +) -> tuple[TeamNotification, ...]: + """One digest per team of the milestones not yet emailed, dropping teams with nobody to email""" + teams_by_id: Final = MappingProxyType({team.team_id: team for team in teams}) + candidates: Final = tuple( + [ + await _notification_for(teams_by_id[team_id], models, cache, prisma_client) + for team_id, models in affected.items() + if team_id in teams_by_id + ] + ) + return tuple(notification for notification in candidates if notification is not None) + + def _days_left_label(days_until: int) -> str: if days_until < 0: return f"deprecated {abs(days_until)}d ago" diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py index 89ce0a1443b..a8db957f5e0 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py @@ -4,15 +4,19 @@ from __future__ import annotations from collections.abc import Mapping, Sequence from datetime import date +from types import SimpleNamespace from typing import Final from unittest.mock import MagicMock import pytest +from litellm.caching.caching import DualCache from litellm.models.team import LiteLLM_TeamTable from litellm.proxy.common_utils.model_deprecation_notifications import ( AffectedModel, TeamNotification, + build_team_notifications, + email_sent_key, render_model_deprecation_email, resolve_affected_teams, select_milestone, @@ -262,3 +266,66 @@ class TestRenderModelDeprecationEmail: notification, email_logo_url="https://logo", email_support_contact="h@x.io" ) assert subject == "[LiteLLM] 1 model(s) deprecating for team Data Team" + + +def _prisma_with_users(rows: Sequence[Mapping[str, object]]) -> SimpleNamespace: + async def find_many(where=None, take=None, skip=None, order=None): + wanted: Final = where["user_id"]["in"] + return [row for row in rows if row["user_id"] in wanted] + + return SimpleNamespace(db=SimpleNamespace(litellm_usertable=SimpleNamespace(find_many=find_many))) + + +ADMIN_USERS: Final = ({"user_id": "u1", "user_email": "admin@example.com"},) + + +def _admin_team(team_id: str, alias: str | None = None) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable(team_id=team_id, team_alias=alias, members_with_roles=[{"user_id": "u1", "role": "admin"}]) + + +class TestBuildTeamNotifications: + @pytest.mark.asyncio + async def test_should_build_one_digest_per_team_with_admin_recipients(self): + models: Final = (AffectedModel(info=_info("gpt-old", 5), display_name="gpt-old", milestone=7),) + notifications: Final = await build_team_notifications( + {"t1": models}, (_admin_team("t1", "Data"),), DualCache(), _prisma_with_users(ADMIN_USERS) + ) + assert notifications == ( + TeamNotification(team_id="t1", team_alias="Data", recipients=("admin@example.com",), models=models), + ) + + @pytest.mark.asyncio + async def test_should_skip_milestones_already_sent(self): + cache: Final = DualCache() + await cache.async_set_cache(key=email_sent_key("t1", "gpt-old", 7), value=1.0) + sent: Final = AffectedModel(info=_info("gpt-old", 5), display_name="gpt-old", milestone=7) + fresh: Final = AffectedModel(info=_info("gpt-older", -1), display_name="gpt-older", milestone=0) + notifications: Final = await build_team_notifications( + {"t1": (sent, fresh)}, (_admin_team("t1"),), cache, _prisma_with_users(ADMIN_USERS) + ) + assert notifications[0].models == (fresh,) + + @pytest.mark.asyncio + async def test_should_drop_team_when_everything_was_already_sent(self): + cache: Final = DualCache() + await cache.async_set_cache(key=email_sent_key("t1", "gpt-old", 7), value=1.0) + models: Final = (AffectedModel(info=_info("gpt-old", 5), display_name="gpt-old", milestone=7),) + notifications: Final = await build_team_notifications( + {"t1": models}, (_admin_team("t1"),), cache, _prisma_with_users(ADMIN_USERS) + ) + assert notifications == () + + @pytest.mark.asyncio + async def test_should_drop_team_without_admin_emails(self): + models: Final = (AffectedModel(info=_info("gpt-old", 5), display_name="gpt-old", milestone=7),) + no_admins: Final = LiteLLM_TeamTable(team_id="t1", members_with_roles=[{"user_id": "u1", "role": "user"}]) + notifications: Final = await build_team_notifications( + {"t1": models}, (no_admins,), DualCache(), _prisma_with_users(ADMIN_USERS) + ) + assert notifications == () + + @pytest.mark.asyncio + async def test_should_key_the_sent_marker_by_team_model_and_milestone(self): + assert email_sent_key("t1", "gpt-old", 7) == "model_deprecation_email:t1:gpt-old:7" + assert email_sent_key("t1", "gpt-old", 7) != email_sent_key("t2", "gpt-old", 7) + assert email_sent_key("t1", "gpt-old", 7) != email_sent_key("t1", "gpt-old", 0) From 45582059658fb6c0e616b7efdead8caf438f6ceb Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 15:31:22 +0530 Subject: [PATCH 14/37] fix(proxy): make the deprecation email cache protocol match dualcache --- .../proxy/common_utils/model_deprecation_notifications.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index 968cf38fdc7..cc654cf3683 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -50,9 +50,9 @@ class TeamNotification: class DeprecationEmailCache(Protocol): - async def async_get_cache(self, key: str) -> object | None: ... + async def async_get_cache(self, *, key: str) -> object: ... - async def async_set_cache(self, key: str, value: float, ttl: int) -> None: ... + async def async_set_cache(self, *, key: str, value: float, ttl: float) -> None: ... def select_milestone(days_until: int, thresholds: Sequence[int]) -> int | None: @@ -198,7 +198,6 @@ async def build_team_notifications( [ await _notification_for(teams_by_id[team_id], models, cache, prisma_client) for team_id, models in affected.items() - if team_id in teams_by_id ] ) return tuple(notification for notification in candidates if notification is not None) From 9c5f0c5a3c0b31f02c689c2df309b1077d18cede Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 15:36:54 +0530 Subject: [PATCH 15/37] feat(proxy): send model deprecation digests to team admins --- .../model_deprecation_notifications.py | 134 +++++++++++- .../test_model_deprecation_notifications.py | 199 +++++++++++++++++- 2 files changed, 329 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index cc654cf3683..756d0660c8c 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -3,7 +3,9 @@ from __future__ import annotations import html -from collections.abc import Mapping, Sequence +import os +import time +from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Final, Protocol @@ -11,7 +13,12 @@ from typing import TYPE_CHECKING, Final, Protocol from pydantic import BaseModel, Field, ValidationError from litellm._logging import verbose_proxy_logger -from litellm.integrations.email_alerting import get_team_admin_emails +from litellm.constants import EMAIL_MODEL_DEPRECATION_LOCK_ID +from litellm.integrations.email_alerting import ( + LITELLM_LOGO_URL, + LITELLM_SUPPORT_CONTACT, + get_team_admin_emails, +) from litellm.integrations.email_templates.email_footer import EMAIL_FOOTER from litellm.integrations.email_templates.model_deprecation_email import ( MODEL_DEPRECATION_EMAIL_ROW_TEMPLATE, @@ -20,9 +27,16 @@ from litellm.integrations.email_templates.model_deprecation_email import ( from litellm.models.team import LiteLLM_TeamTable from litellm.proxy._types import ProxyException from litellm.proxy.auth.auth_checks import can_team_access_model -from litellm.types.proxy.model_deprecation import ModelDeprecationInfo +from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations +from litellm.repositories.team_repository import TeamRepository +from litellm.types.integrations.slack_alerting import SlackAlertingArgs, SlackAlertingCacheKeys +from litellm.types.proxy.model_deprecation import ( + DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, + ModelDeprecationInfo, +) if TYPE_CHECKING: + from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager from litellm.proxy.utils import PrismaClient from litellm.router import Router @@ -55,6 +69,26 @@ class DeprecationEmailCache(Protocol): async def async_set_cache(self, *, key: str, value: float, ttl: float) -> None: ... +class EmailSender(Protocol): + DEFAULT_LITELLM_EMAIL: str + + async def send_email(self, from_email: str, to_email: Sequence[str], subject: str, html_body: str) -> None: ... + + +class SmtpSend(Protocol): + def __call__(self, *, receiver_email: str, subject: str, html: str) -> Awaitable[object]: ... + + +@dataclass(frozen=True, slots=True) +class DeprecationEmailContext: + llm_router: Router + prisma_client: PrismaClient + cache: DeprecationEmailCache + alerting_args: SlackAlertingArgs + pod_lock_manager: PodLockManager | None + deliver: Callable[[Sequence[str], str, str], Awaitable[None]] + + def select_milestone(days_until: int, thresholds: Sequence[int]) -> int | None: """The most urgent threshold already reached, None while the first one is still ahead""" return min((threshold for threshold in thresholds if days_until <= threshold), default=None) @@ -236,3 +270,97 @@ def render_model_deprecation_email( email_support_contact=html.escape(email_support_contact), ) return subject, body + EMAIL_FOOTER + + +def make_email_deliverer( + email_logger: EmailSender | None, smtp_send: SmtpSend | None = None +) -> Callable[[Sequence[str], str, str], Awaitable[None]]: + """The proxy's configured email provider when there is one, else the OSS SMTP helper per recipient""" + if email_logger is not None: + + async def deliver_via_logger(recipients: Sequence[str], subject: str, html_body: str) -> None: + await email_logger.send_email( + from_email=email_logger.DEFAULT_LITELLM_EMAIL, + to_email=tuple(recipients), + subject=subject, + html_body=html_body, + ) + + return deliver_via_logger + + async def deliver_over_smtp(recipients: Sequence[str], subject: str, html_body: str) -> None: + from litellm.proxy.utils import send_email + + send: Final = smtp_send or send_email + for recipient in recipients: + await send(receiver_email=recipient, subject=subject, html=html_body) + + return deliver_over_smtp + + +async def _load_teams(prisma_client: PrismaClient) -> tuple[LiteLLM_TeamTable, ...]: + return tuple(await TeamRepository(prisma_client).find_many()) + + +async def _stamp_pass(cache: DeprecationEmailCache) -> None: + await cache.async_set_cache( + key=SlackAlertingCacheKeys.deprecation_email_pass_key.value, + value=time.time(), + ttl=DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, + ) + + +async def _claimed_email_window(pod_lock_manager: PodLockManager | None) -> bool: + """Without a redis backed lock there is no fleet to coordinate, so a lone pod always sends""" + if pod_lock_manager is None: + return True + return ( + await pod_lock_manager.acquire_lock( + cronjob_id=EMAIL_MODEL_DEPRECATION_LOCK_ID, + ttl=DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, + allow_reentrant=False, + ) + ) is not False + + +async def _send_team_notification(notification: TeamNotification, ctx: DeprecationEmailContext) -> bool: + subject, html_body = render_model_deprecation_email( + notification, + email_logo_url=os.getenv("SMTP_SENDER_LOGO", os.getenv("EMAIL_LOGO_URL", LITELLM_LOGO_URL)), + email_support_contact=os.getenv("EMAIL_SUPPORT_CONTACT", LITELLM_SUPPORT_CONTACT), + ) + try: + await ctx.deliver(notification.recipients, subject, html_body) + except Exception as e: # noqa: BLE001 # one team's mail failure must not block the rest; unstamped keys retry tomorrow + verbose_proxy_logger.exception("model_deprecation: email to team %s failed: %s", notification.team_id, e) + return False + for model in notification.models: + await ctx.cache.async_set_cache( + key=email_sent_key(notification.team_id, model.info.model_name, model.milestone), + value=time.time(), + ttl=ctx.alerting_args.model_deprecation_email_ttl, + ) + return True + + +async def send_model_deprecation_emails(ctx: DeprecationEmailContext) -> int: + """One pass: resolve affected teams, drop what was already sent, deliver one digest per team + + The pass stamp is set whenever a resolution completed, sent or not, so the DB is consulted at most + once a day; the fleet lock is only claimed once there is something to send + """ + snapshot: Final = collect_model_deprecations(llm_router=ctx.llm_router) + infos: Final = (*snapshot.deprecated, *snapshot.imminent, *snapshot.upcoming) + if not infos: + await _stamp_pass(ctx.cache) + return 0 + teams: Final = await _load_teams(ctx.prisma_client) + affected: Final = await resolve_affected_teams( + infos, ctx.llm_router, teams, ctx.alerting_args.model_deprecation_email_thresholds + ) + notifications: Final = await build_team_notifications(affected, teams, ctx.cache, ctx.prisma_client) + await _stamp_pass(ctx.cache) + if not notifications or not await _claimed_email_window(ctx.pod_lock_manager): + return 0 + results: Final = tuple([await _send_team_notification(notification, ctx) for notification in notifications]) + return sum(results) diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py index a8db957f5e0..c47837e7eaf 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py @@ -11,17 +11,22 @@ from unittest.mock import MagicMock import pytest from litellm.caching.caching import DualCache +from litellm.constants import EMAIL_MODEL_DEPRECATION_LOCK_ID from litellm.models.team import LiteLLM_TeamTable from litellm.proxy.common_utils.model_deprecation_notifications import ( AffectedModel, + DeprecationEmailContext, TeamNotification, build_team_notifications, email_sent_key, + make_email_deliverer, render_model_deprecation_email, resolve_affected_teams, select_milestone, + send_model_deprecation_emails, ) -from litellm.types.proxy.model_deprecation import ModelDeprecationInfo +from litellm.types.integrations.slack_alerting import SlackAlertingArgs, SlackAlertingCacheKeys +from litellm.types.proxy.model_deprecation import DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, ModelDeprecationInfo class TestSelectMilestone: @@ -329,3 +334,195 @@ class TestBuildTeamNotifications: assert email_sent_key("t1", "gpt-old", 7) == "model_deprecation_email:t1:gpt-old:7" assert email_sent_key("t1", "gpt-old", 7) != email_sent_key("t2", "gpt-old", 7) assert email_sent_key("t1", "gpt-old", 7) != email_sent_key("t1", "gpt-old", 0) + + +DEAD_DEPLOYMENT: Final = { + "model_name": "dead-alias", + "litellm_params": {"model": "openai/dead-model"}, + "model_info": {"id": "1", "deprecation_date": "2020-01-01", "litellm_provider": "openai"}, +} +FRESH_DEPLOYMENT: Final = { + "model_name": "fresh", + "litellm_params": {"model": "openai/fresh-model-with-no-cost-map-entry"}, + "model_info": {"id": "x"}, +} + + +class _Deliverer: + def __init__(self, fail_for: Sequence[str] = ()): + self.fail_for = fail_for + self.sent = [] + + async def __call__(self, recipients: Sequence[str], subject: str, html_body: str) -> None: + if any(recipient in self.fail_for for recipient in recipients): + raise ConnectionError("smtp down") + self.sent.append((tuple(recipients), subject)) + + +class _Lock: + def __init__(self, result: bool | None): + self.result = result + self.calls = [] + + async def acquire_lock(self, cronjob_id: str, ttl: int | None = None, allow_reentrant: bool = True) -> bool | None: + self.calls.append({"cronjob_id": cronjob_id, "ttl": ttl, "allow_reentrant": allow_reentrant}) + return self.result + + +def _team_row(team_id: str, alias: str = "Team") -> Mapping[str, object]: + return { + "team_id": team_id, + "team_alias": alias, + "models": [], + "members_with_roles": [{"user_id": f"admin-{team_id}", "role": "admin"}], + } + + +def _prisma(team_rows: Sequence[Mapping[str, object]], user_rows: Sequence[Mapping[str, object]]) -> SimpleNamespace: + async def find_many_teams(where=None, take=None, skip=None, order=None): + return list(team_rows) + + async def find_many_users(where=None, take=None, skip=None, order=None): + wanted: Final = where["user_id"]["in"] + return [row for row in user_rows if row["user_id"] in wanted] + + return SimpleNamespace( + db=SimpleNamespace( + litellm_teamtable=SimpleNamespace(find_many=find_many_teams), + litellm_usertable=SimpleNamespace(find_many=find_many_users), + ) + ) + + +TWO_TEAMS: Final = (_team_row("t1", "Alpha"), _team_row("t2", "Beta")) +TWO_ADMINS: Final = ( + {"user_id": "admin-t1", "user_email": "a@x.io"}, + {"user_id": "admin-t2", "user_email": "b@x.io"}, +) + + +def _context(router, prisma, deliver, cache=None, pod_lock_manager=None) -> DeprecationEmailContext: + return DeprecationEmailContext( + llm_router=router, + prisma_client=prisma, + cache=cache or DualCache(), + alerting_args=SlackAlertingArgs(), + pod_lock_manager=pod_lock_manager, + deliver=deliver, + ) + + +class TestSendModelDeprecationEmails: + @pytest.mark.asyncio + async def test_should_email_each_affected_team_once_and_stamp_the_milestones(self): + cache: Final = DualCache() + deliverer: Final = _Deliverer() + ctx: Final = _context(_router([DEAD_DEPLOYMENT]), _prisma(TWO_TEAMS, TWO_ADMINS), deliverer, cache) + + assert await send_model_deprecation_emails(ctx) == 2 + assert sorted(recipients for recipients, _ in deliverer.sent) == [("a@x.io",), ("b@x.io",)] + assert all("deprecated" in subject for _, subject in deliverer.sent) + assert await cache.async_get_cache(key=email_sent_key("t1", "dead-alias", 0)) is not None + assert await cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value) is not None + + assert await send_model_deprecation_emails(ctx) == 0 + assert len(deliverer.sent) == 2 + + @pytest.mark.asyncio + async def test_should_stamp_the_pass_and_touch_nothing_else_when_no_model_deprecates(self): + cache: Final = DualCache() + deliverer: Final = _Deliverer() + lock: Final = _Lock(True) + + async def explode(where=None, take=None, skip=None, order=None): + raise AssertionError("teams must not be loaded when nothing deprecates") + + prisma: Final = _prisma((), ()) + prisma.db.litellm_teamtable.find_many = explode + + assert ( + await send_model_deprecation_emails(_context(_router([FRESH_DEPLOYMENT]), prisma, deliverer, cache, lock)) + == 0 + ) + assert deliverer.sent == [] + assert lock.calls == [] + assert await cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value) is not None + + @pytest.mark.asyncio + async def test_should_not_claim_the_lock_when_every_milestone_was_already_sent(self): + cache: Final = DualCache() + await cache.async_set_cache(key=email_sent_key("t1", "dead-alias", 0), value=1.0) + lock: Final = _Lock(True) + deliverer: Final = _Deliverer() + ctx: Final = _context(_router([DEAD_DEPLOYMENT]), _prisma(TWO_TEAMS[:1], TWO_ADMINS), deliverer, cache, lock) + + assert await send_model_deprecation_emails(ctx) == 0 + assert deliverer.sent == [] + assert lock.calls == [] + + @pytest.mark.parametrize( + ("lock_result", "expected_sent"), + [(True, 1), (None, 1), (False, 0)], + ids=["lock won", "no redis lock", "another pod holds the lock"], + ) + @pytest.mark.asyncio + async def test_should_send_only_from_the_pod_holding_the_daily_lock(self, lock_result, expected_sent): + lock: Final = _Lock(lock_result) + deliverer: Final = _Deliverer() + ctx: Final = _context(_router([DEAD_DEPLOYMENT]), _prisma(TWO_TEAMS[:1], TWO_ADMINS), deliverer, None, lock) + + assert await send_model_deprecation_emails(ctx) == expected_sent + assert len(deliverer.sent) == expected_sent + assert lock.calls == [ + { + "cronjob_id": EMAIL_MODEL_DEPRECATION_LOCK_ID, + "ttl": DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, + "allow_reentrant": False, + } + ] + + @pytest.mark.asyncio + async def test_should_keep_going_and_leave_keys_unstamped_when_one_team_fails(self): + cache: Final = DualCache() + deliverer: Final = _Deliverer(fail_for=("a@x.io",)) + ctx: Final = _context(_router([DEAD_DEPLOYMENT]), _prisma(TWO_TEAMS, TWO_ADMINS), deliverer, cache) + + assert await send_model_deprecation_emails(ctx) == 1 + assert [recipients for recipients, _ in deliverer.sent] == [("b@x.io",)] + assert await cache.async_get_cache(key=email_sent_key("t1", "dead-alias", 0)) is None + assert await cache.async_get_cache(key=email_sent_key("t2", "dead-alias", 0)) is not None + + +class TestMakeEmailDeliverer: + @pytest.mark.asyncio + async def test_should_use_the_configured_email_logger_when_present(self): + calls: Final = [] + + async def send_email(from_email, to_email, subject, html_body): + calls.append((from_email, tuple(to_email), subject, html_body)) + + logger: Final = SimpleNamespace(DEFAULT_LITELLM_EMAIL="noreply@litellm.ai", send_email=send_email) + deliver: Final = make_email_deliverer(logger) + + await deliver(("a@x.io", "b@x.io"), "subj", "

hi

") + assert calls == [("noreply@litellm.ai", ("a@x.io", "b@x.io"), "subj", "

hi

")] + + @pytest.mark.asyncio + async def test_should_fall_back_to_smtp_per_recipient_without_a_logger(self): + calls: Final = [] + + async def smtp_send(*, receiver_email, subject, html): + calls.append((receiver_email, subject, html)) + + deliver: Final = make_email_deliverer(None, smtp_send=smtp_send) + + await deliver(("a@x.io", "b@x.io"), "subj", "

hi

") + assert calls == [("a@x.io", "subj", "

hi

"), ("b@x.io", "subj", "

hi

")] + + @pytest.mark.asyncio + async def test_should_raise_when_smtp_is_not_configured(self, monkeypatch): + monkeypatch.delenv("SMTP_SENDER_EMAIL", raising=False) + deliver: Final = make_email_deliverer(None) + + with pytest.raises(ValueError, match="SMTP_SENDER_EMAIL"): + await deliver(("a@x.io",), "subj", "

hi

") From 9bfda023afdb7765400d148e8199c77ef08a86ae Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 15:47:30 +0530 Subject: [PATCH 16/37] feat(alerting): email team admins from the scheduled model deprecation check --- .../SlackAlerting/slack_alerting.py | 120 ++++++++++-- .../test_model_deprecation_alert.py | 180 +++++++++++++++++- 2 files changed, 288 insertions(+), 12 deletions(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index caac8e888fd..5b3ca4128d2 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -5,7 +5,7 @@ import datetime import os import random import time -from collections.abc import Callable +from collections.abc import Awaitable, Callable from datetime import timedelta from typing import TYPE_CHECKING, Any, Final, Literal @@ -67,6 +67,10 @@ from .ms_teams import ( from .utils import process_slack_alerting_variables if TYPE_CHECKING: + from litellm.proxy.common_utils.model_deprecation_notifications import ( + DeprecationEmailContext, + EmailSender, + ) from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager from litellm.proxy.utils import PrismaClient from litellm.router import Router as _Router @@ -82,6 +86,18 @@ def _proxy_llm_router() -> Router | None: return llm_router +def _proxy_prisma_client() -> "PrismaClient | None": + from litellm.proxy.proxy_server import prisma_client + + return prisma_client + + +def _proxy_email_logger() -> "EmailSender | None": + from litellm.proxy.proxy_server import proxy_logging_obj + + return proxy_logging_obj.email_logging_instance + + class SlackAlerting(CustomBatchLogger): """ Class for sending Slack Alerts @@ -1082,7 +1098,11 @@ Model Info: pass def _deprecation_alerts_enabled(self) -> bool: - return self.alerting is not None and AlertType.model_deprecation_warnings in self.alert_types + return ( + self.alerting is not None + and ("slack" in self.alerting or MS_TEAMS_ALERTING_DESTINATION in self.alerting) + and AlertType.model_deprecation_warnings in self.alert_types + ) async def send_model_deprecation_alert( self, @@ -1154,25 +1174,103 @@ Model Info: return False return await self.send_model_deprecation_alert(llm_router=llm_router, pod_lock_manager=pod_lock_manager) + def _deprecation_emails_enabled(self) -> bool: + return ( + self.alerting is not None + and "email" in self.alerting + and AlertType.model_deprecation_warnings in self.alert_types + and bool(self.alerting_args.model_deprecation_email_thresholds) + ) + + async def _run_deprecation_email_pass( + self, + llm_router: Router | None, + pod_lock_manager: "PodLockManager | None", + get_prisma_client: Callable[[], "PrismaClient | None"], + get_email_logger: Callable[[], "EmailSender | None"], + send_emails: "Callable[[DeprecationEmailContext], Awaitable[int]] | None", + ) -> int: + """Email team admins the milestones crossed, resolving against the DB at most once a day""" + if llm_router is None or not self._deprecation_emails_enabled(): + return 0 + if ( + await self.internal_usage_cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value) + ) is not None: + return 0 + prisma_client: Final = get_prisma_client() + if prisma_client is None: + verbose_proxy_logger.debug("model_deprecation: no database connected, skipping email pass") + return 0 + + from litellm.proxy.common_utils.model_deprecation_notifications import ( + DeprecationEmailContext, + make_email_deliverer, + send_model_deprecation_emails, + ) + + run: Final = send_emails or send_model_deprecation_emails + return await run( + DeprecationEmailContext( + llm_router=llm_router, + prisma_client=prisma_client, + cache=self.internal_usage_cache, + alerting_args=self.alerting_args, + pod_lock_manager=pod_lock_manager, + deliver=make_email_deliverer(get_email_logger()), + ) + ) + + async def _pass_failed(self, pass_run: Awaitable[object], name: str) -> bool: + try: + await pass_run + except Exception as e: # noqa: BLE001 # a failed pass must not kill the loop + verbose_proxy_logger.exception("Error in model deprecation %s loop: %s", name, e) + return True + return False + + async def _run_deprecation_passes( + self, + get_llm_router: Callable[[], Router | None], + pod_lock_manager: "PodLockManager | None", + get_prisma_client: Callable[[], "PrismaClient | None"], + get_email_logger: Callable[[], "EmailSender | None"], + send_emails: "Callable[[DeprecationEmailContext], Awaitable[int]] | None", + ) -> bool: + """Run the Slack pass then the email pass, each guarded on its own; True when either raised""" + llm_router: Final = get_llm_router() + alert_failed: Final = await self._pass_failed( + self._run_deprecation_alert_pass(llm_router, pod_lock_manager), "alert" + ) + email_failed: Final = await self._pass_failed( + self._run_deprecation_email_pass( + llm_router, pod_lock_manager, get_prisma_client, get_email_logger, send_emails + ), + "email", + ) + return alert_failed or email_failed + async def run_scheduled_deprecation_check( self, get_llm_router: Callable[[], Router | None] = _proxy_llm_router, pod_lock_manager: "PodLockManager | None" = None, + get_prisma_client: Callable[[], "PrismaClient | None"] = _proxy_prisma_client, + get_email_logger: Callable[[], "EmailSender | None"] = _proxy_email_logger, + send_emails: "Callable[[DeprecationEmailContext], Awaitable[int]] | None" = None, ) -> None: - """Poll every pass for a loaded router, the alert being on, and no alert in the last day, then alert + """Poll for a loaded router and run both passes; a pass that raised backs the loop off a full day - A pass that could not alert (no router yet, alert type off, a sibling pod holds the daily lock, or a + A pass that could not act (no router yet, alert type off, a sibling pod holds the daily lock, or a redis blip at claim time) is retried on the next poll instead of costing a day, while a pass that raised (a missing webhook, say) backs off a full day so a misconfiguration logs once, not every poll """ while True: - try: - await self._run_deprecation_alert_pass(get_llm_router(), pod_lock_manager) - except Exception as e: # noqa: BLE001 # a failed alert must not kill the loop - verbose_proxy_logger.exception("Error in model deprecation alert loop: %s", e) - await asyncio.sleep(DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS) - continue - await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS) + await asyncio.sleep( + DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS + if await self._run_deprecation_passes( + get_llm_router, pod_lock_manager, get_prisma_client, get_email_logger, send_emails + ) + else DEPRECATION_IDLE_POLL_SECONDS + ) async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool: """ diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py index 997e80b45df..fd0b4ddc266 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py @@ -2,11 +2,12 @@ import asyncio from itertools import chain, repeat +from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest - import litellm from litellm.constants import SLACK_MODEL_DEPRECATION_LOCK_ID from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting @@ -390,3 +391,180 @@ async def test_should_back_off_a_full_day_after_a_pass_raises(monkeypatch): assert slept == [DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS] * 2 assert mock_send_alert.await_count == 2 + + +def _email_only_alerting() -> SlackAlerting: + return SlackAlerting(alerting=["email"], alert_types=[AlertType.model_deprecation_warnings]) + + +class _EmailRun: + def __init__(self, result: int = 1, fail: bool = False): + self.result = result + self.fail = fail + self.contexts = [] + + async def __call__(self, ctx): + self.contexts.append(ctx) + if self.fail: + raise ValueError("smtp exploded") + return self.result + + +@pytest.mark.parametrize( + ("alerting", "alert_types", "thresholds", "expected"), + [ + (None, [AlertType.model_deprecation_warnings], (30,), False), + (["slack"], [AlertType.model_deprecation_warnings], (30,), False), + (["email"], [AlertType.llm_exceptions], (30,), False), + (["email"], [AlertType.model_deprecation_warnings], (), False), + (["email"], [AlertType.model_deprecation_warnings], (30,), True), + (["slack", "email"], [AlertType.model_deprecation_warnings], (30, 7, 0), True), + ], +) +def test_deprecation_emails_enabled_truth_table(alerting, alert_types, thresholds, expected): + alerting_obj: Final = SlackAlerting( + alerting=alerting, alert_types=alert_types, alerting_args={"model_deprecation_email_thresholds": thresholds} + ) + assert alerting_obj._deprecation_emails_enabled() is expected + + +@pytest.mark.asyncio +async def test_should_not_run_the_slack_pass_without_a_slack_or_teams_channel(monkeypatch): + monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) + pod_lock_manager: Final = MagicMock() + pod_lock_manager.acquire_lock = AsyncMock(return_value=True) + + sent: Final = await _email_only_alerting().send_model_deprecation_alert( + llm_router=_make_router([DEAD_ALIAS_DEPLOYMENT]), pod_lock_manager=pod_lock_manager + ) + + assert sent is False + pod_lock_manager.acquire_lock.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_should_run_the_email_pass_with_email_only_alerting(monkeypatch): + monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) + alerting: Final = _email_only_alerting() + prisma: Final = SimpleNamespace() + run: Final = _EmailRun(result=2) + + failed: Final = await alerting._run_deprecation_passes( + get_llm_router=lambda: _make_router([DEAD_ALIAS_DEPLOYMENT]), + pod_lock_manager=None, + get_prisma_client=lambda: prisma, + get_email_logger=lambda: None, + send_emails=run, + ) + + assert failed is False + assert len(run.contexts) == 1 + assert run.contexts[0].prisma_client is prisma + assert run.contexts[0].alerting_args.model_deprecation_email_thresholds == (30, 7, 0) + assert run.contexts[0].cache is alerting.internal_usage_cache + assert callable(run.contexts[0].deliver) + + +@pytest.mark.asyncio +async def test_should_still_email_when_the_slack_pass_raises_and_report_the_failure(monkeypatch): + monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) + alerting: Final = SlackAlerting(alerting=["slack", "email"], alert_types=[AlertType.model_deprecation_warnings]) + run: Final = _EmailRun() + + with patch.object(alerting, "send_alert", new_callable=AsyncMock, side_effect=ValueError("no webhook")): + failed: Final = await alerting._run_deprecation_passes( + get_llm_router=lambda: _make_router([DEAD_ALIAS_DEPLOYMENT]), + pod_lock_manager=None, + get_prisma_client=SimpleNamespace, + get_email_logger=lambda: None, + send_emails=run, + ) + + assert failed is True + assert len(run.contexts) == 1 + + +@pytest.mark.asyncio +async def test_should_report_failure_when_the_email_pass_raises(monkeypatch): + monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) + run: Final = _EmailRun(fail=True) + + failed: Final = await _email_only_alerting()._run_deprecation_passes( + get_llm_router=lambda: _make_router([DEAD_ALIAS_DEPLOYMENT]), + pod_lock_manager=None, + get_prisma_client=SimpleNamespace, + get_email_logger=lambda: None, + send_emails=run, + ) + + assert failed is True + + +@pytest.mark.asyncio +async def test_should_skip_the_email_pass_within_a_day_of_the_last_one(monkeypatch): + monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) + alerting: Final = _email_only_alerting() + await alerting.internal_usage_cache.async_set_cache( + key=SlackAlertingCacheKeys.deprecation_email_pass_key.value, value=1.0 + ) + run: Final = _EmailRun() + + sent: Final = await alerting._run_deprecation_email_pass( + _make_router([DEAD_ALIAS_DEPLOYMENT]), None, SimpleNamespace, lambda: None, run + ) + + assert sent == 0 + assert run.contexts == [] + + +@pytest.mark.asyncio +async def test_should_skip_the_email_pass_without_a_database(monkeypatch): + monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) + run: Final = _EmailRun() + + sent: Final = await _email_only_alerting()._run_deprecation_email_pass( + _make_router([DEAD_ALIAS_DEPLOYMENT]), None, lambda: None, lambda: None, run + ) + + assert sent == 0 + assert run.contexts == [] + + +@pytest.mark.asyncio +async def test_should_skip_the_email_pass_without_a_router(): + run: Final = _EmailRun() + prisma_lookups: Final = [] + + def get_prisma_client(): + prisma_lookups.append(True) + return SimpleNamespace() + + sent: Final = await _email_only_alerting()._run_deprecation_email_pass(None, None, get_prisma_client, lambda: None, run) + + assert sent == 0 + assert prisma_lookups == [] + assert run.contexts == [] + + +@pytest.mark.asyncio +async def test_should_back_off_a_day_when_only_the_email_pass_raises(monkeypatch): + monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) + alerting: Final = _email_only_alerting() + slept: list[float] = [] + + async def stop_after_first_pass(seconds): + slept.append(seconds) + raise asyncio.CancelledError + + with ( + patch("litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep", side_effect=stop_after_first_pass), + pytest.raises(asyncio.CancelledError), + ): + await alerting.run_scheduled_deprecation_check( + get_llm_router=lambda: _make_router([DEAD_ALIAS_DEPLOYMENT]), + get_prisma_client=SimpleNamespace, + get_email_logger=lambda: None, + send_emails=_EmailRun(fail=True), + ) + + assert slept == [DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS] From 6d1e42cac21f86c81ce93d02abebd44d8a86e8b7 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 15:57:48 +0530 Subject: [PATCH 17/37] refactor(alerting): type the alerting channels and cache reads in the deprecation email pass --- .../SlackAlerting/slack_alerting.py | 35 ++++++++++--------- .../model_deprecation_notifications.py | 4 +++ .../test_model_deprecation_notifications.py | 9 +++++ 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 5b3ca4128d2..1c5ab9c1833 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -98,6 +98,9 @@ def _proxy_email_logger() -> "EmailSender | None": return proxy_logging_obj.email_logging_instance +_CHANNELS_ADAPTER: Final = TypeAdapter(tuple[str, ...]) + + class SlackAlerting(CustomBatchLogger): """ Class for sending Slack Alerts @@ -1097,12 +1100,14 @@ Model Info: async def model_removed_alert(self, model_name: str): pass + def _channels(self) -> tuple[str, ...]: + return _CHANNELS_ADAPTER.validate_python(self.alerting or ()) + def _deprecation_alerts_enabled(self) -> bool: + channels: Final = self._channels() return ( - self.alerting is not None - and ("slack" in self.alerting or MS_TEAMS_ALERTING_DESTINATION in self.alerting) - and AlertType.model_deprecation_warnings in self.alert_types - ) + "slack" in channels or MS_TEAMS_ALERTING_DESTINATION in channels + ) and AlertType.model_deprecation_warnings in self.alert_types async def send_model_deprecation_alert( self, @@ -1176,8 +1181,7 @@ Model Info: def _deprecation_emails_enabled(self) -> bool: return ( - self.alerting is not None - and "email" in self.alerting + "email" in self._channels() and AlertType.model_deprecation_warnings in self.alert_types and bool(self.alerting_args.model_deprecation_email_thresholds) ) @@ -1193,21 +1197,20 @@ Model Info: """Email team admins the milestones crossed, resolving against the DB at most once a day""" if llm_router is None or not self._deprecation_emails_enabled(): return 0 - if ( - await self.internal_usage_cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value) - ) is not None: + + from litellm.proxy.common_utils.model_deprecation_notifications import ( + DeprecationEmailContext, + email_pass_done_today, + make_email_deliverer, + send_model_deprecation_emails, + ) + + if await email_pass_done_today(self.internal_usage_cache): return 0 prisma_client: Final = get_prisma_client() if prisma_client is None: verbose_proxy_logger.debug("model_deprecation: no database connected, skipping email pass") return 0 - - from litellm.proxy.common_utils.model_deprecation_notifications import ( - DeprecationEmailContext, - make_email_deliverer, - send_model_deprecation_emails, - ) - run: Final = send_emails or send_model_deprecation_emails return await run( DeprecationEmailContext( diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index 756d0660c8c..a482b6a5d9e 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -310,6 +310,10 @@ async def _stamp_pass(cache: DeprecationEmailCache) -> None: ) +async def email_pass_done_today(cache: DeprecationEmailCache) -> bool: + return (await cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value)) is not None + + async def _claimed_email_window(pod_lock_manager: PodLockManager | None) -> bool: """Without a redis backed lock there is no fleet to coordinate, so a lone pod always sends""" if pod_lock_manager is None: diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py index c47837e7eaf..bfc4edcc0f7 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py @@ -18,6 +18,7 @@ from litellm.proxy.common_utils.model_deprecation_notifications import ( DeprecationEmailContext, TeamNotification, build_team_notifications, + email_pass_done_today, email_sent_key, make_email_deliverer, render_model_deprecation_email, @@ -493,6 +494,14 @@ class TestSendModelDeprecationEmails: assert await cache.async_get_cache(key=email_sent_key("t2", "dead-alias", 0)) is not None +@pytest.mark.asyncio +async def test_should_report_whether_the_daily_pass_already_ran(): + cache: Final = DualCache() + assert await email_pass_done_today(cache) is False + await cache.async_set_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value, value=1.0) + assert await email_pass_done_today(cache) is True + + class TestMakeEmailDeliverer: @pytest.mark.asyncio async def test_should_use_the_configured_email_logger_when_present(self): From 8b1da5c9855669619115837171ac98da7660e43d Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 16:02:55 +0530 Subject: [PATCH 18/37] fix(proxy): claim the pod lock before resolving and stamp the deprecation email pass last --- .../model_deprecation_notifications.py | 13 ++- .../test_model_deprecation_notifications.py | 82 +++++++++++++++++-- 2 files changed, 81 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index a482b6a5d9e..28a5971941a 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -348,23 +348,20 @@ async def _send_team_notification(notification: TeamNotification, ctx: Deprecati async def send_model_deprecation_emails(ctx: DeprecationEmailContext) -> int: - """One pass: resolve affected teams, drop what was already sent, deliver one digest per team + """One pass: claim the day, resolve affected teams, deliver one digest per team, then stamp the pass - The pass stamp is set whenever a resolution completed, sent or not, so the DB is consulted at most - once a day; the fleet lock is only claimed once there is something to send + An empty snapshot returns before the lock or the DB is touched, so a model added later is not + silenced for a day, and the stamp is written last, so a lost lock claim costs one poll, not a day """ snapshot: Final = collect_model_deprecations(llm_router=ctx.llm_router) infos: Final = (*snapshot.deprecated, *snapshot.imminent, *snapshot.upcoming) - if not infos: - await _stamp_pass(ctx.cache) + if not infos or not await _claimed_email_window(ctx.pod_lock_manager): return 0 teams: Final = await _load_teams(ctx.prisma_client) affected: Final = await resolve_affected_teams( infos, ctx.llm_router, teams, ctx.alerting_args.model_deprecation_email_thresholds ) notifications: Final = await build_team_notifications(affected, teams, ctx.cache, ctx.prisma_client) - await _stamp_pass(ctx.cache) - if not notifications or not await _claimed_email_window(ctx.pod_lock_manager): - return 0 results: Final = tuple([await _send_team_notification(notification, ctx) for notification in notifications]) + await _stamp_pass(ctx.cache) return sum(results) diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py index bfc4edcc0f7..1b23458706a 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Mapping, Sequence -from datetime import date +from datetime import date, datetime, timedelta, timezone from types import SimpleNamespace from typing import Final from unittest.mock import MagicMock @@ -342,6 +342,15 @@ DEAD_DEPLOYMENT: Final = { "litellm_params": {"model": "openai/dead-model"}, "model_info": {"id": "1", "deprecation_date": "2020-01-01", "litellm_provider": "openai"}, } +UPCOMING_DEPLOYMENT: Final = { + "model_name": "sunset-alias", + "litellm_params": {"model": "openai/sunset-model"}, + "model_info": { + "id": "2", + "deprecation_date": (datetime.now(timezone.utc).date() + timedelta(days=20)).isoformat(), + "litellm_provider": "openai", + }, +} FRESH_DEPLOYMENT: Final = { "model_name": "fresh", "litellm_params": {"model": "openai/fresh-model-with-no-cost-map-entry"}, @@ -370,6 +379,19 @@ class _Lock: return self.result +class _RecordingCache: + def __init__(self): + self.values = {} + self.ttls = {} + + async def async_get_cache(self, *, key: str) -> object: + return self.values.get(key) + + async def async_set_cache(self, *, key: str, value: float, ttl: float) -> None: + self.values[key] = value + self.ttls[key] = ttl + + def _team_row(team_id: str, alias: str = "Team") -> Mapping[str, object]: return { "team_id": team_id, @@ -430,7 +452,7 @@ class TestSendModelDeprecationEmails: assert len(deliverer.sent) == 2 @pytest.mark.asyncio - async def test_should_stamp_the_pass_and_touch_nothing_else_when_no_model_deprecates(self): + async def test_should_touch_nothing_when_no_model_deprecates(self): cache: Final = DualCache() deliverer: Final = _Deliverer() lock: Final = _Lock(True) @@ -447,10 +469,10 @@ class TestSendModelDeprecationEmails: ) assert deliverer.sent == [] assert lock.calls == [] - assert await cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value) is not None + assert await cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value) is None @pytest.mark.asyncio - async def test_should_not_claim_the_lock_when_every_milestone_was_already_sent(self): + async def test_should_stamp_the_pass_but_send_nothing_when_every_milestone_was_already_sent(self): cache: Final = DualCache() await cache.async_set_cache(key=email_sent_key("t1", "dead-alias", 0), value=1.0) lock: Final = _Lock(True) @@ -459,7 +481,8 @@ class TestSendModelDeprecationEmails: assert await send_model_deprecation_emails(ctx) == 0 assert deliverer.sent == [] - assert lock.calls == [] + assert len(lock.calls) == 1 + assert await cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value) is not None @pytest.mark.parametrize( ("lock_result", "expected_sent"), @@ -468,9 +491,10 @@ class TestSendModelDeprecationEmails: ) @pytest.mark.asyncio async def test_should_send_only_from_the_pod_holding_the_daily_lock(self, lock_result, expected_sent): + cache: Final = DualCache() lock: Final = _Lock(lock_result) deliverer: Final = _Deliverer() - ctx: Final = _context(_router([DEAD_DEPLOYMENT]), _prisma(TWO_TEAMS[:1], TWO_ADMINS), deliverer, None, lock) + ctx: Final = _context(_router([DEAD_DEPLOYMENT]), _prisma(TWO_TEAMS[:1], TWO_ADMINS), deliverer, cache, lock) assert await send_model_deprecation_emails(ctx) == expected_sent assert len(deliverer.sent) == expected_sent @@ -481,6 +505,52 @@ class TestSendModelDeprecationEmails: "allow_reentrant": False, } ] + stamped: Final = await cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value) + assert (stamped is not None) is (expected_sent == 1) + + @pytest.mark.asyncio + async def test_should_not_load_teams_or_stamp_when_the_lock_is_lost(self): + cache: Final = DualCache() + deliverer: Final = _Deliverer() + + async def explode(where=None, take=None, skip=None, order=None): + raise AssertionError("teams must not be loaded without the lock") + + prisma: Final = _prisma((), ()) + prisma.db.litellm_teamtable.find_many = explode + + assert ( + await send_model_deprecation_emails( + _context(_router([DEAD_DEPLOYMENT]), prisma, deliverer, cache, _Lock(False)) + ) + == 0 + ) + assert deliverer.sent == [] + assert await cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value) is None + + @pytest.mark.asyncio + async def test_should_email_the_thirty_day_milestone_for_an_upcoming_model(self): + cache: Final = DualCache() + deliverer: Final = _Deliverer() + ctx: Final = _context(_router([UPCOMING_DEPLOYMENT]), _prisma(TWO_TEAMS[:1], TWO_ADMINS), deliverer, cache) + + assert await send_model_deprecation_emails(ctx) == 1 + assert deliverer.sent[0][1] == "[LiteLLM] 1 model(s) deprecating for team Alpha" + assert await cache.async_get_cache(key=email_sent_key("t1", "sunset-alias", 30)) is not None + assert await cache.async_get_cache(key=email_sent_key("t1", "sunset-alias", 7)) is None + + @pytest.mark.asyncio + async def test_should_stamp_sent_keys_with_the_configured_ttl_and_the_pass_for_a_day(self): + cache: Final = _RecordingCache() + deliverer: Final = _Deliverer() + ctx: Final = _context(_router([DEAD_DEPLOYMENT]), _prisma(TWO_TEAMS[:1], TWO_ADMINS), deliverer, cache) + + assert await send_model_deprecation_emails(ctx) == 1 + assert cache.ttls[email_sent_key("t1", "dead-alias", 0)] == SlackAlertingArgs().model_deprecation_email_ttl + assert ( + cache.ttls[SlackAlertingCacheKeys.deprecation_email_pass_key.value] + == DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS + ) @pytest.mark.asyncio async def test_should_keep_going_and_leave_keys_unstamped_when_one_team_fails(self): From 686f727618af278b70e6a06d3e7103d2f19cadc1 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 16:14:29 +0530 Subject: [PATCH 19/37] test(proxy): cover the upcoming bucket of the deprecation email pass --- .../test_model_deprecation_notifications.py | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py index 1b23458706a..ee013845e8e 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py @@ -356,6 +356,15 @@ FRESH_DEPLOYMENT: Final = { "litellm_params": {"model": "openai/fresh-model-with-no-cost-map-entry"}, "model_info": {"id": "x"}, } +FAR_DEPLOYMENT: Final = { + "model_name": "far-alias", + "litellm_params": {"model": "openai/far-model"}, + "model_info": { + "id": "3", + "deprecation_date": (datetime.now(timezone.utc).date() + timedelta(days=40)).isoformat(), + "litellm_provider": "openai", + }, +} class _Deliverer: @@ -424,12 +433,12 @@ TWO_ADMINS: Final = ( ) -def _context(router, prisma, deliver, cache=None, pod_lock_manager=None) -> DeprecationEmailContext: +def _context(router, prisma, deliver, cache=None, pod_lock_manager=None, alerting_args=None) -> DeprecationEmailContext: return DeprecationEmailContext( llm_router=router, prisma_client=prisma, cache=cache or DualCache(), - alerting_args=SlackAlertingArgs(), + alerting_args=alerting_args or SlackAlertingArgs(), pod_lock_manager=pod_lock_manager, deliver=deliver, ) @@ -539,6 +548,22 @@ class TestSendModelDeprecationEmails: assert await cache.async_get_cache(key=email_sent_key("t1", "sunset-alias", 30)) is not None assert await cache.async_get_cache(key=email_sent_key("t1", "sunset-alias", 7)) is None + @pytest.mark.asyncio + async def test_should_email_an_upcoming_model_when_a_threshold_reaches_past_the_slack_window(self): + cache: Final = DualCache() + deliverer: Final = _Deliverer() + ctx: Final = _context( + _router([FAR_DEPLOYMENT]), + _prisma(TWO_TEAMS[:1], TWO_ADMINS), + deliverer, + cache, + alerting_args=SlackAlertingArgs(model_deprecation_email_thresholds=(45, 7, 0)), + ) + + assert await send_model_deprecation_emails(ctx) == 1 + assert deliverer.sent[0][1] == "[LiteLLM] 1 model(s) deprecating for team Alpha" + assert await cache.async_get_cache(key=email_sent_key("t1", "far-alias", 45)) is not None + @pytest.mark.asyncio async def test_should_stamp_sent_keys_with_the_configured_ttl_and_the_pass_for_a_day(self): cache: Final = _RecordingCache() From 2f3b511430e093fb0c74ebc8f0f2a4eac896e001 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 16:20:24 +0530 Subject: [PATCH 20/37] fix(alerting): back off a raising deprecation pass without stalling the other --- .../SlackAlerting/slack_alerting.py | 52 +++--- .../test_model_deprecation_alert.py | 168 +++++++++++------- 2 files changed, 134 insertions(+), 86 deletions(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 1c5ab9c1833..b22483eba01 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -98,6 +98,15 @@ def _proxy_email_logger() -> "EmailSender | None": return proxy_logging_obj.email_logging_instance +async def _pass_failed(pass_run: Awaitable[object], name: str) -> bool: + try: + await pass_run + except Exception as e: # noqa: BLE001 # a failed pass must not kill the loop + verbose_proxy_logger.exception("Error in model deprecation %s loop: %s", name, e) + return True + return False + + _CHANNELS_ADAPTER: Final = TypeAdapter(tuple[str, ...]) @@ -142,6 +151,8 @@ class SlackAlerting(CustomBatchLogger): self.alert_type_config[key] = AlertTypeConfig(**val) if isinstance(val, dict) else val self.digest_buckets: dict[str, DigestEntry] = {} self.digest_lock = asyncio.Lock() + self.deprecation_alert_backoff_until: float = 0.0 + self.deprecation_email_backoff_until: float = 0.0 super().__init__(**kwargs, flush_lock=self.flush_lock) def update_values( @@ -1223,14 +1234,6 @@ Model Info: ) ) - async def _pass_failed(self, pass_run: Awaitable[object], name: str) -> bool: - try: - await pass_run - except Exception as e: # noqa: BLE001 # a failed pass must not kill the loop - verbose_proxy_logger.exception("Error in model deprecation %s loop: %s", name, e) - return True - return False - async def _run_deprecation_passes( self, get_llm_router: Callable[[], Router | None], @@ -1238,19 +1241,22 @@ Model Info: get_prisma_client: Callable[[], "PrismaClient | None"], get_email_logger: Callable[[], "EmailSender | None"], send_emails: "Callable[[DeprecationEmailContext], Awaitable[int]] | None", - ) -> bool: - """Run the Slack pass then the email pass, each guarded on its own; True when either raised""" + now: Callable[[], float] = time.time, + ) -> None: + """Run the Slack pass then the email pass; a pass that raised is skipped for a day while the other keeps polling""" llm_router: Final = get_llm_router() - alert_failed: Final = await self._pass_failed( + current: Final = now() + if current >= self.deprecation_alert_backoff_until and await _pass_failed( self._run_deprecation_alert_pass(llm_router, pod_lock_manager), "alert" - ) - email_failed: Final = await self._pass_failed( + ): + self.deprecation_alert_backoff_until = current + DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS + if current >= self.deprecation_email_backoff_until and await _pass_failed( self._run_deprecation_email_pass( llm_router, pod_lock_manager, get_prisma_client, get_email_logger, send_emails ), "email", - ) - return alert_failed or email_failed + ): + self.deprecation_email_backoff_until = current + DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS async def run_scheduled_deprecation_check( self, @@ -1260,20 +1266,18 @@ Model Info: get_email_logger: Callable[[], "EmailSender | None"] = _proxy_email_logger, send_emails: "Callable[[DeprecationEmailContext], Awaitable[int]] | None" = None, ) -> None: - """Poll for a loaded router and run both passes; a pass that raised backs the loop off a full day + """Poll for a loaded router and run both passes every poll interval A pass that could not act (no router yet, alert type off, a sibling pod holds the daily lock, or a - redis blip at claim time) is retried on the next poll instead of costing a day, while a pass that - raised (a missing webhook, say) backs off a full day so a misconfiguration logs once, not every poll + redis blip at claim time) simply runs again next poll, while a pass that raised (a missing webhook, + say) is skipped for a full day so a misconfiguration logs once, not every poll, without stalling + the other pass """ while True: - await asyncio.sleep( - DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS - if await self._run_deprecation_passes( - get_llm_router, pod_lock_manager, get_prisma_client, get_email_logger, send_emails - ) - else DEPRECATION_IDLE_POLL_SECONDS + await self._run_deprecation_passes( + get_llm_router, pod_lock_manager, get_prisma_client, get_email_logger, send_emails ) + await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS) async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool: """ diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py index fd0b4ddc266..371e921da9e 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py @@ -10,17 +10,17 @@ import pytest import litellm from litellm.constants import SLACK_MODEL_DEPRECATION_LOCK_ID +from litellm.integrations.SlackAlerting.ms_teams import MS_TEAMS_ALERTING_DESTINATION from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.proxy._types import AlertType +from litellm.proxy.common_utils.model_deprecation_notifications import DeprecationEmailContext from litellm.types.integrations.slack_alerting import SlackAlertingCacheKeys from litellm.types.proxy.model_deprecation import ( DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, DEPRECATION_IDLE_POLL_SECONDS, ) -DEAD_MODEL_COST = { - "dead-model": {"deprecation_date": "2020-01-01", "litellm_provider": "openai"} -} +DEAD_MODEL_COST = {"dead-model": {"deprecation_date": "2020-01-01", "litellm_provider": "openai"}} DEAD_ALIAS_DEPLOYMENT = { "model_name": "dead-alias", "litellm_params": {"model": "dead-model"}, @@ -100,9 +100,7 @@ async def test_should_dispatch_high_severity_when_deprecated(monkeypatch): ] ) - with patch.object( - alerting, "send_alert", new_callable=AsyncMock - ) as mock_send_alert: + with patch.object(alerting, "send_alert", new_callable=AsyncMock) as mock_send_alert: sent = await alerting.send_model_deprecation_alert(llm_router=router) assert sent is True @@ -176,9 +174,7 @@ async def test_should_wait_for_the_router_instead_of_sleeping_a_full_day(monkeyp "model_cost", {"dead-model": {"deprecation_date": "2020-01-01", "litellm_provider": "openai"}}, ) - alerting = SlackAlerting( - alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings] - ) + alerting = SlackAlerting(alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings]) router = _make_router( [ { @@ -205,9 +201,7 @@ async def test_should_wait_for_the_router_instead_of_sleeping_a_full_day(monkeyp ), pytest.raises(asyncio.CancelledError), ): - await alerting.run_scheduled_deprecation_check( - get_llm_router=lambda: next(routers) - ) + await alerting.run_scheduled_deprecation_check(get_llm_router=lambda: next(routers)) assert slept == [DEPRECATION_IDLE_POLL_SECONDS] * (router_absent_passes + 1) mock_send_alert.assert_awaited_once() @@ -220,18 +214,14 @@ async def test_should_wait_for_the_router_instead_of_sleeping_a_full_day(monkeyp ids=["lock won", "no redis lock", "another pod holds the lock"], ) @pytest.mark.asyncio -async def test_should_alert_only_from_the_pod_holding_the_daily_lock( - monkeypatch, lock_acquired, expect_alert -): +async def test_should_alert_only_from_the_pod_holding_the_daily_lock(monkeypatch, lock_acquired, expect_alert): """Every pod runs the loop, so a fleet must not send one identical alert per replica""" monkeypatch.setattr( litellm, "model_cost", {"dead-model": {"deprecation_date": "2020-01-01", "litellm_provider": "openai"}}, ) - alerting = SlackAlerting( - alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings] - ) + alerting = SlackAlerting(alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings]) router = _make_router( [ { @@ -252,9 +242,7 @@ async def test_should_alert_only_from_the_pod_holding_the_daily_lock( ), pytest.raises(asyncio.CancelledError), ): - await alerting.run_scheduled_deprecation_check( - get_llm_router=lambda: router, pod_lock_manager=pod_lock_manager - ) + await alerting.run_scheduled_deprecation_check(get_llm_router=lambda: router, pod_lock_manager=pod_lock_manager) assert mock_send_alert.await_count == int(expect_alert) assert pod_lock_manager.acquire_lock.await_args.kwargs == { @@ -268,9 +256,7 @@ async def test_should_alert_only_from_the_pod_holding_the_daily_lock( async def test_should_retry_on_the_next_poll_when_the_lock_claim_fails(monkeypatch): """A redis blip at claim time returns False like a held lock, and must not cost every pod a day of alerts""" monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) - alerting = SlackAlerting( - alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings] - ) + alerting = SlackAlerting(alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings]) router = _make_router([DEAD_ALIAS_DEPLOYMENT]) pod_lock_manager = MagicMock() pod_lock_manager.acquire_lock = AsyncMock(side_effect=[False, True]) @@ -289,9 +275,7 @@ async def test_should_retry_on_the_next_poll_when_the_lock_claim_fails(monkeypat ), pytest.raises(asyncio.CancelledError), ): - await alerting.run_scheduled_deprecation_check( - get_llm_router=lambda: router, pod_lock_manager=pod_lock_manager - ) + await alerting.run_scheduled_deprecation_check(get_llm_router=lambda: router, pod_lock_manager=pod_lock_manager) assert slept == [DEPRECATION_IDLE_POLL_SECONDS] * 2 assert pod_lock_manager.acquire_lock.await_count == 2 @@ -302,9 +286,7 @@ async def test_should_retry_on_the_next_poll_when_the_lock_claim_fails(monkeypat async def test_should_not_claim_the_lock_when_there_is_nothing_to_report(monkeypatch): """An empty pass must not hold the daily lock, or a sunset added later waits out the whole window""" monkeypatch.setattr(litellm, "model_cost", {}) - alerting = SlackAlerting( - alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings] - ) + alerting = SlackAlerting(alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings]) router = _make_router( [ { @@ -318,9 +300,7 @@ async def test_should_not_claim_the_lock_when_there_is_nothing_to_report(monkeyp pod_lock_manager.acquire_lock = AsyncMock(return_value=True) with patch.object(alerting, "send_alert", new_callable=AsyncMock) as mock_send_alert: - sent = await alerting.send_model_deprecation_alert( - llm_router=router, pod_lock_manager=pod_lock_manager - ) + sent = await alerting.send_model_deprecation_alert(llm_router=router, pod_lock_manager=pod_lock_manager) assert sent is False pod_lock_manager.acquire_lock.assert_not_awaited() @@ -331,9 +311,7 @@ async def test_should_not_claim_the_lock_when_there_is_nothing_to_report(monkeyp async def test_should_not_alert_or_claim_the_lock_within_a_day_of_a_sent_alert(monkeypatch): """The shared sent stamp keeps sibling pods and restarts from re-alerting or re-asking redis for a day""" monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) - alerting = SlackAlerting( - alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings] - ) + alerting = SlackAlerting(alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings]) await alerting.internal_usage_cache.async_set_cache( key=SlackAlertingCacheKeys.deprecation_alert_sent_key.value, value=1.0, @@ -351,21 +329,17 @@ async def test_should_not_alert_or_claim_the_lock_within_a_day_of_a_sent_alert(m ), pytest.raises(asyncio.CancelledError), ): - await alerting.run_scheduled_deprecation_check( - get_llm_router=lambda: router, pod_lock_manager=pod_lock_manager - ) + await alerting.run_scheduled_deprecation_check(get_llm_router=lambda: router, pod_lock_manager=pod_lock_manager) pod_lock_manager.acquire_lock.assert_not_awaited() mock_send_alert.assert_not_awaited() @pytest.mark.asyncio -async def test_should_back_off_a_full_day_after_a_pass_raises(monkeypatch): +async def test_should_skip_a_raising_pass_for_a_day_and_keep_polling(monkeypatch): """A misconfigured webhook raises on every send, which must log once a day rather than every poll""" monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) - alerting = SlackAlerting( - alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings] - ) + alerting = SlackAlerting(alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings]) router = _make_router([DEAD_ALIAS_DEPLOYMENT]) slept: list[float] = [] @@ -389,8 +363,8 @@ async def test_should_back_off_a_full_day_after_a_pass_raises(monkeypatch): ): await alerting.run_scheduled_deprecation_check(get_llm_router=lambda: router) - assert slept == [DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS] * 2 - assert mock_send_alert.await_count == 2 + assert slept == [DEPRECATION_IDLE_POLL_SECONDS] * 2 + assert mock_send_alert.await_count == 1 def _email_only_alerting() -> SlackAlerting: @@ -403,7 +377,7 @@ class _EmailRun: self.fail = fail self.contexts = [] - async def __call__(self, ctx): + async def __call__(self, ctx: DeprecationEmailContext) -> int: self.contexts.append(ctx) if self.fail: raise ValueError("smtp exploded") @@ -449,7 +423,7 @@ async def test_should_run_the_email_pass_with_email_only_alerting(monkeypatch): prisma: Final = SimpleNamespace() run: Final = _EmailRun(result=2) - failed: Final = await alerting._run_deprecation_passes( + await alerting._run_deprecation_passes( get_llm_router=lambda: _make_router([DEAD_ALIAS_DEPLOYMENT]), pod_lock_manager=None, get_prisma_client=lambda: prisma, @@ -457,22 +431,21 @@ async def test_should_run_the_email_pass_with_email_only_alerting(monkeypatch): send_emails=run, ) - assert failed is False assert len(run.contexts) == 1 assert run.contexts[0].prisma_client is prisma assert run.contexts[0].alerting_args.model_deprecation_email_thresholds == (30, 7, 0) assert run.contexts[0].cache is alerting.internal_usage_cache - assert callable(run.contexts[0].deliver) + assert alerting.deprecation_email_backoff_until == 0.0 @pytest.mark.asyncio -async def test_should_still_email_when_the_slack_pass_raises_and_report_the_failure(monkeypatch): +async def test_should_still_email_when_the_slack_pass_raises_and_back_off_only_slack(monkeypatch): monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) alerting: Final = SlackAlerting(alerting=["slack", "email"], alert_types=[AlertType.model_deprecation_warnings]) run: Final = _EmailRun() with patch.object(alerting, "send_alert", new_callable=AsyncMock, side_effect=ValueError("no webhook")): - failed: Final = await alerting._run_deprecation_passes( + await alerting._run_deprecation_passes( get_llm_router=lambda: _make_router([DEAD_ALIAS_DEPLOYMENT]), pod_lock_manager=None, get_prisma_client=SimpleNamespace, @@ -480,16 +453,18 @@ async def test_should_still_email_when_the_slack_pass_raises_and_report_the_fail send_emails=run, ) - assert failed is True assert len(run.contexts) == 1 + assert alerting.deprecation_alert_backoff_until > 0.0 + assert alerting.deprecation_email_backoff_until == 0.0 @pytest.mark.asyncio -async def test_should_report_failure_when_the_email_pass_raises(monkeypatch): +async def test_should_back_off_only_the_email_pass_when_it_raises(monkeypatch): monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) + alerting: Final = _email_only_alerting() run: Final = _EmailRun(fail=True) - failed: Final = await _email_only_alerting()._run_deprecation_passes( + await alerting._run_deprecation_passes( get_llm_router=lambda: _make_router([DEAD_ALIAS_DEPLOYMENT]), pod_lock_manager=None, get_prisma_client=SimpleNamespace, @@ -497,7 +472,8 @@ async def test_should_report_failure_when_the_email_pass_raises(monkeypatch): send_emails=run, ) - assert failed is True + assert alerting.deprecation_email_backoff_until > 0.0 + assert alerting.deprecation_alert_backoff_until == 0.0 @pytest.mark.asyncio @@ -539,7 +515,9 @@ async def test_should_skip_the_email_pass_without_a_router(): prisma_lookups.append(True) return SimpleNamespace() - sent: Final = await _email_only_alerting()._run_deprecation_email_pass(None, None, get_prisma_client, lambda: None, run) + sent: Final = await _email_only_alerting()._run_deprecation_email_pass( + None, None, get_prisma_client, lambda: None, run + ) assert sent == 0 assert prisma_lookups == [] @@ -547,24 +525,90 @@ async def test_should_skip_the_email_pass_without_a_router(): @pytest.mark.asyncio -async def test_should_back_off_a_day_when_only_the_email_pass_raises(monkeypatch): +async def test_should_keep_polling_every_thirty_seconds_after_an_email_failure(monkeypatch): monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) alerting: Final = _email_only_alerting() slept: list[float] = [] - async def stop_after_first_pass(seconds): + async def stop_after_second_pass(seconds): slept.append(seconds) - raise asyncio.CancelledError + if len(slept) == 2: + raise asyncio.CancelledError + run: Final = _EmailRun(fail=True) with ( - patch("litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep", side_effect=stop_after_first_pass), + patch("litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep", side_effect=stop_after_second_pass), pytest.raises(asyncio.CancelledError), ): await alerting.run_scheduled_deprecation_check( get_llm_router=lambda: _make_router([DEAD_ALIAS_DEPLOYMENT]), get_prisma_client=SimpleNamespace, get_email_logger=lambda: None, - send_emails=_EmailRun(fail=True), + send_emails=run, ) - assert slept == [DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS] + assert slept == [DEPRECATION_IDLE_POLL_SECONDS] * 2 + assert len(run.contexts) == 1 + + +@pytest.mark.parametrize( + ("alerting", "expected"), + [ + (None, False), + ([], False), + (["email"], False), + (["webhook"], False), + (["slack"], True), + ([MS_TEAMS_ALERTING_DESTINATION], True), + (["slack", "email"], True), + ], +) +def test_deprecation_alerts_enabled_requires_a_slack_or_teams_channel(alerting, expected): + alerting_obj: Final = SlackAlerting(alerting=alerting, alert_types=[AlertType.model_deprecation_warnings]) + assert alerting_obj._deprecation_alerts_enabled() is expected + + +@pytest.mark.asyncio +async def test_should_keep_emailing_while_a_broken_slack_pass_is_backed_off(monkeypatch): + monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) + alerting: Final = SlackAlerting(alerting=["slack", "email"], alert_types=[AlertType.model_deprecation_warnings]) + run: Final = _EmailRun() + + with patch.object( + alerting, "send_alert", new_callable=AsyncMock, side_effect=ValueError("no webhook") + ) as send_alert: + for _ in range(2): + await alerting._run_deprecation_passes( + get_llm_router=lambda: _make_router([DEAD_ALIAS_DEPLOYMENT]), + pod_lock_manager=None, + get_prisma_client=SimpleNamespace, + get_email_logger=lambda: None, + send_emails=run, + now=lambda: 1_000.0, + ) + + assert send_alert.await_count == 1 + assert len(run.contexts) == 2 + + +@pytest.mark.asyncio +async def test_should_retry_a_backed_off_email_pass_after_a_day(monkeypatch): + monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) + alerting: Final = _email_only_alerting() + run: Final = _EmailRun(fail=True) + + for moment in ( + 1_000.0, + 1_000.0 + DEPRECATION_IDLE_POLL_SECONDS, + 1_000.0 + DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, + ): + await alerting._run_deprecation_passes( + get_llm_router=lambda: _make_router([DEAD_ALIAS_DEPLOYMENT]), + pod_lock_manager=None, + get_prisma_client=SimpleNamespace, + get_email_logger=lambda: None, + send_emails=run, + now=lambda moment=moment: moment, + ) + + assert len(run.contexts) == 2 From d65c74b443bf70675910a6ff66db2fd0bf00ece9 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 17:01:07 +0530 Subject: [PATCH 21/37] refactor(alerting): validate alerting_args through pydantic instead of an untyped spread --- litellm/integrations/SlackAlerting/slack_alerting.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index b22483eba01..1b471b4f0fc 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -5,7 +5,7 @@ import datetime import os import random import time -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Mapping from datetime import timedelta from typing import TYPE_CHECKING, Any, Final, Literal @@ -124,7 +124,7 @@ class SlackAlerting(CustomBatchLogger): alert_types: list[AlertType] = DEFAULT_ALERT_TYPES, alert_to_webhook_url: dict[AlertType, list[str] | str] | None = None, # if user wants to separate alerts to diff channels - alerting_args={}, + alerting_args: Mapping[str, object] | None = None, default_webhook_url: str | None = None, alert_type_config: dict[str, dict] | None = None, **kwargs, @@ -138,7 +138,9 @@ class SlackAlerting(CustomBatchLogger): self.async_http_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback) self.alert_to_webhook_url = process_slack_alerting_variables(alert_to_webhook_url=alert_to_webhook_url) self.is_running = False - self.alerting_args = SlackAlertingArgs(**alerting_args) + self.alerting_args = ( + SlackAlertingArgs() if alerting_args is None else SlackAlertingArgs.model_validate(alerting_args) + ) self.default_webhook_url = default_webhook_url self.flush_lock = asyncio.Lock() self.periodic_started = False @@ -161,7 +163,7 @@ class SlackAlerting(CustomBatchLogger): alerting_threshold: float | None = None, alert_types: list[AlertType] | None = None, alert_to_webhook_url: dict[AlertType, list[str] | str] | None = None, - alerting_args: dict | None = None, + alerting_args: Mapping[str, object] | None = None, llm_router: Router | None = None, alert_type_config: dict[str, dict] | None = None, ): @@ -174,7 +176,7 @@ class SlackAlerting(CustomBatchLogger): if alert_types is not None: self.alert_types = alert_types if alerting_args is not None: - self.alerting_args = SlackAlertingArgs(**alerting_args) + self.alerting_args = SlackAlertingArgs.model_validate(alerting_args) if not self.periodic_started: asyncio.create_task(self.periodic_flush()) self.periodic_started = True From a0fecbdeec45d8d0afc55393b28cd0c00fb1f581 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 17:06:13 +0530 Subject: [PATCH 22/37] test(alerting): make the types integrations test dir a package to avoid a module name clash --- tests/test_litellm/types/integrations/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/test_litellm/types/integrations/__init__.py diff --git a/tests/test_litellm/types/integrations/__init__.py b/tests/test_litellm/types/integrations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From ed46a3c7f27555d8a76701ace7b351213ed3789e Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 14 Sep 2026 17:24:31 +0530 Subject: [PATCH 23/37] fix(proxy): fall back to smtp for the no-op email logger and claim the lock after resolving --- .../model_deprecation_notifications.py | 29 ++++++-- .../test_model_deprecation_notifications.py | 67 ++++++++++++++++--- 2 files changed, 81 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index 28a5971941a..74216325a41 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -258,7 +258,7 @@ def render_model_deprecation_email( ) -> tuple[str, str]: """(subject, html) for one team's digest, every model-sourced string escaped""" alias: Final = " ".join((notification.team_alias or "").split()) - team_name: Final = alias or notification.team_id + team_name: Final = alias or " ".join(notification.team_id.split()) verb: Final = ( "deprecated" if any(model.info.days_until_deprecation < 0 for model in notification.models) else "deprecating" ) @@ -272,11 +272,20 @@ def render_model_deprecation_email( return subject, body + EMAIL_FOOTER +def _is_noop_email_logger(email_logger: object) -> bool: + """The proxy hands out the bare enterprise base logger when no email provider is configured""" + try: + from litellm_enterprise.enterprise_callbacks.send_emails.base_email import BaseEmailLogger + except ImportError: + return False + return type(email_logger) is BaseEmailLogger + + def make_email_deliverer( email_logger: EmailSender | None, smtp_send: SmtpSend | None = None ) -> Callable[[Sequence[str], str, str], Awaitable[None]]: - """The proxy's configured email provider when there is one, else the OSS SMTP helper per recipient""" - if email_logger is not None: + """A configured email provider when the proxy has one, else the OSS SMTP helper once per recipient""" + if email_logger is not None and not _is_noop_email_logger(email_logger): async def deliver_via_logger(recipients: Sequence[str], subject: str, html_body: str) -> None: await email_logger.send_email( @@ -348,20 +357,26 @@ async def _send_team_notification(notification: TeamNotification, ctx: Deprecati async def send_model_deprecation_emails(ctx: DeprecationEmailContext) -> int: - """One pass: claim the day, resolve affected teams, deliver one digest per team, then stamp the pass + """One pass: resolve affected teams, claim the day only when there is something to send, deliver, then stamp - An empty snapshot returns before the lock or the DB is touched, so a model added later is not - silenced for a day, and the stamp is written last, so a lost lock claim costs one poll, not a day + An empty snapshot returns before the DB is touched. A lost lock claim (a sibling pod, or a redis blip) + returns without a stamp, so it costs one poll rather than a day, and a failing resolution raises + before the lock is held, so it cannot silence the other pods """ snapshot: Final = collect_model_deprecations(llm_router=ctx.llm_router) infos: Final = (*snapshot.deprecated, *snapshot.imminent, *snapshot.upcoming) - if not infos or not await _claimed_email_window(ctx.pod_lock_manager): + if not infos: return 0 teams: Final = await _load_teams(ctx.prisma_client) affected: Final = await resolve_affected_teams( infos, ctx.llm_router, teams, ctx.alerting_args.model_deprecation_email_thresholds ) notifications: Final = await build_team_notifications(affected, teams, ctx.cache, ctx.prisma_client) + if not notifications: + await _stamp_pass(ctx.cache) + return 0 + if not await _claimed_email_window(ctx.pod_lock_manager): + return 0 results: Final = tuple([await _send_team_notification(notification, ctx) for notification in notifications]) await _stamp_pass(ctx.cache) return sum(results) diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py index ee013845e8e..5cbc2a6b2a7 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py @@ -273,6 +273,18 @@ class TestRenderModelDeprecationEmail: ) assert subject == "[LiteLLM] 1 model(s) deprecating for team Data Team" + def test_should_collapse_newlines_in_the_team_id_fallback(self): + notification: Final = TeamNotification( + team_id="t\r\n1", + team_alias=None, + recipients=("a@example.com",), + models=(AffectedModel(info=_info("gpt-old", 5), display_name="gpt-old", milestone=7),), + ) + subject, _ = render_model_deprecation_email( + notification, email_logo_url="https://logo", email_support_contact="h@x.io" + ) + assert subject.endswith("for team t 1") + def _prisma_with_users(rows: Sequence[Mapping[str, object]]) -> SimpleNamespace: async def find_many(where=None, take=None, skip=None, order=None): @@ -490,7 +502,7 @@ class TestSendModelDeprecationEmails: assert await send_model_deprecation_emails(ctx) == 0 assert deliverer.sent == [] - assert len(lock.calls) == 1 + assert lock.calls == [] assert await cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value) is not None @pytest.mark.parametrize( @@ -518,15 +530,10 @@ class TestSendModelDeprecationEmails: assert (stamped is not None) is (expected_sent == 1) @pytest.mark.asyncio - async def test_should_not_load_teams_or_stamp_when_the_lock_is_lost(self): + async def test_should_not_send_or_stamp_when_the_lock_is_lost(self): cache: Final = DualCache() deliverer: Final = _Deliverer() - - async def explode(where=None, take=None, skip=None, order=None): - raise AssertionError("teams must not be loaded without the lock") - - prisma: Final = _prisma((), ()) - prisma.db.litellm_teamtable.find_many = explode + prisma: Final = _prisma(TWO_TEAMS[:1], TWO_ADMINS) assert ( await send_model_deprecation_emails( @@ -537,6 +544,23 @@ class TestSendModelDeprecationEmails: assert deliverer.sent == [] assert await cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value) is None + @pytest.mark.asyncio + async def test_should_raise_before_claiming_the_lock_when_loading_teams_fails(self): + cache: Final = DualCache() + lock: Final = _Lock(True) + + async def explode(where=None, take=None, skip=None, order=None): + raise RuntimeError("db down") + + prisma: Final = _prisma((), ()) + prisma.db.litellm_teamtable.find_many = explode + + with pytest.raises(RuntimeError, match="db down"): + await send_model_deprecation_emails(_context(_router([DEAD_DEPLOYMENT]), prisma, _Deliverer(), cache, lock)) + + assert lock.calls == [] + assert await cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value) is None + @pytest.mark.asyncio async def test_should_email_the_thirty_day_milestone_for_an_upcoming_model(self): cache: Final = DualCache() @@ -630,3 +654,30 @@ class TestMakeEmailDeliverer: with pytest.raises(ValueError, match="SMTP_SENDER_EMAIL"): await deliver(("a@x.io",), "subj", "

hi

") + + @pytest.mark.asyncio + async def test_should_skip_the_no_op_enterprise_base_logger(self): + base_email: Final = pytest.importorskip("litellm_enterprise.enterprise_callbacks.send_emails.base_email") + calls: Final = [] + + async def smtp_send(*, receiver_email, subject, html): + calls.append(receiver_email) + + deliver: Final = make_email_deliverer(base_email.BaseEmailLogger(), smtp_send=smtp_send) + + await deliver(("a@x.io",), "subj", "

hi

") + assert calls == ["a@x.io"] + + @pytest.mark.asyncio + async def test_should_use_a_configured_enterprise_logger_subclass(self): + base_email: Final = pytest.importorskip("litellm_enterprise.enterprise_callbacks.send_emails.base_email") + calls: Final = [] + + class ConfiguredLogger(base_email.BaseEmailLogger): + async def send_email(self, from_email, to_email, subject, html_body): + calls.append((from_email, tuple(to_email))) + + deliver: Final = make_email_deliverer(ConfiguredLogger()) + + await deliver(("a@x.io",), "subj", "

hi

") + assert calls == [(base_email.BaseEmailLogger.DEFAULT_LITELLM_EMAIL, ("a@x.io",))] From 3e4366e72854be6369abeaecef644ee822b8028b Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Tue, 15 Sep 2026 03:51:23 +0530 Subject: [PATCH 24/37] fix(proxy): notify email-only team admins about deprecating models --- litellm/integrations/email_alerting.py | 28 +++++++++---------- .../integrations/test_email_alerting.py | 18 ++++++++++-- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/litellm/integrations/email_alerting.py b/litellm/integrations/email_alerting.py index dff8d980430..6b5b9c8718d 100644 --- a/litellm/integrations/email_alerting.py +++ b/litellm/integrations/email_alerting.py @@ -3,7 +3,7 @@ Functions for sending Email Alerts """ import os -from typing import TYPE_CHECKING, Final +from typing import Final from litellm._logging import verbose_logger, verbose_proxy_logger from litellm.models.team import LiteLLM_TeamTable @@ -11,9 +11,6 @@ from litellm.proxy._types import WebhookEvent from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository -if TYPE_CHECKING: - from litellm.proxy.utils import PrismaClient - # we use this for the email header, please send a test email if you change this. verify it looks good on email LITELLM_LOGO_URL: Final = "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" LITELLM_SUPPORT_CONTACT: Final = "support@berri.ai" @@ -70,20 +67,21 @@ async def get_all_team_member_emails(team_id: str | None = None) -> list: return emails -async def get_team_admin_emails(team: LiteLLM_TeamTable, prisma_client: "PrismaClient") -> tuple[str, ...]: - """Emails of the team's admins: members_with_roles admins plus the legacy admins list, deduped""" +async def get_team_admin_emails(team: LiteLLM_TeamTable, prisma_client: object) -> tuple[str, ...]: + """Emails of the team's admins: members_with_roles admins by user id or inline email, plus the legacy admins list, deduped""" + admins: Final = tuple(member for member in team.members_with_roles if member.role == "admin") + inline_emails: Final = tuple(member.user_email for member in admins if member.user_email and not member.user_id) admin_ids: Final = frozenset( - ( - *(member.user_id for member in team.members_with_roles if member.role == "admin" and member.user_id), - *(user_id for user_id in team.admins if user_id), + (*(member.user_id for member in admins if member.user_id), *(user_id for user_id in team.admins if user_id)) + ) + rows: Final = ( + await UserRepository(prisma_client).find_many( + where={"user_id": {"in": sorted(admin_ids)}} # mutable-ok: prisma filter payloads are plain dict/list ) + if admin_ids + else () ) - if not admin_ids: - return () - rows: Final = await UserRepository(prisma_client).find_many( - where={"user_id": {"in": sorted(admin_ids)}} # mutable-ok: prisma filter payloads are plain dict/list - ) - return tuple(dict.fromkeys(row.user_email for row in rows if row.user_email)) + return tuple(dict.fromkeys((*inline_emails, *(row.user_email for row in rows if row.user_email)))) async def send_team_budget_alert(webhook_event: WebhookEvent) -> bool: diff --git a/tests/test_litellm/integrations/test_email_alerting.py b/tests/test_litellm/integrations/test_email_alerting.py index de72cce9bac..ba418582335 100644 --- a/tests/test_litellm/integrations/test_email_alerting.py +++ b/tests/test_litellm/integrations/test_email_alerting.py @@ -70,7 +70,7 @@ async def test_should_not_query_when_team_has_no_admins(): @pytest.mark.asyncio -async def test_should_skip_email_only_members_and_blank_legacy_ids_and_dedupe(): +async def test_should_include_email_only_admins_and_dedupe_ids_and_emails(): prisma: Final = _prisma( ( {"user_id": "u-admin", "user_email": "admin@example.com"}, @@ -85,5 +85,19 @@ async def test_should_skip_email_only_members_and_blank_legacy_ids_and_dedupe(): Member(user_email="mail-only@example.com", role="admin"), ], ) - assert await get_team_admin_emails(team, prisma) == ("admin@example.com",) + assert await get_team_admin_emails(team, prisma) == ("mail-only@example.com", "admin@example.com") assert prisma.db.litellm_usertable.queries == [{"user_id": {"in": ["u-admin", "u-legacy"]}}] + + +@pytest.mark.asyncio +async def test_should_email_an_email_only_admin_without_querying_the_database(): + prisma: Final = _prisma(USERS) + team: Final = LiteLLM_TeamTable( + team_id="t1", + members_with_roles=[ + Member(user_email="mail-only@example.com", role="admin"), + Member(user_id="u-member", role="user"), + ], + ) + assert await get_team_admin_emails(team, prisma) == ("mail-only@example.com",) + assert prisma.db.litellm_usertable.queries == [] From 64c53ddc3adfb92bb8040cffacc82dac25ed24ec Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Tue, 15 Sep 2026 03:56:43 +0530 Subject: [PATCH 25/37] fix(proxy): send deprecation digests per recipient and surface smtp failures --- .../model_deprecation_notifications.py | 31 +++++++++++------ litellm/proxy/utils.py | 3 ++ .../test_model_deprecation_notifications.py | 34 +++++++++++++++---- 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index 74216325a41..c856d170745 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -76,7 +76,8 @@ class EmailSender(Protocol): class SmtpSend(Protocol): - def __call__(self, *, receiver_email: str, subject: str, html: str) -> Awaitable[object]: ... + async def __call__(self, *, receiver_email: str, subject: str, html: str, raise_on_error: bool) -> object: + """Send one HTML email over SMTP, raising on failure when asked to""" @dataclass(frozen=True, slots=True) @@ -281,19 +282,29 @@ def _is_noop_email_logger(email_logger: object) -> bool: return type(email_logger) is BaseEmailLogger +def _is_smtp_email_logger(email_logger: object) -> bool: + """The enterprise SMTP logger fires one task per recipient without awaiting it, so its failures are invisible""" + try: + from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import SMTPEmailLogger + except ImportError: + return False + return isinstance(email_logger, SMTPEmailLogger) + + def make_email_deliverer( email_logger: EmailSender | None, smtp_send: SmtpSend | None = None ) -> Callable[[Sequence[str], str, str], Awaitable[None]]: - """A configured email provider when the proxy has one, else the OSS SMTP helper once per recipient""" - if email_logger is not None and not _is_noop_email_logger(email_logger): + """A configured non-SMTP provider, one email per recipient; SMTP always goes through the OSS helper so failures raise""" + if email_logger is not None and not _is_noop_email_logger(email_logger) and not _is_smtp_email_logger(email_logger): async def deliver_via_logger(recipients: Sequence[str], subject: str, html_body: str) -> None: - await email_logger.send_email( - from_email=email_logger.DEFAULT_LITELLM_EMAIL, - to_email=tuple(recipients), - subject=subject, - html_body=html_body, - ) + for recipient in recipients: + await email_logger.send_email( + from_email=email_logger.DEFAULT_LITELLM_EMAIL, + to_email=(recipient,), + subject=subject, + html_body=html_body, + ) return deliver_via_logger @@ -302,7 +313,7 @@ def make_email_deliverer( send: Final = smtp_send or send_email for recipient in recipients: - await send(receiver_email=recipient, subject=subject, html=html_body) + await send(receiver_email=recipient, subject=subject, html=html_body, raise_on_error=True) return deliver_over_smtp diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index c095586b6c9..9c442aab096 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -6567,6 +6567,7 @@ async def send_email( receiver_email: str | None = None, subject: str | None = None, html: str | None = None, + raise_on_error: bool = False, ): """ smtp_host, @@ -6621,6 +6622,8 @@ async def send_email( except Exception as e: verbose_proxy_logger.exception("An error occurred while sending the email:" + str(e)) + if raise_on_error: + raise def hash_token(token: str): diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py index 5cbc2a6b2a7..84ba8ce2cb3 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py @@ -623,7 +623,7 @@ async def test_should_report_whether_the_daily_pass_already_ran(): class TestMakeEmailDeliverer: @pytest.mark.asyncio - async def test_should_use_the_configured_email_logger_when_present(self): + async def test_should_use_the_configured_email_logger_once_per_recipient(self): calls: Final = [] async def send_email(from_email, to_email, subject, html_body): @@ -633,13 +633,17 @@ class TestMakeEmailDeliverer: deliver: Final = make_email_deliverer(logger) await deliver(("a@x.io", "b@x.io"), "subj", "

hi

") - assert calls == [("noreply@litellm.ai", ("a@x.io", "b@x.io"), "subj", "

hi

")] + assert calls == [ + ("noreply@litellm.ai", ("a@x.io",), "subj", "

hi

"), + ("noreply@litellm.ai", ("b@x.io",), "subj", "

hi

"), + ] @pytest.mark.asyncio async def test_should_fall_back_to_smtp_per_recipient_without_a_logger(self): calls: Final = [] - async def smtp_send(*, receiver_email, subject, html): + async def smtp_send(*, receiver_email, subject, html, raise_on_error): + assert raise_on_error is True calls.append((receiver_email, subject, html)) deliver: Final = make_email_deliverer(None, smtp_send=smtp_send) @@ -647,6 +651,20 @@ class TestMakeEmailDeliverer: await deliver(("a@x.io", "b@x.io"), "subj", "

hi

") assert calls == [("a@x.io", "subj", "

hi

"), ("b@x.io", "subj", "

hi

")] + @pytest.mark.asyncio + async def test_should_route_the_enterprise_smtp_logger_through_the_oss_helper(self): + smtp_email: Final = pytest.importorskip("litellm_enterprise.enterprise_callbacks.send_emails.smtp_email") + calls: Final = [] + + async def smtp_send(*, receiver_email, subject, html, raise_on_error): + assert raise_on_error is True + calls.append(receiver_email) + + deliver: Final = make_email_deliverer(smtp_email.SMTPEmailLogger(), smtp_send=smtp_send) + + await deliver(("a@x.io", "b@x.io"), "subj", "

hi

") + assert calls == ["a@x.io", "b@x.io"] + @pytest.mark.asyncio async def test_should_raise_when_smtp_is_not_configured(self, monkeypatch): monkeypatch.delenv("SMTP_SENDER_EMAIL", raising=False) @@ -660,7 +678,8 @@ class TestMakeEmailDeliverer: base_email: Final = pytest.importorskip("litellm_enterprise.enterprise_callbacks.send_emails.base_email") calls: Final = [] - async def smtp_send(*, receiver_email, subject, html): + async def smtp_send(*, receiver_email, subject, html, raise_on_error): + assert raise_on_error is True calls.append(receiver_email) deliver: Final = make_email_deliverer(base_email.BaseEmailLogger(), smtp_send=smtp_send) @@ -679,5 +698,8 @@ class TestMakeEmailDeliverer: deliver: Final = make_email_deliverer(ConfiguredLogger()) - await deliver(("a@x.io",), "subj", "

hi

") - assert calls == [(base_email.BaseEmailLogger.DEFAULT_LITELLM_EMAIL, ("a@x.io",))] + await deliver(("a@x.io", "b@x.io"), "subj", "

hi

") + assert calls == [ + (base_email.BaseEmailLogger.DEFAULT_LITELLM_EMAIL, ("a@x.io",)), + (base_email.BaseEmailLogger.DEFAULT_LITELLM_EMAIL, ("b@x.io",)), + ] From e57c957fccdff5cae99143e3f9743eaece6155c6 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Tue, 15 Sep 2026 04:03:46 +0530 Subject: [PATCH 26/37] fix(proxy): key sent markers by deprecation date and stop notifying long-deprecated models --- .../model_deprecation_notifications.py | 35 ++++- .../test_model_deprecation_notifications.py | 129 +++++++++++++----- 2 files changed, 123 insertions(+), 41 deletions(-) diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index c856d170745..1b63487520c 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -7,6 +7,7 @@ import os import time from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass +from datetime import date from types import MappingProxyType from typing import TYPE_CHECKING, Final, Protocol @@ -95,7 +96,15 @@ def select_milestone(days_until: int, thresholds: Sequence[int]) -> int | None: return min((threshold for threshold in thresholds if days_until <= threshold), default=None) -def _reached(info: ModelDeprecationInfo, thresholds: Sequence[int]) -> tuple[ModelDeprecationInfo, int] | None: +def _max_age_days(alerting_args: SlackAlertingArgs) -> int: + return alerting_args.model_deprecation_email_ttl // (24 * 60 * 60) + + +def _reached( + info: ModelDeprecationInfo, thresholds: Sequence[int], max_age_days: int +) -> tuple[ModelDeprecationInfo, int] | None: + if info.days_until_deprecation < -max_age_days: + return None milestone: Final = select_milestone(info.days_until_deprecation, thresholds) return None if milestone is None else (info, milestone) @@ -178,9 +187,12 @@ async def resolve_affected_teams( llm_router: Router, teams: Sequence[LiteLLM_TeamTable], thresholds: Sequence[int], + max_age_days: int, ) -> Mapping[str, tuple[AffectedModel, ...]]: """Per team id, the deprecating models it can reach that have crossed a threshold""" - reached: Final = tuple(pair for pair in (_reached(info, thresholds) for info in infos) if pair is not None) + reached: Final = tuple( + pair for pair in (_reached(info, thresholds, max_age_days) for info in infos) if pair is not None + ) owners: Final = _deployment_owners(llm_router) per_team: Final = MappingProxyType( { @@ -192,8 +204,8 @@ async def resolve_affected_teams( return MappingProxyType({team_id: models for team_id, models in per_team.items() if models}) -def email_sent_key(team_id: str, model_name: str, milestone: int) -> str: - return f"model_deprecation_email:{team_id}:{model_name}:{milestone}" +def email_sent_key(team_id: str, model_name: str, deprecation_date: date, milestone: int) -> str: + return f"model_deprecation_email:{team_id}:{model_name}:{deprecation_date.isoformat()}:{milestone}" async def _unsent( @@ -201,7 +213,10 @@ async def _unsent( ) -> tuple[AffectedModel, ...]: flags: Final = tuple( [ - await cache.async_get_cache(key=email_sent_key(team_id, model.info.model_name, model.milestone)) is None + await cache.async_get_cache( + key=email_sent_key(team_id, model.info.model_name, model.info.deprecation_date, model.milestone) + ) + is None for model in models ] ) @@ -360,7 +375,9 @@ async def _send_team_notification(notification: TeamNotification, ctx: Deprecati return False for model in notification.models: await ctx.cache.async_set_cache( - key=email_sent_key(notification.team_id, model.info.model_name, model.milestone), + key=email_sent_key( + notification.team_id, model.info.model_name, model.info.deprecation_date, model.milestone + ), value=time.time(), ttl=ctx.alerting_args.model_deprecation_email_ttl, ) @@ -380,7 +397,11 @@ async def send_model_deprecation_emails(ctx: DeprecationEmailContext) -> int: return 0 teams: Final = await _load_teams(ctx.prisma_client) affected: Final = await resolve_affected_teams( - infos, ctx.llm_router, teams, ctx.alerting_args.model_deprecation_email_thresholds + infos, + ctx.llm_router, + teams, + ctx.alerting_args.model_deprecation_email_thresholds, + _max_age_days(ctx.alerting_args), ) notifications: Final = await build_team_notifications(affected, teams, ctx.cache, ctx.prisma_client) if not notifications: diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py index 84ba8ce2cb3..6c0923130fd 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py @@ -45,11 +45,16 @@ class TestSelectMilestone: assert select_milestone(5, (0, 7, 30)) == 7 -def _info(model_name: str, days_until: int, provider: str | None = "openai") -> ModelDeprecationInfo: +def _info( + model_name: str, + days_until: int, + provider: str | None = "openai", + deprecation_date: date = date(2026, 10, 1), +) -> ModelDeprecationInfo: return ModelDeprecationInfo( model_name=model_name, litellm_model=model_name, - deprecation_date=date(2026, 10, 1), + deprecation_date=deprecation_date, days_until_deprecation=days_until, status="deprecated" if days_until < 0 else ("imminent" if days_until <= 30 else "upcoming"), litellm_provider=provider, @@ -75,14 +80,16 @@ class TestResolveAffectedTeams: @pytest.mark.asyncio async def test_should_include_team_listing_the_model_explicitly(self): affected: Final = await resolve_affected_teams( - (_info("gpt-old", 5),), _router([]), (_team("t1", ("gpt-old",)), _team("t2", ("other",))), THRESHOLDS + (_info("gpt-old", 5),), _router([]), (_team("t1", ("gpt-old",)), _team("t2", ("other",))), THRESHOLDS, 90 ) assert set(affected) == {"t1"} assert affected["t1"] == (AffectedModel(info=_info("gpt-old", 5), display_name="gpt-old", milestone=7),) @pytest.mark.asyncio async def test_should_include_unrestricted_team(self): - affected: Final = await resolve_affected_teams((_info("gpt-old", 5),), _router([]), (_team("t1"),), THRESHOLDS) + affected: Final = await resolve_affected_teams( + (_info("gpt-old", 5),), _router([]), (_team("t1"),), THRESHOLDS, 90 + ) assert set(affected) == {"t1"} @pytest.mark.asyncio @@ -92,19 +99,22 @@ class TestResolveAffectedTeams: _router([]), (_team("t1", ("openai/*",)), _team("t2", ("azure/*",))), THRESHOLDS, + 90, ) assert set(affected) == {"t1"} @pytest.mark.asyncio async def test_should_skip_blocked_teams(self): affected: Final = await resolve_affected_teams( - (_info("gpt-old", 5),), _router([]), (_team("t1", blocked=True),), THRESHOLDS + (_info("gpt-old", 5),), _router([]), (_team("t1", blocked=True),), THRESHOLDS, 90 ) assert dict(affected) == {} @pytest.mark.asyncio async def test_should_skip_models_that_have_not_reached_a_threshold(self): - affected: Final = await resolve_affected_teams((_info("gpt-old", 45),), _router([]), (_team("t1"),), THRESHOLDS) + affected: Final = await resolve_affected_teams( + (_info("gpt-old", 45),), _router([]), (_team("t1"),), THRESHOLDS, 90 + ) assert dict(affected) == {} @pytest.mark.asyncio @@ -117,7 +127,7 @@ class TestResolveAffectedTeams: } ] affected: Final = await resolve_affected_teams( - (_info("model_name_t2_abc", -3),), _router(deployments), (_team("t1"), _team("t2")), THRESHOLDS + (_info("model_name_t2_abc", -3),), _router(deployments), (_team("t1"), _team("t2")), THRESHOLDS, 90 ) assert set(affected) == {"t2"} assert affected["t2"][0].display_name == "gpt-old" @@ -130,6 +140,7 @@ class TestResolveAffectedTeams: _router([]), (_team("t1", ("gpt-old", "gpt-older")),), THRESHOLDS, + 90, ) assert [m.info.model_name for m in affected["t1"]] == ["gpt-old", "gpt-older"] @@ -139,7 +150,7 @@ class TestResolveAffectedTeams: {"model_name": "gpt-old", "litellm_params": {"model": "openai/gpt-old"}, "model_info": {"id": "1"}} ] affected: Final = await resolve_affected_teams( - (_info("gpt-old", 5),), _router(deployments), (_team("t1", ("gpt-old",)),), THRESHOLDS + (_info("gpt-old", 5),), _router(deployments), (_team("t1", ("gpt-old",)),), THRESHOLDS, 90 ) assert set(affected) == {"t1"} assert affected["t1"][0].display_name == "gpt-old" @@ -154,10 +165,17 @@ class TestResolveAffectedTeams: } ] affected: Final = await resolve_affected_teams( - (_info("model_name_t2_abc", -3),), _router(deployments), (_team("t2"),), THRESHOLDS + (_info("model_name_t2_abc", -3),), _router(deployments), (_team("t2"),), THRESHOLDS, 90 ) assert affected["t2"][0].display_name == "model_name_t2_abc" + @pytest.mark.asyncio + async def test_should_skip_models_deprecated_longer_ago_than_the_marker_lifetime(self): + affected: Final = await resolve_affected_teams( + (_info("ancient", -200), _info("gpt-old", -10)), _router([]), (_team("t1"),), THRESHOLDS, 90 + ) + assert [model.info.model_name for model in affected["t1"]] == ["gpt-old"] + @pytest.mark.asyncio async def test_should_deny_a_team_whose_allowlist_is_malformed_and_keep_going(self): affected: Final = await resolve_affected_teams( @@ -165,6 +183,7 @@ class TestResolveAffectedTeams: _router([]), (_team("t1", ("openai/[*",)), _team("t2", ("openai/gpt-old",))), THRESHOLDS, + 90, ) assert set(affected) == {"t2"} @@ -315,7 +334,7 @@ class TestBuildTeamNotifications: @pytest.mark.asyncio async def test_should_skip_milestones_already_sent(self): cache: Final = DualCache() - await cache.async_set_cache(key=email_sent_key("t1", "gpt-old", 7), value=1.0) + await cache.async_set_cache(key=email_sent_key("t1", "gpt-old", date(2026, 10, 1), 7), value=1.0) sent: Final = AffectedModel(info=_info("gpt-old", 5), display_name="gpt-old", milestone=7) fresh: Final = AffectedModel(info=_info("gpt-older", -1), display_name="gpt-older", milestone=0) notifications: Final = await build_team_notifications( @@ -326,7 +345,7 @@ class TestBuildTeamNotifications: @pytest.mark.asyncio async def test_should_drop_team_when_everything_was_already_sent(self): cache: Final = DualCache() - await cache.async_set_cache(key=email_sent_key("t1", "gpt-old", 7), value=1.0) + await cache.async_set_cache(key=email_sent_key("t1", "gpt-old", date(2026, 10, 1), 7), value=1.0) models: Final = (AffectedModel(info=_info("gpt-old", 5), display_name="gpt-old", milestone=7),) notifications: Final = await build_team_notifications( {"t1": models}, (_admin_team("t1"),), cache, _prisma_with_users(ADMIN_USERS) @@ -343,25 +362,54 @@ class TestBuildTeamNotifications: assert notifications == () @pytest.mark.asyncio - async def test_should_key_the_sent_marker_by_team_model_and_milestone(self): - assert email_sent_key("t1", "gpt-old", 7) == "model_deprecation_email:t1:gpt-old:7" - assert email_sent_key("t1", "gpt-old", 7) != email_sent_key("t2", "gpt-old", 7) - assert email_sent_key("t1", "gpt-old", 7) != email_sent_key("t1", "gpt-old", 0) + async def test_should_send_a_second_deployment_sharing_a_model_name_with_a_later_date(self): + cache: Final = DualCache() + await cache.async_set_cache(key=email_sent_key("t1", "gpt-old", date(2026, 10, 1), 7), value=1.0) + sent: Final = AffectedModel(info=_info("gpt-old", 5), display_name="gpt-old", milestone=7) + other_date: Final = AffectedModel( + info=_info("gpt-old", 5, deprecation_date=date(2027, 3, 1)), display_name="gpt-old", milestone=7 + ) + notifications: Final = await build_team_notifications( + {"t1": (sent, other_date)}, (_admin_team("t1"),), cache, _prisma_with_users(ADMIN_USERS) + ) + assert notifications[0].models == (other_date,) + @pytest.mark.asyncio + async def test_should_key_the_sent_marker_by_team_model_date_and_milestone(self): + assert ( + email_sent_key("t1", "gpt-old", date(2026, 10, 1), 7) == "model_deprecation_email:t1:gpt-old:2026-10-01:7" + ) + assert email_sent_key("t1", "gpt-old", date(2026, 10, 1), 7) != email_sent_key( + "t2", "gpt-old", date(2026, 10, 1), 7 + ) + assert email_sent_key("t1", "gpt-old", date(2026, 10, 1), 7) != email_sent_key( + "t1", "gpt-old", date(2026, 10, 1), 0 + ) + assert email_sent_key("t1", "gpt-old", date(2026, 10, 1), 7) != email_sent_key( + "t1", "gpt-old", date(2027, 3, 1), 7 + ) + + +TODAY: Final = datetime.now(timezone.utc).date() +DEAD_DATE: Final = TODAY - timedelta(days=10) +LONG_DEAD_DATE: Final = TODAY - timedelta(days=200) +UPCOMING_DATE: Final = TODAY + timedelta(days=20) +FAR_DATE: Final = TODAY + timedelta(days=40) DEAD_DEPLOYMENT: Final = { "model_name": "dead-alias", "litellm_params": {"model": "openai/dead-model"}, - "model_info": {"id": "1", "deprecation_date": "2020-01-01", "litellm_provider": "openai"}, + "model_info": {"id": "1", "deprecation_date": DEAD_DATE.isoformat(), "litellm_provider": "openai"}, +} +LONG_DEAD_DEPLOYMENT: Final = { + "model_name": "ancient-alias", + "litellm_params": {"model": "openai/ancient-model"}, + "model_info": {"id": "4", "deprecation_date": LONG_DEAD_DATE.isoformat(), "litellm_provider": "openai"}, } UPCOMING_DEPLOYMENT: Final = { "model_name": "sunset-alias", "litellm_params": {"model": "openai/sunset-model"}, - "model_info": { - "id": "2", - "deprecation_date": (datetime.now(timezone.utc).date() + timedelta(days=20)).isoformat(), - "litellm_provider": "openai", - }, + "model_info": {"id": "2", "deprecation_date": UPCOMING_DATE.isoformat(), "litellm_provider": "openai"}, } FRESH_DEPLOYMENT: Final = { "model_name": "fresh", @@ -371,11 +419,7 @@ FRESH_DEPLOYMENT: Final = { FAR_DEPLOYMENT: Final = { "model_name": "far-alias", "litellm_params": {"model": "openai/far-model"}, - "model_info": { - "id": "3", - "deprecation_date": (datetime.now(timezone.utc).date() + timedelta(days=40)).isoformat(), - "litellm_provider": "openai", - }, + "model_info": {"id": "3", "deprecation_date": FAR_DATE.isoformat(), "litellm_provider": "openai"}, } @@ -466,7 +510,7 @@ class TestSendModelDeprecationEmails: assert await send_model_deprecation_emails(ctx) == 2 assert sorted(recipients for recipients, _ in deliverer.sent) == [("a@x.io",), ("b@x.io",)] assert all("deprecated" in subject for _, subject in deliverer.sent) - assert await cache.async_get_cache(key=email_sent_key("t1", "dead-alias", 0)) is not None + assert await cache.async_get_cache(key=email_sent_key("t1", "dead-alias", DEAD_DATE, 0)) is not None assert await cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value) is not None assert await send_model_deprecation_emails(ctx) == 0 @@ -495,7 +539,7 @@ class TestSendModelDeprecationEmails: @pytest.mark.asyncio async def test_should_stamp_the_pass_but_send_nothing_when_every_milestone_was_already_sent(self): cache: Final = DualCache() - await cache.async_set_cache(key=email_sent_key("t1", "dead-alias", 0), value=1.0) + await cache.async_set_cache(key=email_sent_key("t1", "dead-alias", DEAD_DATE, 0), value=1.0) lock: Final = _Lock(True) deliverer: Final = _Deliverer() ctx: Final = _context(_router([DEAD_DEPLOYMENT]), _prisma(TWO_TEAMS[:1], TWO_ADMINS), deliverer, cache, lock) @@ -569,8 +613,8 @@ class TestSendModelDeprecationEmails: assert await send_model_deprecation_emails(ctx) == 1 assert deliverer.sent[0][1] == "[LiteLLM] 1 model(s) deprecating for team Alpha" - assert await cache.async_get_cache(key=email_sent_key("t1", "sunset-alias", 30)) is not None - assert await cache.async_get_cache(key=email_sent_key("t1", "sunset-alias", 7)) is None + assert await cache.async_get_cache(key=email_sent_key("t1", "sunset-alias", UPCOMING_DATE, 30)) is not None + assert await cache.async_get_cache(key=email_sent_key("t1", "sunset-alias", UPCOMING_DATE, 7)) is None @pytest.mark.asyncio async def test_should_email_an_upcoming_model_when_a_threshold_reaches_past_the_slack_window(self): @@ -586,7 +630,7 @@ class TestSendModelDeprecationEmails: assert await send_model_deprecation_emails(ctx) == 1 assert deliverer.sent[0][1] == "[LiteLLM] 1 model(s) deprecating for team Alpha" - assert await cache.async_get_cache(key=email_sent_key("t1", "far-alias", 45)) is not None + assert await cache.async_get_cache(key=email_sent_key("t1", "far-alias", FAR_DATE, 45)) is not None @pytest.mark.asyncio async def test_should_stamp_sent_keys_with_the_configured_ttl_and_the_pass_for_a_day(self): @@ -595,12 +639,29 @@ class TestSendModelDeprecationEmails: ctx: Final = _context(_router([DEAD_DEPLOYMENT]), _prisma(TWO_TEAMS[:1], TWO_ADMINS), deliverer, cache) assert await send_model_deprecation_emails(ctx) == 1 - assert cache.ttls[email_sent_key("t1", "dead-alias", 0)] == SlackAlertingArgs().model_deprecation_email_ttl + assert ( + cache.ttls[email_sent_key("t1", "dead-alias", DEAD_DATE, 0)] + == SlackAlertingArgs().model_deprecation_email_ttl + ) assert ( cache.ttls[SlackAlertingCacheKeys.deprecation_email_pass_key.value] == DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS ) + @pytest.mark.asyncio + async def test_should_not_email_a_model_deprecated_longer_ago_than_the_marker_lifetime(self): + cache: Final = DualCache() + deliverer: Final = _Deliverer() + lock: Final = _Lock(True) + ctx: Final = _context( + _router([LONG_DEAD_DEPLOYMENT]), _prisma(TWO_TEAMS[:1], TWO_ADMINS), deliverer, cache, lock + ) + + assert await send_model_deprecation_emails(ctx) == 0 + assert deliverer.sent == [] + assert lock.calls == [] + assert await cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value) is not None + @pytest.mark.asyncio async def test_should_keep_going_and_leave_keys_unstamped_when_one_team_fails(self): cache: Final = DualCache() @@ -609,8 +670,8 @@ class TestSendModelDeprecationEmails: assert await send_model_deprecation_emails(ctx) == 1 assert [recipients for recipients, _ in deliverer.sent] == [("b@x.io",)] - assert await cache.async_get_cache(key=email_sent_key("t1", "dead-alias", 0)) is None - assert await cache.async_get_cache(key=email_sent_key("t2", "dead-alias", 0)) is not None + assert await cache.async_get_cache(key=email_sent_key("t1", "dead-alias", DEAD_DATE, 0)) is None + assert await cache.async_get_cache(key=email_sent_key("t2", "dead-alias", DEAD_DATE, 0)) is not None @pytest.mark.asyncio From 208206b480b550e9ee7708a1b5dbbadcfde5bf2c Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Tue, 15 Sep 2026 04:08:43 +0530 Subject: [PATCH 27/37] refactor(proxy): break the import cycle between alerting and deprecation emails --- .../SlackAlerting/slack_alerting.py | 19 +++++++--------- .../model_deprecation_notifications.py | 22 ++++++++----------- litellm/types/proxy/model_deprecation.py | 12 +++++++++- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 1b471b4f0fc..77928749642 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -53,6 +53,7 @@ from litellm.types.integrations.slack_alerting import * from litellm.types.proxy.model_deprecation import ( DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, DEPRECATION_IDLE_POLL_SECONDS, + EmailSender, ) from ..email_templates.templates import * @@ -67,10 +68,6 @@ from .ms_teams import ( from .utils import process_slack_alerting_variables if TYPE_CHECKING: - from litellm.proxy.common_utils.model_deprecation_notifications import ( - DeprecationEmailContext, - EmailSender, - ) from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager from litellm.proxy.utils import PrismaClient from litellm.router import Router as _Router @@ -92,7 +89,7 @@ def _proxy_prisma_client() -> "PrismaClient | None": return prisma_client -def _proxy_email_logger() -> "EmailSender | None": +def _proxy_email_logger() -> EmailSender | None: from litellm.proxy.proxy_server import proxy_logging_obj return proxy_logging_obj.email_logging_instance @@ -1204,8 +1201,8 @@ Model Info: llm_router: Router | None, pod_lock_manager: "PodLockManager | None", get_prisma_client: Callable[[], "PrismaClient | None"], - get_email_logger: Callable[[], "EmailSender | None"], - send_emails: "Callable[[DeprecationEmailContext], Awaitable[int]] | None", + get_email_logger: Callable[[], EmailSender | None], + send_emails: "Callable[..., Awaitable[int]] | None", ) -> int: """Email team admins the milestones crossed, resolving against the DB at most once a day""" if llm_router is None or not self._deprecation_emails_enabled(): @@ -1241,8 +1238,8 @@ Model Info: get_llm_router: Callable[[], Router | None], pod_lock_manager: "PodLockManager | None", get_prisma_client: Callable[[], "PrismaClient | None"], - get_email_logger: Callable[[], "EmailSender | None"], - send_emails: "Callable[[DeprecationEmailContext], Awaitable[int]] | None", + get_email_logger: Callable[[], EmailSender | None], + send_emails: "Callable[..., Awaitable[int]] | None", now: Callable[[], float] = time.time, ) -> None: """Run the Slack pass then the email pass; a pass that raised is skipped for a day while the other keeps polling""" @@ -1265,8 +1262,8 @@ Model Info: get_llm_router: Callable[[], Router | None] = _proxy_llm_router, pod_lock_manager: "PodLockManager | None" = None, get_prisma_client: Callable[[], "PrismaClient | None"] = _proxy_prisma_client, - get_email_logger: Callable[[], "EmailSender | None"] = _proxy_email_logger, - send_emails: "Callable[[DeprecationEmailContext], Awaitable[int]] | None" = None, + get_email_logger: Callable[[], EmailSender | None] = _proxy_email_logger, + send_emails: "Callable[..., Awaitable[int]] | None" = None, ) -> None: """Poll for a loaded router and run both passes every poll interval diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index 1b63487520c..d8218980402 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -33,12 +33,12 @@ from litellm.repositories.team_repository import TeamRepository from litellm.types.integrations.slack_alerting import SlackAlertingArgs, SlackAlertingCacheKeys from litellm.types.proxy.model_deprecation import ( DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, + EmailSender, ModelDeprecationInfo, ) if TYPE_CHECKING: from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager - from litellm.proxy.utils import PrismaClient from litellm.router import Router @@ -65,15 +65,11 @@ class TeamNotification: class DeprecationEmailCache(Protocol): - async def async_get_cache(self, *, key: str) -> object: ... + async def async_get_cache(self, *, key: str) -> object: + """The value stored under key, or None""" - async def async_set_cache(self, *, key: str, value: float, ttl: float) -> None: ... - - -class EmailSender(Protocol): - DEFAULT_LITELLM_EMAIL: str - - async def send_email(self, from_email: str, to_email: Sequence[str], subject: str, html_body: str) -> None: ... + async def async_set_cache(self, *, key: str, value: float, ttl: float) -> None: + """Store value under key for ttl seconds""" class SmtpSend(Protocol): @@ -84,7 +80,7 @@ class SmtpSend(Protocol): @dataclass(frozen=True, slots=True) class DeprecationEmailContext: llm_router: Router - prisma_client: PrismaClient + prisma_client: object cache: DeprecationEmailCache alerting_args: SlackAlertingArgs pod_lock_manager: PodLockManager | None @@ -224,7 +220,7 @@ async def _unsent( async def _notification_for( - team: LiteLLM_TeamTable, models: Sequence[AffectedModel], cache: DeprecationEmailCache, prisma_client: PrismaClient + team: LiteLLM_TeamTable, models: Sequence[AffectedModel], cache: DeprecationEmailCache, prisma_client: object ) -> TeamNotification | None: unsent: Final = await _unsent(team.team_id, models, cache) if not unsent: @@ -240,7 +236,7 @@ async def build_team_notifications( affected: Mapping[str, Sequence[AffectedModel]], teams: Sequence[LiteLLM_TeamTable], cache: DeprecationEmailCache, - prisma_client: PrismaClient, + prisma_client: object, ) -> tuple[TeamNotification, ...]: """One digest per team of the milestones not yet emailed, dropping teams with nobody to email""" teams_by_id: Final = MappingProxyType({team.team_id: team for team in teams}) @@ -333,7 +329,7 @@ def make_email_deliverer( return deliver_over_smtp -async def _load_teams(prisma_client: PrismaClient) -> tuple[LiteLLM_TeamTable, ...]: +async def _load_teams(prisma_client: object) -> tuple[LiteLLM_TeamTable, ...]: return tuple(await TeamRepository(prisma_client).find_many()) diff --git a/litellm/types/proxy/model_deprecation.py b/litellm/types/proxy/model_deprecation.py index bbad63a278d..1e9f90c9b7e 100644 --- a/litellm/types/proxy/model_deprecation.py +++ b/litellm/types/proxy/model_deprecation.py @@ -1,7 +1,8 @@ from __future__ import annotations +from collections.abc import Sequence from datetime import date, datetime -from typing import Final, Literal +from typing import Final, Literal, Protocol from pydantic import BaseModel, Field @@ -14,6 +15,15 @@ DEPRECATION_IDLE_POLL_SECONDS: Final = 30 DeprecationStatus = Literal["upcoming", "imminent", "deprecated"] +class EmailSender(Protocol): + """The proxy's configured email logger, as far as deprecation digests need it""" + + DEFAULT_LITELLM_EMAIL: str + + async def send_email(self, from_email: str, to_email: Sequence[str], subject: str, html_body: str) -> None: + """Deliver one HTML email to the given recipients""" + + class ModelDeprecationInfo(BaseModel): model_name: str = Field(description="The public name of the model on the proxy (model_group).") litellm_model: str | None = Field( From 3675f185d66f20db9eaee2acbec724387e1bf076 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Tue, 15 Sep 2026 04:10:11 +0530 Subject: [PATCH 28/37] test(alerting): inject the poll sleep instead of patching asyncio --- litellm/integrations/SlackAlerting/slack_alerting.py | 4 +++- .../SlackAlerting/test_model_deprecation_alert.py | 6 ++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 77928749642..d36076b8e8c 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1264,6 +1264,7 @@ Model Info: get_prisma_client: Callable[[], "PrismaClient | None"] = _proxy_prisma_client, get_email_logger: Callable[[], EmailSender | None] = _proxy_email_logger, send_emails: "Callable[..., Awaitable[int]] | None" = None, + sleep: Callable[[float], Awaitable[None]] | None = None, ) -> None: """Poll for a loaded router and run both passes every poll interval @@ -1272,11 +1273,12 @@ Model Info: say) is skipped for a full day so a misconfiguration logs once, not every poll, without stalling the other pass """ + sleep_fn: Final = sleep or asyncio.sleep while True: await self._run_deprecation_passes( get_llm_router, pod_lock_manager, get_prisma_client, get_email_logger, send_emails ) - await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS) + await sleep_fn(DEPRECATION_IDLE_POLL_SECONDS) async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool: """ diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py index 371e921da9e..1c78e0a0d84 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py @@ -536,15 +536,13 @@ async def test_should_keep_polling_every_thirty_seconds_after_an_email_failure(m raise asyncio.CancelledError run: Final = _EmailRun(fail=True) - with ( - patch("litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep", side_effect=stop_after_second_pass), - pytest.raises(asyncio.CancelledError), - ): + with pytest.raises(asyncio.CancelledError): await alerting.run_scheduled_deprecation_check( get_llm_router=lambda: _make_router([DEAD_ALIAS_DEPLOYMENT]), get_prisma_client=SimpleNamespace, get_email_logger=lambda: None, send_emails=run, + sleep=stop_after_second_pass, ) assert slept == [DEPRECATION_IDLE_POLL_SECONDS] * 2 From 7712f4f0ae6b2e1a1c342c90bd715089f17c5439 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Tue, 15 Sep 2026 04:46:51 +0530 Subject: [PATCH 29/37] fix(proxy): test the smtp re-raise, log it once, and close the deprecation age boundary --- litellm/integrations/email_alerting.py | 2 +- .../model_deprecation_notifications.py | 2 +- litellm/proxy/utils.py | 3 +- .../test_model_deprecation_notifications.py | 7 ++++ tests/test_litellm/proxy/test_proxy_utils.py | 37 ++++++++++++------- 5 files changed, 34 insertions(+), 17 deletions(-) diff --git a/litellm/integrations/email_alerting.py b/litellm/integrations/email_alerting.py index 6b5b9c8718d..2f0327a6799 100644 --- a/litellm/integrations/email_alerting.py +++ b/litellm/integrations/email_alerting.py @@ -68,7 +68,7 @@ async def get_all_team_member_emails(team_id: str | None = None) -> list: async def get_team_admin_emails(team: LiteLLM_TeamTable, prisma_client: object) -> tuple[str, ...]: - """Emails of the team's admins: members_with_roles admins by user id or inline email, plus the legacy admins list, deduped""" + """Emails of the team's admins: by user id, inline email, or the legacy admins list, deduped""" admins: Final = tuple(member for member in team.members_with_roles if member.role == "admin") inline_emails: Final = tuple(member.user_email for member in admins if member.user_email and not member.user_id) admin_ids: Final = frozenset( diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index d8218980402..c715a67bd29 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -99,7 +99,7 @@ def _max_age_days(alerting_args: SlackAlertingArgs) -> int: def _reached( info: ModelDeprecationInfo, thresholds: Sequence[int], max_age_days: int ) -> tuple[ModelDeprecationInfo, int] | None: - if info.days_until_deprecation < -max_age_days: + if info.days_until_deprecation <= -max_age_days: return None milestone: Final = select_milestone(info.days_until_deprecation, thresholds) return None if milestone is None else (info, milestone) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 9c442aab096..d585d434e8d 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -6621,9 +6621,10 @@ async def send_email( ) except Exception as e: - verbose_proxy_logger.exception("An error occurred while sending the email:" + str(e)) if raise_on_error: + verbose_proxy_logger.debug("An error occurred while sending the email: %s", e) raise + verbose_proxy_logger.exception("An error occurred while sending the email:" + str(e)) def hash_token(token: str): diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py index 6c0923130fd..b13d305e5a7 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py @@ -176,6 +176,13 @@ class TestResolveAffectedTeams: ) assert [model.info.model_name for model in affected["t1"]] == ["gpt-old"] + @pytest.mark.asyncio + async def test_should_exclude_a_model_exactly_at_the_marker_lifetime_boundary(self): + affected: Final = await resolve_affected_teams( + (_info("edge-out", -90), _info("edge-in", -89)), _router([]), (_team("t1"),), THRESHOLDS, 90 + ) + assert [model.info.model_name for model in affected["t1"]] == ["edge-in"] + @pytest.mark.asyncio async def test_should_deny_a_team_whose_allowlist_is_malformed_and_keep_going(self): affected: Final = await resolve_affected_teams( diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index 9def21c0573..a821747586e 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -9,7 +9,7 @@ from litellm.caching.caching import DualCache from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import ProxyErrorTypes, UserAPIKeyAuth -from litellm.proxy.utils import PrismaClient, ProxyLogging +from litellm.proxy.utils import PrismaClient, ProxyLogging, send_email from litellm.types.guardrails import GuardrailEventHooks @@ -2110,9 +2110,7 @@ def test_create_model_info_response_resolves_alias_to_deployment_model(): ] ) - response = create_model_info_response( - model_id="bedrock-claude-opus-5", provider="openai", llm_router=router - ) + response = create_model_info_response(model_id="bedrock-claude-opus-5", provider="openai", llm_router=router) finally: litellm.model_cost.clear() litellm.model_cost.update(saved_model_cost) @@ -2141,9 +2139,7 @@ def test_create_model_info_response_keeps_exact_alias_over_generalized_deploymen ] ) - response = create_model_info_response( - model_id="claude-opus-5", provider="openai", llm_router=router - ) + response = create_model_info_response(model_id="claude-opus-5", provider="openai", llm_router=router) finally: litellm.model_cost.clear() litellm.model_cost.update(saved_model_cost) @@ -2167,9 +2163,7 @@ def test_create_model_info_response_falls_back_to_alias_for_opaque_deployment_na ] ) - response = create_model_info_response( - model_id="gpt-4o", provider="openai", llm_router=router - ) + response = create_model_info_response(model_id="gpt-4o", provider="openai", llm_router=router) finally: litellm.model_cost.clear() litellm.model_cost.update(saved_model_cost) @@ -2194,9 +2188,7 @@ def test_create_model_info_response_resolves_mode_through_deployment_model(): ] ) - response = create_model_info_response( - model_id="my-embeddings", provider="openai", llm_router=router - ) + response = create_model_info_response(model_id="my-embeddings", provider="openai", llm_router=router) finally: litellm.model_cost.clear() litellm.model_cost.update(saved_model_cost) @@ -2274,7 +2266,9 @@ async def test_post_call_failure_hook_redacts_traceback_before_callbacks(monkeyp with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()): await proxy_logging_obj.post_call_failure_hook( request_data={"metadata": {}}, - original_exception=HTTPException(status_code=400, detail="Upstream passthrough request failed with status 400"), + original_exception=HTTPException( + status_code=400, detail="Upstream passthrough request failed with status 400" + ), user_api_key_dict=UserAPIKeyAuth(), traceback_str=upstream_traceback, ) @@ -2328,3 +2322,18 @@ class TestPrismaClientTokenAuthBehindThePool: assert isinstance(client.db, RoutingPrismaWrapper) assert client.db.writer.iam_token_db_auth is True assert client.db.reader.iam_token_db_auth is True + + +@pytest.mark.asyncio +async def test_send_email_raises_transport_errors_only_when_asked(monkeypatch): + monkeypatch.setenv("SMTP_HOST", "127.0.0.1") + monkeypatch.setenv("SMTP_PORT", "9") + monkeypatch.setenv("SMTP_SENDER_EMAIL", "proxy@example.com") + monkeypatch.setenv("SMTP_TIMEOUT", "5") + monkeypatch.delenv("SMTP_USERNAME", raising=False) + monkeypatch.delenv("SMTP_PASSWORD", raising=False) + + with pytest.raises(ConnectionRefusedError): + await send_email(receiver_email="a@x.io", subject="s", html="

x

", raise_on_error=True) + + assert await send_email(receiver_email="a@x.io", subject="s", html="

x

") is None From a03be776ea96a88030bf77955df4bd8b56074d28 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Tue, 15 Sep 2026 04:49:37 +0530 Subject: [PATCH 30/37] test(proxy): keep the utils test diff to the new send_email case --- tests/test_litellm/proxy/test_proxy_utils.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index a821747586e..f58b67e0e37 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -2110,7 +2110,9 @@ def test_create_model_info_response_resolves_alias_to_deployment_model(): ] ) - response = create_model_info_response(model_id="bedrock-claude-opus-5", provider="openai", llm_router=router) + response = create_model_info_response( + model_id="bedrock-claude-opus-5", provider="openai", llm_router=router + ) finally: litellm.model_cost.clear() litellm.model_cost.update(saved_model_cost) @@ -2139,7 +2141,9 @@ def test_create_model_info_response_keeps_exact_alias_over_generalized_deploymen ] ) - response = create_model_info_response(model_id="claude-opus-5", provider="openai", llm_router=router) + response = create_model_info_response( + model_id="claude-opus-5", provider="openai", llm_router=router + ) finally: litellm.model_cost.clear() litellm.model_cost.update(saved_model_cost) @@ -2163,7 +2167,9 @@ def test_create_model_info_response_falls_back_to_alias_for_opaque_deployment_na ] ) - response = create_model_info_response(model_id="gpt-4o", provider="openai", llm_router=router) + response = create_model_info_response( + model_id="gpt-4o", provider="openai", llm_router=router + ) finally: litellm.model_cost.clear() litellm.model_cost.update(saved_model_cost) @@ -2188,7 +2194,9 @@ def test_create_model_info_response_resolves_mode_through_deployment_model(): ] ) - response = create_model_info_response(model_id="my-embeddings", provider="openai", llm_router=router) + response = create_model_info_response( + model_id="my-embeddings", provider="openai", llm_router=router + ) finally: litellm.model_cost.clear() litellm.model_cost.update(saved_model_cost) @@ -2266,9 +2274,7 @@ async def test_post_call_failure_hook_redacts_traceback_before_callbacks(monkeyp with patch.object(proxy_logging_obj, "update_request_status", new=AsyncMock()): await proxy_logging_obj.post_call_failure_hook( request_data={"metadata": {}}, - original_exception=HTTPException( - status_code=400, detail="Upstream passthrough request failed with status 400" - ), + original_exception=HTTPException(status_code=400, detail="Upstream passthrough request failed with status 400"), user_api_key_dict=UserAPIKeyAuth(), traceback_str=upstream_traceback, ) From 121e3ba7e20d800a1dc229b45a247884a397aea3 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Tue, 15 Sep 2026 05:39:46 +0530 Subject: [PATCH 31/37] fix(proxy): keep day-zero deprecation emails when the marker lifetime is under a day --- .../model_deprecation_notifications.py | 2 +- .../test_model_deprecation_notifications.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index c715a67bd29..1c8bdfe2493 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -93,7 +93,7 @@ def select_milestone(days_until: int, thresholds: Sequence[int]) -> int | None: def _max_age_days(alerting_args: SlackAlertingArgs) -> int: - return alerting_args.model_deprecation_email_ttl // (24 * 60 * 60) + return max(1, alerting_args.model_deprecation_email_ttl // (24 * 60 * 60)) def _reached( diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py index b13d305e5a7..01d6854999f 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation_notifications.py @@ -423,6 +423,11 @@ FRESH_DEPLOYMENT: Final = { "litellm_params": {"model": "openai/fresh-model-with-no-cost-map-entry"}, "model_info": {"id": "x"}, } +TODAY_DEPLOYMENT: Final = { + "model_name": "today-alias", + "litellm_params": {"model": "openai/today-model"}, + "model_info": {"id": "5", "deprecation_date": TODAY.isoformat(), "litellm_provider": "openai"}, +} FAR_DEPLOYMENT: Final = { "model_name": "far-alias", "litellm_params": {"model": "openai/far-model"}, @@ -669,6 +674,19 @@ class TestSendModelDeprecationEmails: assert lock.calls == [] assert await cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value) is not None + @pytest.mark.asyncio + async def test_should_still_email_the_day_zero_milestone_when_the_marker_lifetime_is_under_a_day(self): + deliverer: Final = _Deliverer() + ctx: Final = _context( + _router([TODAY_DEPLOYMENT, DEAD_DEPLOYMENT]), + _prisma(TWO_TEAMS[:1], TWO_ADMINS), + deliverer, + alerting_args=SlackAlertingArgs(model_deprecation_email_ttl=3600), + ) + + assert await send_model_deprecation_emails(ctx) == 1 + assert len(deliverer.sent) == 1 + @pytest.mark.asyncio async def test_should_keep_going_and_leave_keys_unstamped_when_one_team_fails(self): cache: Final = DualCache() From c0305a568d0c792ace441ce40971204eb1b88f9f Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Tue, 15 Sep 2026 05:39:47 +0530 Subject: [PATCH 32/37] fix(proxy): fall back to a team admin's inline email when the user record has none --- litellm/integrations/email_alerting.py | 10 +++++++--- .../integrations/test_email_alerting.py | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/email_alerting.py b/litellm/integrations/email_alerting.py index 2f0327a6799..f0e77e6db78 100644 --- a/litellm/integrations/email_alerting.py +++ b/litellm/integrations/email_alerting.py @@ -3,6 +3,7 @@ Functions for sending Email Alerts """ import os +from types import MappingProxyType from typing import Final from litellm._logging import verbose_logger, verbose_proxy_logger @@ -68,9 +69,8 @@ async def get_all_team_member_emails(team_id: str | None = None) -> list: async def get_team_admin_emails(team: LiteLLM_TeamTable, prisma_client: object) -> tuple[str, ...]: - """Emails of the team's admins: by user id, inline email, or the legacy admins list, deduped""" + """Emails of the team's admins: the user record's email, else the inline one, plus the legacy admins list, deduped""" admins: Final = tuple(member for member in team.members_with_roles if member.role == "admin") - inline_emails: Final = tuple(member.user_email for member in admins if member.user_email and not member.user_id) admin_ids: Final = frozenset( (*(member.user_id for member in admins if member.user_id), *(user_id for user_id in team.admins if user_id)) ) @@ -81,7 +81,11 @@ async def get_team_admin_emails(team: LiteLLM_TeamTable, prisma_client: object) if admin_ids else () ) - return tuple(dict.fromkeys((*inline_emails, *(row.user_email for row in rows if row.user_email)))) + record_emails: Final = MappingProxyType({row.user_id: row.user_email for row in rows if row.user_email}) + inline_emails: Final = tuple( + member.user_email for member in admins if member.user_email and member.user_id not in record_emails + ) + return tuple(dict.fromkeys((*inline_emails, *record_emails.values()))) async def send_team_budget_alert(webhook_event: WebhookEvent) -> bool: diff --git a/tests/test_litellm/integrations/test_email_alerting.py b/tests/test_litellm/integrations/test_email_alerting.py index ba418582335..e819a3d4d59 100644 --- a/tests/test_litellm/integrations/test_email_alerting.py +++ b/tests/test_litellm/integrations/test_email_alerting.py @@ -101,3 +101,21 @@ async def test_should_email_an_email_only_admin_without_querying_the_database(): ) assert await get_team_admin_emails(team, prisma) == ("mail-only@example.com",) assert prisma.db.litellm_usertable.queries == [] + + +@pytest.mark.asyncio +async def test_should_fall_back_to_the_inline_email_when_the_user_record_has_none(): + team: Final = LiteLLM_TeamTable( + team_id="t1", + members_with_roles=[Member(user_id="u-no-email", user_email="inline@example.com", role="admin")], + ) + assert await get_team_admin_emails(team, _prisma(USERS)) == ("inline@example.com",) + + +@pytest.mark.asyncio +async def test_should_prefer_the_user_record_email_over_a_differing_inline_one(): + team: Final = LiteLLM_TeamTable( + team_id="t1", + members_with_roles=[Member(user_id="u-admin", user_email="stale@example.com", role="admin")], + ) + assert await get_team_admin_emails(team, _prisma(USERS)) == ("admin@example.com",) From 708d6fa859656c36b2df0c7d30d2ac66ee3f6cf2 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Tue, 15 Sep 2026 05:45:00 +0530 Subject: [PATCH 33/37] fix(proxy): email only members holding the team admin role, not the legacy admins list --- litellm/integrations/email_alerting.py | 6 ++---- tests/test_litellm/integrations/test_email_alerting.py | 8 +++++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/litellm/integrations/email_alerting.py b/litellm/integrations/email_alerting.py index f0e77e6db78..e237b207974 100644 --- a/litellm/integrations/email_alerting.py +++ b/litellm/integrations/email_alerting.py @@ -69,11 +69,9 @@ async def get_all_team_member_emails(team_id: str | None = None) -> list: async def get_team_admin_emails(team: LiteLLM_TeamTable, prisma_client: object) -> tuple[str, ...]: - """Emails of the team's admins: the user record's email, else the inline one, plus the legacy admins list, deduped""" + """Emails of the members holding the admin role: the user record's email, else the inline one, deduped""" admins: Final = tuple(member for member in team.members_with_roles if member.role == "admin") - admin_ids: Final = frozenset( - (*(member.user_id for member in admins if member.user_id), *(user_id for user_id in team.admins if user_id)) - ) + admin_ids: Final = frozenset(member.user_id for member in admins if member.user_id) rows: Final = ( await UserRepository(prisma_client).find_many( where={"user_id": {"in": sorted(admin_ids)}} # mutable-ok: prisma filter payloads are plain dict/list diff --git a/tests/test_litellm/integrations/test_email_alerting.py b/tests/test_litellm/integrations/test_email_alerting.py index e819a3d4d59..8b13b223239 100644 --- a/tests/test_litellm/integrations/test_email_alerting.py +++ b/tests/test_litellm/integrations/test_email_alerting.py @@ -44,11 +44,13 @@ async def test_should_return_only_admin_emails(): @pytest.mark.asyncio -async def test_should_include_legacy_admins_list(): +async def test_should_ignore_the_legacy_admins_list_the_proxy_does_not_authorize(): + prisma: Final = _prisma(USERS) team: Final = LiteLLM_TeamTable( team_id="t1", admins=["u-legacy"], members_with_roles=[Member(user_id="u-admin", role="admin")] ) - assert sorted(await get_team_admin_emails(team, _prisma(USERS))) == ["admin@example.com", "legacy@example.com"] + assert await get_team_admin_emails(team, prisma) == ("admin@example.com",) + assert prisma.db.litellm_usertable.queries == [{"user_id": {"in": ["u-admin"]}}] @pytest.mark.asyncio @@ -79,9 +81,9 @@ async def test_should_include_email_only_admins_and_dedupe_ids_and_emails(): ) team: Final = LiteLLM_TeamTable( team_id="t1", - admins=["", "u-legacy", "u-admin"], members_with_roles=[ Member(user_id="u-admin", role="admin"), + Member(user_id="u-legacy", role="admin"), Member(user_email="mail-only@example.com", role="admin"), ], ) From 32fa6f752e5a38ec165750ba27e4b6601f570eda Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Sat, 17 Oct 2026 04:17:43 +0530 Subject: [PATCH 34/37] fix(proxy): look up team admin emails with the chunked user query --- litellm/integrations/email_alerting.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/integrations/email_alerting.py b/litellm/integrations/email_alerting.py index 7b973b254f3..9bff02afd57 100644 --- a/litellm/integrations/email_alerting.py +++ b/litellm/integrations/email_alerting.py @@ -72,9 +72,7 @@ async def get_team_admin_emails(team: LiteLLM_TeamTable, prisma_client: object) """Emails of the members holding the admin role: the user record's email, else the inline one, deduped""" admins: Final = tuple(member for member in team.members_with_roles if member.role == "admin") admin_ids: Final = frozenset(member.user_id for member in admins if member.user_id) - rows: Final = ( - await UserRepository(prisma_client).find_many(where={"user_id": {"in": sorted(admin_ids)}}) if admin_ids else () - ) + rows: Final = await UserRepository(prisma_client).find_many_in("user_id", sorted(admin_ids)) record_emails: Final = MappingProxyType({row.user_id: row.user_email for row in rows if row.user_email}) inline_emails: Final = tuple( member.user_email for member in admins if member.user_email and member.user_id not in record_emails From 65b2c1963e047969230076109a64e92869a411cc Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Sat, 17 Oct 2026 04:17:44 +0530 Subject: [PATCH 35/37] fix(alerting): keep update_values taking the proxy's untyped alerting_args --- litellm/integrations/SlackAlerting/slack_alerting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index b41be4286cf..60e2a9d0ef0 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -174,7 +174,7 @@ class SlackAlerting(CustomBatchLogger): alerting_threshold: float | None = None, alert_types: list[AlertType] | None = None, alert_to_webhook_url: dict[AlertType, list[str] | str] | None = None, - alerting_args: Mapping[str, object] | None = None, + alerting_args: dict | None = None, llm_router: Router | None = None, alert_type_config: dict[str, dict] | None = None, ): From 67d4d0191159e08453e488b1432df4279e0d93ce Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Sat, 17 Oct 2026 13:49:08 +0530 Subject: [PATCH 36/37] fix(proxy): recognize the enterprise email loggers without importing litellm_enterprise --- .../model_deprecation_notifications.py | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index c7f10677a64..d54183fb452 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -288,22 +288,28 @@ def render_model_deprecation_email( return subject, body + EMAIL_FOOTER +_ENTERPRISE_BASE_EMAIL_LOGGER: Final = ( + "litellm_enterprise.enterprise_callbacks.send_emails.base_email", + "BaseEmailLogger", +) +_ENTERPRISE_SMTP_EMAIL_LOGGER: Final = ( + "litellm_enterprise.enterprise_callbacks.send_emails.smtp_email", + "SMTPEmailLogger", +) + + +def _class_identity(cls: type) -> tuple[str, str]: + return (cls.__module__, cls.__qualname__) + + def _is_noop_email_logger(email_logger: object) -> bool: """The proxy hands out the bare enterprise base logger when no email provider is configured""" - try: - from litellm_enterprise.enterprise_callbacks.send_emails.base_email import BaseEmailLogger - except ImportError: - return False - return type(email_logger) is BaseEmailLogger + return _class_identity(type(email_logger)) == _ENTERPRISE_BASE_EMAIL_LOGGER def _is_smtp_email_logger(email_logger: object) -> bool: """The enterprise SMTP logger fires one task per recipient without awaiting it, so its failures are invisible""" - try: - from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import SMTPEmailLogger - except ImportError: - return False - return isinstance(email_logger, SMTPEmailLogger) + return _ENTERPRISE_SMTP_EMAIL_LOGGER in map(_class_identity, type(email_logger).__mro__) def make_email_deliverer( From 50e1bf12ebf7c89bf6b8450cd2977da1e062e909 Mon Sep 17 00:00:00 2001 From: Rameshwaran Mohan Date: Mon, 19 Oct 2026 04:17:04 +0530 Subject: [PATCH 37/37] fix(proxy): tag the deprecation email cache calls with their service family --- .../model_deprecation_notifications.py | 6 +++++ .../test_model_deprecation_notifications.py | 27 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/litellm/proxy/common_utils/model_deprecation_notifications.py b/litellm/proxy/common_utils/model_deprecation_notifications.py index d54183fb452..f912df11138 100644 --- a/litellm/proxy/common_utils/model_deprecation_notifications.py +++ b/litellm/proxy/common_utils/model_deprecation_notifications.py @@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Final, Protocol from pydantic import BaseModel, ConfigDict, Field, ValidationError +from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.constants import EMAIL_MODEL_DEPRECATION_LOCK_ID from litellm.integrations.email_alerting import ( @@ -288,6 +289,9 @@ def render_model_deprecation_email( return subject, body + EMAIL_FOOTER +_DEPRECATION_EMAIL_TARGET: Final = "model_deprecation_emails" + + _ENTERPRISE_BASE_EMAIL_LOGGER: Final = ( "litellm_enterprise.enterprise_callbacks.send_emails.base_email", "BaseEmailLogger", @@ -351,6 +355,7 @@ async def _stamp_pass(cache: DeprecationEmailCache) -> None: ) +@with_service_target(_DEPRECATION_EMAIL_TARGET) async def email_pass_done_today(cache: DeprecationEmailCache) -> bool: return (await cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_email_pass_key.value)) is not None @@ -390,6 +395,7 @@ async def _send_team_notification(notification: TeamNotification, ctx: Deprecati return True +@with_service_target(_DEPRECATION_EMAIL_TARGET) async def send_model_deprecation_emails(ctx: DeprecationEmailContext) -> int: """One pass: resolve affected teams, claim the day only when there is something to send, deliver, then stamp diff --git a/tests/unit/proxy/common_utils/test_model_deprecation_notifications.py b/tests/unit/proxy/common_utils/test_model_deprecation_notifications.py index 01d6854999f..e3bd45f28a0 100644 --- a/tests/unit/proxy/common_utils/test_model_deprecation_notifications.py +++ b/tests/unit/proxy/common_utils/test_model_deprecation_notifications.py @@ -10,6 +10,7 @@ from unittest.mock import MagicMock import pytest +from litellm._internal_context import current_service_target from litellm.caching.caching import DualCache from litellm.constants import EMAIL_MODEL_DEPRECATION_LOCK_ID from litellm.models.team import LiteLLM_TeamTable @@ -469,6 +470,20 @@ class _RecordingCache: self.ttls[key] = ttl +class _TargetRecordingCache(_RecordingCache): + def __init__(self): + super().__init__() + self.targets = [] + + async def async_get_cache(self, *, key: str) -> object: + self.targets.append(current_service_target()) + return await super().async_get_cache(key=key) + + async def async_set_cache(self, *, key: str, value: float, ttl: float) -> None: + self.targets.append(current_service_target()) + await super().async_set_cache(key=key, value=value, ttl=ttl) + + def _team_row(team_id: str, alias: str = "Team") -> Mapping[str, object]: return { "team_id": team_id, @@ -707,6 +722,18 @@ async def test_should_report_whether_the_daily_pass_already_ran(): assert await email_pass_done_today(cache) is True +@pytest.mark.asyncio +async def test_should_tag_every_cache_read_and_write_with_the_deprecation_email_family(): + cache: Final = _TargetRecordingCache() + ctx: Final = _context(_router([DEAD_DEPLOYMENT]), _prisma(TWO_TEAMS[:1], TWO_ADMINS), _Deliverer(), cache) + + assert await email_pass_done_today(cache) is False + assert await send_model_deprecation_emails(ctx) == 1 + assert len(cache.targets) >= 3 + assert set(cache.targets) == {"model_deprecation_emails"} + assert current_service_target() is None + + class TestMakeEmailDeliverer: @pytest.mark.asyncio async def test_should_use_the_configured_email_logger_once_per_recipient(self):