From f249356e1674da786a91f8ade36f6e62b255b05b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 30 Apr 2026 17:47:18 +0000 Subject: [PATCH 01/16] feat(proxy): proactive model deprecation alerts and /model/deprecations endpoint Surfaces deprecation_date metadata that is already shipped in model_prices_and_context_window.json so operators get lead time to migrate before a provider sunsets a model. - New helper litellm.proxy.common_utils.model_deprecation classifies the router's configured models into deprecated / imminent / upcoming buckets. Resolution order: explicit model_info.deprecation_date > model_info.base_model > litellm_params.model. - New GET /model/deprecations (and /v1/model/deprecations) endpoint returns a ModelDeprecationResponse, gated by user_api_key_auth. - New AlertType.model_deprecation_warnings (in DEFAULT_ALERT_TYPES) plus SlackAlerting.send_model_deprecation_alert dispatches a Slack message for deprecated/imminent models. Severity is High when any model is already past its date, Medium when only imminent. - ProxyLogging.startup_event schedules a daily background task (_run_scheduled_deprecation_check) when the alert type is enabled. The interval is configurable via LITELLM_MODEL_DEPRECATION_CHECK_INTERVAL and the warn window via LITELLM_MODEL_DEPRECATION_WARN_DAYS. - Tests: 16 unit tests for the helper plus 4 for the Slack hook in tests/test_litellm/. Co-authored-by: Mateo Wang --- .../SlackAlerting/slack_alerting.py | 75 +++++ .../proxy/common_utils/model_deprecation.py | 247 +++++++++++++++ litellm/proxy/proxy_server.py | 51 ++++ litellm/proxy/utils.py | 14 + litellm/types/integrations/slack_alerting.py | 2 + litellm/types/proxy/model_deprecation.py | 93 ++++++ .../test_model_deprecation_alert.py | 100 +++++++ .../common_utils/test_model_deprecation.py | 280 ++++++++++++++++++ 8 files changed, 862 insertions(+) create mode 100644 litellm/proxy/common_utils/model_deprecation.py create mode 100644 litellm/types/proxy/model_deprecation.py create mode 100644 tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py create mode 100644 tests/test_litellm/proxy/common_utils/test_model_deprecation.py diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 771d7876fea..12b5d7525dc 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1038,6 +1038,81 @@ Model Info: async def model_removed_alert(self, model_name: str): pass + async def send_model_deprecation_alert( + self, llm_router: Optional[Any] = None + ) -> bool: + """Aggregate deprecation metadata for the configured models and alert. + + Returns ``True`` when an alert payload was dispatched, ``False`` + otherwise. The ``send_alert`` helper itself is responsible for honoring + the user's webhook configuration; this method only owns producing the + message and choosing whether to send it. + """ + if ( + self.alerting is None + or AlertType.model_deprecation_warnings not in self.alert_types + ): + return False + + from litellm.proxy.common_utils.model_deprecation import ( + collect_model_deprecations, + format_deprecation_alert_message, + ) + + try: + snapshot = collect_model_deprecations(llm_router=llm_router) + except Exception as e: + verbose_proxy_logger.exception( + "Error collecting model deprecation snapshot: %s", e + ) + return False + + message = format_deprecation_alert_message(snapshot) + if message is None: + return False + + level: Literal["Low", "Medium", "High"] = ( + "High" if snapshot.deprecated else "Medium" + ) + + await self.send_alert( + message=message, + level=level, + alert_type=AlertType.model_deprecation_warnings, + alerting_metadata={ + "deprecated_count": len(snapshot.deprecated), + "imminent_count": len(snapshot.imminent), + "upcoming_count": len(snapshot.upcoming), + }, + ) + return True + + async def _run_scheduled_deprecation_check(self, llm_router: Optional[Any] = None): + """Periodic background task that emits a model deprecation alert. + + Runs immediately on startup (so operators see the current state in + Slack) and then sleeps ``DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS`` + between runs. Exits silently if the alert type is not enabled. + """ + from litellm.types.proxy.model_deprecation import ( + DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, + ) + + if ( + self.alerting is None + or AlertType.model_deprecation_warnings not in self.alert_types + ): + return + + while True: + try: + await self.send_model_deprecation_alert(llm_router=llm_router) + except Exception as e: + verbose_proxy_logger.exception( + "Error in model deprecation alert loop: %s", e + ) + await asyncio.sleep(DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS) + async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool: """ Sends structured alert to webhook, if set. diff --git a/litellm/proxy/common_utils/model_deprecation.py b/litellm/proxy/common_utils/model_deprecation.py new file mode 100644 index 00000000000..1b11fa5abd7 --- /dev/null +++ b/litellm/proxy/common_utils/model_deprecation.py @@ -0,0 +1,247 @@ +"""Helpers for surfacing model deprecation/sunset information. + +This module reads ``deprecation_date`` metadata that is bundled in +``model_prices_and_context_window.json`` (exposed at runtime via +``litellm.model_cost``) and classifies the proxy's configured models into +``upcoming``, ``imminent`` and ``deprecated`` buckets. It is the single +source of truth used by both the ``/model/deprecations`` endpoint and the +proactive Slack alert. + +Resolution order for a deployment's deprecation date: + +1. ``model_info.deprecation_date`` – an explicit override on the deployment. +2. ``model_info.base_model`` looked up in ``litellm.model_cost``. +3. The ``litellm_params.model`` string looked up in ``litellm.model_cost``. + +Models without any deprecation metadata are skipped silently (most models +are not deprecated, and we don't want to pollute the response). +""" + +from __future__ import annotations + +from datetime import date, datetime, timezone +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +import litellm +from litellm._logging import verbose_logger +from litellm.types.proxy.model_deprecation import ( + DEFAULT_DEPRECATION_WARN_DAYS, + ModelDeprecationInfo, + ModelDeprecationResponse, +) + +if TYPE_CHECKING: + from litellm.router import Router as _Router + + Router = _Router +else: + Router = Any + + +def _parse_deprecation_date(raw_value: Any) -> Optional[date]: + """Parse a ``deprecation_date`` string in YYYY-MM-DD form. + + Returns ``None`` for missing, malformed, or sentinel placeholder values + (the JSON map ships a documentation sentinel of the form ``"date when..."``). + """ + if raw_value is None: + return None + if isinstance(raw_value, date): + return raw_value + if not isinstance(raw_value, str): + return None + try: + return datetime.strptime(raw_value.strip(), "%Y-%m-%d").date() + except ValueError: + return None + + +def _lookup_deprecation_date_from_cost_map( + model_key: Optional[str], +) -> Tuple[Optional[date], Optional[str]]: + """Look up a deprecation date in ``litellm.model_cost`` for ``model_key``. + + Returns a tuple of (deprecation_date, litellm_provider). + """ + if not model_key: + return None, None + entry = litellm.model_cost.get(model_key) + if not isinstance(entry, dict): + return None, None + return ( + _parse_deprecation_date(entry.get("deprecation_date")), + entry.get("litellm_provider"), + ) + + +def _resolve_deployment_deprecation( + deployment: Dict[str, Any], +) -> Tuple[Optional[date], Optional[str], Optional[str]]: + """Resolve a deployment's deprecation metadata. + + Returns a tuple of (deprecation_date, litellm_model, litellm_provider). + """ + model_info = deployment.get("model_info") or {} + explicit = _parse_deprecation_date(model_info.get("deprecation_date")) + if explicit is not None: + litellm_params = deployment.get("litellm_params") or {} + return ( + explicit, + litellm_params.get("model"), + model_info.get("litellm_provider"), + ) + + base_model = model_info.get("base_model") + dep_date, provider = _lookup_deprecation_date_from_cost_map(base_model) + if dep_date is not None: + return dep_date, base_model, provider + + litellm_params = deployment.get("litellm_params") or {} + raw_model = litellm_params.get("model") + dep_date, provider = _lookup_deprecation_date_from_cost_map(raw_model) + if dep_date is not None: + return dep_date, raw_model, provider + + if isinstance(raw_model, str) and "/" in raw_model: + # Try the un-prefixed lookup (e.g. "openai/gpt-4o" → "gpt-4o"). + bare = raw_model.split("/", 1)[1] + dep_date, provider = _lookup_deprecation_date_from_cost_map(bare) + if dep_date is not None: + return dep_date, bare, provider + + return None, raw_model, model_info.get("litellm_provider") + + +def _classify(days_until: int, warn_within_days: int) -> str: + if days_until < 0: + return "deprecated" + if days_until <= warn_within_days: + return "imminent" + return "upcoming" + + +def _model_dump_compat(deployment: Any) -> Dict[str, Any]: + """Return a plain dict for both pydantic models and dicts.""" + if isinstance(deployment, dict): + return deployment + if hasattr(deployment, "model_dump"): + return deployment.model_dump(exclude_none=True) + if hasattr(deployment, "dict"): + return deployment.dict() + return dict(deployment) + + +def collect_model_deprecations( + llm_router: Optional[Router], + warn_within_days: int = DEFAULT_DEPRECATION_WARN_DAYS, + today: Optional[date] = None, +) -> ModelDeprecationResponse: + """Aggregate deprecation info for all deployments configured on the router. + + De-duplicates by ``(model_name, deprecation_date)`` so multi-deployment + model groups (load-balanced across regions) only surface once per + deprecation date. + """ + snapshot_time = datetime.now(timezone.utc) + today = today or snapshot_time.date() + + response = ModelDeprecationResponse( + warn_within_days=warn_within_days, + checked_at=snapshot_time, + ) + + if llm_router is None: + return response + + seen: set = set() + deployments = llm_router.get_model_list() or [] + for deployment in deployments: + deployment_dict = _model_dump_compat(deployment) + model_name = deployment_dict.get("model_name") + if not model_name: + continue + + dep_date, litellm_model, provider = _resolve_deployment_deprecation( + deployment_dict + ) + if dep_date is None: + continue + + dedup_key = (model_name, dep_date.isoformat()) + if dedup_key in seen: + continue + seen.add(dedup_key) + + days_until = (dep_date - today).days + status = _classify(days_until, warn_within_days) + + info = ModelDeprecationInfo( + model_name=model_name, + litellm_model=litellm_model, + deprecation_date=dep_date, + days_until_deprecation=days_until, + status=status, + litellm_provider=provider, + ) + + if status == "deprecated": + response.deprecated.append(info) + elif status == "imminent": + response.imminent.append(info) + else: + response.upcoming.append(info) + + response.deprecated.sort(key=lambda m: m.deprecation_date) + response.imminent.sort(key=lambda m: m.deprecation_date) + response.upcoming.sort(key=lambda m: m.deprecation_date) + + verbose_logger.debug( + "model_deprecation: deprecated=%d imminent=%d upcoming=%d", + len(response.deprecated), + len(response.imminent), + len(response.upcoming), + ) + + return response + + +def format_deprecation_alert_message( + snapshot: ModelDeprecationResponse, +) -> Optional[str]: + """Format a Slack-friendly alert message for the warning buckets. + + Only ``deprecated`` and ``imminent`` models are included; ``upcoming`` + models are intentionally omitted to avoid alert fatigue. Returns + ``None`` when there is nothing to alert on. + """ + if not snapshot.deprecated and not snapshot.imminent: + return None + + lines: List[str] = ["*⚠️ Model Deprecation Warning*"] + + def _format_entry(info: ModelDeprecationInfo) -> str: + suffix = ( + f"already deprecated {abs(info.days_until_deprecation)}d ago" + if info.days_until_deprecation < 0 + else f"in {info.days_until_deprecation}d" + ) + return ( + f"• `{info.model_name}` " + f"(provider: {info.litellm_provider or 'unknown'}, " + f"deprecates {info.deprecation_date.isoformat()} – {suffix})" + ) + + if snapshot.deprecated: + lines.append("\n*Already deprecated:*") + lines.extend(_format_entry(i) for i in snapshot.deprecated) + + if snapshot.imminent: + lines.append(f"\n*Deprecating within {snapshot.warn_within_days} days:*") + lines.extend(_format_entry(i) for i in snapshot.imminent) + + lines.append( + "\nPlan migrations to a supported model. See " + "https://docs.litellm.ai/docs/proxy/model_management for guidance." + ) + + return "\n".join(lines) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bc980934f9f..3d137732075 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -319,6 +319,7 @@ from litellm.proxy.common_utils.load_config_utils import ( get_config_file_contents_from_gcs, get_file_contents_from_s3, ) +from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator from litellm.proxy.common_utils.openai_endpoint_utils import ( remove_sensitive_info_from_deployment, @@ -624,6 +625,10 @@ from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry from litellm.types.proxy.management_endpoints.model_management_endpoints import ( ModelGroupInfoProxy, ) +from litellm.types.proxy.model_deprecation import ( + DEFAULT_DEPRECATION_WARN_DAYS, + ModelDeprecationResponse, +) from litellm.types.proxy.management_endpoints.ui_sso import ( DefaultTeamSSOParams, LiteLLM_UpperboundKeyGenerateParams, @@ -13436,6 +13441,52 @@ async def model_info_v1( return {"data": all_models} +@router.get( + "/model/deprecations", + tags=["model management"], + dependencies=[Depends(user_api_key_auth)], + response_model=ModelDeprecationResponse, +) +@router.get( + "/v1/model/deprecations", + tags=["model management"], + dependencies=[Depends(user_api_key_auth)], + response_model=ModelDeprecationResponse, +) +async def model_deprecations( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + warn_within_days: int = DEFAULT_DEPRECATION_WARN_DAYS, +) -> ModelDeprecationResponse: + """List models with known deprecation/sunset dates, bucketed by urgency. + + Reads `deprecation_date` metadata from `model_prices_and_context_window.json` + (and any per-deployment `model_info.deprecation_date` overrides) for the + models configured on this proxy. + + Parameters: + warn_within_days: Window (in days) used to bucket "imminent" models. + Defaults to `LITELLM_MODEL_DEPRECATION_WARN_DAYS` env var (or 30). + + Returns: + A payload with three lists of `ModelDeprecationInfo` entries: + + - `deprecated`: deprecation date is in the past — these requests may + fail at any time. + - `imminent`: deprecation date is within `warn_within_days` from today. + - `upcoming`: deprecation date is further out. + + Example: + ```shell + curl -X GET 'http://localhost:4000/model/deprecations' \\ + -H 'Authorization: Bearer sk-1234' + ``` + """ + global llm_router + return collect_model_deprecations( + llm_router=llm_router, warn_within_days=warn_within_days + ) + + def _get_model_group_info( llm_router: Router, all_models_str: list[str], model_group: str | None ) -> list[ModelGroupInfoProxy]: diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index dd0c57aa911..f98df15a346 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -442,6 +442,7 @@ class ProxyLogging: # Guard flags to prevent duplicate background tasks self.daily_report_started: bool = False self.hanging_requests_check_started: bool = False + self.deprecation_check_started: bool = False def startup_event( self, @@ -481,6 +482,19 @@ class ProxyLogging: ) # RUN HANGING REQUEST CHECK (if user wants to alert on hanging requests) self.hanging_requests_check_started = True + if ( + self.slack_alerting_instance is not None + and AlertType.model_deprecation_warnings + in self.slack_alerting_instance.alert_types + and not self.deprecation_check_started + ): + asyncio.create_task( + self.slack_alerting_instance._run_scheduled_deprecation_check( + llm_router=llm_router + ) + ) # RUN MODEL DEPRECATION ALERT LOOP (if scheduled) + self.deprecation_check_started = True + def update_values( self, alerting: list | None = None, diff --git a/litellm/types/integrations/slack_alerting.py b/litellm/types/integrations/slack_alerting.py index 56616c00aa0..768b5d35597 100644 --- a/litellm/types/integrations/slack_alerting.py +++ b/litellm/types/integrations/slack_alerting.py @@ -147,6 +147,7 @@ class AlertType(str, Enum): # Deployment alerts cooldown_deployment = "cooldown_deployment" new_model_added = "new_model_added" + model_deprecation_warnings = "model_deprecation_warnings" # Outage alerts outage_alerts = "outage_alerts" @@ -187,6 +188,7 @@ DEFAULT_ALERT_TYPES: Final[list[AlertType]] = [ # Deployment alerts AlertType.cooldown_deployment, AlertType.new_model_added, + AlertType.model_deprecation_warnings, # Outage alerts AlertType.outage_alerts, AlertType.region_outage_alerts, diff --git a/litellm/types/proxy/model_deprecation.py b/litellm/types/proxy/model_deprecation.py new file mode 100644 index 00000000000..72ccb48fbfc --- /dev/null +++ b/litellm/types/proxy/model_deprecation.py @@ -0,0 +1,93 @@ +"""Type definitions for model deprecation tracking and proactive alerts. + +The proxy reads deprecation/sunset metadata from +``litellm.model_cost`` (sourced from ``model_prices_and_context_window.json``) +and surfaces it through the ``/model/deprecations`` endpoint and Slack +alerting. These types describe the response payload and the alert payload. +""" + +from __future__ import annotations + +import os +from datetime import date, datetime +from typing import List, Optional + +from pydantic import BaseModel, Field + + +DEFAULT_DEPRECATION_WARN_DAYS = int( + os.getenv("LITELLM_MODEL_DEPRECATION_WARN_DAYS", "30") +) +"""Number of days before the deprecation date to start raising warnings. + +Configurable via the ``LITELLM_MODEL_DEPRECATION_WARN_DAYS`` environment +variable. Defaults to 30 days, matching the typical migration window most +LLM providers offer between announcement and removal. +""" + +DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS = int( + os.getenv("LITELLM_MODEL_DEPRECATION_CHECK_INTERVAL", str(24 * 60 * 60)) +) +"""How often the periodic background check runs. Defaults to once per day.""" + + +DeprecationStatusLiteral = str +"""One of ``"upcoming"``, ``"imminent"``, ``"deprecated"``. + +* ``upcoming`` – deprecation is scheduled but more than the warn window away. +* ``imminent`` – deprecation date is within ``warn_within_days`` from today. +* ``deprecated`` – deprecation date has already passed. +""" + + +class ModelDeprecationInfo(BaseModel): + """Per-model deprecation metadata returned by ``/model/deprecations``.""" + + model_name: str = Field( + description="The public name of the model on the proxy (model_group)." + ) + litellm_model: Optional[str] = Field( + default=None, + description="The underlying litellm model string the deprecation date is sourced from.", + ) + deprecation_date: date = Field( + description="The date (UTC) when the model becomes deprecated." + ) + days_until_deprecation: int = Field( + description=( + "Days remaining until the deprecation date. Negative if the model " + "is already deprecated." + ), + ) + status: DeprecationStatusLiteral = Field( + description="One of 'upcoming', 'imminent', or 'deprecated'.", + ) + litellm_provider: Optional[str] = Field( + default=None, description="The provider this model belongs to." + ) + + +class ModelDeprecationResponse(BaseModel): + """Response payload for ``GET /model/deprecations``.""" + + deprecated: List[ModelDeprecationInfo] = Field( + default_factory=list, + description="Models whose deprecation date has already passed.", + ) + imminent: List[ModelDeprecationInfo] = Field( + default_factory=list, + description=( + "Models whose deprecation date is within ``warn_within_days`` from " + "today and require immediate migration planning." + ), + ) + upcoming: List[ModelDeprecationInfo] = Field( + default_factory=list, + description="Models with a future deprecation date outside the warn window.", + ) + warn_within_days: int = Field( + description="The window (in days) used to bucket 'imminent' models." + ) + checked_at: datetime = Field( + description="UTC timestamp when the deprecation snapshot was generated." + ) diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py new file mode 100644 index 00000000000..1cf6bbe0354 --- /dev/null +++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py @@ -0,0 +1,100 @@ +"""Tests for the Slack alerting model deprecation hook.""" + +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../../..")) + +import litellm +from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.proxy._types import AlertType + + +def _make_router(deployments): + router = MagicMock() + router.get_model_list.return_value = deployments + return router + + +@pytest.mark.asyncio +async def test_should_skip_when_alert_type_disabled(): + alerting = SlackAlerting( + alerting=["slack"], + alert_types=[AlertType.llm_exceptions], + ) + sent = await alerting.send_model_deprecation_alert(llm_router=MagicMock()) + assert sent is False + + +@pytest.mark.asyncio +async def test_should_skip_when_no_alerting_configured(): + alerting = SlackAlerting( + alerting=None, + alert_types=[AlertType.model_deprecation_warnings], + ) + sent = await alerting.send_model_deprecation_alert(llm_router=MagicMock()) + assert sent is False + + +@pytest.mark.asyncio +async def test_should_skip_when_no_deprecations_found(monkeypatch): + monkeypatch.setattr(litellm, "model_cost", {}) + alerting = SlackAlerting( + alerting=["slack"], + alert_types=[AlertType.model_deprecation_warnings], + ) + router = _make_router( + [ + { + "model_name": "fresh", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": {"id": "x"}, + } + ] + ) + sent = await alerting.send_model_deprecation_alert(llm_router=router) + assert sent is False + + +@pytest.mark.asyncio +async def test_should_dispatch_high_severity_when_deprecated(monkeypatch): + monkeypatch.setattr( + litellm, + "model_cost", + { + "dead-model": { + "deprecation_date": "2020-01-01", + "litellm_provider": "openai", + } + }, + ) + alerting = SlackAlerting( + alerting=["slack"], + alert_types=[AlertType.model_deprecation_warnings], + ) + router = _make_router( + [ + { + "model_name": "dead-alias", + "litellm_params": {"model": "dead-model"}, + "model_info": {"id": "1"}, + } + ] + ) + + with patch.object( + alerting, "send_alert", new_callable=AsyncMock + ) as mock_send_alert: + sent = await alerting.send_model_deprecation_alert(llm_router=router) + + assert sent is True + mock_send_alert.assert_awaited_once() + call_kwargs = mock_send_alert.await_args.kwargs + assert call_kwargs["alert_type"] == AlertType.model_deprecation_warnings + assert call_kwargs["level"] == "High" + assert call_kwargs["alerting_metadata"]["deprecated_count"] == 1 + assert call_kwargs["alerting_metadata"]["imminent_count"] == 0 + assert "dead-alias" in call_kwargs["message"] diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation.py new file mode 100644 index 00000000000..c873e2494f9 --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation.py @@ -0,0 +1,280 @@ +"""Tests for the model deprecation helper module. + +These tests focus on the helper itself — not on the proxy endpoint or +Slack integration — so they can run without the full proxy stack. +""" + +import os +import sys +from datetime import date +from unittest.mock import MagicMock + + +sys.path.insert(0, os.path.abspath("../../../..")) + +import litellm +from litellm.proxy.common_utils.model_deprecation import ( + _classify, + _parse_deprecation_date, + collect_model_deprecations, + format_deprecation_alert_message, +) + + +def _make_router(deployments): + router = MagicMock() + router.get_model_list.return_value = deployments + return router + + +class TestParseDeprecationDate: + def test_should_parse_iso_string(self): + assert _parse_deprecation_date("2026-12-31") == date(2026, 12, 31) + + def test_should_pass_through_date_object(self): + d = date(2026, 1, 1) + assert _parse_deprecation_date(d) == d + + def test_should_return_none_for_documentation_sentinel(self): + # The JSON map ships a sentinel string under the "sample_spec" key. + assert ( + _parse_deprecation_date( + "date when the model becomes deprecated in the format YYYY-MM-DD" + ) + is None + ) + + def test_should_return_none_for_none(self): + assert _parse_deprecation_date(None) is None + + def test_should_return_none_for_unsupported_type(self): + assert _parse_deprecation_date(12345) is None + + +class TestClassify: + def test_should_classify_past_dates_as_deprecated(self): + assert _classify(-1, warn_within_days=30) == "deprecated" + assert _classify(-365, warn_within_days=30) == "deprecated" + + def test_should_classify_inside_window_as_imminent(self): + assert _classify(0, warn_within_days=30) == "imminent" + assert _classify(15, warn_within_days=30) == "imminent" + assert _classify(30, warn_within_days=30) == "imminent" + + def test_should_classify_outside_window_as_upcoming(self): + assert _classify(31, warn_within_days=30) == "upcoming" + assert _classify(365, warn_within_days=30) == "upcoming" + + +class TestCollectModelDeprecations: + def test_should_return_empty_response_when_router_is_none(self): + snapshot = collect_model_deprecations(llm_router=None) + assert snapshot.deprecated == [] + assert snapshot.imminent == [] + assert snapshot.upcoming == [] + + def test_should_skip_models_without_deprecation_metadata(self, monkeypatch): + monkeypatch.setattr(litellm, "model_cost", {}) + router = _make_router( + [ + { + "model_name": "gpt-4o", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": {"id": "abc"}, + } + ] + ) + snapshot = collect_model_deprecations(llm_router=router) + assert snapshot.deprecated == [] + assert snapshot.imminent == [] + assert snapshot.upcoming == [] + + def test_should_classify_into_three_buckets(self, monkeypatch): + today = date(2026, 6, 1) + monkeypatch.setattr( + litellm, + "model_cost", + { + "deprecated-model": { + "deprecation_date": "2026-01-01", + "litellm_provider": "openai", + }, + "imminent-model": { + "deprecation_date": "2026-06-15", + "litellm_provider": "openai", + }, + "upcoming-model": { + "deprecation_date": "2027-01-01", + "litellm_provider": "openai", + }, + }, + ) + router = _make_router( + [ + { + "model_name": "deprecated-alias", + "litellm_params": {"model": "openai/deprecated-model"}, + "model_info": {"id": "1"}, + }, + { + "model_name": "imminent-alias", + "litellm_params": {"model": "imminent-model"}, + "model_info": {"id": "2"}, + }, + { + "model_name": "upcoming-alias", + "litellm_params": {"model": "openai/upcoming-model"}, + "model_info": {"id": "3"}, + }, + ] + ) + + snapshot = collect_model_deprecations( + llm_router=router, warn_within_days=30, today=today + ) + + assert [m.model_name for m in snapshot.deprecated] == ["deprecated-alias"] + assert [m.model_name for m in snapshot.imminent] == ["imminent-alias"] + assert [m.model_name for m in snapshot.upcoming] == ["upcoming-alias"] + + assert snapshot.deprecated[0].days_until_deprecation < 0 + assert snapshot.imminent[0].days_until_deprecation == 14 + assert snapshot.upcoming[0].days_until_deprecation > 30 + + def test_should_prefer_explicit_deployment_override(self, monkeypatch): + today = date(2026, 6, 1) + monkeypatch.setattr( + litellm, + "model_cost", + {"some-model": {"deprecation_date": "2030-01-01"}}, + ) + router = _make_router( + [ + { + "model_name": "my-alias", + "litellm_params": {"model": "some-model"}, + "model_info": { + "id": "x", + "deprecation_date": "2026-06-10", + }, + } + ] + ) + + snapshot = collect_model_deprecations( + llm_router=router, warn_within_days=30, today=today + ) + + assert len(snapshot.imminent) == 1 + assert snapshot.imminent[0].deprecation_date == date(2026, 6, 10) + + def test_should_dedupe_duplicate_deployments_in_same_group(self, monkeypatch): + today = date(2026, 6, 1) + monkeypatch.setattr( + litellm, + "model_cost", + {"shared-model": {"deprecation_date": "2026-06-10"}}, + ) + router = _make_router( + [ + { + "model_name": "alias", + "litellm_params": {"model": "shared-model"}, + "model_info": {"id": "1"}, + }, + { + "model_name": "alias", + "litellm_params": {"model": "shared-model"}, + "model_info": {"id": "2"}, + }, + ] + ) + + snapshot = collect_model_deprecations( + llm_router=router, warn_within_days=30, today=today + ) + + assert len(snapshot.imminent) == 1 + + def test_should_resolve_via_base_model(self, monkeypatch): + today = date(2026, 6, 1) + monkeypatch.setattr( + litellm, + "model_cost", + {"base-thing": {"deprecation_date": "2026-06-10"}}, + ) + router = _make_router( + [ + { + "model_name": "alias", + "litellm_params": {"model": "azure/some-deployment-name"}, + "model_info": {"id": "1", "base_model": "base-thing"}, + } + ] + ) + + snapshot = collect_model_deprecations( + llm_router=router, warn_within_days=30, today=today + ) + + assert len(snapshot.imminent) == 1 + assert snapshot.imminent[0].litellm_model == "base-thing" + + +class TestFormatDeprecationAlertMessage: + def test_should_return_none_when_nothing_to_alert(self): + snapshot = collect_model_deprecations(llm_router=None) + assert format_deprecation_alert_message(snapshot) is None + + def test_should_render_imminent_and_deprecated_sections(self, monkeypatch): + today = date(2026, 6, 1) + monkeypatch.setattr( + litellm, + "model_cost", + { + "dead-model": { + "deprecation_date": "2026-01-01", + "litellm_provider": "openai", + }, + "soon-model": { + "deprecation_date": "2026-06-15", + "litellm_provider": "anthropic", + }, + "later-model": { + "deprecation_date": "2027-01-01", + "litellm_provider": "anthropic", + }, + }, + ) + router = _make_router( + [ + { + "model_name": "dead", + "litellm_params": {"model": "dead-model"}, + "model_info": {"id": "1"}, + }, + { + "model_name": "soon", + "litellm_params": {"model": "soon-model"}, + "model_info": {"id": "2"}, + }, + { + "model_name": "later", + "litellm_params": {"model": "later-model"}, + "model_info": {"id": "3"}, + }, + ] + ) + + snapshot = collect_model_deprecations( + llm_router=router, warn_within_days=30, today=today + ) + message = format_deprecation_alert_message(snapshot) + + assert message is not None + assert "Already deprecated" in message + assert "Deprecating within 30 days" in message + assert "`dead`" in message + assert "`soon`" in message + # Upcoming models must NOT be in the alert (avoid alert fatigue). + assert "`later`" not in message From 2d7350412440f07535768bc6b7bc522bf55bc678 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 30 Apr 2026 17:51:54 +0000 Subject: [PATCH 02/16] fix(model_deprecation): drop env-var overrides to satisfy docs validation The proxy documentation lives in BerriAI/litellm-docs and any new env key flagged by os.getenv() must be added there before the test_env_keys.py CI check passes. Rather than fork the docs repo for two niche tunables, hard-code the defaults: - DEFAULT_DEPRECATION_WARN_DAYS = 30 (already overridable per-request via ?warn_within_days=N on /model/deprecations). - DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS = 24h. Both can still be raised as env-var follow-ups together with their docs update if operators ask for it. Co-authored-by: Mateo Wang --- litellm/types/proxy/model_deprecation.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/litellm/types/proxy/model_deprecation.py b/litellm/types/proxy/model_deprecation.py index 72ccb48fbfc..8cf98f7a1d5 100644 --- a/litellm/types/proxy/model_deprecation.py +++ b/litellm/types/proxy/model_deprecation.py @@ -8,27 +8,23 @@ alerting. These types describe the response payload and the alert payload. from __future__ import annotations -import os from datetime import date, datetime from typing import List, Optional from pydantic import BaseModel, Field -DEFAULT_DEPRECATION_WARN_DAYS = int( - os.getenv("LITELLM_MODEL_DEPRECATION_WARN_DAYS", "30") -) -"""Number of days before the deprecation date to start raising warnings. +DEFAULT_DEPRECATION_WARN_DAYS = 30 +"""Default warning window (in days) for the ``imminent`` bucket. -Configurable via the ``LITELLM_MODEL_DEPRECATION_WARN_DAYS`` environment -variable. Defaults to 30 days, matching the typical migration window most -LLM providers offer between announcement and removal. +Matches the typical migration window most LLM providers offer between +deprecation announcement and removal. Callers of ``/model/deprecations`` +can override this per-request via the ``?warn_within_days=N`` query +parameter without restarting the proxy. """ -DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS = int( - os.getenv("LITELLM_MODEL_DEPRECATION_CHECK_INTERVAL", str(24 * 60 * 60)) -) -"""How often the periodic background check runs. Defaults to once per day.""" +DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS = 24 * 60 * 60 +"""How often the periodic background check runs. Once per day.""" DeprecationStatusLiteral = str From 590fa227a1e5a787502ea47f86b0bb476d6b7abb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 30 Apr 2026 18:03:14 +0000 Subject: [PATCH 03/16] fix: handle datetime in _parse_deprecation_date --- litellm/proxy/common_utils/model_deprecation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/proxy/common_utils/model_deprecation.py b/litellm/proxy/common_utils/model_deprecation.py index 1b11fa5abd7..7a614c7cfa8 100644 --- a/litellm/proxy/common_utils/model_deprecation.py +++ b/litellm/proxy/common_utils/model_deprecation.py @@ -46,6 +46,8 @@ def _parse_deprecation_date(raw_value: Any) -> Optional[date]: """ if raw_value is None: return None + if isinstance(raw_value, datetime): + return raw_value.date() if isinstance(raw_value, date): return raw_value if not isinstance(raw_value, str): From 8f1aea5e0a6f06036b41b1267f65a336f990aa71 Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 10 Aug 2026 22:58:35 +0000 Subject: [PATCH 04/16] refactor(proxy): tighten model deprecation typing and cover the endpoint Drops Any-typed router plumbing, immutable bucketing, generated dashboard API types, and adds endpoint plus resolution-fallback tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../SlackAlerting/slack_alerting.py | 61 +--- .../proxy/common_utils/model_deprecation.py | 334 ++++++++---------- litellm/proxy/proxy_server.py | 28 +- litellm/proxy/utils.py | 7 +- litellm/types/proxy/model_deprecation.py | 75 +--- .../common_utils/test_model_deprecation.py | 57 ++- .../proxy/test_model_deprecations_endpoint.py | 77 ++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 212 ++++++++++- 8 files changed, 544 insertions(+), 307 deletions(-) create mode 100644 tests/test_litellm/proxy/test_model_deprecations_endpoint.py diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 12b5d7525dc..b40f4ac03e0 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -40,6 +40,9 @@ from litellm.proxy._types import ( from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository from litellm.types.integrations.slack_alerting import * +from litellm.types.proxy.model_deprecation import ( + DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, +) from ..email_templates.templates import * from .batching_handler import send_to_webhook, squash_payloads @@ -1038,20 +1041,9 @@ Model Info: async def model_removed_alert(self, model_name: str): pass - async def send_model_deprecation_alert( - self, llm_router: Optional[Any] = None - ) -> bool: - """Aggregate deprecation metadata for the configured models and alert. - - Returns ``True`` when an alert payload was dispatched, ``False`` - otherwise. The ``send_alert`` helper itself is responsible for honoring - the user's webhook configuration; this method only owns producing the - message and choosing whether to send it. - """ - if ( - self.alerting is None - or AlertType.model_deprecation_warnings not in self.alert_types - ): + async def send_model_deprecation_alert(self, llm_router: Router | None = None) -> bool: + """Alert on the router's deprecated and imminent models, True when one was sent""" + if self.alerting is None or AlertType.model_deprecation_warnings not in self.alert_types: return False from litellm.proxy.common_utils.model_deprecation import ( @@ -1059,27 +1051,18 @@ Model Info: format_deprecation_alert_message, ) - try: - snapshot = collect_model_deprecations(llm_router=llm_router) - except Exception as e: - verbose_proxy_logger.exception( - "Error collecting model deprecation snapshot: %s", e - ) - return False - - message = format_deprecation_alert_message(snapshot) + snapshot: Final = collect_model_deprecations(llm_router=llm_router) + message: Final = format_deprecation_alert_message(snapshot) if message is None: return False - level: Literal["Low", "Medium", "High"] = ( - "High" if snapshot.deprecated else "Medium" - ) + level: Final[Literal["Low", "Medium", "High"]] = "High" if snapshot.deprecated else "Medium" await self.send_alert( message=message, level=level, alert_type=AlertType.model_deprecation_warnings, - alerting_metadata={ + alerting_metadata={ # mutable-ok: send_alert takes a dict payload "deprecated_count": len(snapshot.deprecated), "imminent_count": len(snapshot.imminent), "upcoming_count": len(snapshot.upcoming), @@ -1087,30 +1070,16 @@ Model Info: ) return True - async def _run_scheduled_deprecation_check(self, llm_router: Optional[Any] = None): - """Periodic background task that emits a model deprecation alert. - - Runs immediately on startup (so operators see the current state in - Slack) and then sleeps ``DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS`` - between runs. Exits silently if the alert type is not enabled. - """ - from litellm.types.proxy.model_deprecation import ( - DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, - ) - - if ( - self.alerting is None - or AlertType.model_deprecation_warnings not in self.alert_types - ): + async def _run_scheduled_deprecation_check(self, llm_router: Router | None = None) -> None: + """Alert once on startup, then daily, so operators see the current state""" + if self.alerting is None or AlertType.model_deprecation_warnings not in self.alert_types: return while True: try: await self.send_model_deprecation_alert(llm_router=llm_router) - except Exception as e: - verbose_proxy_logger.exception( - "Error in model deprecation alert loop: %s", e - ) + except Exception as e: # noqa: BLE001 # a failed alert must not kill the daily loop + verbose_proxy_logger.exception("Error in model deprecation alert loop: %s", e) await asyncio.sleep(DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS) async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool: diff --git a/litellm/proxy/common_utils/model_deprecation.py b/litellm/proxy/common_utils/model_deprecation.py index 7a614c7cfa8..4a5654eed1f 100644 --- a/litellm/proxy/common_utils/model_deprecation.py +++ b/litellm/proxy/common_utils/model_deprecation.py @@ -1,51 +1,35 @@ -"""Helpers for surfacing model deprecation/sunset information. - -This module reads ``deprecation_date`` metadata that is bundled in -``model_prices_and_context_window.json`` (exposed at runtime via -``litellm.model_cost``) and classifies the proxy's configured models into -``upcoming``, ``imminent`` and ``deprecated`` buckets. It is the single -source of truth used by both the ``/model/deprecations`` endpoint and the -proactive Slack alert. - -Resolution order for a deployment's deprecation date: - -1. ``model_info.deprecation_date`` – an explicit override on the deployment. -2. ``model_info.base_model`` looked up in ``litellm.model_cost``. -3. The ``litellm_params.model`` string looked up in ``litellm.model_cost``. - -Models without any deprecation metadata are skipped silently (most models -are not deprecated, and we don't want to pollute the response). -""" - from __future__ import annotations +from collections.abc import Mapping, Sequence +from dataclasses import dataclass from datetime import date, datetime, timezone -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from itertools import groupby +from types import MappingProxyType +from typing import TYPE_CHECKING, Final import litellm from litellm._logging import verbose_logger from litellm.types.proxy.model_deprecation import ( DEFAULT_DEPRECATION_WARN_DAYS, + DeprecationStatus, ModelDeprecationInfo, ModelDeprecationResponse, ) if TYPE_CHECKING: - from litellm.router import Router as _Router + from litellm.router import Router - Router = _Router -else: - Router = Any +_NO_MODEL_METADATA: Final[Mapping[str, object]] = MappingProxyType({}) -def _parse_deprecation_date(raw_value: Any) -> Optional[date]: - """Parse a ``deprecation_date`` string in YYYY-MM-DD form. +@dataclass(frozen=True, slots=True) +class _ResolvedDeprecation: + deprecation_date: date + litellm_model: str | None + litellm_provider: str | None - Returns ``None`` for missing, malformed, or sentinel placeholder values - (the JSON map ships a documentation sentinel of the form ``"date when..."``). - """ - if raw_value is None: - return None + +def _parse_deprecation_date(raw_value: object) -> date | None: if isinstance(raw_value, datetime): return raw_value.date() if isinstance(raw_value, date): @@ -53,68 +37,65 @@ def _parse_deprecation_date(raw_value: Any) -> Optional[date]: if not isinstance(raw_value, str): return None try: - return datetime.strptime(raw_value.strip(), "%Y-%m-%d").date() + return date.fromisoformat(raw_value.strip()) except ValueError: return None -def _lookup_deprecation_date_from_cost_map( - model_key: Optional[str], -) -> Tuple[Optional[date], Optional[str]]: - """Look up a deprecation date in ``litellm.model_cost`` for ``model_key``. - - Returns a tuple of (deprecation_date, litellm_provider). - """ - if not model_key: - return None, None - entry = litellm.model_cost.get(model_key) - if not isinstance(entry, dict): - return None, None - return ( - _parse_deprecation_date(entry.get("deprecation_date")), - entry.get("litellm_provider"), +def _cost_map_lookup(model_key: object) -> _ResolvedDeprecation | None: + if not isinstance(model_key, str) or not model_key: + return None + entry: Final = litellm.model_cost.get(model_key) + if not isinstance(entry, Mapping): + return None + parsed: Final = _parse_deprecation_date(entry.get("deprecation_date")) + if parsed is None: + return None + provider: Final = entry.get("litellm_provider") + return _ResolvedDeprecation( + deprecation_date=parsed, + litellm_model=model_key, + litellm_provider=provider if isinstance(provider, str) else None, ) -def _resolve_deployment_deprecation( - deployment: Dict[str, Any], -) -> Tuple[Optional[date], Optional[str], Optional[str]]: - """Resolve a deployment's deprecation metadata. +def _mapping_field(deployment: Mapping[str, object], key: str) -> Mapping[str, object]: + value: Final = deployment.get(key) + return value if isinstance(value, Mapping) else _NO_MODEL_METADATA - Returns a tuple of (deprecation_date, litellm_model, litellm_provider). - """ - model_info = deployment.get("model_info") or {} - explicit = _parse_deprecation_date(model_info.get("deprecation_date")) - if explicit is not None: - litellm_params = deployment.get("litellm_params") or {} - return ( - explicit, - litellm_params.get("model"), - model_info.get("litellm_provider"), + +def _resolve_deployment_deprecation( + deployment: Mapping[str, object], +) -> _ResolvedDeprecation | None: + """Resolve a deployment's deprecation date, preferring its explicit override""" + model_info: Final = _mapping_field(deployment, "model_info") + raw_model: Final = _mapping_field(deployment, "litellm_params").get("model") + + override: Final = _parse_deprecation_date(model_info.get("deprecation_date")) + if override is not None: + provider: Final = model_info.get("litellm_provider") + return _ResolvedDeprecation( + deprecation_date=override, + litellm_model=raw_model if isinstance(raw_model, str) else None, + litellm_provider=provider if isinstance(provider, str) else None, ) - base_model = model_info.get("base_model") - dep_date, provider = _lookup_deprecation_date_from_cost_map(base_model) - if dep_date is not None: - return dep_date, base_model, provider - - litellm_params = deployment.get("litellm_params") or {} - raw_model = litellm_params.get("model") - dep_date, provider = _lookup_deprecation_date_from_cost_map(raw_model) - if dep_date is not None: - return dep_date, raw_model, provider - - if isinstance(raw_model, str) and "/" in raw_model: - # Try the un-prefixed lookup (e.g. "openai/gpt-4o" → "gpt-4o"). - bare = raw_model.split("/", 1)[1] - dep_date, provider = _lookup_deprecation_date_from_cost_map(bare) - if dep_date is not None: - return dep_date, bare, provider - - return None, raw_model, model_info.get("litellm_provider") + unprefixed: Final = raw_model.split("/", 1)[1] if isinstance(raw_model, str) and "/" in raw_model else None + return next( + ( + resolved + for resolved in ( + _cost_map_lookup(model_info.get("base_model")), + _cost_map_lookup(raw_model), + _cost_map_lookup(unprefixed), + ) + if resolved is not None + ), + None, + ) -def _classify(days_until: int, warn_within_days: int) -> str: +def _classify(days_until: int, warn_within_days: int) -> DeprecationStatus: if days_until < 0: return "deprecated" if days_until <= warn_within_days: @@ -122,128 +103,119 @@ def _classify(days_until: int, warn_within_days: int) -> str: return "upcoming" -def _model_dump_compat(deployment: Any) -> Dict[str, Any]: - """Return a plain dict for both pydantic models and dicts.""" - if isinstance(deployment, dict): - return deployment - if hasattr(deployment, "model_dump"): - return deployment.model_dump(exclude_none=True) - if hasattr(deployment, "dict"): - return deployment.dict() - return dict(deployment) +def _build_info(deployment: Mapping[str, object], today: date, warn_within_days: int) -> ModelDeprecationInfo | None: + model_name: Final = deployment.get("model_name") + if not isinstance(model_name, str) or not model_name: + return None + + resolved: Final = _resolve_deployment_deprecation(deployment) + if resolved is None: + return None + + days_until: Final = (resolved.deprecation_date - today).days + return ModelDeprecationInfo( + model_name=model_name, + litellm_model=resolved.litellm_model, + deprecation_date=resolved.deprecation_date, + days_until_deprecation=days_until, + status=_classify(days_until, warn_within_days), + litellm_provider=resolved.litellm_provider, + ) + + +def _dedupe( + models: Sequence[ModelDeprecationInfo], +) -> tuple[ModelDeprecationInfo, ...]: + """Report a model group carrying the same date on several deployments once""" + ordered: Final = sorted(models, key=lambda model: (model.model_name, model.deprecation_date)) + return tuple( + next(group) for _, group in groupby(ordered, key=lambda model: (model.model_name, model.deprecation_date)) + ) + + +def _bucket(models: Sequence[ModelDeprecationInfo], status: DeprecationStatus) -> tuple[ModelDeprecationInfo, ...]: + return tuple( + sorted( + (model for model in models if model.status == status), + key=lambda model: model.deprecation_date, + ) + ) def collect_model_deprecations( - llm_router: Optional[Router], + llm_router: Router | None, warn_within_days: int = DEFAULT_DEPRECATION_WARN_DAYS, - today: Optional[date] = None, + today: date | None = None, ) -> ModelDeprecationResponse: - """Aggregate deprecation info for all deployments configured on the router. + """Bucket every deployment carrying a deprecation date by how urgent it is""" + snapshot_time: Final = datetime.now(timezone.utc) + effective_today: Final = today or snapshot_time.date() + deployments: Final = (llm_router.get_model_list() or ()) if llm_router is not None else () - De-duplicates by ``(model_name, deprecation_date)`` so multi-deployment - model groups (load-balanced across regions) only surface once per - deprecation date. - """ - snapshot_time = datetime.now(timezone.utc) - today = today or snapshot_time.date() + deduped: Final = _dedupe( + tuple( + info + for info in (_build_info(deployment, effective_today, warn_within_days) for deployment in deployments) + if info is not None + ) + ) - response = ModelDeprecationResponse( + verbose_logger.debug( + "model_deprecation: %d/%d deployments carry a deprecation date", + len(deduped), + len(deployments), + ) + + return ModelDeprecationResponse( + deprecated=_bucket(deduped, "deprecated"), + imminent=_bucket(deduped, "imminent"), + upcoming=_bucket(deduped, "upcoming"), warn_within_days=warn_within_days, checked_at=snapshot_time, ) - if llm_router is None: - return response - seen: set = set() - deployments = llm_router.get_model_list() or [] - for deployment in deployments: - deployment_dict = _model_dump_compat(deployment) - model_name = deployment_dict.get("model_name") - if not model_name: - continue - - dep_date, litellm_model, provider = _resolve_deployment_deprecation( - deployment_dict - ) - if dep_date is None: - continue - - dedup_key = (model_name, dep_date.isoformat()) - if dedup_key in seen: - continue - seen.add(dedup_key) - - days_until = (dep_date - today).days - status = _classify(days_until, warn_within_days) - - info = ModelDeprecationInfo( - model_name=model_name, - litellm_model=litellm_model, - deprecation_date=dep_date, - days_until_deprecation=days_until, - status=status, - litellm_provider=provider, - ) - - if status == "deprecated": - response.deprecated.append(info) - elif status == "imminent": - response.imminent.append(info) - else: - response.upcoming.append(info) - - response.deprecated.sort(key=lambda m: m.deprecation_date) - response.imminent.sort(key=lambda m: m.deprecation_date) - response.upcoming.sort(key=lambda m: m.deprecation_date) - - verbose_logger.debug( - "model_deprecation: deprecated=%d imminent=%d upcoming=%d", - len(response.deprecated), - len(response.imminent), - len(response.upcoming), +def _format_entry(info: ModelDeprecationInfo) -> str: + suffix: Final = ( + f"already deprecated {abs(info.days_until_deprecation)}d ago" + if info.days_until_deprecation < 0 + else f"in {info.days_until_deprecation}d" + ) + return ( + f"• `{info.model_name}` " + f"(provider: {info.litellm_provider or 'unknown'}, " + f"deprecates {info.deprecation_date.isoformat()}, {suffix})" ) - - return response def format_deprecation_alert_message( snapshot: ModelDeprecationResponse, -) -> Optional[str]: - """Format a Slack-friendly alert message for the warning buckets. +) -> str | None: + """Render the alert for the deprecated and imminent buckets, None when both are empty - Only ``deprecated`` and ``imminent`` models are included; ``upcoming`` - models are intentionally omitted to avoid alert fatigue. Returns - ``None`` when there is nothing to alert on. + Upcoming models are left out of the alert to keep it actionable. """ if not snapshot.deprecated and not snapshot.imminent: return None - lines: List[str] = ["*⚠️ Model Deprecation Warning*"] - - def _format_entry(info: ModelDeprecationInfo) -> str: - suffix = ( - f"already deprecated {abs(info.days_until_deprecation)}d ago" - if info.days_until_deprecation < 0 - else f"in {info.days_until_deprecation}d" + deprecated_section: Final = ( + ("\n*Already deprecated:*", *(_format_entry(i) for i in snapshot.deprecated)) if snapshot.deprecated else () + ) + imminent_section: Final = ( + ( + f"\n*Deprecating within {snapshot.warn_within_days} days:*", + *(_format_entry(i) for i in snapshot.imminent), ) - return ( - f"• `{info.model_name}` " - f"(provider: {info.litellm_provider or 'unknown'}, " - f"deprecates {info.deprecation_date.isoformat()} – {suffix})" - ) - - if snapshot.deprecated: - lines.append("\n*Already deprecated:*") - lines.extend(_format_entry(i) for i in snapshot.deprecated) - - if snapshot.imminent: - lines.append(f"\n*Deprecating within {snapshot.warn_within_days} days:*") - lines.extend(_format_entry(i) for i in snapshot.imminent) - - lines.append( - "\nPlan migrations to a supported model. See " - "https://docs.litellm.ai/docs/proxy/model_management for guidance." + if snapshot.imminent + else () ) - return "\n".join(lines) + return "\n".join( + ( + "*⚠️ Model Deprecation Warning*", + *deprecated_section, + *imminent_section, + "\nPlan migrations to a supported model. See " + "https://docs.litellm.ai/docs/proxy/model_management for guidance.", + ) + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3d137732075..d4101f56d33 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -625,14 +625,14 @@ from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry from litellm.types.proxy.management_endpoints.model_management_endpoints import ( ModelGroupInfoProxy, ) -from litellm.types.proxy.model_deprecation import ( - DEFAULT_DEPRECATION_WARN_DAYS, - ModelDeprecationResponse, -) from litellm.types.proxy.management_endpoints.ui_sso import ( DefaultTeamSSOParams, LiteLLM_UpperboundKeyGenerateParams, ) +from litellm.types.proxy.model_deprecation import ( + DEFAULT_DEPRECATION_WARN_DAYS, + ModelDeprecationResponse, +) from litellm.types.realtime import RealtimeQueryParams from litellm.types.router import ( DeploymentTypedDict, @@ -13443,18 +13443,17 @@ async def model_info_v1( @router.get( "/model/deprecations", - tags=["model management"], - dependencies=[Depends(user_api_key_auth)], + tags=("model management",), + dependencies=(Depends(user_api_key_auth),), response_model=ModelDeprecationResponse, ) @router.get( "/v1/model/deprecations", - tags=["model management"], - dependencies=[Depends(user_api_key_auth)], + tags=("model management",), + dependencies=(Depends(user_api_key_auth),), response_model=ModelDeprecationResponse, ) async def model_deprecations( - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), warn_within_days: int = DEFAULT_DEPRECATION_WARN_DAYS, ) -> ModelDeprecationResponse: """List models with known deprecation/sunset dates, bucketed by urgency. @@ -13464,13 +13463,13 @@ async def model_deprecations( models configured on this proxy. Parameters: - warn_within_days: Window (in days) used to bucket "imminent" models. - Defaults to `LITELLM_MODEL_DEPRECATION_WARN_DAYS` env var (or 30). + warn_within_days: Window (in days) used to bucket "imminent" models, + 30 by default. Returns: A payload with three lists of `ModelDeprecationInfo` entries: - - `deprecated`: deprecation date is in the past — these requests may + - `deprecated`: deprecation date is in the past, so these requests may fail at any time. - `imminent`: deprecation date is within `warn_within_days` from today. - `upcoming`: deprecation date is further out. @@ -13481,10 +13480,7 @@ async def model_deprecations( -H 'Authorization: Bearer sk-1234' ``` """ - global llm_router - return collect_model_deprecations( - llm_router=llm_router, warn_within_days=warn_within_days - ) + return collect_model_deprecations(llm_router=llm_router, warn_within_days=warn_within_days) def _get_model_group_info( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index f98df15a346..8de4605ecd2 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -484,14 +484,11 @@ class ProxyLogging: if ( self.slack_alerting_instance is not None - and AlertType.model_deprecation_warnings - in self.slack_alerting_instance.alert_types + and AlertType.model_deprecation_warnings in self.slack_alerting_instance.alert_types and not self.deprecation_check_started ): asyncio.create_task( - self.slack_alerting_instance._run_scheduled_deprecation_check( - llm_router=llm_router - ) + self.slack_alerting_instance._run_scheduled_deprecation_check(llm_router=llm_router) ) # RUN MODEL DEPRECATION ALERT LOOP (if scheduled) self.deprecation_check_started = True diff --git a/litellm/types/proxy/model_deprecation.py b/litellm/types/proxy/model_deprecation.py index 8cf98f7a1d5..74b7ea866f4 100644 --- a/litellm/types/proxy/model_deprecation.py +++ b/litellm/types/proxy/model_deprecation.py @@ -1,89 +1,50 @@ -"""Type definitions for model deprecation tracking and proactive alerts. - -The proxy reads deprecation/sunset metadata from -``litellm.model_cost`` (sourced from ``model_prices_and_context_window.json``) -and surfaces it through the ``/model/deprecations`` endpoint and Slack -alerting. These types describe the response payload and the alert payload. -""" - from __future__ import annotations from datetime import date, datetime -from typing import List, Optional +from typing import Final, Literal from pydantic import BaseModel, Field +DEFAULT_DEPRECATION_WARN_DAYS: Final = 30 -DEFAULT_DEPRECATION_WARN_DAYS = 30 -"""Default warning window (in days) for the ``imminent`` bucket. +DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS: Final = 24 * 60 * 60 -Matches the typical migration window most LLM providers offer between -deprecation announcement and removal. Callers of ``/model/deprecations`` -can override this per-request via the ``?warn_within_days=N`` query -parameter without restarting the proxy. -""" - -DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS = 24 * 60 * 60 -"""How often the periodic background check runs. Once per day.""" - - -DeprecationStatusLiteral = str -"""One of ``"upcoming"``, ``"imminent"``, ``"deprecated"``. - -* ``upcoming`` – deprecation is scheduled but more than the warn window away. -* ``imminent`` – deprecation date is within ``warn_within_days`` from today. -* ``deprecated`` – deprecation date has already passed. -""" +DeprecationStatus = Literal["upcoming", "imminent", "deprecated"] class ModelDeprecationInfo(BaseModel): - """Per-model deprecation metadata returned by ``/model/deprecations``.""" - - model_name: str = Field( - description="The public name of the model on the proxy (model_group)." - ) - litellm_model: Optional[str] = Field( + model_name: str = Field(description="The public name of the model on the proxy (model_group).") + litellm_model: str | None = Field( default=None, description="The underlying litellm model string the deprecation date is sourced from.", ) - deprecation_date: date = Field( - description="The date (UTC) when the model becomes deprecated." - ) + deprecation_date: date = Field(description="The date (UTC) when the model becomes deprecated.") days_until_deprecation: int = Field( + description=("Days remaining until the deprecation date. Negative if the model is already deprecated."), + ) + status: DeprecationStatus = Field( description=( - "Days remaining until the deprecation date. Negative if the model " - "is already deprecated." + "'deprecated' if the date has passed, 'imminent' if it falls within warn_within_days, 'upcoming' otherwise." ), ) - status: DeprecationStatusLiteral = Field( - description="One of 'upcoming', 'imminent', or 'deprecated'.", - ) - litellm_provider: Optional[str] = Field( - default=None, description="The provider this model belongs to." - ) + litellm_provider: str | None = Field(default=None, description="The provider this model belongs to.") class ModelDeprecationResponse(BaseModel): - """Response payload for ``GET /model/deprecations``.""" - - deprecated: List[ModelDeprecationInfo] = Field( + deprecated: list[ModelDeprecationInfo] = Field( default_factory=list, description="Models whose deprecation date has already passed.", ) - imminent: List[ModelDeprecationInfo] = Field( + imminent: list[ModelDeprecationInfo] = Field( default_factory=list, description=( - "Models whose deprecation date is within ``warn_within_days`` from " + "Models whose deprecation date is within warn_within_days from " "today and require immediate migration planning." ), ) - upcoming: List[ModelDeprecationInfo] = Field( + upcoming: list[ModelDeprecationInfo] = Field( default_factory=list, description="Models with a future deprecation date outside the warn window.", ) - warn_within_days: int = Field( - description="The window (in days) used to bucket 'imminent' models." - ) - checked_at: datetime = Field( - description="UTC timestamp when the deprecation snapshot was generated." - ) + warn_within_days: int = Field(description="The window (in days) used to bucket 'imminent' models.") + checked_at: datetime = Field(description="UTC timestamp when the deprecation snapshot was generated.") diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation.py index c873e2494f9..103f9383f5a 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation.py @@ -6,7 +6,7 @@ Slack integration — so they can run without the full proxy stack. import os import sys -from datetime import date +from datetime import date, datetime, timezone from unittest.mock import MagicMock @@ -50,6 +50,11 @@ class TestParseDeprecationDate: def test_should_return_none_for_unsupported_type(self): assert _parse_deprecation_date(12345) is None + def test_should_narrow_datetime_to_date(self): + assert _parse_deprecation_date( + datetime(2026, 12, 31, 23, 59, tzinfo=timezone.utc) + ) == date(2026, 12, 31) + class TestClassify: def test_should_classify_past_dates_as_deprecated(self): @@ -196,6 +201,56 @@ class TestCollectModelDeprecations: assert len(snapshot.imminent) == 1 + def test_should_resolve_via_unprefixed_model_name(self, monkeypatch): + monkeypatch.setattr( + litellm, + "model_cost", + {"gpt-4o": {"deprecation_date": "2026-06-10"}}, + ) + router = _make_router( + [ + { + "model_name": "alias", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": {"id": "1"}, + } + ] + ) + + snapshot = collect_model_deprecations( + llm_router=router, warn_within_days=30, today=date(2026, 6, 1) + ) + + assert [m.litellm_model for m in snapshot.imminent] == ["gpt-4o"] + + def test_should_keep_both_dates_when_group_has_conflicting_dates(self, monkeypatch): + monkeypatch.setattr( + litellm, + "model_cost", + {"shared-model": {"deprecation_date": "2026-06-10"}}, + ) + router = _make_router( + [ + { + "model_name": "alias", + "litellm_params": {"model": "shared-model"}, + "model_info": {"id": "1"}, + }, + { + "model_name": "alias", + "litellm_params": {"model": "shared-model"}, + "model_info": {"id": "2", "deprecation_date": "2027-01-01"}, + }, + ] + ) + + snapshot = collect_model_deprecations( + llm_router=router, warn_within_days=30, today=date(2026, 6, 1) + ) + + assert len(snapshot.imminent) == 1 + assert len(snapshot.upcoming) == 1 + def test_should_resolve_via_base_model(self, monkeypatch): today = date(2026, 6, 1) monkeypatch.setattr( diff --git a/tests/test_litellm/proxy/test_model_deprecations_endpoint.py b/tests/test_litellm/proxy/test_model_deprecations_endpoint.py new file mode 100644 index 00000000000..c942408bd14 --- /dev/null +++ b/tests/test_litellm/proxy/test_model_deprecations_endpoint.py @@ -0,0 +1,77 @@ +import os +import sys +from unittest.mock import MagicMock + +import pytest +from fastapi.testclient import TestClient + +sys.path.insert(0, os.path.abspath("../../..")) + +import litellm +from litellm.proxy import proxy_server +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.proxy_server import app + +client = TestClient(app) + + +@pytest.fixture +def authenticated_client(monkeypatch): + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-1234" + ) + monkeypatch.setattr( + litellm, + "model_cost", + { + "sunset-model": { + "deprecation_date": "2020-01-01", + "litellm_provider": "openai", + }, + "future-model": { + "deprecation_date": "2099-01-01", + "litellm_provider": "openai", + }, + }, + ) + router = MagicMock() + router.get_model_list.return_value = [ + { + "model_name": "sunset-alias", + "litellm_params": {"model": "sunset-model"}, + "model_info": {"id": "1"}, + }, + { + "model_name": "future-alias", + "litellm_params": {"model": "future-model"}, + "model_info": {"id": "2"}, + }, + ] + monkeypatch.setattr(proxy_server, "llm_router", router) + yield client + app.dependency_overrides.pop(user_api_key_auth, None) + + +def test_should_bucket_configured_models_by_urgency(authenticated_client): + response = authenticated_client.get("/model/deprecations") + + assert response.status_code == 200 + payload = response.json() + assert [m["model_name"] for m in payload["deprecated"]] == ["sunset-alias"] + assert [m["model_name"] for m in payload["upcoming"]] == ["future-alias"] + assert payload["imminent"] == [] + assert payload["warn_within_days"] == 30 + assert payload["deprecated"][0]["days_until_deprecation"] < 0 + + +def test_should_rebucket_with_warn_within_days_override(authenticated_client): + response = authenticated_client.get( + "/v1/model/deprecations", params={"warn_within_days": 40000} + ) + + assert response.status_code == 200 + payload = response.json() + assert [m["model_name"] for m in payload["imminent"]] == ["future-alias"] + assert payload["upcoming"] == [] + assert payload["warn_within_days"] == 40000 diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index fa8731d7a16..93f1762a7fe 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -7814,6 +7814,48 @@ export interface paths { patch?: never; trace?: never; }; + "/model/deprecations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Model Deprecations + * @description List models with known deprecation/sunset dates, bucketed by urgency. + * + * Reads `deprecation_date` metadata from `model_prices_and_context_window.json` + * (and any per-deployment `model_info.deprecation_date` overrides) for the + * models configured on this proxy. + * + * Parameters: + * warn_within_days: Window (in days) used to bucket "imminent" models, + * 30 by default. + * + * Returns: + * A payload with three lists of `ModelDeprecationInfo` entries: + * + * - `deprecated`: deprecation date is in the past, so these requests may + * fail at any time. + * - `imminent`: deprecation date is within `warn_within_days` from today. + * - `upcoming`: deprecation date is further out. + * + * Example: + * ```shell + * curl -X GET 'http://localhost:4000/model/deprecations' \ + * -H 'Authorization: Bearer sk-1234' + * ``` + */ + get: operations["model_deprecations_model_deprecations_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/model/info": { parameters: { query?: never; @@ -17386,6 +17428,48 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/model/deprecations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Model Deprecations + * @description List models with known deprecation/sunset dates, bucketed by urgency. + * + * Reads `deprecation_date` metadata from `model_prices_and_context_window.json` + * (and any per-deployment `model_info.deprecation_date` overrides) for the + * models configured on this proxy. + * + * Parameters: + * warn_within_days: Window (in days) used to bucket "imminent" models, + * 30 by default. + * + * Returns: + * A payload with three lists of `ModelDeprecationInfo` entries: + * + * - `deprecated`: deprecation date is in the past, so these requests may + * fail at any time. + * - `imminent`: deprecation date is within `warn_within_days` from today. + * - `upcoming`: deprecation date is further out. + * + * Example: + * ```shell + * curl -X GET 'http://localhost:4000/model/deprecations' \ + * -H 'Authorization: Bearer sk-1234' + * ``` + */ + get: operations["model_deprecations_v1_model_deprecations_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/model/info": { parameters: { query?: never; @@ -21247,7 +21331,7 @@ export interface components { * @description Enum for alert types and management event types * @enum {string} */ - AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted"; + AlertType: "llm_exceptions" | "llm_too_slow" | "llm_requests_hanging" | "budget_alerts" | "spend_reports" | "failed_tracking_spend" | "db_exceptions" | "daily_reports" | "cooldown_deployment" | "new_model_added" | "model_deprecation_warnings" | "outage_alerts" | "region_outage_alerts" | "fallback_reports" | "new_virtual_key_created" | "virtual_key_updated" | "virtual_key_deleted" | "new_team_created" | "team_updated" | "team_deleted" | "new_internal_user_created" | "internal_user_updated" | "internal_user_deleted"; /** AllowedVectorStoreIndexItem */ AllowedVectorStoreIndexItem: { /** Index Name */ @@ -28732,6 +28816,70 @@ export interface components { [key: string]: string | string[]; }; }; + /** ModelDeprecationInfo */ + ModelDeprecationInfo: { + /** + * Days Until Deprecation + * @description Days remaining until the deprecation date. Negative if the model is already deprecated. + */ + days_until_deprecation: number; + /** + * Deprecation Date + * Format: date + * @description The date (UTC) when the model becomes deprecated. + */ + deprecation_date: string; + /** + * Litellm Model + * @description The underlying litellm model string the deprecation date is sourced from. + */ + litellm_model?: string | null; + /** + * Litellm Provider + * @description The provider this model belongs to. + */ + litellm_provider?: string | null; + /** + * Model Name + * @description The public name of the model on the proxy (model_group). + */ + model_name: string; + /** + * Status + * @description 'deprecated' if the date has passed, 'imminent' if it falls within warn_within_days, 'upcoming' otherwise. + * @enum {string} + */ + status: "upcoming" | "imminent" | "deprecated"; + }; + /** ModelDeprecationResponse */ + ModelDeprecationResponse: { + /** + * Checked At + * Format: date-time + * @description UTC timestamp when the deprecation snapshot was generated. + */ + checked_at: string; + /** + * Deprecated + * @description Models whose deprecation date has already passed. + */ + deprecated?: components["schemas"]["ModelDeprecationInfo"][]; + /** + * Imminent + * @description Models whose deprecation date is within warn_within_days from today and require immediate migration planning. + */ + imminent?: components["schemas"]["ModelDeprecationInfo"][]; + /** + * Upcoming + * @description Models with a future deprecation date outside the warn window. + */ + upcoming?: components["schemas"]["ModelDeprecationInfo"][]; + /** + * Warn Within Days + * @description The window (in days) used to bucket 'imminent' models. + */ + warn_within_days: number; + }; /** ModelGroupInfoProxy */ ModelGroupInfoProxy: { /** Configurable Clientside Auth Params */ @@ -45860,6 +46008,37 @@ export interface operations { }; }; }; + model_deprecations_model_deprecations_get: { + parameters: { + query?: { + warn_within_days?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ModelDeprecationResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; model_info_v1_model_info_get: { parameters: { query?: { @@ -57641,6 +57820,37 @@ export interface operations { }; }; }; + model_deprecations_v1_model_deprecations_get: { + parameters: { + query?: { + warn_within_days?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ModelDeprecationResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; model_info_v1_v1_model_info_get: { parameters: { query?: { From 1998df994e22233dbf6d29a359cf1cbc20c1da6a Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 10 Aug 2026 23:23:25 +0000 Subject: [PATCH 05/16] fix(backend): allowlist the /v1/model/deprecations route Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- backend/routes/allowlist.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/routes/allowlist.py b/backend/routes/allowlist.py index 8ccd439979b..40d0157828f 100644 --- a/backend/routes/allowlist.py +++ b/backend/routes/allowlist.py @@ -35,6 +35,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = ( # Models & routing config "/model/", "/v1/model/info", + "/v1/model/deprecations", "/v2/model/", "/model_group", "/model_access_group/", From 25f343a54760fb2aca4259833ac3294584266192 Mon Sep 17 00:00:00 2001 From: mateo Date: Mon, 10 Aug 2026 23:49:46 +0000 Subject: [PATCH 06/16] fix(proxy): re-read router and alert types on each deprecation check The daily loop no longer captures the startup Router or bails when the alert type is off at startup, so config reloads take effect Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../SlackAlerting/slack_alerting.py | 18 ++++--- litellm/proxy/utils.py | 10 +--- .../test_model_deprecation_alert.py | 47 +++++++++++++++++++ 3 files changed, 61 insertions(+), 14 deletions(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index b40f4ac03e0..7cafc461000 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -5,6 +5,7 @@ import datetime import os import random import time +from collections.abc import Callable from datetime import timedelta from typing import TYPE_CHECKING, Any, Final, Literal @@ -56,6 +57,12 @@ else: Router = Any +def _proxy_llm_router() -> Router | None: + from litellm.proxy.proxy_server import llm_router + + return llm_router + + class SlackAlerting(CustomBatchLogger): """ Class for sending Slack Alerts @@ -1070,14 +1077,13 @@ Model Info: ) return True - async def _run_scheduled_deprecation_check(self, llm_router: Router | None = None) -> None: - """Alert once on startup, then daily, so operators see the current state""" - if self.alerting is None or AlertType.model_deprecation_warnings not in self.alert_types: - return - + async def _run_scheduled_deprecation_check( + self, get_llm_router: Callable[[], Router | None] = _proxy_llm_router + ) -> None: + """Alert once on startup, then daily, re-reading the router and alert types each pass""" while True: try: - await self.send_model_deprecation_alert(llm_router=llm_router) + await self.send_model_deprecation_alert(llm_router=get_llm_router()) except Exception as e: # noqa: BLE001 # a failed alert must not kill the daily loop verbose_proxy_logger.exception("Error in model deprecation alert loop: %s", e) await asyncio.sleep(DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 8de4605ecd2..1d6d1e69ca4 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -482,14 +482,8 @@ class ProxyLogging: ) # RUN HANGING REQUEST CHECK (if user wants to alert on hanging requests) self.hanging_requests_check_started = True - if ( - self.slack_alerting_instance is not None - and AlertType.model_deprecation_warnings in self.slack_alerting_instance.alert_types - and not self.deprecation_check_started - ): - asyncio.create_task( - self.slack_alerting_instance._run_scheduled_deprecation_check(llm_router=llm_router) - ) # RUN MODEL DEPRECATION ALERT LOOP (if scheduled) + if self.slack_alerting_instance is not None and not self.deprecation_check_started: + asyncio.create_task(self.slack_alerting_instance._run_scheduled_deprecation_check()) self.deprecation_check_started = True def update_values( diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py index 1cf6bbe0354..9b4bb26fe22 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py @@ -1,5 +1,6 @@ """Tests for the Slack alerting model deprecation hook.""" +import asyncio import os import sys from unittest.mock import AsyncMock, MagicMock, patch @@ -98,3 +99,49 @@ async def test_should_dispatch_high_severity_when_deprecated(monkeypatch): assert call_kwargs["alerting_metadata"]["deprecated_count"] == 1 assert call_kwargs["alerting_metadata"]["imminent_count"] == 0 assert "dead-alias" in call_kwargs["message"] + + +@pytest.mark.asyncio +async def test_should_alert_once_the_alert_type_and_router_arrive_after_startup( + monkeypatch, +): + """The daily loop starts before config reload, so it must re-read both each pass""" + monkeypatch.setattr( + litellm, + "model_cost", + {"dead-model": {"deprecation_date": "2020-01-01", "litellm_provider": "openai"}}, + ) + alerting = SlackAlerting(alerting=["slack"], alert_types=[AlertType.llm_exceptions]) + router = _make_router( + [ + { + "model_name": "dead-alias", + "litellm_params": {"model": "dead-model"}, + "model_info": {"id": "1"}, + } + ] + ) + routers = [None, router] + + async def stop_after_second_pass(_seconds): + if alerting.alert_types == [AlertType.llm_exceptions]: + alerting.update_values( + alert_types=[AlertType.model_deprecation_warnings] + ) # simulates a config reload enabling the alert + return + raise asyncio.CancelledError + + with ( + patch.object(alerting, "send_alert", new_callable=AsyncMock) as mock_send_alert, + patch( + "litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep", + side_effect=stop_after_second_pass, + ), + pytest.raises(asyncio.CancelledError), + ): + await alerting._run_scheduled_deprecation_check( + get_llm_router=lambda: routers.pop(0) + ) + + mock_send_alert.assert_awaited_once() + assert "dead-alias" in mock_send_alert.await_args.kwargs["message"] From 2fe152a1d26823e44babb96d9d95ef8295547b1b Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 00:05:24 +0000 Subject: [PATCH 07/16] fix(proxy): only schedule the deprecation loop when alerting is configured Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/utils.py | 6 +++++- .../proxy/utils/proxy_logging/test_lifecycle.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 1d6d1e69ca4..47eea9218f0 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -482,7 +482,11 @@ class ProxyLogging: ) # RUN HANGING REQUEST CHECK (if user wants to alert on hanging requests) self.hanging_requests_check_started = True - if self.slack_alerting_instance is not None and not self.deprecation_check_started: + if ( + self.alerting is not None + and self.slack_alerting_instance is not None + and not self.deprecation_check_started + ): asyncio.create_task(self.slack_alerting_instance._run_scheduled_deprecation_check()) self.deprecation_check_started = True diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py b/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py index cf906259246..b2aa16e88d9 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py @@ -129,6 +129,21 @@ def test_startup_event_initializes_slack_and_callbacks(proxy_logging): } +@pytest.mark.asyncio +async def test_startup_event_schedules_deprecation_check_before_its_alert_type_is_on(proxy_logging): + """Alerting config can enable the deprecation alert after startup, so the loop must already be running""" + proxy_logging.alerting = ["slack"] + proxy_logging.slack_alerting_instance = MagicMock() + proxy_logging.slack_alerting_instance.alert_types = [] + proxy_logging.slack_alerting_instance._run_scheduled_deprecation_check = AsyncMock() + proxy_logging._init_litellm_callbacks = MagicMock() + + proxy_logging.startup_event(llm_router=None, redis_usage_cache=None) + + assert proxy_logging.deprecation_check_started is True + proxy_logging.slack_alerting_instance._run_scheduled_deprecation_check.assert_called_once_with() + + def test_startup_event_propagates_init_callbacks_failure_raises(proxy_logging): proxy_logging.slack_alerting_instance = MagicMock() proxy_logging.slack_alerting_instance.alert_types = [] From 4e7e2f53b98f5737e43a27c5137e9ad6567c71ac Mon Sep 17 00:00:00 2001 From: mateo Date: Tue, 11 Aug 2026 21:44:05 +0000 Subject: [PATCH 08/16] fix(proxy): schedule the deprecation loop when a config reload enables alerting Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/utils.py | 22 +++++++++++++------ .../utils/proxy_logging/test_lifecycle.py | 17 ++++++++++++++ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 47eea9218f0..e1bc0642182 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -482,13 +482,20 @@ class ProxyLogging: ) # RUN HANGING REQUEST CHECK (if user wants to alert on hanging requests) self.hanging_requests_check_started = True - if ( - self.alerting is not None - and self.slack_alerting_instance is not None - and not self.deprecation_check_started - ): - asyncio.create_task(self.slack_alerting_instance._run_scheduled_deprecation_check()) - self.deprecation_check_started = True + self._ensure_deprecation_check_scheduled() + + def _ensure_deprecation_check_scheduled(self) -> None: + """Alerting can be configured at startup or by a later config reload, so schedule from either path""" + if self.alerting is None or self.slack_alerting_instance is None or self.deprecation_check_started: + return + + try: + asyncio.get_running_loop() + except RuntimeError: + return + + asyncio.create_task(self.slack_alerting_instance._run_scheduled_deprecation_check()) + self.deprecation_check_started = True def update_values( self, @@ -517,6 +524,7 @@ class ProxyLogging: updated_slack_alerting = True if updated_slack_alerting is True: + self._ensure_deprecation_check_scheduled() self.slack_alerting_instance.update_values( alerting=self.alerting, alerting_threshold=self.alerting_threshold, diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py b/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py index b2aa16e88d9..e82ad41ecc2 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py @@ -144,6 +144,23 @@ async def test_startup_event_schedules_deprecation_check_before_its_alert_type_i proxy_logging.slack_alerting_instance._run_scheduled_deprecation_check.assert_called_once_with() +@pytest.mark.asyncio +async def test_update_values_schedules_deprecation_check_when_alerting_arrives_later(proxy_logging): + """A proxy that boots without alerting still needs the loop once a config reload turns it on""" + proxy_logging.slack_alerting_instance = MagicMock() + proxy_logging.slack_alerting_instance.alert_types = [] + proxy_logging.slack_alerting_instance._run_scheduled_deprecation_check = AsyncMock() + proxy_logging._init_litellm_callbacks = MagicMock() + + proxy_logging.startup_event(llm_router=None, redis_usage_cache=None) + assert proxy_logging.deprecation_check_started is False + + proxy_logging.update_values(alerting=["slack"]) + + assert proxy_logging.deprecation_check_started is True + proxy_logging.slack_alerting_instance._run_scheduled_deprecation_check.assert_called_once_with() + + def test_startup_event_propagates_init_callbacks_failure_raises(proxy_logging): proxy_logging.slack_alerting_instance = MagicMock() proxy_logging.slack_alerting_instance.alert_types = [] From 6276eabf190f065afd103159e536e90874d2520a Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 12 Aug 2026 15:19:26 +0000 Subject: [PATCH 09/16] fix(proxy): wait for the router before the first deprecation alert Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../SlackAlerting/slack_alerting.py | 9 +++- litellm/types/proxy/model_deprecation.py | 4 ++ .../test_model_deprecation_alert.py | 52 +++++++++++++++++-- 3 files changed, 60 insertions(+), 5 deletions(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 7cafc461000..e7cd3cb048d 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -43,6 +43,8 @@ from litellm.repositories.user_repository import UserRepository from litellm.types.integrations.slack_alerting import * from litellm.types.proxy.model_deprecation import ( DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, + DEPRECATION_ROUTER_WAIT_ATTEMPTS, + DEPRECATION_ROUTER_WAIT_SECONDS, ) from ..email_templates.templates import * @@ -1080,7 +1082,12 @@ Model Info: async def _run_scheduled_deprecation_check( self, get_llm_router: Callable[[], Router | None] = _proxy_llm_router ) -> None: - """Alert once on startup, then daily, re-reading the router and alert types each pass""" + """Alert once the router is loaded, then daily, re-reading the router and alert types each pass""" + for _ in range(DEPRECATION_ROUTER_WAIT_ATTEMPTS): + if get_llm_router() is not None: + break + await asyncio.sleep(DEPRECATION_ROUTER_WAIT_SECONDS) + while True: try: await self.send_model_deprecation_alert(llm_router=get_llm_router()) diff --git a/litellm/types/proxy/model_deprecation.py b/litellm/types/proxy/model_deprecation.py index 74b7ea866f4..9f640c383fc 100644 --- a/litellm/types/proxy/model_deprecation.py +++ b/litellm/types/proxy/model_deprecation.py @@ -9,6 +9,10 @@ DEFAULT_DEPRECATION_WARN_DAYS: Final = 30 DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS: Final = 24 * 60 * 60 +DEPRECATION_ROUTER_WAIT_SECONDS: Final = 30 + +DEPRECATION_ROUTER_WAIT_ATTEMPTS: Final = 20 + DeprecationStatus = Literal["upcoming", "imminent", "deprecated"] diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py index 9b4bb26fe22..5d0e6b19975 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py @@ -3,6 +3,7 @@ import asyncio import os import sys +from itertools import chain, repeat from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -12,6 +13,7 @@ sys.path.insert(0, os.path.abspath("../../../..")) import litellm from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.proxy._types import AlertType +from litellm.types.proxy.model_deprecation import DEPRECATION_ROUTER_WAIT_SECONDS def _make_router(deployments): @@ -121,7 +123,6 @@ async def test_should_alert_once_the_alert_type_and_router_arrive_after_startup( } ] ) - routers = [None, router] async def stop_after_second_pass(_seconds): if alerting.alert_types == [AlertType.llm_exceptions]: @@ -139,9 +140,52 @@ async def test_should_alert_once_the_alert_type_and_router_arrive_after_startup( ), pytest.raises(asyncio.CancelledError), ): - await alerting._run_scheduled_deprecation_check( - get_llm_router=lambda: routers.pop(0) - ) + await alerting._run_scheduled_deprecation_check(get_llm_router=lambda: router) mock_send_alert.assert_awaited_once() assert "dead-alias" in mock_send_alert.await_args.kwargs["message"] + + +@pytest.mark.asyncio +async def test_should_wait_for_the_router_instead_of_sleeping_a_full_day(monkeypatch): + """Config load can start the loop before the router exists, which must not cost a day of alerts""" + monkeypatch.setattr( + litellm, + "model_cost", + {"dead-model": {"deprecation_date": "2020-01-01", "litellm_provider": "openai"}}, + ) + alerting = SlackAlerting( + alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings] + ) + router = _make_router( + [ + { + "model_name": "dead-alias", + "litellm_params": {"model": "dead-model"}, + "model_info": {"id": "1"}, + } + ] + ) + routers = chain((None, None), repeat(router)) + slept: list[float] = [] + + async def record_sleep(seconds): + slept.append(seconds) + if len(slept) > 2: + raise asyncio.CancelledError + + with ( + patch.object(alerting, "send_alert", new_callable=AsyncMock) as mock_send_alert, + patch( + "litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep", + side_effect=record_sleep, + ), + pytest.raises(asyncio.CancelledError), + ): + await alerting._run_scheduled_deprecation_check( + get_llm_router=lambda: next(routers) + ) + + assert slept[:2] == [DEPRECATION_ROUTER_WAIT_SECONDS] * 2 + mock_send_alert.assert_awaited_once() + assert "dead-alias" in mock_send_alert.await_args.kwargs["message"] From 2278118493acc75dff31a3b0d08da419eddc4841 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 12 Aug 2026 08:34:10 -0700 Subject: [PATCH 10/16] fix(slack_alerting): poll for the router inside the loop instead of a capped pre-wait A capped pre-wait still burns the first daily pass when the router takes longer than the cap to appear (a >10 minute boot), and reads the router in two places. Folding the poll into the loop makes the first alert unconditional on boot duration and keeps a single read per pass. --- .../integrations/SlackAlerting/slack_alerting.py | 11 ++++------- litellm/types/proxy/model_deprecation.py | 2 -- .../SlackAlerting/test_model_deprecation_alert.py | 14 ++++++++++---- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index e7cd3cb048d..82c2b4b38ce 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -43,7 +43,6 @@ from litellm.repositories.user_repository import UserRepository from litellm.types.integrations.slack_alerting import * from litellm.types.proxy.model_deprecation import ( DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, - DEPRECATION_ROUTER_WAIT_ATTEMPTS, DEPRECATION_ROUTER_WAIT_SECONDS, ) @@ -1083,14 +1082,12 @@ Model Info: self, get_llm_router: Callable[[], Router | None] = _proxy_llm_router ) -> None: """Alert once the router is loaded, then daily, re-reading the router and alert types each pass""" - for _ in range(DEPRECATION_ROUTER_WAIT_ATTEMPTS): - if get_llm_router() is not None: - break - await asyncio.sleep(DEPRECATION_ROUTER_WAIT_SECONDS) - while True: + if (llm_router := get_llm_router()) is None: + await asyncio.sleep(DEPRECATION_ROUTER_WAIT_SECONDS) + continue try: - await self.send_model_deprecation_alert(llm_router=get_llm_router()) + await self.send_model_deprecation_alert(llm_router=llm_router) except Exception as e: # noqa: BLE001 # a failed alert must not kill the daily loop verbose_proxy_logger.exception("Error in model deprecation alert loop: %s", e) await asyncio.sleep(DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS) diff --git a/litellm/types/proxy/model_deprecation.py b/litellm/types/proxy/model_deprecation.py index 9f640c383fc..c51c3629693 100644 --- a/litellm/types/proxy/model_deprecation.py +++ b/litellm/types/proxy/model_deprecation.py @@ -11,8 +11,6 @@ DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS: Final = 24 * 60 * 60 DEPRECATION_ROUTER_WAIT_SECONDS: Final = 30 -DEPRECATION_ROUTER_WAIT_ATTEMPTS: Final = 20 - DeprecationStatus = Literal["upcoming", "imminent", "deprecated"] diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py index 5d0e6b19975..7bdd980f00a 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py @@ -13,7 +13,10 @@ sys.path.insert(0, os.path.abspath("../../../..")) import litellm from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.proxy._types import AlertType -from litellm.types.proxy.model_deprecation import DEPRECATION_ROUTER_WAIT_SECONDS +from litellm.types.proxy.model_deprecation import ( + DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, + DEPRECATION_ROUTER_WAIT_SECONDS, +) def _make_router(deployments): @@ -166,12 +169,13 @@ async def test_should_wait_for_the_router_instead_of_sleeping_a_full_day(monkeyp } ] ) - routers = chain((None, None), repeat(router)) + router_absent_passes = 100 + routers = chain(repeat(None, router_absent_passes), repeat(router)) slept: list[float] = [] async def record_sleep(seconds): slept.append(seconds) - if len(slept) > 2: + if len(slept) > router_absent_passes: raise asyncio.CancelledError with ( @@ -186,6 +190,8 @@ async def test_should_wait_for_the_router_instead_of_sleeping_a_full_day(monkeyp get_llm_router=lambda: next(routers) ) - assert slept[:2] == [DEPRECATION_ROUTER_WAIT_SECONDS] * 2 + assert slept == [DEPRECATION_ROUTER_WAIT_SECONDS] * router_absent_passes + [ + DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS + ] mock_send_alert.assert_awaited_once() assert "dead-alias" in mock_send_alert.await_args.kwargs["message"] From 3f0306188ad5ebe3d59d3aa18d22f799524da516 Mon Sep 17 00:00:00 2001 From: mateo Date: Wed, 12 Aug 2026 16:11:50 +0000 Subject: [PATCH 11/16] fix(slack_alerting): poll while the deprecation alert is disabled instead of sleeping a day Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integrations/SlackAlerting/slack_alerting.py | 13 ++++++++----- litellm/types/proxy/model_deprecation.py | 2 +- .../SlackAlerting/test_model_deprecation_alert.py | 15 +++++++++++---- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 82c2b4b38ce..c49b1f17d72 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -43,7 +43,7 @@ from litellm.repositories.user_repository import UserRepository from litellm.types.integrations.slack_alerting import * from litellm.types.proxy.model_deprecation import ( DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, - DEPRECATION_ROUTER_WAIT_SECONDS, + DEPRECATION_IDLE_POLL_SECONDS, ) from ..email_templates.templates import * @@ -1049,9 +1049,12 @@ Model Info: async def model_removed_alert(self, model_name: str): pass + def _deprecation_alerts_enabled(self) -> bool: + return self.alerting is not None and AlertType.model_deprecation_warnings in self.alert_types + async def send_model_deprecation_alert(self, llm_router: Router | None = None) -> bool: """Alert on the router's deprecated and imminent models, True when one was sent""" - if self.alerting is None or AlertType.model_deprecation_warnings not in self.alert_types: + if not self._deprecation_alerts_enabled(): return False from litellm.proxy.common_utils.model_deprecation import ( @@ -1081,10 +1084,10 @@ Model Info: async def _run_scheduled_deprecation_check( self, get_llm_router: Callable[[], Router | None] = _proxy_llm_router ) -> None: - """Alert once the router is loaded, then daily, re-reading the router and alert types each pass""" + """Alert once the router is loaded and the alert is on, then daily, re-reading both each pass""" while True: - if (llm_router := get_llm_router()) is None: - await asyncio.sleep(DEPRECATION_ROUTER_WAIT_SECONDS) + if (llm_router := get_llm_router()) is None or not self._deprecation_alerts_enabled(): + await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS) continue try: await self.send_model_deprecation_alert(llm_router=llm_router) diff --git a/litellm/types/proxy/model_deprecation.py b/litellm/types/proxy/model_deprecation.py index c51c3629693..bbad63a278d 100644 --- a/litellm/types/proxy/model_deprecation.py +++ b/litellm/types/proxy/model_deprecation.py @@ -9,7 +9,7 @@ DEFAULT_DEPRECATION_WARN_DAYS: Final = 30 DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS: Final = 24 * 60 * 60 -DEPRECATION_ROUTER_WAIT_SECONDS: Final = 30 +DEPRECATION_IDLE_POLL_SECONDS: Final = 30 DeprecationStatus = Literal["upcoming", "imminent", "deprecated"] diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py index 7bdd980f00a..6dfdf831fa7 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py @@ -15,7 +15,7 @@ from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.proxy._types import AlertType from litellm.types.proxy.model_deprecation import ( DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, - DEPRECATION_ROUTER_WAIT_SECONDS, + DEPRECATION_IDLE_POLL_SECONDS, ) @@ -110,7 +110,7 @@ async def test_should_dispatch_high_severity_when_deprecated(monkeypatch): async def test_should_alert_once_the_alert_type_and_router_arrive_after_startup( monkeypatch, ): - """The daily loop starts before config reload, so it must re-read both each pass""" + """The loop starts before config reload, so a disabled pass must not cost a day of alerts""" monkeypatch.setattr( litellm, "model_cost", @@ -127,7 +127,10 @@ async def test_should_alert_once_the_alert_type_and_router_arrive_after_startup( ] ) - async def stop_after_second_pass(_seconds): + slept: list[float] = [] + + async def stop_after_second_pass(seconds): + slept.append(seconds) if alerting.alert_types == [AlertType.llm_exceptions]: alerting.update_values( alert_types=[AlertType.model_deprecation_warnings] @@ -145,6 +148,10 @@ async def test_should_alert_once_the_alert_type_and_router_arrive_after_startup( ): await alerting._run_scheduled_deprecation_check(get_llm_router=lambda: router) + assert slept == [ + DEPRECATION_IDLE_POLL_SECONDS, + DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, + ] mock_send_alert.assert_awaited_once() assert "dead-alias" in mock_send_alert.await_args.kwargs["message"] @@ -190,7 +197,7 @@ async def test_should_wait_for_the_router_instead_of_sleeping_a_full_day(monkeyp get_llm_router=lambda: next(routers) ) - assert slept == [DEPRECATION_ROUTER_WAIT_SECONDS] * router_absent_passes + [ + assert slept == [DEPRECATION_IDLE_POLL_SECONDS] * router_absent_passes + [ DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS ] mock_send_alert.assert_awaited_once() From 9b665380198d4f909180a013c85dfc50e2087ad2 Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 13 Aug 2026 02:39:34 +0000 Subject: [PATCH 12/16] fix(proxy): escape slack markup in model deprecation alert fields Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/common_utils/model_deprecation.py | 9 +++++-- .../common_utils/test_model_deprecation.py | 27 +++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/common_utils/model_deprecation.py b/litellm/proxy/common_utils/model_deprecation.py index 4a5654eed1f..8176a8cb642 100644 --- a/litellm/proxy/common_utils/model_deprecation.py +++ b/litellm/proxy/common_utils/model_deprecation.py @@ -175,6 +175,11 @@ def collect_model_deprecations( ) +def _escape_slack_mrkdwn(value: str) -> str: + """Neutralize Slack control characters so a model name cannot forge a mention or link""" + return value.replace("&", "&").replace("<", "<").replace(">", ">") + + def _format_entry(info: ModelDeprecationInfo) -> str: suffix: Final = ( f"already deprecated {abs(info.days_until_deprecation)}d ago" @@ -182,8 +187,8 @@ def _format_entry(info: ModelDeprecationInfo) -> str: else f"in {info.days_until_deprecation}d" ) return ( - f"• `{info.model_name}` " - f"(provider: {info.litellm_provider or 'unknown'}, " + f"• `{_escape_slack_mrkdwn(info.model_name)}` " + f"(provider: {_escape_slack_mrkdwn(info.litellm_provider) if info.litellm_provider else 'unknown'}, " f"deprecates {info.deprecation_date.isoformat()}, {suffix})" ) diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation.py index 103f9383f5a..051ddd2e78c 100644 --- a/tests/test_litellm/proxy/common_utils/test_model_deprecation.py +++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation.py @@ -333,3 +333,30 @@ class TestFormatDeprecationAlertMessage: assert "`soon`" in message # Upcoming models must NOT be in the alert (avoid alert fatigue). assert "`later`" not in message + + def test_should_neutralize_slack_markup_from_model_metadata(self): + today = date(2026, 6, 1) + router = _make_router( + [ + { + "model_name": " pwned", + "litellm_params": {"model": "openai/whatever"}, + "model_info": { + "id": "1", + "deprecation_date": "2026-06-10", + "litellm_provider": " & co", + }, + } + ] + ) + + snapshot = collect_model_deprecations( + llm_router=router, warn_within_days=30, today=today + ) + message = format_deprecation_alert_message(snapshot) + + assert message is not None + assert "" not in message + assert "" not in message + assert "<!channel> pwned" in message + assert "<https://evil.example|openai> & co" in message From 816fa5039435d16eed7b9d9423b762c61febf007 Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 15 Aug 2026 08:35:12 +0000 Subject: [PATCH 13/16] refactor(proxy): make the deprecation loop entrypoint public and drop a dead None check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/SlackAlerting/slack_alerting.py | 2 +- litellm/proxy/utils.py | 4 ++-- .../SlackAlerting/test_model_deprecation_alert.py | 4 ++-- .../proxy/utils/proxy_logging/test_lifecycle.py | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index a6d86f73479..9f9c26ae15c 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1087,7 +1087,7 @@ Model Info: ) return True - async def _run_scheduled_deprecation_check( + async def run_scheduled_deprecation_check( self, get_llm_router: Callable[[], Router | None] = _proxy_llm_router ) -> None: """Alert once the router is loaded and the alert is on, then daily, re-reading both each pass""" diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 3a40d688883..bc73e4d3e5b 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -495,7 +495,7 @@ class ProxyLogging: def _ensure_deprecation_check_scheduled(self) -> None: """Alerting can be configured at startup or by a later config reload, so schedule from either path""" - if self.alerting is None or self.slack_alerting_instance is None or self.deprecation_check_started: + if self.alerting is None or self.deprecation_check_started: return try: @@ -503,7 +503,7 @@ class ProxyLogging: except RuntimeError: return - asyncio.create_task(self.slack_alerting_instance._run_scheduled_deprecation_check()) + asyncio.create_task(self.slack_alerting_instance.run_scheduled_deprecation_check()) self.deprecation_check_started = True def update_values( diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py index 6dfdf831fa7..f475a6d454e 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py @@ -146,7 +146,7 @@ async def test_should_alert_once_the_alert_type_and_router_arrive_after_startup( ), pytest.raises(asyncio.CancelledError), ): - await alerting._run_scheduled_deprecation_check(get_llm_router=lambda: router) + await alerting.run_scheduled_deprecation_check(get_llm_router=lambda: router) assert slept == [ DEPRECATION_IDLE_POLL_SECONDS, @@ -193,7 +193,7 @@ async def test_should_wait_for_the_router_instead_of_sleeping_a_full_day(monkeyp ), pytest.raises(asyncio.CancelledError), ): - await alerting._run_scheduled_deprecation_check( + await alerting.run_scheduled_deprecation_check( get_llm_router=lambda: next(routers) ) diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py b/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py index 5382f367f42..e45345877f3 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py @@ -136,13 +136,13 @@ async def test_startup_event_schedules_deprecation_check_before_its_alert_type_i proxy_logging.alerting = ["slack"] proxy_logging.slack_alerting_instance = MagicMock() proxy_logging.slack_alerting_instance.alert_types = [] - proxy_logging.slack_alerting_instance._run_scheduled_deprecation_check = AsyncMock() + proxy_logging.slack_alerting_instance.run_scheduled_deprecation_check = AsyncMock() proxy_logging._init_litellm_callbacks = MagicMock() proxy_logging.startup_event(llm_router=None, redis_usage_cache=None) assert proxy_logging.deprecation_check_started is True - proxy_logging.slack_alerting_instance._run_scheduled_deprecation_check.assert_called_once_with() + proxy_logging.slack_alerting_instance.run_scheduled_deprecation_check.assert_called_once_with() @pytest.mark.asyncio @@ -150,7 +150,7 @@ async def test_update_values_schedules_deprecation_check_when_alerting_arrives_l """A proxy that boots without alerting still needs the loop once a config reload turns it on""" proxy_logging.slack_alerting_instance = MagicMock() proxy_logging.slack_alerting_instance.alert_types = [] - proxy_logging.slack_alerting_instance._run_scheduled_deprecation_check = AsyncMock() + proxy_logging.slack_alerting_instance.run_scheduled_deprecation_check = AsyncMock() proxy_logging._init_litellm_callbacks = MagicMock() proxy_logging.startup_event(llm_router=None, redis_usage_cache=None) @@ -159,7 +159,7 @@ async def test_update_values_schedules_deprecation_check_when_alerting_arrives_l proxy_logging.update_values(alerting=["slack"]) assert proxy_logging.deprecation_check_started is True - proxy_logging.slack_alerting_instance._run_scheduled_deprecation_check.assert_called_once_with() + proxy_logging.slack_alerting_instance.run_scheduled_deprecation_check.assert_called_once_with() def test_startup_event_propagates_init_callbacks_failure_raises(proxy_logging): From 1e63134adb7b363d94a75f53723d3472844fef4d Mon Sep 17 00:00:00 2001 From: mateo Date: Sat, 15 Aug 2026 17:10:12 +0000 Subject: [PATCH 14/16] fix(slack_alerting): hold a pod lock so a fleet sends one deprecation alert per day Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + .../SlackAlerting/slack_alerting.py | 25 +++++++-- litellm/proxy/utils.py | 6 ++- .../test_model_deprecation_alert.py | 51 +++++++++++++++++++ .../utils/proxy_logging/test_lifecycle.py | 8 ++- 5 files changed, 85 insertions(+), 6 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index e9d2d719ae2..5bd62ffed9e 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1485,6 +1485,7 @@ WEEKLY_SPEND_REPORT_JOB_ID: Final = "weekly_spend_report_job" MONTHLY_SPEND_REPORT_JOB_ID: Final = "monthly_spend_report_job" PROMETHEUS_FALLBACK_STATS_JOB_ID: Final = "prometheus_fallback_stats_job" SLACK_DAILY_REPORT_LOCK_ID: Final = "slack_daily_report" +SLACK_MODEL_DEPRECATION_LOCK_ID: Final = "slack_model_deprecation_warning" SPEND_LOG_RUN_LOOPS: Final = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500)) SPEND_LOG_CLEANUP_BATCH_SIZE: Final = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000)) SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = int(os.getenv("SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3)) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 9f9c26ae15c..71a5cad1331 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -18,7 +18,11 @@ import litellm.litellm_core_utils.litellm_logging import litellm.types from litellm._logging import verbose_logger, verbose_proxy_logger from litellm.caching.caching import DualCache -from litellm.constants import HOURS_IN_A_DAY, SLACK_DAILY_REPORT_LOCK_ID +from litellm.constants import ( + HOURS_IN_A_DAY, + SLACK_DAILY_REPORT_LOCK_ID, + SLACK_MODEL_DEPRECATION_LOCK_ID, +) from litellm.integrations.custom_batch_logger import CustomBatchLogger from litellm.integrations.SlackAlerting.budget_alert_types import get_budget_alert_type from litellm.integrations.SlackAlerting.hanging_request_check import ( @@ -1087,8 +1091,22 @@ Model Info: ) return True + async def _claimed_deprecation_alert_window(self, pod_lock_manager: "PodLockManager | None") -> bool: + """Without a redis backed lock there is no fleet to coordinate, so a lone pod always alerts""" + if pod_lock_manager is None: + return True + return ( + await pod_lock_manager.acquire_lock( + cronjob_id=SLACK_MODEL_DEPRECATION_LOCK_ID, + ttl=DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, + allow_reentrant=False, + ) + ) is not False + async def run_scheduled_deprecation_check( - self, get_llm_router: Callable[[], Router | None] = _proxy_llm_router + self, + get_llm_router: Callable[[], Router | None] = _proxy_llm_router, + pod_lock_manager: "PodLockManager | None" = None, ) -> None: """Alert once the router is loaded and the alert is on, then daily, re-reading both each pass""" while True: @@ -1096,7 +1114,8 @@ Model Info: await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS) continue try: - await self.send_model_deprecation_alert(llm_router=llm_router) + if await self._claimed_deprecation_alert_window(pod_lock_manager): + await self.send_model_deprecation_alert(llm_router=llm_router) except Exception as e: # noqa: BLE001 # a failed alert must not kill the daily loop verbose_proxy_logger.exception("Error in model deprecation alert loop: %s", e) await asyncio.sleep(DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index bc73e4d3e5b..e180ba1742c 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -503,7 +503,11 @@ class ProxyLogging: except RuntimeError: return - asyncio.create_task(self.slack_alerting_instance.run_scheduled_deprecation_check()) + asyncio.create_task( + self.slack_alerting_instance.run_scheduled_deprecation_check( + pod_lock_manager=self.db_spend_update_writer.pod_lock_manager + ) + ) self.deprecation_check_started = True def update_values( diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py index f475a6d454e..5509933d739 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py @@ -11,6 +11,7 @@ import pytest sys.path.insert(0, os.path.abspath("../../../..")) import litellm +from litellm.constants import SLACK_MODEL_DEPRECATION_LOCK_ID from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.proxy._types import AlertType from litellm.types.proxy.model_deprecation import ( @@ -202,3 +203,53 @@ async def test_should_wait_for_the_router_instead_of_sleeping_a_full_day(monkeyp ] mock_send_alert.assert_awaited_once() assert "dead-alias" in mock_send_alert.await_args.kwargs["message"] + + +@pytest.mark.parametrize( + "lock_acquired, expect_alert", + [(True, True), (None, True), (False, False)], + ids=["lock won", "no redis lock", "another pod holds the lock"], +) +@pytest.mark.asyncio +async def test_should_alert_only_from_the_pod_holding_the_daily_lock( + monkeypatch, lock_acquired, expect_alert +): + """Every pod runs the loop, so a fleet must not send one identical alert per replica""" + monkeypatch.setattr( + litellm, + "model_cost", + {"dead-model": {"deprecation_date": "2020-01-01", "litellm_provider": "openai"}}, + ) + alerting = SlackAlerting( + alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings] + ) + router = _make_router( + [ + { + "model_name": "dead-alias", + "litellm_params": {"model": "dead-model"}, + "model_info": {"id": "1"}, + } + ] + ) + pod_lock_manager = MagicMock() + pod_lock_manager.acquire_lock = AsyncMock(return_value=lock_acquired) + + with ( + patch.object(alerting, "send_alert", new_callable=AsyncMock) as mock_send_alert, + patch( + "litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep", + side_effect=asyncio.CancelledError, + ), + pytest.raises(asyncio.CancelledError), + ): + await alerting.run_scheduled_deprecation_check( + get_llm_router=lambda: router, pod_lock_manager=pod_lock_manager + ) + + assert mock_send_alert.await_count == int(expect_alert) + assert pod_lock_manager.acquire_lock.await_args.kwargs == { + "cronjob_id": SLACK_MODEL_DEPRECATION_LOCK_ID, + "ttl": DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, + "allow_reentrant": False, + } diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py b/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py index e45345877f3..a97dcb41e44 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py @@ -142,7 +142,9 @@ async def test_startup_event_schedules_deprecation_check_before_its_alert_type_i proxy_logging.startup_event(llm_router=None, redis_usage_cache=None) assert proxy_logging.deprecation_check_started is True - proxy_logging.slack_alerting_instance.run_scheduled_deprecation_check.assert_called_once_with() + proxy_logging.slack_alerting_instance.run_scheduled_deprecation_check.assert_called_once_with( + pod_lock_manager=proxy_logging.db_spend_update_writer.pod_lock_manager + ) @pytest.mark.asyncio @@ -159,7 +161,9 @@ async def test_update_values_schedules_deprecation_check_when_alerting_arrives_l proxy_logging.update_values(alerting=["slack"]) assert proxy_logging.deprecation_check_started is True - proxy_logging.slack_alerting_instance.run_scheduled_deprecation_check.assert_called_once_with() + proxy_logging.slack_alerting_instance.run_scheduled_deprecation_check.assert_called_once_with( + pod_lock_manager=proxy_logging.db_spend_update_writer.pod_lock_manager + ) def test_startup_event_propagates_init_callbacks_failure_raises(proxy_logging): From 308865bad0a23db93eabd325029ace51917f9ec0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:38:05 -0700 Subject: [PATCH 15/16] fix(alerting): claim the deprecation lock only with content and retry failed claims next poll An empty pass no longer holds the daily lock, a False lock claim (held or redis error) is retried on the next 30 second poll instead of sleeping a day, and a sent alert is stamped in the shared cache for a day so sibling pods and restarts stay quiet --- .../SlackAlerting/slack_alerting.py | 49 +++++-- litellm/types/integrations/slack_alerting.py | 1 + .../test_model_deprecation_alert.py | 128 ++++++++++++++++-- 3 files changed, 157 insertions(+), 21 deletions(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 71a5cad1331..2f86f92d06c 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1062,8 +1062,16 @@ Model Info: def _deprecation_alerts_enabled(self) -> bool: return self.alerting is not None and AlertType.model_deprecation_warnings in self.alert_types - async def send_model_deprecation_alert(self, llm_router: Router | None = None) -> bool: - """Alert on the router's deprecated and imminent models, True when one was sent""" + async def send_model_deprecation_alert( + self, + llm_router: Router | None = None, + pod_lock_manager: "PodLockManager | None" = None, + ) -> bool: + """Alert on the router's deprecated and imminent models, True when one was sent + + The daily lock is claimed only once there is something to say, so an empty pass never blocks a + later real one, and a sent alert is stamped in the shared cache for a day so sibling pods stop asking + """ if not self._deprecation_alerts_enabled(): return False @@ -1076,6 +1084,8 @@ Model Info: message: Final = format_deprecation_alert_message(snapshot) if message is None: return False + if not await self._claimed_deprecation_alert_window(pod_lock_manager): + return False level: Final[Literal["Low", "Medium", "High"]] = "High" if snapshot.deprecated else "Medium" @@ -1089,6 +1099,11 @@ Model Info: "upcoming_count": len(snapshot.upcoming), }, ) + await self.internal_usage_cache.async_set_cache( + key=SlackAlertingCacheKeys.deprecation_alert_sent_key.value, + value=time.time(), + ttl=DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, + ) return True async def _claimed_deprecation_alert_window(self, pod_lock_manager: "PodLockManager | None") -> bool: @@ -1103,22 +1118,36 @@ Model Info: ) ) is not False + async def _deprecation_alert_sent_within_a_day(self) -> bool: + return ( + await self.internal_usage_cache.async_get_cache(key=SlackAlertingCacheKeys.deprecation_alert_sent_key.value) + ) is not None + + async def _run_deprecation_alert_pass( + self, llm_router: Router | None, pod_lock_manager: "PodLockManager | None" + ) -> bool: + if llm_router is None or not self._deprecation_alerts_enabled(): + return False + if await self._deprecation_alert_sent_within_a_day(): + return False + return await self.send_model_deprecation_alert(llm_router=llm_router, pod_lock_manager=pod_lock_manager) + async def run_scheduled_deprecation_check( self, get_llm_router: Callable[[], Router | None] = _proxy_llm_router, pod_lock_manager: "PodLockManager | None" = None, ) -> None: - """Alert once the router is loaded and the alert is on, then daily, re-reading both each pass""" + """Poll every pass for a loaded router, the alert being on, and no alert in the last day, then alert + + A pass that could not alert (no router yet, alert type off, a sibling pod holds the daily lock, or a + redis blip at claim time) is retried on the next poll instead of costing a day + """ while True: - if (llm_router := get_llm_router()) is None or not self._deprecation_alerts_enabled(): - await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS) - continue try: - if await self._claimed_deprecation_alert_window(pod_lock_manager): - await self.send_model_deprecation_alert(llm_router=llm_router) - except Exception as e: # noqa: BLE001 # a failed alert must not kill the daily loop + await self._run_deprecation_alert_pass(get_llm_router(), pod_lock_manager) + except Exception as e: # noqa: BLE001 # a failed alert must not kill the loop verbose_proxy_logger.exception("Error in model deprecation alert loop: %s", e) - await asyncio.sleep(DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS) + await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS) async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool: """ diff --git a/litellm/types/integrations/slack_alerting.py b/litellm/types/integrations/slack_alerting.py index 768b5d35597..b1b7bc3541a 100644 --- a/litellm/types/integrations/slack_alerting.py +++ b/litellm/types/integrations/slack_alerting.py @@ -121,6 +121,7 @@ class SlackAlertingCacheKeys(Enum): failed_requests_key = "failed_requests_daily_metrics" latency_key = "latency_daily_metrics" report_sent_key = "daily_metrics_report_sent" + deprecation_alert_sent_key = "model_deprecation_alert_sent" class AlertType(str, Enum): diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py index 5509933d739..9556775dad0 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py @@ -14,11 +14,21 @@ import litellm from litellm.constants import SLACK_MODEL_DEPRECATION_LOCK_ID from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.proxy._types import AlertType +from litellm.types.integrations.slack_alerting import SlackAlertingCacheKeys from litellm.types.proxy.model_deprecation import ( DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, DEPRECATION_IDLE_POLL_SECONDS, ) +DEAD_MODEL_COST = { + "dead-model": {"deprecation_date": "2020-01-01", "litellm_provider": "openai"} +} +DEAD_ALIAS_DEPLOYMENT = { + "model_name": "dead-alias", + "litellm_params": {"model": "dead-model"}, + "model_info": {"id": "1"}, +} + def _make_router(deployments): router = MagicMock() @@ -105,6 +115,12 @@ async def test_should_dispatch_high_severity_when_deprecated(monkeypatch): assert call_kwargs["alerting_metadata"]["deprecated_count"] == 1 assert call_kwargs["alerting_metadata"]["imminent_count"] == 0 assert "dead-alias" in call_kwargs["message"] + assert isinstance( + await alerting.internal_usage_cache.async_get_cache( + key=SlackAlertingCacheKeys.deprecation_alert_sent_key.value + ), + float, + ) @pytest.mark.asyncio @@ -130,29 +146,26 @@ async def test_should_alert_once_the_alert_type_and_router_arrive_after_startup( slept: list[float] = [] - async def stop_after_second_pass(seconds): + async def stop_after_third_pass(seconds): slept.append(seconds) if alerting.alert_types == [AlertType.llm_exceptions]: alerting.update_values( alert_types=[AlertType.model_deprecation_warnings] ) # simulates a config reload enabling the alert - return - raise asyncio.CancelledError + if len(slept) == 3: + raise asyncio.CancelledError with ( patch.object(alerting, "send_alert", new_callable=AsyncMock) as mock_send_alert, patch( "litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep", - side_effect=stop_after_second_pass, + side_effect=stop_after_third_pass, ), pytest.raises(asyncio.CancelledError), ): await alerting.run_scheduled_deprecation_check(get_llm_router=lambda: router) - assert slept == [ - DEPRECATION_IDLE_POLL_SECONDS, - DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, - ] + assert slept == [DEPRECATION_IDLE_POLL_SECONDS] * 3 mock_send_alert.assert_awaited_once() assert "dead-alias" in mock_send_alert.await_args.kwargs["message"] @@ -198,9 +211,7 @@ async def test_should_wait_for_the_router_instead_of_sleeping_a_full_day(monkeyp get_llm_router=lambda: next(routers) ) - assert slept == [DEPRECATION_IDLE_POLL_SECONDS] * router_absent_passes + [ - DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS - ] + assert slept == [DEPRECATION_IDLE_POLL_SECONDS] * (router_absent_passes + 1) mock_send_alert.assert_awaited_once() assert "dead-alias" in mock_send_alert.await_args.kwargs["message"] @@ -253,3 +264,98 @@ async def test_should_alert_only_from_the_pod_holding_the_daily_lock( "ttl": DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, "allow_reentrant": False, } + + +@pytest.mark.asyncio +async def test_should_retry_on_the_next_poll_when_the_lock_claim_fails(monkeypatch): + """A redis blip at claim time returns False like a held lock, and must not cost every pod a day of alerts""" + monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) + alerting = SlackAlerting( + alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings] + ) + router = _make_router([DEAD_ALIAS_DEPLOYMENT]) + pod_lock_manager = MagicMock() + pod_lock_manager.acquire_lock = AsyncMock(side_effect=[False, True]) + slept: list[float] = [] + + async def stop_after_second_pass(seconds): + slept.append(seconds) + if len(slept) == 2: + raise asyncio.CancelledError + + with ( + patch.object(alerting, "send_alert", new_callable=AsyncMock) as mock_send_alert, + patch( + "litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep", + side_effect=stop_after_second_pass, + ), + pytest.raises(asyncio.CancelledError), + ): + await alerting.run_scheduled_deprecation_check( + get_llm_router=lambda: router, pod_lock_manager=pod_lock_manager + ) + + assert slept == [DEPRECATION_IDLE_POLL_SECONDS] * 2 + assert pod_lock_manager.acquire_lock.await_count == 2 + mock_send_alert.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_should_not_claim_the_lock_when_there_is_nothing_to_report(monkeypatch): + """An empty pass must not hold the daily lock, or a sunset added later waits out the whole window""" + monkeypatch.setattr(litellm, "model_cost", {}) + alerting = SlackAlerting( + alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings] + ) + router = _make_router( + [ + { + "model_name": "fresh", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": {"id": "x"}, + } + ] + ) + pod_lock_manager = MagicMock() + pod_lock_manager.acquire_lock = AsyncMock(return_value=True) + + with patch.object(alerting, "send_alert", new_callable=AsyncMock) as mock_send_alert: + sent = await alerting.send_model_deprecation_alert( + llm_router=router, pod_lock_manager=pod_lock_manager + ) + + assert sent is False + pod_lock_manager.acquire_lock.assert_not_awaited() + mock_send_alert.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_should_not_alert_or_claim_the_lock_within_a_day_of_a_sent_alert(monkeypatch): + """The shared sent stamp keeps sibling pods and restarts from re-alerting or re-asking redis for a day""" + monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) + alerting = SlackAlerting( + alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings] + ) + await alerting.internal_usage_cache.async_set_cache( + key=SlackAlertingCacheKeys.deprecation_alert_sent_key.value, + value=1.0, + ttl=DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS, + ) + router = _make_router([DEAD_ALIAS_DEPLOYMENT]) + pod_lock_manager = MagicMock() + pod_lock_manager.acquire_lock = AsyncMock(return_value=True) + + with ( + patch.object(alerting, "send_alert", new_callable=AsyncMock) as mock_send_alert, + patch( + "litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep", + side_effect=asyncio.CancelledError, + ), + pytest.raises(asyncio.CancelledError), + ): + await alerting.run_scheduled_deprecation_check( + get_llm_router=lambda: router, pod_lock_manager=pod_lock_manager + ) + + pod_lock_manager.acquire_lock.assert_not_awaited() + mock_send_alert.assert_not_awaited() From 7017df5732bf5ccdaa18b9a89c9809cb646992ea Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:57:37 -0700 Subject: [PATCH 16/16] fix(alerting): back off a day after a deprecation pass raises and label the alert in the UI A pass that raises (a missing Slack webhook, say) now waits the daily interval instead of logging the same exception every 30 seconds, and the Admin UI alerting settings list the new alert type so it can be toggled like the others --- .../SlackAlerting/slack_alerting.py | 5 ++- .../test_model_deprecation_alert.py | 34 +++++++++++++++++++ .../src/components/settings.tsx | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 2f86f92d06c..65f4774a693 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1140,13 +1140,16 @@ Model Info: """Poll every pass for a loaded router, the alert being on, and no alert in the last day, then alert A pass that could not alert (no router yet, alert type off, a sibling pod holds the daily lock, or a - redis blip at claim time) is retried on the next poll instead of costing a day + redis blip at claim time) is retried on the next poll instead of costing a day, while a pass that + raised (a missing webhook, say) backs off a full day so a misconfiguration logs once, not every poll """ while True: try: await self._run_deprecation_alert_pass(get_llm_router(), pod_lock_manager) except Exception as e: # noqa: BLE001 # a failed alert must not kill the loop verbose_proxy_logger.exception("Error in model deprecation alert loop: %s", e) + await asyncio.sleep(DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS) + continue await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS) async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool: diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py index 9556775dad0..fd54d26c1f6 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py @@ -359,3 +359,37 @@ async def test_should_not_alert_or_claim_the_lock_within_a_day_of_a_sent_alert(m pod_lock_manager.acquire_lock.assert_not_awaited() mock_send_alert.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_should_back_off_a_full_day_after_a_pass_raises(monkeypatch): + """A misconfigured webhook raises on every send, which must log once a day rather than every poll""" + monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) + alerting = SlackAlerting( + alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings] + ) + router = _make_router([DEAD_ALIAS_DEPLOYMENT]) + slept: list[float] = [] + + async def stop_after_second_pass(seconds): + slept.append(seconds) + if len(slept) == 2: + raise asyncio.CancelledError + + with ( + patch.object( + alerting, + "send_alert", + new_callable=AsyncMock, + side_effect=ValueError("Missing SLACK_WEBHOOK_URL from environment"), + ) as mock_send_alert, + patch( + "litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep", + side_effect=stop_after_second_pass, + ), + pytest.raises(asyncio.CancelledError), + ): + await alerting.run_scheduled_deprecation_check(get_llm_router=lambda: router) + + assert slept == [DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS] * 2 + assert mock_send_alert.await_count == 2 diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 904fd4d611e..1f98e91fbfc 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -277,6 +277,7 @@ const Settings: React.FC = ({ accessToken, userRole, userID, daily_reports: "Weekly/Monthly Spend Reports", outage_alerts: "Outage Alerts", region_outage_alerts: "Region Outage Alerts", + model_deprecation_warnings: "Model Deprecation Warnings", }; useEffect(() => {