From 7017df5732bf5ccdaa18b9a89c9809cb646992ea Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:57:37 -0700 Subject: [PATCH] fix(alerting): back off a day after a deprecation pass raises and label the alert in the UI A pass that raises (a missing Slack webhook, say) now waits the daily interval instead of logging the same exception every 30 seconds, and the Admin UI alerting settings list the new alert type so it can be toggled like the others --- .../SlackAlerting/slack_alerting.py | 5 ++- .../test_model_deprecation_alert.py | 34 +++++++++++++++++++ .../src/components/settings.tsx | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 2f86f92d06c..65f4774a693 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1140,13 +1140,16 @@ Model Info: """Poll every pass for a loaded router, the alert being on, and no alert in the last day, then alert A pass that could not alert (no router yet, alert type off, a sibling pod holds the daily lock, or a - redis blip at claim time) is retried on the next poll instead of costing a day + redis blip at claim time) is retried on the next poll instead of costing a day, while a pass that + raised (a missing webhook, say) backs off a full day so a misconfiguration logs once, not every poll """ while True: try: await self._run_deprecation_alert_pass(get_llm_router(), pod_lock_manager) except Exception as e: # noqa: BLE001 # a failed alert must not kill the loop verbose_proxy_logger.exception("Error in model deprecation alert loop: %s", e) + await asyncio.sleep(DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS) + continue await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS) async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool: diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py index 9556775dad0..fd54d26c1f6 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py @@ -359,3 +359,37 @@ async def test_should_not_alert_or_claim_the_lock_within_a_day_of_a_sent_alert(m pod_lock_manager.acquire_lock.assert_not_awaited() mock_send_alert.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_should_back_off_a_full_day_after_a_pass_raises(monkeypatch): + """A misconfigured webhook raises on every send, which must log once a day rather than every poll""" + monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) + alerting = SlackAlerting( + alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings] + ) + router = _make_router([DEAD_ALIAS_DEPLOYMENT]) + slept: list[float] = [] + + async def stop_after_second_pass(seconds): + slept.append(seconds) + if len(slept) == 2: + raise asyncio.CancelledError + + with ( + patch.object( + alerting, + "send_alert", + new_callable=AsyncMock, + side_effect=ValueError("Missing SLACK_WEBHOOK_URL from environment"), + ) as mock_send_alert, + patch( + "litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep", + side_effect=stop_after_second_pass, + ), + pytest.raises(asyncio.CancelledError), + ): + await alerting.run_scheduled_deprecation_check(get_llm_router=lambda: router) + + assert slept == [DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS] * 2 + assert mock_send_alert.await_count == 2 diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 904fd4d611e..1f98e91fbfc 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -277,6 +277,7 @@ const Settings: React.FC = ({ accessToken, userRole, userID, daily_reports: "Weekly/Monthly Spend Reports", outage_alerts: "Outage Alerts", region_outage_alerts: "Region Outage Alerts", + model_deprecation_warnings: "Model Deprecation Warnings", }; useEffect(() => {