diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 2f86f92d06c..65f4774a693 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1140,13 +1140,16 @@ Model Info: """Poll every pass for a loaded router, the alert being on, and no alert in the last day, then alert A pass that could not alert (no router yet, alert type off, a sibling pod holds the daily lock, or a - redis blip at claim time) is retried on the next poll instead of costing a day + redis blip at claim time) is retried on the next poll instead of costing a day, while a pass that + raised (a missing webhook, say) backs off a full day so a misconfiguration logs once, not every poll """ while True: try: await self._run_deprecation_alert_pass(get_llm_router(), pod_lock_manager) except Exception as e: # noqa: BLE001 # a failed alert must not kill the loop verbose_proxy_logger.exception("Error in model deprecation alert loop: %s", e) + await asyncio.sleep(DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS) + continue await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS) async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool: diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py index 9556775dad0..fd54d26c1f6 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py @@ -359,3 +359,37 @@ async def test_should_not_alert_or_claim_the_lock_within_a_day_of_a_sent_alert(m pod_lock_manager.acquire_lock.assert_not_awaited() mock_send_alert.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_should_back_off_a_full_day_after_a_pass_raises(monkeypatch): + """A misconfigured webhook raises on every send, which must log once a day rather than every poll""" + monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST) + alerting = SlackAlerting( + alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings] + ) + router = _make_router([DEAD_ALIAS_DEPLOYMENT]) + slept: list[float] = [] + + async def stop_after_second_pass(seconds): + slept.append(seconds) + if len(slept) == 2: + raise asyncio.CancelledError + + with ( + patch.object( + alerting, + "send_alert", + new_callable=AsyncMock, + side_effect=ValueError("Missing SLACK_WEBHOOK_URL from environment"), + ) as mock_send_alert, + patch( + "litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep", + side_effect=stop_after_second_pass, + ), + pytest.raises(asyncio.CancelledError), + ): + await alerting.run_scheduled_deprecation_check(get_llm_router=lambda: router) + + assert slept == [DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS] * 2 + assert mock_send_alert.await_count == 2 diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 904fd4d611e..1f98e91fbfc 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -277,6 +277,7 @@ const Settings: React.FC = ({ accessToken, userRole, userID, daily_reports: "Weekly/Monthly Spend Reports", outage_alerts: "Outage Alerts", region_outage_alerts: "Region Outage Alerts", + model_deprecation_warnings: "Model Deprecation Warnings", }; useEffect(() => {