fix(alerting): back off a day after a deprecation pass raises and label the alert in the UI

A pass that raises (a missing Slack webhook, say) now waits the daily interval instead of logging the
same exception every 30 seconds, and the Admin UI alerting settings list the new alert type so it can be
toggled like the others
This commit is contained in:
mateo-berri 2026-08-17 17:57:37 -07:00
parent 308865bad0
commit 7017df5732
3 changed files with 39 additions and 1 deletions

View file

@ -1140,13 +1140,16 @@ Model Info:
"""Poll every pass for a loaded router, the alert being on, and no alert in the last day, then alert
A pass that could not alert (no router yet, alert type off, a sibling pod holds the daily lock, or a
redis blip at claim time) is retried on the next poll instead of costing a day
redis blip at claim time) is retried on the next poll instead of costing a day, while a pass that
raised (a missing webhook, say) backs off a full day so a misconfiguration logs once, not every poll
"""
while True:
try:
await self._run_deprecation_alert_pass(get_llm_router(), pod_lock_manager)
except Exception as e: # noqa: BLE001 # a failed alert must not kill the loop
verbose_proxy_logger.exception("Error in model deprecation alert loop: %s", e)
await asyncio.sleep(DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS)
continue
await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS)
async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool:

View file

@ -359,3 +359,37 @@ async def test_should_not_alert_or_claim_the_lock_within_a_day_of_a_sent_alert(m
pod_lock_manager.acquire_lock.assert_not_awaited()
mock_send_alert.assert_not_awaited()
@pytest.mark.asyncio
async def test_should_back_off_a_full_day_after_a_pass_raises(monkeypatch):
"""A misconfigured webhook raises on every send, which must log once a day rather than every poll"""
monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST)
alerting = SlackAlerting(
alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings]
)
router = _make_router([DEAD_ALIAS_DEPLOYMENT])
slept: list[float] = []
async def stop_after_second_pass(seconds):
slept.append(seconds)
if len(slept) == 2:
raise asyncio.CancelledError
with (
patch.object(
alerting,
"send_alert",
new_callable=AsyncMock,
side_effect=ValueError("Missing SLACK_WEBHOOK_URL from environment"),
) as mock_send_alert,
patch(
"litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep",
side_effect=stop_after_second_pass,
),
pytest.raises(asyncio.CancelledError),
):
await alerting.run_scheduled_deprecation_check(get_llm_router=lambda: router)
assert slept == [DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS] * 2
assert mock_send_alert.await_count == 2

View file

@ -277,6 +277,7 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID,
daily_reports: "Weekly/Monthly Spend Reports",
outage_alerts: "Outage Alerts",
region_outage_alerts: "Region Outage Alerts",
model_deprecation_warnings: "Model Deprecation Warnings",
};
useEffect(() => {