mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
fix(alerting): back off a day after a deprecation pass raises and label the alert in the UI
A pass that raises (a missing Slack webhook, say) now waits the daily interval instead of logging the same exception every 30 seconds, and the Admin UI alerting settings list the new alert type so it can be toggled like the others
This commit is contained in:
parent
308865bad0
commit
7017df5732
3 changed files with 39 additions and 1 deletions
|
|
@ -1140,13 +1140,16 @@ Model Info:
|
|||
"""Poll every pass for a loaded router, the alert being on, and no alert in the last day, then alert
|
||||
|
||||
A pass that could not alert (no router yet, alert type off, a sibling pod holds the daily lock, or a
|
||||
redis blip at claim time) is retried on the next poll instead of costing a day
|
||||
redis blip at claim time) is retried on the next poll instead of costing a day, while a pass that
|
||||
raised (a missing webhook, say) backs off a full day so a misconfiguration logs once, not every poll
|
||||
"""
|
||||
while True:
|
||||
try:
|
||||
await self._run_deprecation_alert_pass(get_llm_router(), pod_lock_manager)
|
||||
except Exception as e: # noqa: BLE001 # a failed alert must not kill the loop
|
||||
verbose_proxy_logger.exception("Error in model deprecation alert loop: %s", e)
|
||||
await asyncio.sleep(DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS)
|
||||
continue
|
||||
await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS)
|
||||
|
||||
async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool:
|
||||
|
|
|
|||
|
|
@ -359,3 +359,37 @@ async def test_should_not_alert_or_claim_the_lock_within_a_day_of_a_sent_alert(m
|
|||
|
||||
pod_lock_manager.acquire_lock.assert_not_awaited()
|
||||
mock_send_alert.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_should_back_off_a_full_day_after_a_pass_raises(monkeypatch):
|
||||
"""A misconfigured webhook raises on every send, which must log once a day rather than every poll"""
|
||||
monkeypatch.setattr(litellm, "model_cost", DEAD_MODEL_COST)
|
||||
alerting = SlackAlerting(
|
||||
alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings]
|
||||
)
|
||||
router = _make_router([DEAD_ALIAS_DEPLOYMENT])
|
||||
slept: list[float] = []
|
||||
|
||||
async def stop_after_second_pass(seconds):
|
||||
slept.append(seconds)
|
||||
if len(slept) == 2:
|
||||
raise asyncio.CancelledError
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
alerting,
|
||||
"send_alert",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=ValueError("Missing SLACK_WEBHOOK_URL from environment"),
|
||||
) as mock_send_alert,
|
||||
patch(
|
||||
"litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep",
|
||||
side_effect=stop_after_second_pass,
|
||||
),
|
||||
pytest.raises(asyncio.CancelledError),
|
||||
):
|
||||
await alerting.run_scheduled_deprecation_check(get_llm_router=lambda: router)
|
||||
|
||||
assert slept == [DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS] * 2
|
||||
assert mock_send_alert.await_count == 2
|
||||
|
|
|
|||
|
|
@ -277,6 +277,7 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID,
|
|||
daily_reports: "Weekly/Monthly Spend Reports",
|
||||
outage_alerts: "Outage Alerts",
|
||||
region_outage_alerts: "Region Outage Alerts",
|
||||
model_deprecation_warnings: "Model Deprecation Warnings",
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue