diff --git a/litellm/constants.py b/litellm/constants.py index cc6db6c10cc..9872783bfab 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1728,6 +1728,7 @@ SENTRY_DENYLIST: Final = [ "jwt_token", "private_key", "SLACK_WEBHOOK_URL", + "ALERTING_WEBHOOK_URL", "webhook_url", "LANGFUSE_SECRET_KEY", # Email Configuration diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index d7d06387d85..94d734546be 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1485,9 +1485,9 @@ Model Info: elif self.default_webhook_url is not None: _digest_webhook = self.default_webhook_url else: - _digest_webhook = os.getenv("SLACK_WEBHOOK_URL", None) + _digest_webhook = os.getenv("SLACK_WEBHOOK_URL") or os.getenv("ALERTING_WEBHOOK_URL") if _digest_webhook is None: - raise ValueError("Missing SLACK_WEBHOOK_URL from environment") + raise ValueError("Missing SLACK_WEBHOOK_URL / ALERTING_WEBHOOK_URL from environment") digest_key: Final = f"{alert_type_name_str}:{request_model or ''}:{api_base or ''}" @@ -1516,10 +1516,10 @@ Model Info: elif self.default_webhook_url is not None: slack_webhook_url = self.default_webhook_url else: - slack_webhook_url = os.getenv("SLACK_WEBHOOK_URL", None) + slack_webhook_url = os.getenv("SLACK_WEBHOOK_URL") or os.getenv("ALERTING_WEBHOOK_URL") if slack_webhook_url is None: - raise ValueError("Missing SLACK_WEBHOOK_URL from environment") + raise ValueError("Missing SLACK_WEBHOOK_URL / ALERTING_WEBHOOK_URL from environment") payload: Final = {"text": formatted_message} headers: Final = {"Content-type": "application/json"} diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5ba5e8fa1aa..0f2d97b8b1c 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2541,7 +2541,7 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): ) alerting: list | None = Field( None, - description="List of alerting integrations. Today, just slack - `alerting: ['slack']`", + description="List of alerting integrations - e.g. `alerting: ['slack', 'webhook', 'email']`. 'slack' posts Slack-format messages to any Slack-compatible webhook (Slack, Rocket.Chat, Mattermost); 'webhook' posts structured JSON budget alerts to WEBHOOK_URL", ) alert_types: list[AlertType] | None = Field( None, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 887716a383a..da2e09bd7f6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1363,7 +1363,7 @@ _OPENAPI_HTTP_METHODS: Final = { # the UI. Kept here at module scope to match the analogous descriptor # `is_secret` flags in litellm.proxy.config_resolvers and the # `_CACHE_SENSITIVE_FIELDS` constant in the cache endpoint file. -_ALERTING_SENSITIVE_VARS: Final[set[str]] = {"SLACK_WEBHOOK_URL", "SMTP_PASSWORD"} +_ALERTING_SENSITIVE_VARS: Final[set[str]] = {"ALERTING_WEBHOOK_URL", "SLACK_WEBHOOK_URL", "SMTP_PASSWORD"} def _strip_operation_id_method_suffix(operation_id: str) -> str: @@ -16566,6 +16566,7 @@ async def create_config_audit_log( _EXTRA_SECRET_CALLBACK_ENV_VARS: Final = frozenset( { + "ALERTING_WEBHOOK_URL", "GALILEO_USERNAME", "GENERIC_LOGGER_HEADERS", "OTEL_HEADERS", diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index cf56fc0b1dd..051d36c4d0f 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -645,7 +645,7 @@ class ProxyLogging: self.max_parallel_request_limiter = _PROXY_MaxParallelRequestsHandler(self.internal_usage_cache) self.max_budget_limiter = _PROXY_MaxBudgetLimiter() self.cache_control_check = _PROXY_CacheControlCheck() - self.alerting: list | None = None + self.alerting: list[str] | None = None self.alerting_threshold: float = 300 # default to 5 min. threshold self.alert_types: list[AlertType] = DEFAULT_ALERT_TYPES self.alert_to_webhook_url: dict | None = None @@ -2364,7 +2364,9 @@ class ProxyLogging: # do nothing if alerting is not switched on (unless it's a soft_budget alert with team-specific emails) return - if self.alerting is not None and ("slack" in self.alerting or "ms_teams" in self.alerting): + if self.alerting is not None and ( + "slack" in self.alerting or "ms_teams" in self.alerting or "webhook" in self.alerting + ): if self.slack_alerting_instance is not None: await self.slack_alerting_instance.budget_alerts( type=type, diff --git a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py index cfbd3e76a88..55e2dcdc270 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py @@ -12,7 +12,7 @@ import litellm from litellm.caching.caching import DualCache from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.proxy._types import CallInfo, Litellm_EntityType -from litellm.types.integrations.slack_alerting import SlackAlertingCacheKeys +from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingCacheKeys class TestSlackAlerting(unittest.TestCase): @@ -366,3 +366,56 @@ async def test_scheduled_daily_report_threads_the_pod_lock_manager_through(): _, kwargs = slack_alerting._run_scheduler_helper.await_args assert kwargs["pod_lock_manager"] is pod_lock_manager + + +def _slack_alerting_with_env_resolution() -> SlackAlerting: + slack_alerting: Final = SlackAlerting(alerting=["slack"], internal_usage_cache=DualCache()) + slack_alerting.periodic_started = True + return slack_alerting + + +@pytest.mark.asyncio +async def test_send_alert_falls_back_to_alerting_webhook_url_env(monkeypatch): + monkeypatch.delenv("SLACK_WEBHOOK_URL", raising=False) + monkeypatch.setenv("ALERTING_WEBHOOK_URL", "https://chat.example.com/hooks/abc") + slack_alerting: Final = _slack_alerting_with_env_resolution() + + await slack_alerting.send_alert( + message="budget crossed", + level="High", + alert_type=AlertType.budget_alerts, + alerting_metadata={}, + ) + + assert slack_alerting.log_queue[0]["url"] == "https://chat.example.com/hooks/abc" + + +@pytest.mark.asyncio +async def test_send_alert_prefers_slack_webhook_url_over_fallback(monkeypatch): + monkeypatch.setenv("SLACK_WEBHOOK_URL", "https://hooks.slack.com/services/T0/B0/X0") + monkeypatch.setenv("ALERTING_WEBHOOK_URL", "https://chat.example.com/hooks/abc") + slack_alerting: Final = _slack_alerting_with_env_resolution() + + await slack_alerting.send_alert( + message="budget crossed", + level="High", + alert_type=AlertType.budget_alerts, + alerting_metadata={}, + ) + + assert slack_alerting.log_queue[0]["url"] == "https://hooks.slack.com/services/T0/B0/X0" + + +@pytest.mark.asyncio +async def test_send_alert_raises_when_no_webhook_url_configured(monkeypatch): + monkeypatch.delenv("SLACK_WEBHOOK_URL", raising=False) + monkeypatch.delenv("ALERTING_WEBHOOK_URL", raising=False) + slack_alerting: Final = _slack_alerting_with_env_resolution() + + with pytest.raises(ValueError, match="SLACK_WEBHOOK_URL / ALERTING_WEBHOOK_URL"): + await slack_alerting.send_alert( + message="budget crossed", + level="High", + alert_type=AlertType.budget_alerts, + alerting_metadata={}, + ) diff --git a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_digest.py b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_digest.py index edce5c5f3a2..d614823c0ef 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_digest.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_digest.py @@ -79,6 +79,23 @@ class TestDigestMode(unittest.IsolatedAsyncioTestCase): self.assertEqual(len(self.slack_alerting.digest_buckets), 2) + async def test_digest_falls_back_to_alerting_webhook_url_env(self): + """With SLACK_WEBHOOK_URL unset, the digest entry resolves ALERTING_WEBHOOK_URL instead.""" + env = {k: v for k, v in os.environ.items() if k != "SLACK_WEBHOOK_URL"} + env["ALERTING_WEBHOOK_URL"] = "https://chat.example.com/hooks/abc" + with unittest.mock.patch.dict(os.environ, env, clear=True): + await self.slack_alerting.send_alert( + message="`Requests are hanging`", + level="Medium", + alert_type=AlertType.llm_requests_hanging, + alerting_metadata={}, + request_model="gemini-2.5-flash", + api_base="None", + ) + + bucket = list(self.slack_alerting.digest_buckets.values())[0] + self.assertEqual(bucket["webhook_url"], "https://chat.example.com/hooks/abc") + async def test_non_digest_alert_goes_to_queue(self): """Alert types without digest enabled should go straight to the log queue.""" message = "Budget exceeded" diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_alerting.py b/tests/test_litellm/proxy/utils/proxy_logging/test_alerting.py index cede859cb38..77c0f71dbf9 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_alerting.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_alerting.py @@ -115,6 +115,35 @@ async def test_budget_alerts_slack_when_slack_alerting(proxy_logging): assert snapshot == {"type": "user_budget", "user_info_is_callinfo": True, "user_id": "u1"} +@pytest.mark.asyncio +async def test_budget_alerts_webhook_only_forwards_to_slack_alerting_instance(proxy_logging): + proxy_logging.alerting = ["webhook"] + captured: Dict[str, Any] = {} + + async def fake_alert(**kwargs): + captured.update(kwargs) + + proxy_logging.slack_alerting_instance = MagicMock(budget_alerts=fake_alert) + proxy_logging.email_logging_instance = None + await proxy_logging.budget_alerts(type="user_budget", user_info=_user_info()) + snapshot = { + "type": captured["type"], + "user_info_is_callinfo": isinstance(captured["user_info"], CallInfo), + "user_id": captured["user_info"].user_id, + } + assert snapshot == {"type": "user_budget", "user_info_is_callinfo": True, "user_id": "u1"} + + +@pytest.mark.asyncio +async def test_budget_alerts_email_only_skips_slack_alerting_instance(proxy_logging): + proxy_logging.alerting = ["email"] + proxy_logging.slack_alerting_instance = MagicMock(budget_alerts=AsyncMock()) + proxy_logging.email_logging_instance = MagicMock(budget_alerts=AsyncMock()) + await proxy_logging.budget_alerts(type="user_budget", user_info=_user_info()) + proxy_logging.slack_alerting_instance.budget_alerts.assert_not_called() + proxy_logging.email_logging_instance.budget_alerts.assert_called_once() + + @pytest.mark.asyncio async def test_budget_alerts_soft_budget_with_alert_emails_bypasses_global(proxy_logging): proxy_logging.alerting = None diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 05e66985e6f..72da01d0918 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -522,7 +522,8 @@ const Settings: React.FC = ({ accessToken, userRole, userID,

- Alerts are only supported for Slack Webhook URLs. Get your webhook urls from{" "} + Alerts are sent to any Slack-compatible incoming webhook URL (Slack, Rocket.Chat, Mattermost, etc.). Get + Slack webhook urls from{" "} here @@ -532,7 +533,7 @@ const Settings: React.FC = ({ accessToken, userRole, userID, - Slack Webhook URL + Webhook URL (Slack-compatible) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 137c67e837c..8a564a07489 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25182,7 +25182,7 @@ export interface components { alert_types?: components["schemas"]["AlertType"][] | null; /** * Alerting - * @description List of alerting integrations. Today, just slack - `alerting: ['slack']` + * @description List of alerting integrations - e.g. `alerting: ['slack', 'webhook', 'email']`. 'slack' posts Slack-format messages to any Slack-compatible webhook (Slack, Rocket.Chat, Mattermost); 'webhook' posts structured JSON budget alerts to WEBHOOK_URL */ alerting?: unknown[] | null; /**