scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-06 08:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

fix(proxy): deliver budget alerts on webhook-only alerting and accept ALERTING_WEBHOOK_URL (#38441)

Browse source 
* fix(proxy): deliver budget alerts on webhook-only alerting and accept ALERTING_WEBHOOK_URL

ProxyLogging.budget_alerts forwarded to the alerting pipeline only when
'slack' was in general_settings.alerting, so alerting: ['webhook'] plus
WEBHOOK_URL silently never delivered a budget alert (the config
/health/services?service=webhook exists to test). Forward when 'webhook'
is present too; SlackAlerting.send_alert already fans out per channel.

Also accept a provider-neutral ALERTING_WEBHOOK_URL env fallback for the
Slack-format channel (any Slack-compatible receiver works), mark it as a
sensitive var, and de-brand the admin UI alerting copy.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(ui): format settings.tsx with prettier

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* chore(ui): regenerate schema.d.ts for updated alerting description

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* ci: retrigger checks after ALERTING_WEBHOOK_URL docs merged

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-08-31 09:22:36 -07:00 • committed by GitHub
parent 32291c9ad2
commit f079e4061b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 116 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
litellm/constants.py
View file

@ -1728,6 +1728,7 @@ SENTRY_DENYLIST: Final = [
"jwt_token",
"private_key",
"SLACK_WEBHOOK_URL",
"ALERTING_WEBHOOK_URL",
"webhook_url",
"LANGFUSE_SECRET_KEY",
# Email Configuration

8
litellm/integrations/SlackAlerting/slack_alerting.py
View file

@ -1485,9 +1485,9 @@ Model Info:
elif self.default_webhook_url is not None:
_digest_webhook = self.default_webhook_url
else:
_digest_webhook = os.getenv("SLACK_WEBHOOK_URL", None)
_digest_webhook = os.getenv("SLACK_WEBHOOK_URL") or os.getenv("ALERTING_WEBHOOK_URL")
if _digest_webhook is None:
raise ValueError("Missing SLACK_WEBHOOK_URL from environment")
raise ValueError("Missing SLACK_WEBHOOK_URL / ALERTING_WEBHOOK_URL from environment")
digest_key: Final = f"{alert_type_name_str}:{request_model or ''}:{api_base or ''}"
@ -1516,10 +1516,10 @@ Model Info:
elif self.default_webhook_url is not None:
slack_webhook_url = self.default_webhook_url
else:
slack_webhook_url = os.getenv("SLACK_WEBHOOK_URL", None)
slack_webhook_url = os.getenv("SLACK_WEBHOOK_URL") or os.getenv("ALERTING_WEBHOOK_URL")
if slack_webhook_url is None:
raise ValueError("Missing SLACK_WEBHOOK_URL from environment")
raise ValueError("Missing SLACK_WEBHOOK_URL / ALERTING_WEBHOOK_URL from environment")
payload: Final = {"text": formatted_message}
headers: Final = {"Content-type": "application/json"}

2
litellm/proxy/_types.py
View file

@ -2541,7 +2541,7 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
)
alerting: list | None = Field(
None,
description="List of alerting integrations. Today, just slack - `alerting: ['slack']`",
description="List of alerting integrations - e.g. `alerting: ['slack', 'webhook', 'email']`. 'slack' posts Slack-format messages to any Slack-compatible webhook (Slack, Rocket.Chat, Mattermost); 'webhook' posts structured JSON budget alerts to WEBHOOK_URL",
)
alert_types: list[AlertType] | None = Field(
None,

3
litellm/proxy/proxy_server.py
View file

@ -1363,7 +1363,7 @@ _OPENAPI_HTTP_METHODS: Final = {
# the UI. Kept here at module scope to match the analogous descriptor
# `is_secret` flags in litellm.proxy.config_resolvers and the
# `_CACHE_SENSITIVE_FIELDS` constant in the cache endpoint file.
_ALERTING_SENSITIVE_VARS: Final[set[str]] = {"SLACK_WEBHOOK_URL", "SMTP_PASSWORD"}
_ALERTING_SENSITIVE_VARS: Final[set[str]] = {"ALERTING_WEBHOOK_URL", "SLACK_WEBHOOK_URL", "SMTP_PASSWORD"}
def _strip_operation_id_method_suffix(operation_id: str) -> str:
@ -16566,6 +16566,7 @@ async def create_config_audit_log(
_EXTRA_SECRET_CALLBACK_ENV_VARS: Final = frozenset(
{
"ALERTING_WEBHOOK_URL",
"GALILEO_USERNAME",
"GENERIC_LOGGER_HEADERS",
"OTEL_HEADERS",

6
litellm/proxy/utils.py
View file

@ -645,7 +645,7 @@ class ProxyLogging:
self.max_parallel_request_limiter = _PROXY_MaxParallelRequestsHandler(self.internal_usage_cache)
self.max_budget_limiter = _PROXY_MaxBudgetLimiter()
self.cache_control_check = _PROXY_CacheControlCheck()
self.alerting: list | None = None
self.alerting: list[str] | None = None
self.alerting_threshold: float = 300 # default to 5 min. threshold
self.alert_types: list[AlertType] = DEFAULT_ALERT_TYPES
self.alert_to_webhook_url: dict | None = None
@ -2364,7 +2364,9 @@ class ProxyLogging:
# do nothing if alerting is not switched on (unless it's a soft_budget alert with team-specific emails)
return
if self.alerting is not None and ("slack" in self.alerting or "ms_teams" in self.alerting):
if self.alerting is not None and (
"slack" in self.alerting or "ms_teams" in self.alerting or "webhook" in self.alerting
):
if self.slack_alerting_instance is not None:
await self.slack_alerting_instance.budget_alerts(
type=type,

55
tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py
View file

@ -12,7 +12,7 @@ import litellm
from litellm.caching.caching import DualCache
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.proxy._types import CallInfo, Litellm_EntityType
from litellm.types.integrations.slack_alerting import SlackAlertingCacheKeys
from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingCacheKeys
class TestSlackAlerting(unittest.TestCase):
@ -366,3 +366,56 @@ async def test_scheduled_daily_report_threads_the_pod_lock_manager_through():
_, kwargs = slack_alerting._run_scheduler_helper.await_args
assert kwargs["pod_lock_manager"] is pod_lock_manager
def _slack_alerting_with_env_resolution() -> SlackAlerting:
slack_alerting: Final = SlackAlerting(alerting=["slack"], internal_usage_cache=DualCache())
slack_alerting.periodic_started = True
return slack_alerting
@pytest.mark.asyncio
async def test_send_alert_falls_back_to_alerting_webhook_url_env(monkeypatch):
monkeypatch.delenv("SLACK_WEBHOOK_URL", raising=False)
monkeypatch.setenv("ALERTING_WEBHOOK_URL", "https://chat.example.com/hooks/abc")
slack_alerting: Final = _slack_alerting_with_env_resolution()
await slack_alerting.send_alert(
message="budget crossed",
level="High",
alert_type=AlertType.budget_alerts,
alerting_metadata={},
)
assert slack_alerting.log_queue[0]["url"] == "https://chat.example.com/hooks/abc"
@pytest.mark.asyncio
async def test_send_alert_prefers_slack_webhook_url_over_fallback(monkeypatch):
monkeypatch.setenv("SLACK_WEBHOOK_URL", "https://hooks.slack.com/services/T0/B0/X0")
monkeypatch.setenv("ALERTING_WEBHOOK_URL", "https://chat.example.com/hooks/abc")
slack_alerting: Final = _slack_alerting_with_env_resolution()
await slack_alerting.send_alert(
message="budget crossed",
level="High",
alert_type=AlertType.budget_alerts,
alerting_metadata={},
)
assert slack_alerting.log_queue[0]["url"] == "https://hooks.slack.com/services/T0/B0/X0"
@pytest.mark.asyncio
async def test_send_alert_raises_when_no_webhook_url_configured(monkeypatch):
monkeypatch.delenv("SLACK_WEBHOOK_URL", raising=False)
monkeypatch.delenv("ALERTING_WEBHOOK_URL", raising=False)
slack_alerting: Final = _slack_alerting_with_env_resolution()
with pytest.raises(ValueError, match="SLACK_WEBHOOK_URL / ALERTING_WEBHOOK_URL"):
await slack_alerting.send_alert(
message="budget crossed",
level="High",
alert_type=AlertType.budget_alerts,
alerting_metadata={},
)

17
tests/test_litellm/integrations/SlackAlerting/test_slack_alerting_digest.py
View file

@ -79,6 +79,23 @@ class TestDigestMode(unittest.IsolatedAsyncioTestCase):
self.assertEqual(len(self.slack_alerting.digest_buckets), 2)
async def test_digest_falls_back_to_alerting_webhook_url_env(self):
"""With SLACK_WEBHOOK_URL unset, the digest entry resolves ALERTING_WEBHOOK_URL instead."""
env = {k: v for k, v in os.environ.items() if k != "SLACK_WEBHOOK_URL"}
env["ALERTING_WEBHOOK_URL"] = "https://chat.example.com/hooks/abc"
with unittest.mock.patch.dict(os.environ, env, clear=True):
await self.slack_alerting.send_alert(
message="`Requests are hanging`",
level="Medium",
alert_type=AlertType.llm_requests_hanging,
alerting_metadata={},
request_model="gemini-2.5-flash",
api_base="None",
)
bucket = list(self.slack_alerting.digest_buckets.values())[0]
self.assertEqual(bucket["webhook_url"], "https://chat.example.com/hooks/abc")
async def test_non_digest_alert_goes_to_queue(self):
"""Alert types without digest enabled should go straight to the log queue."""
message = "Budget exceeded"

29
tests/test_litellm/proxy/utils/proxy_logging/test_alerting.py
View file

@ -115,6 +115,35 @@ async def test_budget_alerts_slack_when_slack_alerting(proxy_logging):
assert snapshot == {"type": "user_budget", "user_info_is_callinfo": True, "user_id": "u1"}
@pytest.mark.asyncio
async def test_budget_alerts_webhook_only_forwards_to_slack_alerting_instance(proxy_logging):
proxy_logging.alerting = ["webhook"]
captured: Dict[str, Any] = {}
async def fake_alert(**kwargs):
captured.update(kwargs)
proxy_logging.slack_alerting_instance = MagicMock(budget_alerts=fake_alert)
proxy_logging.email_logging_instance = None
await proxy_logging.budget_alerts(type="user_budget", user_info=_user_info())
snapshot = {
"type": captured["type"],
"user_info_is_callinfo": isinstance(captured["user_info"], CallInfo),
"user_id": captured["user_info"].user_id,
}
assert snapshot == {"type": "user_budget", "user_info_is_callinfo": True, "user_id": "u1"}
@pytest.mark.asyncio
async def test_budget_alerts_email_only_skips_slack_alerting_instance(proxy_logging):
proxy_logging.alerting = ["email"]
proxy_logging.slack_alerting_instance = MagicMock(budget_alerts=AsyncMock())
proxy_logging.email_logging_instance = MagicMock(budget_alerts=AsyncMock())
await proxy_logging.budget_alerts(type="user_budget", user_info=_user_info())
proxy_logging.slack_alerting_instance.budget_alerts.assert_not_called()
proxy_logging.email_logging_instance.budget_alerts.assert_called_once()
@pytest.mark.asyncio
async def test_budget_alerts_soft_budget_with_alert_emails_bypasses_global(proxy_logging):
proxy_logging.alerting = None

5
ui/litellm-dashboard/src/components/settings.tsx
View file

@ -522,7 +522,8 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID,
<TabsContent value="alerting-types" keepMounted>
<Card className="p-6">
<p className="my-2">
Alerts are only supported for Slack Webhook URLs. Get your webhook urls from{" "}
Alerts are sent to any Slack-compatible incoming webhook URL (Slack, Rocket.Chat, Mattermost, etc.). Get
Slack webhook urls from{" "}
<a href="https://api.slack.com/messaging/webhooks" target="_blank" style={{ color: "blue" }}>
here
</a>
@ -532,7 +533,7 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID,
<TableRow>
<TableHead></TableHead>
<TableHead></TableHead>
<TableHead>Slack Webhook URL</TableHead>
<TableHead>Webhook URL (Slack-compatible)</TableHead>
</TableRow>
</TableHeader>

2
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -25182,7 +25182,7 @@ export interface components {
alert_types?: components["schemas"]["AlertType"][] | null;
/**
* Alerting
* @description List of alerting integrations. Today, just slack - `alerting: ['slack']`
* @description List of alerting integrations - e.g. `alerting: ['slack', 'webhook', 'email']`. 'slack' posts Slack-format messages to any Slack-compatible webhook (Slack, Rocket.Chat, Mattermost); 'webhook' posts structured JSON budget alerts to WEBHOOK_URL
*/
alerting?: unknown[] | null;
/**