diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 65f4774a693..f93a8abe6bc 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -1809,7 +1809,7 @@ Model Info: _team_spend = round(float(spend["total_spend"]), 4) _spend_message += f"Team: `{spend['team_alias']}` | Spend: `${_team_spend}`\n" - if spend_per_tag is not None: + if spend_per_tag is not None and self.alerting_args.spend_report_include_tags: _spend_message += "\n*Tag Spend Report:*\n" for spend in spend_per_tag: _tag_spend = round(float(spend["total_spend"]), 4) @@ -1872,7 +1872,7 @@ Model Info: _team_spend = round(_team_spend, 4) _spend_message += f"Team: `{spend['team_alias']}` | Spend: `${_team_spend}`\n" - if monthly_spend_per_tag is not None: + if monthly_spend_per_tag is not None and self.alerting_args.spend_report_include_tags: _spend_message += "\n*Tag Spend Report:*\n" for spend in monthly_spend_per_tag: _tag_spend = spend["total_spend"] diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9b5d1b9bfea..2638918f2f8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14495,6 +14495,22 @@ async def model_settings(): #### ALERTING MANAGEMENT ENDPOINTS #### +_ALERTING_SETTINGS_FIELD_TYPES: Final[Mapping[str, str]] = MappingProxyType( + { + "slack_alerting": "Boolean", + "daily_report_frequency": "Integer", + "report_check_interval": "Integer", + "budget_alert_ttl": "Integer", + "outage_alert_ttl": "Integer", + "region_outage_alert_ttl": "Integer", + "minor_outage_alert_threshold": "Integer", + "major_outage_alert_threshold": "Integer", + "max_outage_alert_list_size": "Integer", + "spend_report_include_tags": "Boolean", + } +) + + @router.get( "/alerting/settings", description="Return the configurable alerting param, description, and current value", @@ -14510,7 +14526,7 @@ async def alerting_settings( Used by UI to generate 'alerting settings' page { field_name=field_name, - field_type=allowed_args[field_name]["type"], # string/int + field_type=allowed_args[field_name], # string/int field_description=field_info.description or "", # human-friendly description field_value=general_settings.get(field_name, None), # example value } @@ -14544,17 +14560,7 @@ async def alerting_settings( alerting_args_dict = {} alerting_values = None - allowed_args: Final = { - "slack_alerting": {"type": "Boolean"}, - "daily_report_frequency": {"type": "Integer"}, - "report_check_interval": {"type": "Integer"}, - "budget_alert_ttl": {"type": "Integer"}, - "outage_alert_ttl": {"type": "Integer"}, - "region_outage_alert_ttl": {"type": "Integer"}, - "minor_outage_alert_threshold": {"type": "Integer"}, - "major_outage_alert_threshold": {"type": "Integer"}, - "max_outage_alert_list_size": {"type": "Integer"}, - } + allowed_args: Final = _ALERTING_SETTINGS_FIELD_TYPES _slack_alerting: Final[SlackAlerting] = proxy_logging_obj.slack_alerting_instance _slack_alerting_args_dict: Final = _slack_alerting.alerting_args.model_dump() @@ -14569,7 +14575,7 @@ async def alerting_settings( _response_obj = ConfigList( field_name="slack_alerting", - field_type=allowed_args["slack_alerting"]["type"], + field_type=allowed_args["slack_alerting"], field_description="Enable slack alerting for monitoring proxy in production: llm outages, budgets, spend tracking failures.", field_value=is_slack_enabled, stored_in_db=True if alerting_values is not None else False, @@ -14588,7 +14594,7 @@ async def alerting_settings( _response_obj = ConfigList( field_name=field_name, - field_type=allowed_args[field_name]["type"], + field_type=allowed_args[field_name], field_description=field_info.description or "", field_value=_slack_alerting_args_dict.get(field_name, None), stored_in_db=_stored_in_db, diff --git a/litellm/types/integrations/slack_alerting.py b/litellm/types/integrations/slack_alerting.py index b1b7bc3541a..bdab417d952 100644 --- a/litellm/types/integrations/slack_alerting.py +++ b/litellm/types/integrations/slack_alerting.py @@ -91,6 +91,10 @@ class SlackAlertingArgs(LiteLLMPydanticObjectBase): default=False, description="If true, the alerting payload will be printed to the console.", ) + spend_report_include_tags: bool = Field( + default=True, + description="If false, spend reports drop the per-tag breakdown and keep the per-team one. Tags stay tracked.", + ) class DeploymentMetrics(LiteLLMPydanticObjectBase): diff --git a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py index cfbd3e76a88..b67509991bd 100644 --- a/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py +++ b/tests/test_litellm/integrations/SlackAlerting/test_slack_alerting.py @@ -3,7 +3,9 @@ import datetime import json import time import unittest -from typing import Final, List, Optional, Tuple +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, List, Literal, Optional, Tuple from unittest.mock import ANY, AsyncMock, MagicMock, Mock, patch import pytest @@ -12,7 +14,7 @@ import litellm from litellm.caching.caching import DualCache from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.proxy._types import CallInfo, Litellm_EntityType -from litellm.types.integrations.slack_alerting import SlackAlertingCacheKeys +from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingCacheKeys class TestSlackAlerting(unittest.TestCase): @@ -366,3 +368,68 @@ async def test_scheduled_daily_report_threads_the_pod_lock_manager_through(): _, kwargs = slack_alerting._run_scheduler_helper.await_args assert kwargs["pod_lock_manager"] is pod_lock_manager + + +_SPEND_PER_TEAM: Final = (MappingProxyType({"team_alias": "eng", "total_spend": 12.3456789}),) +_SPEND_PER_TAG: Final = (MappingProxyType({"individual_request_tag": "prod", "total_spend": 4.2}),) +_SPEND_REPORT_WEBHOOK: Final = "https://hooks.slack.example/spend-report" +_SPEND_REPORT_BATCH_SIZE: Final = 2 # pins the flush threshold above 1 so DEFAULT_BATCH_SIZE can't trigger a real POST + + +async def _delivered_spend_report( + monkeypatch: pytest.MonkeyPatch, + alerting_args: Mapping[str, bool], + report_type: Literal["weekly", "monthly"], +) -> tuple[str, AsyncMock]: + monkeypatch.delenv("PROXY_BASE_URL", raising=False) # send_alert appends it to the payload + slack_alerting: Final = SlackAlerting( + alerting=["slack"], + alert_types=[AlertType.spend_reports], + internal_usage_cache=DualCache(), + alerting_args=alerting_args, + default_webhook_url=_SPEND_REPORT_WEBHOOK, + batch_size=_SPEND_REPORT_BATCH_SIZE, + ) + slack_alerting.periodic_started = True # keeps send_alert from spawning an unawaited flush task + get_report: Final = AsyncMock(return_value=(_SPEND_PER_TEAM, _SPEND_PER_TAG)) + with patch( # test-quality-ok: lazily imported module function, no injection seam; the boundary is the DB + "litellm.proxy.spend_tracking.spend_management_endpoints._get_spend_report_for_time_range", + new=get_report, + ): + if report_type == "weekly": + await slack_alerting.send_weekly_spend_report() + else: + await slack_alerting.send_monthly_spend_report() + + assert len(slack_alerting.log_queue) == 1 + assert slack_alerting.log_queue[0]["url"] == _SPEND_REPORT_WEBHOOK + return slack_alerting.log_queue[0]["payload"]["text"], get_report + + +@pytest.mark.parametrize("report_type", ("weekly", "monthly")) +@pytest.mark.parametrize("alerting_args", (MappingProxyType({}), MappingProxyType({"spend_report_include_tags": True}))) +@pytest.mark.asyncio +async def test_spend_report_includes_tag_breakdown_by_default( + monkeypatch: pytest.MonkeyPatch, report_type: Literal["weekly", "monthly"], alerting_args: Mapping[str, bool] +) -> None: + message, _ = await _delivered_spend_report(monkeypatch, alerting_args, report_type) + + assert "*Team Spend Report:*" in message + assert "Team: `eng` | Spend: `$12.3457`" in message + assert "*Tag Spend Report:*" in message + assert "Tag: `prod` | Spend: `$4.2`" in message + + +@pytest.mark.parametrize("report_type", ("weekly", "monthly")) +@pytest.mark.asyncio +async def test_spend_report_omits_tag_breakdown_when_disabled( + monkeypatch: pytest.MonkeyPatch, report_type: Literal["weekly", "monthly"] +) -> None: + message, get_report = await _delivered_spend_report( + monkeypatch, MappingProxyType({"spend_report_include_tags": False}), report_type + ) + + assert "*Team Spend Report:*" in message + assert "Team: `eng` | Spend: `$12.3457`" in message + assert "Tag" not in message + get_report.assert_awaited_once() diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index afc42e8db45..42c0bb52ae4 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -9427,6 +9427,51 @@ def test_get_config_list_includes_apply_user_budget_to_team_keys(monkeypatch): app.dependency_overrides.clear() +def test_alerting_settings_exposes_spend_report_include_tags(monkeypatch: pytest.MonkeyPatch) -> None: + """The Admin UI form rebuilds alerting_args from exactly the fields /alerting/settings returns, + and /config/field/update replaces the whole blob, so a field missing from allowed_args is + silently reset to its default the next time anyone saves that form.""" + import types + from unittest.mock import AsyncMock, MagicMock + + from fastapi.testclient import TestClient + + import litellm.proxy.proxy_server as ps + from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.proxy_server import app + + mock_prisma = MagicMock() + mock_config_table = MagicMock() + mock_config_table.find_first = AsyncMock( + return_value=types.SimpleNamespace(param_value={"alerting_args": {"spend_report_include_tags": False}}) + ) + mock_prisma.db = types.SimpleNamespace(litellm_config=mock_config_table) + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + monkeypatch.setattr( + ps, + "proxy_logging_obj", + types.SimpleNamespace( + slack_alerting_instance=SlackAlerting(alerting_args={"spend_report_include_tags": False}) + ), + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN + ) + try: + client = TestClient(app) + resp = client.get("/alerting/settings") + assert resp.status_code == 200, resp.text + fields = {item["field_name"]: item for item in resp.json()} + assert "spend_report_include_tags" in fields + assert fields["spend_report_include_tags"]["field_type"] == "Boolean" + assert fields["spend_report_include_tags"]["field_value"] is False + assert fields["spend_report_include_tags"]["field_default_value"] is True + assert fields["spend_report_include_tags"]["stored_in_db"] is True + finally: + app.dependency_overrides.clear() + + def test_get_config_list_includes_budget_exceeded_throttle_percentage(monkeypatch): """The throttle fraction is a litellm_settings scalar surfaced on the General Settings table as a Float field so it sits with the other global limits; it diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 31f3b29286a..a11c1d72153 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -3,7 +3,7 @@ "limit": 22749 }, "LIT002": { - "limit": 26866 + "limit": 26856 }, "LIT003": { "limit": 269