mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
feat(slack_alerting): add spend_report_include_tags to hide tag breakdown in spend reports
This commit is contained in:
parent
168a0055a2
commit
c3e241caee
5 changed files with 121 additions and 4 deletions
|
|
@ -1857,7 +1857,7 @@ Model Info:
|
|||
_team_spend = round(float(spend["total_spend"]), 4)
|
||||
_spend_message += f"Team: `{spend['team_alias']}` | Spend: `${_team_spend}`\n"
|
||||
|
||||
if spend_per_tag is not None:
|
||||
if spend_per_tag is not None and self.alerting_args.spend_report_include_tags:
|
||||
_spend_message += "\n*Tag Spend Report:*\n"
|
||||
for spend in spend_per_tag:
|
||||
_tag_spend = round(float(spend["total_spend"]), 4)
|
||||
|
|
@ -1920,7 +1920,7 @@ Model Info:
|
|||
_team_spend = round(_team_spend, 4)
|
||||
_spend_message += f"Team: `{spend['team_alias']}` | Spend: `${_team_spend}`\n"
|
||||
|
||||
if monthly_spend_per_tag is not None:
|
||||
if monthly_spend_per_tag is not None and self.alerting_args.spend_report_include_tags:
|
||||
_spend_message += "\n*Tag Spend Report:*\n"
|
||||
for spend in monthly_spend_per_tag:
|
||||
_tag_spend = spend["total_spend"]
|
||||
|
|
|
|||
|
|
@ -15243,7 +15243,7 @@ async def alerting_settings(
|
|||
Used by UI to generate 'alerting settings' page
|
||||
{
|
||||
field_name=field_name,
|
||||
field_type=allowed_args[field_name]["type"], # string/int
|
||||
field_type=allowed_args[field_name], # string/int
|
||||
field_description=field_info.description or "", # human-friendly description
|
||||
field_value=general_settings.get(field_name, None), # example value
|
||||
}
|
||||
|
|
@ -15294,6 +15294,7 @@ async def alerting_settings(
|
|||
"spend_anomaly_baseline_days": "Integer",
|
||||
"spend_anomaly_min_spend": "Float",
|
||||
"user_spend_check_interval": "Integer",
|
||||
"spend_report_include_tags": "Boolean",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -125,6 +125,10 @@ class SlackAlertingArgs(LiteLLMPydanticObjectBase):
|
|||
ge=60,
|
||||
description="How often (in seconds) to check per-user spend thresholds and anomalies. Default is hourly.",
|
||||
)
|
||||
spend_report_include_tags: bool = Field(
|
||||
default=True,
|
||||
description="If false, spend reports drop the per-tag breakdown and keep the per-team one. Tags stay tracked.",
|
||||
)
|
||||
|
||||
|
||||
class DeploymentMetrics(LiteLLMPydanticObjectBase):
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@ import datetime
|
|||
import json
|
||||
import time
|
||||
import unittest
|
||||
from typing import Final, List, Optional, Tuple
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final, List, Literal, Optional, Tuple
|
||||
from unittest.mock import ANY, AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -419,3 +421,68 @@ async def test_send_alert_raises_when_no_webhook_url_configured(monkeypatch):
|
|||
alert_type=AlertType.budget_alerts,
|
||||
alerting_metadata={},
|
||||
)
|
||||
|
||||
|
||||
_SPEND_PER_TEAM: Final = (MappingProxyType({"team_alias": "eng", "total_spend": 12.3456789}),)
|
||||
_SPEND_PER_TAG: Final = (MappingProxyType({"individual_request_tag": "prod", "total_spend": 4.2}),)
|
||||
_SPEND_REPORT_WEBHOOK: Final = "https://hooks.slack.example/spend-report"
|
||||
_SPEND_REPORT_BATCH_SIZE: Final = 2 # pins the flush threshold above 1 so DEFAULT_BATCH_SIZE can't trigger a real POST
|
||||
|
||||
|
||||
async def _delivered_spend_report(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
alerting_args: Mapping[str, bool],
|
||||
report_type: Literal["weekly", "monthly"],
|
||||
) -> tuple[str, AsyncMock]:
|
||||
monkeypatch.delenv("PROXY_BASE_URL", raising=False) # send_alert appends it to the payload
|
||||
slack_alerting: Final = SlackAlerting(
|
||||
alerting=["slack"],
|
||||
alert_types=[AlertType.spend_reports],
|
||||
internal_usage_cache=DualCache(),
|
||||
alerting_args=alerting_args,
|
||||
default_webhook_url=_SPEND_REPORT_WEBHOOK,
|
||||
batch_size=_SPEND_REPORT_BATCH_SIZE,
|
||||
)
|
||||
slack_alerting.periodic_started = True # keeps send_alert from spawning an unawaited flush task
|
||||
get_report: Final = AsyncMock(return_value=(_SPEND_PER_TEAM, _SPEND_PER_TAG))
|
||||
with patch( # test-quality-ok: lazily imported module function, no injection seam; the boundary is the DB
|
||||
"litellm.proxy.spend_tracking.spend_management_endpoints._get_spend_report_for_time_range",
|
||||
new=get_report,
|
||||
):
|
||||
if report_type == "weekly":
|
||||
await slack_alerting.send_weekly_spend_report()
|
||||
else:
|
||||
await slack_alerting.send_monthly_spend_report()
|
||||
|
||||
assert len(slack_alerting.log_queue) == 1
|
||||
assert slack_alerting.log_queue[0]["url"] == _SPEND_REPORT_WEBHOOK
|
||||
return slack_alerting.log_queue[0]["payload"]["text"], get_report
|
||||
|
||||
|
||||
@pytest.mark.parametrize("report_type", ("weekly", "monthly"))
|
||||
@pytest.mark.parametrize("alerting_args", (MappingProxyType({}), MappingProxyType({"spend_report_include_tags": True})))
|
||||
@pytest.mark.asyncio
|
||||
async def test_spend_report_includes_tag_breakdown_by_default(
|
||||
monkeypatch: pytest.MonkeyPatch, report_type: Literal["weekly", "monthly"], alerting_args: Mapping[str, bool]
|
||||
) -> None:
|
||||
message, _ = await _delivered_spend_report(monkeypatch, alerting_args, report_type)
|
||||
|
||||
assert "*Team Spend Report:*" in message
|
||||
assert "Team: `eng` | Spend: `$12.3457`" in message
|
||||
assert "*Tag Spend Report:*" in message
|
||||
assert "Tag: `prod` | Spend: `$4.2`" in message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("report_type", ("weekly", "monthly"))
|
||||
@pytest.mark.asyncio
|
||||
async def test_spend_report_omits_tag_breakdown_when_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch, report_type: Literal["weekly", "monthly"]
|
||||
) -> None:
|
||||
message, get_report = await _delivered_spend_report(
|
||||
monkeypatch, MappingProxyType({"spend_report_include_tags": False}), report_type
|
||||
)
|
||||
|
||||
assert "*Team Spend Report:*" in message
|
||||
assert "Team: `eng` | Spend: `$12.3457`" in message
|
||||
assert "Tag" not in message
|
||||
get_report.assert_awaited_once()
|
||||
|
|
|
|||
|
|
@ -10050,6 +10050,51 @@ def test_get_config_list_includes_apply_user_budget_to_team_keys(monkeypatch):
|
|||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def test_alerting_settings_exposes_spend_report_include_tags(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The Admin UI form rebuilds alerting_args from exactly the fields /alerting/settings returns,
|
||||
and /config/field/update replaces the whole blob, so a field missing from allowed_args is
|
||||
silently reset to its default the next time anyone saves that form."""
|
||||
import types
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.proxy_server import app
|
||||
|
||||
mock_prisma = MagicMock()
|
||||
mock_config_table = MagicMock()
|
||||
mock_config_table.find_first = AsyncMock(
|
||||
return_value=types.SimpleNamespace(param_value={"alerting_args": {"spend_report_include_tags": False}})
|
||||
)
|
||||
mock_prisma.db = types.SimpleNamespace(litellm_config=mock_config_table)
|
||||
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
|
||||
monkeypatch.setattr(
|
||||
ps,
|
||||
"proxy_logging_obj",
|
||||
types.SimpleNamespace(
|
||||
slack_alerting_instance=SlackAlerting(alerting_args={"spend_report_include_tags": False})
|
||||
),
|
||||
)
|
||||
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN
|
||||
)
|
||||
try:
|
||||
client = TestClient(app)
|
||||
resp = client.get("/alerting/settings")
|
||||
assert resp.status_code == 200, resp.text
|
||||
fields = {item["field_name"]: item for item in resp.json()}
|
||||
assert "spend_report_include_tags" in fields
|
||||
assert fields["spend_report_include_tags"]["field_type"] == "Boolean"
|
||||
assert fields["spend_report_include_tags"]["field_value"] is False
|
||||
assert fields["spend_report_include_tags"]["field_default_value"] is True
|
||||
assert fields["spend_report_include_tags"]["stored_in_db"] is True
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def test_get_config_list_includes_budget_exceeded_throttle_percentage(monkeypatch):
|
||||
"""The throttle fraction is a litellm_settings scalar surfaced on the General
|
||||
Settings table as a Float field so it sits with the other global limits; it
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue