mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-08 22:21:35 +00:00
feat(slack_alerting): add spend_report_include_tags to hide tag breakdown in spend reports
This commit is contained in:
parent
40423e6ec0
commit
b0c3914fa4
6 changed files with 141 additions and 19 deletions
|
|
@ -1809,7 +1809,7 @@ Model Info:
|
|||
_team_spend = round(float(spend["total_spend"]), 4)
|
||||
_spend_message += f"Team: `{spend['team_alias']}` | Spend: `${_team_spend}`\n"
|
||||
|
||||
if spend_per_tag is not None:
|
||||
if spend_per_tag is not None and self.alerting_args.spend_report_include_tags:
|
||||
_spend_message += "\n*Tag Spend Report:*\n"
|
||||
for spend in spend_per_tag:
|
||||
_tag_spend = round(float(spend["total_spend"]), 4)
|
||||
|
|
@ -1872,7 +1872,7 @@ Model Info:
|
|||
_team_spend = round(_team_spend, 4)
|
||||
_spend_message += f"Team: `{spend['team_alias']}` | Spend: `${_team_spend}`\n"
|
||||
|
||||
if monthly_spend_per_tag is not None:
|
||||
if monthly_spend_per_tag is not None and self.alerting_args.spend_report_include_tags:
|
||||
_spend_message += "\n*Tag Spend Report:*\n"
|
||||
for spend in monthly_spend_per_tag:
|
||||
_tag_spend = spend["total_spend"]
|
||||
|
|
|
|||
|
|
@ -14495,6 +14495,22 @@ async def model_settings():
|
|||
#### ALERTING MANAGEMENT ENDPOINTS ####
|
||||
|
||||
|
||||
_ALERTING_SETTINGS_FIELD_TYPES: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
"slack_alerting": "Boolean",
|
||||
"daily_report_frequency": "Integer",
|
||||
"report_check_interval": "Integer",
|
||||
"budget_alert_ttl": "Integer",
|
||||
"outage_alert_ttl": "Integer",
|
||||
"region_outage_alert_ttl": "Integer",
|
||||
"minor_outage_alert_threshold": "Integer",
|
||||
"major_outage_alert_threshold": "Integer",
|
||||
"max_outage_alert_list_size": "Integer",
|
||||
"spend_report_include_tags": "Boolean",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/alerting/settings",
|
||||
description="Return the configurable alerting param, description, and current value",
|
||||
|
|
@ -14510,7 +14526,7 @@ async def alerting_settings(
|
|||
Used by UI to generate 'alerting settings' page
|
||||
{
|
||||
field_name=field_name,
|
||||
field_type=allowed_args[field_name]["type"], # string/int
|
||||
field_type=allowed_args[field_name], # string/int
|
||||
field_description=field_info.description or "", # human-friendly description
|
||||
field_value=general_settings.get(field_name, None), # example value
|
||||
}
|
||||
|
|
@ -14544,17 +14560,7 @@ async def alerting_settings(
|
|||
alerting_args_dict = {}
|
||||
alerting_values = None
|
||||
|
||||
allowed_args: Final = {
|
||||
"slack_alerting": {"type": "Boolean"},
|
||||
"daily_report_frequency": {"type": "Integer"},
|
||||
"report_check_interval": {"type": "Integer"},
|
||||
"budget_alert_ttl": {"type": "Integer"},
|
||||
"outage_alert_ttl": {"type": "Integer"},
|
||||
"region_outage_alert_ttl": {"type": "Integer"},
|
||||
"minor_outage_alert_threshold": {"type": "Integer"},
|
||||
"major_outage_alert_threshold": {"type": "Integer"},
|
||||
"max_outage_alert_list_size": {"type": "Integer"},
|
||||
}
|
||||
allowed_args: Final = _ALERTING_SETTINGS_FIELD_TYPES
|
||||
|
||||
_slack_alerting: Final[SlackAlerting] = proxy_logging_obj.slack_alerting_instance
|
||||
_slack_alerting_args_dict: Final = _slack_alerting.alerting_args.model_dump()
|
||||
|
|
@ -14569,7 +14575,7 @@ async def alerting_settings(
|
|||
|
||||
_response_obj = ConfigList(
|
||||
field_name="slack_alerting",
|
||||
field_type=allowed_args["slack_alerting"]["type"],
|
||||
field_type=allowed_args["slack_alerting"],
|
||||
field_description="Enable slack alerting for monitoring proxy in production: llm outages, budgets, spend tracking failures.",
|
||||
field_value=is_slack_enabled,
|
||||
stored_in_db=True if alerting_values is not None else False,
|
||||
|
|
@ -14588,7 +14594,7 @@ async def alerting_settings(
|
|||
|
||||
_response_obj = ConfigList(
|
||||
field_name=field_name,
|
||||
field_type=allowed_args[field_name]["type"],
|
||||
field_type=allowed_args[field_name],
|
||||
field_description=field_info.description or "",
|
||||
field_value=_slack_alerting_args_dict.get(field_name, None),
|
||||
stored_in_db=_stored_in_db,
|
||||
|
|
|
|||
|
|
@ -91,6 +91,10 @@ class SlackAlertingArgs(LiteLLMPydanticObjectBase):
|
|||
default=False,
|
||||
description="If true, the alerting payload will be printed to the console.",
|
||||
)
|
||||
spend_report_include_tags: bool = Field(
|
||||
default=True,
|
||||
description="If false, spend reports drop the per-tag breakdown and keep the per-team one. Tags stay tracked.",
|
||||
)
|
||||
|
||||
|
||||
class DeploymentMetrics(LiteLLMPydanticObjectBase):
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@ import datetime
|
|||
import json
|
||||
import time
|
||||
import unittest
|
||||
from typing import Final, List, Optional, Tuple
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final, List, Literal, Optional, Tuple
|
||||
from unittest.mock import ANY, AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -12,7 +14,7 @@ import litellm
|
|||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.proxy._types import CallInfo, Litellm_EntityType
|
||||
from litellm.types.integrations.slack_alerting import SlackAlertingCacheKeys
|
||||
from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingCacheKeys
|
||||
|
||||
|
||||
class TestSlackAlerting(unittest.TestCase):
|
||||
|
|
@ -366,3 +368,68 @@ async def test_scheduled_daily_report_threads_the_pod_lock_manager_through():
|
|||
|
||||
_, kwargs = slack_alerting._run_scheduler_helper.await_args
|
||||
assert kwargs["pod_lock_manager"] is pod_lock_manager
|
||||
|
||||
|
||||
_SPEND_PER_TEAM: Final = (MappingProxyType({"team_alias": "eng", "total_spend": 12.3456789}),)
|
||||
_SPEND_PER_TAG: Final = (MappingProxyType({"individual_request_tag": "prod", "total_spend": 4.2}),)
|
||||
_SPEND_REPORT_WEBHOOK: Final = "https://hooks.slack.example/spend-report"
|
||||
_SPEND_REPORT_BATCH_SIZE: Final = 2 # pins the flush threshold above 1 so DEFAULT_BATCH_SIZE can't trigger a real POST
|
||||
|
||||
|
||||
async def _delivered_spend_report(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
alerting_args: Mapping[str, bool],
|
||||
report_type: Literal["weekly", "monthly"],
|
||||
) -> tuple[str, AsyncMock]:
|
||||
monkeypatch.delenv("PROXY_BASE_URL", raising=False) # send_alert appends it to the payload
|
||||
slack_alerting: Final = SlackAlerting(
|
||||
alerting=["slack"],
|
||||
alert_types=[AlertType.spend_reports],
|
||||
internal_usage_cache=DualCache(),
|
||||
alerting_args=alerting_args,
|
||||
default_webhook_url=_SPEND_REPORT_WEBHOOK,
|
||||
batch_size=_SPEND_REPORT_BATCH_SIZE,
|
||||
)
|
||||
slack_alerting.periodic_started = True # keeps send_alert from spawning an unawaited flush task
|
||||
get_report: Final = AsyncMock(return_value=(_SPEND_PER_TEAM, _SPEND_PER_TAG))
|
||||
with patch( # test-quality-ok: lazily imported module function, no injection seam; the boundary is the DB
|
||||
"litellm.proxy.spend_tracking.spend_management_endpoints._get_spend_report_for_time_range",
|
||||
new=get_report,
|
||||
):
|
||||
if report_type == "weekly":
|
||||
await slack_alerting.send_weekly_spend_report()
|
||||
else:
|
||||
await slack_alerting.send_monthly_spend_report()
|
||||
|
||||
assert len(slack_alerting.log_queue) == 1
|
||||
assert slack_alerting.log_queue[0]["url"] == _SPEND_REPORT_WEBHOOK
|
||||
return slack_alerting.log_queue[0]["payload"]["text"], get_report
|
||||
|
||||
|
||||
@pytest.mark.parametrize("report_type", ("weekly", "monthly"))
|
||||
@pytest.mark.parametrize("alerting_args", (MappingProxyType({}), MappingProxyType({"spend_report_include_tags": True})))
|
||||
@pytest.mark.asyncio
|
||||
async def test_spend_report_includes_tag_breakdown_by_default(
|
||||
monkeypatch: pytest.MonkeyPatch, report_type: Literal["weekly", "monthly"], alerting_args: Mapping[str, bool]
|
||||
) -> None:
|
||||
message, _ = await _delivered_spend_report(monkeypatch, alerting_args, report_type)
|
||||
|
||||
assert "*Team Spend Report:*" in message
|
||||
assert "Team: `eng` | Spend: `$12.3457`" in message
|
||||
assert "*Tag Spend Report:*" in message
|
||||
assert "Tag: `prod` | Spend: `$4.2`" in message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("report_type", ("weekly", "monthly"))
|
||||
@pytest.mark.asyncio
|
||||
async def test_spend_report_omits_tag_breakdown_when_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch, report_type: Literal["weekly", "monthly"]
|
||||
) -> None:
|
||||
message, get_report = await _delivered_spend_report(
|
||||
monkeypatch, MappingProxyType({"spend_report_include_tags": False}), report_type
|
||||
)
|
||||
|
||||
assert "*Team Spend Report:*" in message
|
||||
assert "Team: `eng` | Spend: `$12.3457`" in message
|
||||
assert "Tag" not in message
|
||||
get_report.assert_awaited_once()
|
||||
|
|
|
|||
|
|
@ -9427,6 +9427,51 @@ def test_get_config_list_includes_apply_user_budget_to_team_keys(monkeypatch):
|
|||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def test_alerting_settings_exposes_spend_report_include_tags(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The Admin UI form rebuilds alerting_args from exactly the fields /alerting/settings returns,
|
||||
and /config/field/update replaces the whole blob, so a field missing from allowed_args is
|
||||
silently reset to its default the next time anyone saves that form."""
|
||||
import types
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.proxy_server import app
|
||||
|
||||
mock_prisma = MagicMock()
|
||||
mock_config_table = MagicMock()
|
||||
mock_config_table.find_first = AsyncMock(
|
||||
return_value=types.SimpleNamespace(param_value={"alerting_args": {"spend_report_include_tags": False}})
|
||||
)
|
||||
mock_prisma.db = types.SimpleNamespace(litellm_config=mock_config_table)
|
||||
monkeypatch.setattr(ps, "prisma_client", mock_prisma)
|
||||
monkeypatch.setattr(
|
||||
ps,
|
||||
"proxy_logging_obj",
|
||||
types.SimpleNamespace(
|
||||
slack_alerting_instance=SlackAlerting(alerting_args={"spend_report_include_tags": False})
|
||||
),
|
||||
)
|
||||
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN
|
||||
)
|
||||
try:
|
||||
client = TestClient(app)
|
||||
resp = client.get("/alerting/settings")
|
||||
assert resp.status_code == 200, resp.text
|
||||
fields = {item["field_name"]: item for item in resp.json()}
|
||||
assert "spend_report_include_tags" in fields
|
||||
assert fields["spend_report_include_tags"]["field_type"] == "Boolean"
|
||||
assert fields["spend_report_include_tags"]["field_value"] is False
|
||||
assert fields["spend_report_include_tags"]["field_default_value"] is True
|
||||
assert fields["spend_report_include_tags"]["stored_in_db"] is True
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def test_get_config_list_includes_budget_exceeded_throttle_percentage(monkeypatch):
|
||||
"""The throttle fraction is a litellm_settings scalar surfaced on the General
|
||||
Settings table as a Float field so it sits with the other global limits; it
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
"limit": 22749
|
||||
},
|
||||
"LIT002": {
|
||||
"limit": 26866
|
||||
"limit": 26856
|
||||
},
|
||||
"LIT003": {
|
||||
"limit": 269
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue