scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-12 08:42:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 207

feat(daily-activity): add a per-guardrail spend breakdown to /user/daily/activity/aggregated

Browse source 
Each day now has breakdown.guardrails[<guardrail_id>] with metrics.spend and an api_key_breakdown carrying each key's spend and metadata. Guardrail entries only report spend, since that is the one metric stored per key. The breakdown is on the user endpoint only, is empty when a model filter is set because guardrail cost is not stored per model, and lists only keys within api_key_limit while the guardrail total keeps every key. spend and budgets are unchanged

Resolves LIT-9785
This commit is contained in:
Mrinal Chanshetty 2026-10-27 18:30:06 -07:00
parent c864255f33
commit 68571af5da
8 changed files with 233 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -3487,6 +3487,13 @@
"title": "Entities",
"type": "object"
},
"guardrails": {
"additionalProperties": {
"$ref": "#/components/schemas/GuardrailMetric"
},
"title": "Guardrails",
"type": "object"
},
"mcp_servers": {
"additionalProperties": {
"$ref": "#/components/schemas/MetricWithMetadata"
@ -3802,6 +3809,52 @@
"title": "ExportType",
"type": "string"
},
"GuardrailKeyMetric": {
"properties": {
"metadata": {
"$ref": "#/components/schemas/KeyMetadata"
},
"metrics": {
"$ref": "#/components/schemas/GuardrailSpendMetrics"
}
},
"required": [
"metrics",
"metadata"
],
"title": "GuardrailKeyMetric",
"type": "object"
},
"GuardrailMetric": {
"properties": {
"api_key_breakdown": {
"additionalProperties": {
"$ref": "#/components/schemas/GuardrailKeyMetric"
},
"title": "Api Key Breakdown",
"type": "object"
},
"metrics": {
"$ref": "#/components/schemas/GuardrailSpendMetrics"
}
},
"required": [
"metrics"
],
"title": "GuardrailMetric",
"type": "object"
},
"GuardrailSpendMetrics": {
"properties": {
"spend": {
"default": 0.0,
"title": "Spend",
"type": "number"
}
},
"title": "GuardrailSpendMetrics",
"type": "object"
},
"HTTPAuthSecurityScheme": {
"description": "Defines a security scheme using HTTP authentication.",
"properties": {

48
litellm/proxy/management_endpoints/common_daily_activity.py
View file

@ -3,6 +3,7 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence
from collections.abc import Set as AbstractSet
from dataclasses import dataclass, replace
from datetime import date, datetime, timedelta
from itertools import groupby
from types import MappingProxyType
from typing import Final, Literal, NoReturn, Protocol
@ -28,6 +29,9 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import (
DailySpendData,
DailySpendMetadata,
GroupedData,
GuardrailKeyMetric,
GuardrailMetric,
GuardrailSpendMetrics,
KeyMetadata,
KeyMetricWithMetadata,
MetricWithMetadata,
@ -39,6 +43,7 @@ from litellm.types.repositories.daily_activity import (
DailyActivityTable,
EntityRollupRow,
GroupingSetsRow,
GuardrailSpendRow,
KeyMetadataRow,
RollupMetricsRow,
SpendLogsWindow,
@ -1013,17 +1018,52 @@ def _fold_entity_rollups_sync(
)
def _guardrail_metric(day: DailySpendData, rows: Sequence[GuardrailSpendRow]) -> GuardrailMetric:
return GuardrailMetric(
metrics=GuardrailSpendMetrics(spend=sum(row.spend for row in rows)),
api_key_breakdown={
row.api_key: GuardrailKeyMetric(
metrics=GuardrailSpendMetrics(spend=row.spend), metadata=day.breakdown.api_keys[row.api_key].metadata
)
for row in rows
if row.api_key in day.breakdown.api_keys
},
)
def _day_with_guardrails(day: DailySpendData, rows: Sequence[GuardrailSpendRow]) -> DailySpendData:
by_guardrail: Final = groupby(sorted(rows, key=lambda row: row.guardrail_id), key=lambda row: row.guardrail_id)
guardrails: Final = {guardrail_id: _guardrail_metric(day, tuple(group)) for guardrail_id, group in by_guardrail}
return day.model_copy(update={"breakdown": day.breakdown.model_copy(update={"guardrails": guardrails})})
def _with_guardrail_breakdown(
results: Sequence[DailySpendData], rows: Sequence[GuardrailSpendRow]
) -> tuple[DailySpendData, ...]:
"""API keys outside the top api_key_limit stay in each guardrail's spend but not in its api_key_breakdown."""
rows_by_date: Final = MappingProxyType(
{day: tuple(group) for day, group in groupby(sorted(rows, key=lambda row: row.date), key=lambda row: row.date)}
)
return tuple(_day_with_guardrails(day, rows_by_date.get(day.date.strftime("%Y-%m-%d"), ())) for day in results)
async def get_daily_activity_aggregated(
repository: DailyActivityRepository,
scope: DailyActivityScope,
*,
entity_metadata_field: Mapping[str, dict[str, object]] | None = None,
include_entity_breakdown: bool = False,
include_guardrail_breakdown: bool = False,
api_key_limit: int = constants.USAGE_TOP_API_KEYS_DEFAULT,
) -> SpendAnalyticsPaginatedResponse:
try:
aggregated_rows: Final = await repository.aggregated(
scope, include_entity_breakdown=include_entity_breakdown, api_key_limit=api_key_limit
aggregated_rows, guardrail_rows = await asyncio.gather(
repository.aggregated(
scope, include_entity_breakdown=include_entity_breakdown, api_key_limit=api_key_limit
),
repository.guardrail_spend(scope)
if include_guardrail_breakdown and scope.model is None
else asyncio.sleep(0, result=()),
)
records: Final = aggregated_rows.grouping_rows
aggregated: Final = await _aggregate_grouping_sets_records(
@ -1059,7 +1099,9 @@ async def get_daily_activity_aggregated(
entity_metadata_field=entity_metadata_field,
)
return SpendAnalyticsPaginatedResponse(
results=aggregated["results"],
results=list(_with_guardrail_breakdown(aggregated["results"], guardrail_rows))
if guardrail_rows
else aggregated["results"],
metadata=DailySpendMetadata(
total_spend=aggregated["totals"].spend,
total_flat_cost=aggregated["totals"].flat_cost,

1
litellm/proxy/management_endpoints/daily_activity_routes.py
View file

@ -300,6 +300,7 @@ def _register_aggregated_route(router: APIRouter, resolver: EntityScopeResolver,
resolved.scope,
entity_metadata_field=resolved.entity_metadata,
include_entity_breakdown=resolver.include_entity_breakdown,
include_guardrail_breakdown=resolver.include_guardrail_breakdown,
api_key_limit=api_key_limit,
)
if resolver.entity != "team":

2
litellm/proxy/management_endpoints/daily_activity_scopes.py
View file

@ -56,6 +56,7 @@ class EntityScopeResolver:
resolve: EntityScopeResolve
include_entity_breakdown: bool
operation_names: OperationNames
include_guardrail_breakdown: bool = False
def _query_ids(value: str | None) -> tuple[str, ...] | None:
@ -364,6 +365,7 @@ USER_RESOLVER = EntityScopeResolver(
query=_user_query,
resolve=_resolve_user,
include_entity_breakdown=False,
include_guardrail_breakdown=True,
operation_names=MappingProxyType(
{
"aggregated": "get_user_daily_activity_aggregated",

21
litellm/types/proxy/management_endpoints/common_daily_activity.py
View file

@ -65,6 +65,26 @@ class MetricWithMetadata(MetricBase):
api_key_breakdown: dict[str, KeyMetricWithMetadata] = Field(default_factory=dict) # api_key -> {metrics, metadata}
class GuardrailSpendMetrics(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
spend: float = 0.0
class GuardrailKeyMetric(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
metrics: GuardrailSpendMetrics
metadata: KeyMetadata
class GuardrailMetric(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
metrics: GuardrailSpendMetrics
api_key_breakdown: Mapping[str, GuardrailKeyMetric] = Field(default_factory=dict)
class BreakdownMetrics(LiteLLMBaseModel):
"""Breakdown of spend by different dimensions"""
@ -75,6 +95,7 @@ class BreakdownMetrics(LiteLLMBaseModel):
endpoints: dict[str, MetricWithMetadata] = Field(default_factory=dict) # endpoint -> {metrics, metadata}
api_keys: dict[str, KeyMetricWithMetadata] = Field(default_factory=dict) # api_key -> {metrics, metadata}
entities: dict[str, MetricWithMetadata] = Field(default_factory=dict) # entity -> {metrics, metadata}
guardrails: Mapping[str, GuardrailMetric] = Field(default_factory=dict) # guardrail_id -> spend
class DailySpendData(LiteLLMBaseModel):

85
tests/integration/spend/test_daily_activity_aggregated_breakdowns.py
View file

@ -243,3 +243,88 @@ async def test_get_daily_activity_aggregated_model_group_rollups_fall_back_to_mo
assert object_value(object_value(models["gpt-5"])["metrics"])["spend"] == 10.0
finally:
_clean(day)
def _seed_guardrail_units(day: str, rows: Sequence[tuple[str, str, str, float | None]]) -> None:
for guardrail_id, api_key, usage_unit, cost in rows:
write_rows(
'INSERT INTO "LiteLLM_DailyGuardrailUsageUnits" (guardrail_id, date, team_id, api_key, usage_unit,'
" units, cost, updated_at) VALUES (%s, %s, '', %s, %s, 1, %s, now())",
(guardrail_id, day, api_key, usage_unit, None if cost is None else str(cost)),
)
def _guardrail_spend(breakdown: dict[str, JsonValue]) -> dict[str, tuple[float, dict[str, float]]]:
return {
guardrail_id: (
float(object_value(object_value(entry)["metrics"])["spend"]),
{
api_key: float(object_value(object_value(key_entry)["metrics"])["spend"])
for api_key, key_entry in object_value(object_value(entry)["api_key_breakdown"]).items()
},
)
for guardrail_id, entry in object_value(breakdown["guardrails"]).items()
}
def _day_breakdown(gateway: Gateway, day: str, **params: str) -> dict[str, JsonValue]:
results: Final = _RESULTS.validate_python(_activity(gateway, day, **params)["results"])
assert len(results) == 1
return object_value(object_value(results[0])["breakdown"])
@pytest.mark.asyncio
async def test_get_daily_activity_aggregated_breaks_down_guardrail_spend(gateway: Gateway) -> None:
"""Guardrail cost per guardrail and key follows the request's user, model and api_key_limit scope.
orphan-key has guardrail cost but no spend row on the day, so no user scope reaches it. Model spend
keeps the guardrail cost it already carries."""
day: Final = _unique_day()
_seed(
day,
[
(_row_id(), "user-a", day, "key-a", "gpt-5", "", "bedrock", None, "/v1/chat/completions", 10, 1.0, 1, 1),
(_row_id(), "user-b", day, "key-b", "gpt-5", "", "bedrock", None, "/v1/chat/completions", 10, 2.0, 1, 1),
],
)
_seed_guardrail_units(
day,
[
("pii-filter", "key-a", "contentPolicyUnits", 0.10),
("pii-filter", "key-a", "sensitiveInformationPolicyUnits", 0.05),
("pii-filter", "key-b", "contentPolicyUnits", 0.20),
("topic-filter", "key-a", "topicPolicyUnits", 0.01),
("topic-filter", "key-b", "topicPolicyUnits", None),
("pii-filter", "orphan-key", "contentPolicyUnits", 9.0),
],
)
try:
everyone: Final = _day_breakdown(gateway, day)
assert _guardrail_spend(everyone) == {
"pii-filter": (pytest.approx(0.35), {"key-a": pytest.approx(0.15), "key-b": pytest.approx(0.20)}),
"topic-filter": (pytest.approx(0.01), {"key-a": pytest.approx(0.01), "key-b": 0.0}),
}
api_keys: Final = object_value(everyone["api_keys"])
assert object_value(everyone["guardrails"])["topic-filter"] == {
"metrics": {"spend": pytest.approx(0.01)},
"api_key_breakdown": {
"key-a": {
"metrics": {"spend": pytest.approx(0.01)},
"metadata": object_value(api_keys["key-a"])["metadata"],
},
"key-b": {"metrics": {"spend": 0.0}, "metadata": object_value(api_keys["key-b"])["metadata"]},
},
}
assert object_value(object_value(object_value(everyone["models"])["gpt-5"])["metrics"])["spend"] == 3.0
assert _guardrail_spend(_day_breakdown(gateway, day, user_id="user-a")) == {
"pii-filter": (pytest.approx(0.15), {"key-a": pytest.approx(0.15)}),
"topic-filter": (pytest.approx(0.01), {"key-a": pytest.approx(0.01)}),
}
assert _guardrail_spend(_day_breakdown(gateway, day, api_key_limit="1")) == {
"pii-filter": (pytest.approx(0.35), {"key-b": pytest.approx(0.20)}),
"topic-filter": (pytest.approx(0.01), {"key-b": 0.0}),
}
assert _guardrail_spend(_day_breakdown(gateway, day, model="gpt-5")) == {}
finally:
_clean(day)
write_rows('DELETE FROM "LiteLLM_DailyGuardrailUsageUnits" WHERE date = %s', (day,))

1
tests/unit/proxy/management_endpoints/test_daily_activity_routes.py
View file

@ -248,6 +248,7 @@ class _FakeRepository:
def __init__(self, rows: tuple[_Activity, ...]) -> None:
self._rows: Final = rows
self.aggregated = AsyncMock(side_effect=self._aggregated)
self.guardrail_spend = AsyncMock(return_value=())
self.key_page = AsyncMock(side_effect=self._key_page)
self.key_page_call: tuple[DailyActivityScope, int, int] | None = None
self.search_keys = AsyncMock(side_effect=self._search_keys)

25
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -27719,6 +27719,10 @@ export interface components {
entities?: {
[key: string]: components["schemas"]["MetricWithMetadata"];
};
/** Guardrails */
guardrails?: {
[key: string]: components["schemas"]["GuardrailMetric"];
};
/** Mcp Servers */
mcp_servers?: {
[key: string]: components["schemas"]["MetricWithMetadata"];
@ -33222,6 +33226,27 @@ export interface components {
/** Updated At */
updated_at?: string | null;
};
/** GuardrailKeyMetric */
GuardrailKeyMetric: {
metadata: components["schemas"]["KeyMetadata"];
metrics: components["schemas"]["GuardrailSpendMetrics"];
};
/** GuardrailMetric */
GuardrailMetric: {
/** Api Key Breakdown */
api_key_breakdown?: {
[key: string]: components["schemas"]["GuardrailKeyMetric"];
};
metrics: components["schemas"]["GuardrailSpendMetrics"];
};
/** GuardrailSpendMetrics */
GuardrailSpendMetrics: {
/**
* Spend
* @default 0
*/
spend: number;
};
/** GuardrailSubmissionItem */
GuardrailSubmissionItem: {
/** Created At */