scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 280

feat(daily-activity): add the guardrail breakdown to team and organization activity, with guardrail names

Browse source 
/team/daily/activity/aggregated and /organization/daily/activity/aggregated now return breakdown.guardrails the same way the user endpoint does, since each key belongs to one team and one organization. Each guardrail entry also carries metadata.guardrail_name, read from LiteLLM_GuardrailsTable for guardrails created through the UI or API and falling back to the breakdown key for config.yaml guardrails, whose key is already their name
This commit is contained in:
Mrinal Chanshetty 2026-10-28 10:14:14 -07:00
parent 68571af5da
commit 7e4f25a5e2
8 changed files with 108 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -3825,6 +3825,19 @@
"title": "GuardrailKeyMetric",
"type": "object"
},
"GuardrailMetadata": {
"properties": {
"guardrail_name": {
"title": "Guardrail Name",
"type": "string"
}
},
"required": [
"guardrail_name"
],
"title": "GuardrailMetadata",
"type": "object"
},
"GuardrailMetric": {
"properties": {
"api_key_breakdown": {
@ -3834,12 +3847,16 @@
"title": "Api Key Breakdown",
"type": "object"
},
"metadata": {
"$ref": "#/components/schemas/GuardrailMetadata"
},
"metrics": {
"$ref": "#/components/schemas/GuardrailSpendMetrics"
}
},
"required": [
"metrics"
"metrics",
"metadata"
],
"title": "GuardrailMetric",
"type": "object"

2
litellm/proxy/management_endpoints/common_daily_activity.py
View file

@ -30,6 +30,7 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import (
DailySpendMetadata,
GroupedData,
GuardrailKeyMetric,
GuardrailMetadata,
GuardrailMetric,
GuardrailSpendMetrics,
KeyMetadata,
@ -1021,6 +1022,7 @@ def _fold_entity_rollups_sync(
def _guardrail_metric(day: DailySpendData, rows: Sequence[GuardrailSpendRow]) -> GuardrailMetric:
return GuardrailMetric(
metrics=GuardrailSpendMetrics(spend=sum(row.spend for row in rows)),
metadata=GuardrailMetadata(guardrail_name=rows[0].guardrail_name),
api_key_breakdown={
row.api_key: GuardrailKeyMetric(
metrics=GuardrailSpendMetrics(spend=row.spend), metadata=day.breakdown.api_keys[row.api_key].metadata

2
litellm/proxy/management_endpoints/daily_activity_scopes.py
View file

@ -387,6 +387,7 @@ TEAM_RESOLVER = EntityScopeResolver(
query=_team_query,
resolve=_resolve_team,
include_entity_breakdown=True,
include_guardrail_breakdown=True,
operation_names=MappingProxyType(
{
"aggregated": "get_team_daily_activity_aggregated",
@ -425,6 +426,7 @@ ORGANIZATION_RESOLVER = EntityScopeResolver(
query=_organization_query,
resolve=_resolve_organization,
include_entity_breakdown=True,
include_guardrail_breakdown=True,
operation_names=MappingProxyType(
{
"aggregated": "get_organization_daily_activity_aggregated",

6
litellm/repositories/daily_activity_sql.py
View file

@ -265,14 +265,16 @@ def build_guardrail_spend_sql(scope: DailyActivityScope) -> SqlQuery:
pg_table: Final = PRISMA_TO_PG_TABLE[scope.table]
where_clause, where_params = build_where_clause(scope)
sql: Final = f"""
SELECT g.date, g.guardrail_id, g.api_key, COALESCE(SUM(g.cost), 0)::float AS spend
SELECT g.date, g.guardrail_id, COALESCE(t.guardrail_name, g.guardrail_id) AS guardrail_name, g.api_key,
COALESCE(SUM(g.cost), 0)::float AS spend
FROM "LiteLLM_DailyGuardrailUsageUnits" g
LEFT JOIN "LiteLLM_GuardrailsTable" t ON t.guardrail_id = g.guardrail_id
WHERE g.date >= $1 AND g.date <= $2
AND EXISTS (
SELECT 1 FROM "{pg_table}"
WHERE {where_clause} AND "{pg_table}".date = g.date AND "{pg_table}".api_key = g.api_key
)
GROUP BY g.date, g.guardrail_id, g.api_key
GROUP BY g.date, g.guardrail_id, t.guardrail_name, g.api_key
"""
return SqlQuery(sql=sql, params=where_params)

7
litellm/types/proxy/management_endpoints/common_daily_activity.py
View file

@ -78,10 +78,17 @@ class GuardrailKeyMetric(LiteLLMBaseModel):
metadata: KeyMetadata
class GuardrailMetadata(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
guardrail_name: str
class GuardrailMetric(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
metrics: GuardrailSpendMetrics
metadata: GuardrailMetadata
api_key_breakdown: Mapping[str, GuardrailKeyMetric] = Field(default_factory=dict)

1
litellm/types/repositories/daily_activity.py
View file

@ -172,6 +172,7 @@ class EntityRollupRow(RollupMetricsRow):
class GuardrailSpendRow:
date: str
guardrail_id: str
guardrail_name: str
api_key: str
spend: float

68
tests/integration/spend/test_daily_activity_aggregated_breakdowns.py
View file

@ -306,6 +306,7 @@ async def test_get_daily_activity_aggregated_breaks_down_guardrail_spend(gateway
api_keys: Final = object_value(everyone["api_keys"])
assert object_value(everyone["guardrails"])["topic-filter"] == {
"metrics": {"spend": pytest.approx(0.01)},
"metadata": {"guardrail_name": "topic-filter"},
"api_key_breakdown": {
"key-a": {
"metrics": {"spend": pytest.approx(0.01)},
@ -328,3 +329,70 @@ async def test_get_daily_activity_aggregated_breaks_down_guardrail_spend(gateway
finally:
_clean(day)
write_rows('DELETE FROM "LiteLLM_DailyGuardrailUsageUnits" WHERE date = %s', (day,))
def _entity_breakdown(gateway: Gateway, url: str, day: str, **params: str) -> dict[str, JsonValue]:
response: Final = gateway.request("GET", url, params={"start_date": day, "end_date": day, **params})
assert response.status_code == 200, response.text
results: Final = _RESULTS.validate_python(object_value(response.json())["results"])
assert len(results) == 1
return object_value(object_value(results[0])["breakdown"])
@pytest.mark.asyncio
@pytest.mark.parametrize(
("url", "table", "entity_column", "filter_param"),
[
("/team/daily/activity/aggregated", "LiteLLM_DailyTeamSpend", "team_id", "team_ids"),
(
"/organization/daily/activity/aggregated",
"LiteLLM_DailyOrganizationSpend",
"organization_id",
"organization_ids",
),
],
)
async def test_team_and_organization_activity_break_down_guardrail_spend_by_name(
gateway: Gateway, url: str, table: str, entity_column: str, filter_param: str
) -> None:
day: Final = _unique_day()
db_guardrail_id: Final = str(uuid.uuid4())
db_guardrail_name: Final = f"pii-filter-{db_guardrail_id[:8]}"
for entity_id, api_key, spend in (("entity-a", "key-a", 1.0), ("entity-b", "key-b", 2.0)):
write_rows(
f'INSERT INTO "{table}" (id, {entity_column}, date, api_key, model, spend, api_requests,'
" successful_requests, updated_at) VALUES (%s, %s, %s, %s, 'gpt-5', %s, 1, 1, now())",
(_row_id(), entity_id, day, api_key, str(spend)),
)
write_rows(
'INSERT INTO "LiteLLM_GuardrailsTable" (guardrail_id, guardrail_name, litellm_params, updated_at)'
" VALUES (%s, %s, '{}', now())",
(db_guardrail_id, db_guardrail_name),
)
_seed_guardrail_units(
day,
[
(db_guardrail_id, "key-a", "contentPolicyUnits", 0.10),
(db_guardrail_id, "key-b", "contentPolicyUnits", 0.20),
("config-topic-filter", "key-a", "topicPolicyUnits", 0.05),
(db_guardrail_id, "orphan-key", "contentPolicyUnits", 9.0),
],
)
try:
everyone: Final = _entity_breakdown(gateway, url, day)
assert _guardrail_spend(everyone) == {
db_guardrail_id: (pytest.approx(0.30), {"key-a": pytest.approx(0.10), "key-b": pytest.approx(0.20)}),
"config-topic-filter": (pytest.approx(0.05), {"key-a": pytest.approx(0.05)}),
}
guardrails: Final = object_value(everyone["guardrails"])
assert object_value(guardrails[db_guardrail_id])["metadata"] == {"guardrail_name": db_guardrail_name}
assert object_value(guardrails["config-topic-filter"])["metadata"] == {"guardrail_name": "config-topic-filter"}
assert _guardrail_spend(_entity_breakdown(gateway, url, day, **{filter_param: "entity-a"})) == {
db_guardrail_id: (pytest.approx(0.10), {"key-a": pytest.approx(0.10)}),
"config-topic-filter": (pytest.approx(0.05), {"key-a": pytest.approx(0.05)}),
}
finally:
write_rows(f'DELETE FROM "{table}" WHERE date = %s', (day,))
write_rows('DELETE FROM "LiteLLM_DailyGuardrailUsageUnits" WHERE date = %s', (day,))
write_rows('DELETE FROM "LiteLLM_GuardrailsTable" WHERE guardrail_id = %s', (db_guardrail_id,))

6
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -33231,12 +33231,18 @@ export interface components {
metadata: components["schemas"]["KeyMetadata"];
metrics: components["schemas"]["GuardrailSpendMetrics"];
};
/** GuardrailMetadata */
GuardrailMetadata: {
/** Guardrail Name */
guardrail_name: string;
};
/** GuardrailMetric */
GuardrailMetric: {
/** Api Key Breakdown */
api_key_breakdown?: {
[key: string]: components["schemas"]["GuardrailKeyMetric"];
};
metadata: components["schemas"]["GuardrailMetadata"];
metrics: components["schemas"]["GuardrailSpendMetrics"];
};
/** GuardrailSpendMetrics */