diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 9d87671392e..1d61eb9af4d 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -3825,6 +3825,19 @@ "title": "GuardrailKeyMetric", "type": "object" }, + "GuardrailMetadata": { + "properties": { + "guardrail_name": { + "title": "Guardrail Name", + "type": "string" + } + }, + "required": [ + "guardrail_name" + ], + "title": "GuardrailMetadata", + "type": "object" + }, "GuardrailMetric": { "properties": { "api_key_breakdown": { @@ -3834,12 +3847,16 @@ "title": "Api Key Breakdown", "type": "object" }, + "metadata": { + "$ref": "#/components/schemas/GuardrailMetadata" + }, "metrics": { "$ref": "#/components/schemas/GuardrailSpendMetrics" } }, "required": [ - "metrics" + "metrics", + "metadata" ], "title": "GuardrailMetric", "type": "object" diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index de905369ac5..6b51a77b036 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -30,6 +30,7 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import ( DailySpendMetadata, GroupedData, GuardrailKeyMetric, + GuardrailMetadata, GuardrailMetric, GuardrailSpendMetrics, KeyMetadata, @@ -1021,6 +1022,7 @@ def _fold_entity_rollups_sync( def _guardrail_metric(day: DailySpendData, rows: Sequence[GuardrailSpendRow]) -> GuardrailMetric: return GuardrailMetric( metrics=GuardrailSpendMetrics(spend=sum(row.spend for row in rows)), + metadata=GuardrailMetadata(guardrail_name=rows[0].guardrail_name), api_key_breakdown={ row.api_key: GuardrailKeyMetric( metrics=GuardrailSpendMetrics(spend=row.spend), metadata=day.breakdown.api_keys[row.api_key].metadata diff --git a/litellm/proxy/management_endpoints/daily_activity_scopes.py b/litellm/proxy/management_endpoints/daily_activity_scopes.py index af2e2935e49..77246f5dda6 100644 --- a/litellm/proxy/management_endpoints/daily_activity_scopes.py +++ b/litellm/proxy/management_endpoints/daily_activity_scopes.py @@ -387,6 +387,7 @@ TEAM_RESOLVER = EntityScopeResolver( query=_team_query, resolve=_resolve_team, include_entity_breakdown=True, + include_guardrail_breakdown=True, operation_names=MappingProxyType( { "aggregated": "get_team_daily_activity_aggregated", @@ -425,6 +426,7 @@ ORGANIZATION_RESOLVER = EntityScopeResolver( query=_organization_query, resolve=_resolve_organization, include_entity_breakdown=True, + include_guardrail_breakdown=True, operation_names=MappingProxyType( { "aggregated": "get_organization_daily_activity_aggregated", diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index bddc49fdb13..9a3770d7803 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -265,14 +265,16 @@ def build_guardrail_spend_sql(scope: DailyActivityScope) -> SqlQuery: pg_table: Final = PRISMA_TO_PG_TABLE[scope.table] where_clause, where_params = build_where_clause(scope) sql: Final = f""" - SELECT g.date, g.guardrail_id, g.api_key, COALESCE(SUM(g.cost), 0)::float AS spend + SELECT g.date, g.guardrail_id, COALESCE(t.guardrail_name, g.guardrail_id) AS guardrail_name, g.api_key, + COALESCE(SUM(g.cost), 0)::float AS spend FROM "LiteLLM_DailyGuardrailUsageUnits" g + LEFT JOIN "LiteLLM_GuardrailsTable" t ON t.guardrail_id = g.guardrail_id WHERE g.date >= $1 AND g.date <= $2 AND EXISTS ( SELECT 1 FROM "{pg_table}" WHERE {where_clause} AND "{pg_table}".date = g.date AND "{pg_table}".api_key = g.api_key ) - GROUP BY g.date, g.guardrail_id, g.api_key + GROUP BY g.date, g.guardrail_id, t.guardrail_name, g.api_key """ return SqlQuery(sql=sql, params=where_params) diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 593f21aa2c4..728b95cd8c0 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -78,10 +78,17 @@ class GuardrailKeyMetric(LiteLLMBaseModel): metadata: KeyMetadata +class GuardrailMetadata(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + guardrail_name: str + + class GuardrailMetric(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) metrics: GuardrailSpendMetrics + metadata: GuardrailMetadata api_key_breakdown: Mapping[str, GuardrailKeyMetric] = Field(default_factory=dict) diff --git a/litellm/types/repositories/daily_activity.py b/litellm/types/repositories/daily_activity.py index b6c4c1e85ba..8c874a5acaf 100644 --- a/litellm/types/repositories/daily_activity.py +++ b/litellm/types/repositories/daily_activity.py @@ -172,6 +172,7 @@ class EntityRollupRow(RollupMetricsRow): class GuardrailSpendRow: date: str guardrail_id: str + guardrail_name: str api_key: str spend: float diff --git a/tests/integration/spend/test_daily_activity_aggregated_breakdowns.py b/tests/integration/spend/test_daily_activity_aggregated_breakdowns.py index 4eebf6c4a96..2d42a5c9e83 100644 --- a/tests/integration/spend/test_daily_activity_aggregated_breakdowns.py +++ b/tests/integration/spend/test_daily_activity_aggregated_breakdowns.py @@ -306,6 +306,7 @@ async def test_get_daily_activity_aggregated_breaks_down_guardrail_spend(gateway api_keys: Final = object_value(everyone["api_keys"]) assert object_value(everyone["guardrails"])["topic-filter"] == { "metrics": {"spend": pytest.approx(0.01)}, + "metadata": {"guardrail_name": "topic-filter"}, "api_key_breakdown": { "key-a": { "metrics": {"spend": pytest.approx(0.01)}, @@ -328,3 +329,70 @@ async def test_get_daily_activity_aggregated_breaks_down_guardrail_spend(gateway finally: _clean(day) write_rows('DELETE FROM "LiteLLM_DailyGuardrailUsageUnits" WHERE date = %s', (day,)) + + +def _entity_breakdown(gateway: Gateway, url: str, day: str, **params: str) -> dict[str, JsonValue]: + response: Final = gateway.request("GET", url, params={"start_date": day, "end_date": day, **params}) + assert response.status_code == 200, response.text + results: Final = _RESULTS.validate_python(object_value(response.json())["results"]) + assert len(results) == 1 + return object_value(object_value(results[0])["breakdown"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("url", "table", "entity_column", "filter_param"), + [ + ("/team/daily/activity/aggregated", "LiteLLM_DailyTeamSpend", "team_id", "team_ids"), + ( + "/organization/daily/activity/aggregated", + "LiteLLM_DailyOrganizationSpend", + "organization_id", + "organization_ids", + ), + ], +) +async def test_team_and_organization_activity_break_down_guardrail_spend_by_name( + gateway: Gateway, url: str, table: str, entity_column: str, filter_param: str +) -> None: + day: Final = _unique_day() + db_guardrail_id: Final = str(uuid.uuid4()) + db_guardrail_name: Final = f"pii-filter-{db_guardrail_id[:8]}" + for entity_id, api_key, spend in (("entity-a", "key-a", 1.0), ("entity-b", "key-b", 2.0)): + write_rows( + f'INSERT INTO "{table}" (id, {entity_column}, date, api_key, model, spend, api_requests,' + " successful_requests, updated_at) VALUES (%s, %s, %s, %s, 'gpt-5', %s, 1, 1, now())", + (_row_id(), entity_id, day, api_key, str(spend)), + ) + write_rows( + 'INSERT INTO "LiteLLM_GuardrailsTable" (guardrail_id, guardrail_name, litellm_params, updated_at)' + " VALUES (%s, %s, '{}', now())", + (db_guardrail_id, db_guardrail_name), + ) + _seed_guardrail_units( + day, + [ + (db_guardrail_id, "key-a", "contentPolicyUnits", 0.10), + (db_guardrail_id, "key-b", "contentPolicyUnits", 0.20), + ("config-topic-filter", "key-a", "topicPolicyUnits", 0.05), + (db_guardrail_id, "orphan-key", "contentPolicyUnits", 9.0), + ], + ) + try: + everyone: Final = _entity_breakdown(gateway, url, day) + assert _guardrail_spend(everyone) == { + db_guardrail_id: (pytest.approx(0.30), {"key-a": pytest.approx(0.10), "key-b": pytest.approx(0.20)}), + "config-topic-filter": (pytest.approx(0.05), {"key-a": pytest.approx(0.05)}), + } + guardrails: Final = object_value(everyone["guardrails"]) + assert object_value(guardrails[db_guardrail_id])["metadata"] == {"guardrail_name": db_guardrail_name} + assert object_value(guardrails["config-topic-filter"])["metadata"] == {"guardrail_name": "config-topic-filter"} + + assert _guardrail_spend(_entity_breakdown(gateway, url, day, **{filter_param: "entity-a"})) == { + db_guardrail_id: (pytest.approx(0.10), {"key-a": pytest.approx(0.10)}), + "config-topic-filter": (pytest.approx(0.05), {"key-a": pytest.approx(0.05)}), + } + finally: + write_rows(f'DELETE FROM "{table}" WHERE date = %s', (day,)) + write_rows('DELETE FROM "LiteLLM_DailyGuardrailUsageUnits" WHERE date = %s', (day,)) + write_rows('DELETE FROM "LiteLLM_GuardrailsTable" WHERE guardrail_id = %s', (db_guardrail_id,)) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index b136c0dba9d..b8e33369588 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -33231,12 +33231,18 @@ export interface components { metadata: components["schemas"]["KeyMetadata"]; metrics: components["schemas"]["GuardrailSpendMetrics"]; }; + /** GuardrailMetadata */ + GuardrailMetadata: { + /** Guardrail Name */ + guardrail_name: string; + }; /** GuardrailMetric */ GuardrailMetric: { /** Api Key Breakdown */ api_key_breakdown?: { [key: string]: components["schemas"]["GuardrailKeyMetric"]; }; + metadata: components["schemas"]["GuardrailMetadata"]; metrics: components["schemas"]["GuardrailSpendMetrics"]; }; /** GuardrailSpendMetrics */