From 1d375d8ada91eb6f6aeceb8af8bc649a031a5012 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 4 Sep 2026 19:46:14 -0700 Subject: [PATCH] fix(guardrails): flag unpriced units per team and key, sort unknown cost last The detail endpoint now returns untracked_usage_units_by_team and untracked_usage_units_by_key next to the cost breakdowns, and the By team and By key tables show them in an Unpriced Units column, so a row that pairs its total units with a partial cost says how many units that cost leaves out. The overview comparator no longer treats a missing cost as zero: guardrails with no known cost sort last in both directions instead of mixing in with genuinely free ones. Refs LIT-5652 --- litellm/proxy/_lazy_openapi_snapshot.json | 24 ++++++++++- litellm/proxy/guardrails/usage_endpoints.py | 20 ++++++++-- .../proxy/guardrails/test_usage_endpoints.py | 8 ++++ .../_components/GuardrailDetail.test.tsx | 2 + .../GuardrailUsageBreakdown.test.tsx | 11 ++++- .../_components/GuardrailUsageBreakdown.tsx | 40 ++++++++++++------- .../_components/GuardrailsOverview.test.tsx | 16 ++++++-- .../_components/GuardrailsOverview.tsx | 9 +++-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 12 ++++++ 9 files changed, 114 insertions(+), 28 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index c24eea968f8..ddf6a59bea7 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -13218,6 +13218,26 @@ "title": "Untracked Usage Units", "type": "object" }, + "untracked_usage_units_by_key": { + "additionalProperties": { + "additionalProperties": { + "type": "integer" + }, + "type": "object" + }, + "title": "Untracked Usage Units By Key", + "type": "object" + }, + "untracked_usage_units_by_team": { + "additionalProperties": { + "additionalProperties": { + "type": "integer" + }, + "type": "object" + }, + "title": "Untracked Usage Units By Team", + "type": "object" + }, "usage_units": { "additionalProperties": { "type": "integer" @@ -13274,7 +13294,9 @@ "cost_by_unit", "cost_by_team", "cost_by_key", - "untracked_usage_units" + "untracked_usage_units", + "untracked_usage_units_by_team", + "untracked_usage_units_by_key" ], "title": "UsageDetailResponse", "type": "object" diff --git a/litellm/proxy/guardrails/usage_endpoints.py b/litellm/proxy/guardrails/usage_endpoints.py index 0390a2b5013..d3bd09f7d27 100644 --- a/litellm/proxy/guardrails/usage_endpoints.py +++ b/litellm/proxy/guardrails/usage_endpoints.py @@ -156,6 +156,14 @@ def _counter_name(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> str: return row.usage_unit +def _team_of(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> str: + return row.team_id + + +def _key_of(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> str: + return row.api_key + + def _row_untracked_units(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> int: """A row written before the cost column carries NULL cost and is untracked in full.""" return int(row.units) if row.cost is None else int(row.untracked_units) @@ -308,6 +316,8 @@ class UsageDetailResponse(BaseModel): cost_by_team: Mapping[str, float | None] cost_by_key: Mapping[str, float | None] untracked_usage_units: Mapping[str, int] + untracked_usage_units_by_team: Mapping[str, Mapping[str, int]] + untracked_usage_units_by_key: Mapping[str, Mapping[str, int]] class UsageLogEntry(BaseModel): @@ -705,13 +715,15 @@ async def guardrails_usage_detail( time_series=time_series, usage_units=_sum_counter_units(units_rows), usage_units_daily=units_daily, - usage_units_by_team=_by(units_rows, lambda r: r.team_id, _sum_counter_units), - usage_units_by_key=_by(units_rows, lambda r: r.api_key, _sum_counter_units), + usage_units_by_team=_by(units_rows, _team_of, _sum_counter_units), + usage_units_by_key=_by(units_rows, _key_of, _sum_counter_units), cost=_sum_tracked_cost(units_rows), cost_by_unit=_by(units_rows, _counter_name, _sum_tracked_cost), - cost_by_team=_by(units_rows, lambda r: r.team_id, _sum_tracked_cost), - cost_by_key=_by(units_rows, lambda r: r.api_key, _sum_tracked_cost), + cost_by_team=_by(units_rows, _team_of, _sum_tracked_cost), + cost_by_key=_by(units_rows, _key_of, _sum_tracked_cost), untracked_usage_units=_sum_untracked_units(units_rows), + untracked_usage_units_by_team=_by(units_rows, _team_of, _sum_untracked_units), + untracked_usage_units_by_key=_by(units_rows, _key_of, _sum_untracked_units), ) diff --git a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py index ebb2be6edc2..1ff33c76035 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py @@ -430,6 +430,13 @@ async def test_detail_breaks_cost_down_by_unit_day_team_and_key(): assert resp.cost_by_team.keys() == resp.usage_units_by_team.keys() assert resp.cost_by_key.keys() == resp.usage_units_by_key.keys() assert resp.untracked_usage_units == {"contentPolicyUnits": 50, "topicPolicyUnits": 10} + assert resp.untracked_usage_units_by_team == {"team-a": {"topicPolicyUnits": 10}, "": {"contentPolicyUnits": 50}} + assert resp.untracked_usage_units_by_key == { + "hash-1": {"topicPolicyUnits": 10}, + "hash-2": {"contentPolicyUnits": 50}, + } + assert resp.untracked_usage_units_by_team.keys() == resp.usage_units_by_team.keys() + assert resp.untracked_usage_units_by_key.keys() == resp.usage_units_by_key.keys() @pytest.mark.asyncio @@ -451,6 +458,7 @@ async def test_detail_degrades_units_to_empty_when_units_table_is_missing(): ) assert (resp.cost, resp.cost_by_unit, resp.cost_by_team, resp.cost_by_key) == (None, {}, {}, {}) assert resp.untracked_usage_units == {} + assert (resp.untracked_usage_units_by_team, resp.untracked_usage_units_by_key) == ({}, {}) # ---- logs ------------------------------------------------------------------- diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx index bbcb8138d52..3d00e29245d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx @@ -45,6 +45,8 @@ const detail: GuardrailUsageDetail = { cost_by_team: { "": 0.0004 }, cost_by_key: { "hash-1": 0.0004 }, untracked_usage_units: {}, + untracked_usage_units_by_team: {}, + untracked_usage_units_by_key: {}, }; const loaded = (data: GuardrailUsageDetail | undefined) => ({ data, isLoading: false, error: null }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx index 3b24185f0b6..7929b97b000 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx @@ -31,6 +31,8 @@ const detail: GuardrailUsageDetail = { cost_by_team: { "team-a": 0.165, "": 0.015 }, cost_by_key: { "hash-1": 0.18, "hash-2": null }, untracked_usage_units: { someFutureCounter: 7 }, + untracked_usage_units_by_team: { "team-a": {}, "": { someFutureCounter: 7 } }, + untracked_usage_units_by_key: { "hash-1": {}, "hash-2": { someFutureCounter: 7 } }, }; const rowNamed = (name: string) => screen.getByRole("row", { name: new RegExp(name) }); @@ -61,7 +63,7 @@ describe("GuardrailUsageBreakdown", () => { expect(within(future).getByText("—")).toBeInTheDocument(); }); - it("breaks units and cost down by team and by key, naming the rows without one", () => { + it("breaks units and cost down by team and by key, flagging the unpriced share of each row", () => { render(); expect(screen.getByRole("heading", { name: "By team" })).toBeInTheDocument(); @@ -69,14 +71,17 @@ describe("GuardrailUsageBreakdown", () => { const teamA = rowNamed("team-a"); expect(within(teamA).getByText("1,200")).toBeInTheDocument(); expect(within(teamA).getByText("$0.1650")).toBeInTheDocument(); + expect(within(teamA).getByText("—")).toBeInTheDocument(); + expect(within(teamA).queryByText("7")).not.toBeInTheDocument(); const noTeam = rowNamed("No team"); expect(within(noTeam).getByText("107")).toBeInTheDocument(); expect(within(noTeam).getByText("$0.0150")).toBeInTheDocument(); + expect(within(noTeam).getByText("7", { selector: ".text-warning" })).toBeInTheDocument(); const unpricedKey = rowNamed("hash-2"); - expect(within(unpricedKey).getByText("7")).toBeInTheDocument(); expect(within(unpricedKey).getByText("—")).toBeInTheDocument(); + expect(within(unpricedKey).getByText("7", { selector: ".text-warning" })).toBeInTheDocument(); }); it("orders teams and keys by units, largest first", () => { @@ -104,6 +109,8 @@ describe("GuardrailUsageBreakdown", () => { cost_by_team: {}, cost_by_key: {}, untracked_usage_units: {}, + untracked_usage_units_by_team: {}, + untracked_usage_units_by_key: {}, }} />, ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx index 1eaab86c506..6e8b725aa2e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx @@ -19,6 +19,7 @@ interface GroupRow { id: string; units: number; cost: number | null; + unpriced: number; } const counterRows = (detail: GuardrailUsageDetail): CounterRow[] => @@ -32,11 +33,31 @@ const counterRows = (detail: GuardrailUsageDetail): CounterRow[] => const groupRows = ( unitsByGroup: GuardrailUsageDetail["usage_units_by_team"], costByGroup: GuardrailUsageDetail["cost_by_team"], + untrackedByGroup: GuardrailUsageDetail["untracked_usage_units_by_team"], ): GroupRow[] => Object.entries(unitsByGroup) - .map(([id, units]) => ({ id, units: totalUnits(units), cost: costByGroup[id] ?? null })) + .map(([id, units]) => ({ + id, + units: totalUnits(units), + cost: costByGroup[id] ?? null, + unpriced: totalUnits(untrackedByGroup[id] ?? {}), + })) .sort((a, b) => b.units - a.units); +const UnpricedUnitsCell = ({ unpriced }: { unpriced: number }) => + unpriced > 0 ? ( + {unpriced.toLocaleString()} + ) : ( + — + ); + +const unpricedColumn = (): ColumnDef => ({ + header: "Unpriced Units", + accessorKey: "unpriced", + meta: { numeric: true }, + cell: ({ row }) => , +}); + const counterColumns: ColumnDef[] = [ { header: "Counter", accessorKey: "counter", cell: ({ row }) => counterLabel(row.original.counter) }, { @@ -51,17 +72,7 @@ const counterColumns: ColumnDef[] = [ meta: { numeric: true }, cell: ({ row }) => , }, - { - header: "Unpriced Units", - accessorKey: "unpriced", - meta: { numeric: true }, - cell: ({ row }) => - row.original.unpriced > 0 ? ( - {row.original.unpriced.toLocaleString()} - ) : ( - — - ), - }, + unpricedColumn(), ]; const groupColumns = (label: string, emptyLabel: string): ColumnDef[] => [ @@ -87,6 +98,7 @@ const groupColumns = (label: string, emptyLabel: string): ColumnDef[] meta: { numeric: true }, cell: ({ row }) => , }, + unpricedColumn(), ]; const teamColumns = groupColumns("Team", "No team"); @@ -139,14 +151,14 @@ export function GuardrailUsageBreakdown({ detail }: { detail: GuardrailUsageDeta
row.id || "no-team"} size="compact" toolbar={() => } /> row.id || "no-key"} size="compact" toolbar={() => } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx index 3a56667b156..c52645def70 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx @@ -160,14 +160,24 @@ describe("GuardrailsOverview", () => { expect(screen.getByText("Sensitive Information Policy: 250")).toBeInTheDocument(); }); - it("sorts by cost when its header is clicked", async () => { + it("sorts by cost when its header is clicked, keeping guardrails with no known cost last either way", async () => { const user = userEvent.setup(); renderOverview(); + const rowNames = () => + screen + .getAllByRole("row") + .slice(1) + .map((r) => r.textContent ?? ""); await user.click(await screen.findByRole("button", { name: /Cost/ })); + await waitFor(() => expect(rowNames()[0]).toContain("Free Bedrock Guardrail")); + expect(rowNames()[1]).toContain("High Failure Guardrail"); + expect(rowNames()[2]).toContain("Low Failure Guardrail"); - await waitFor(() => expect(screen.getAllByRole("row")[1]).toHaveTextContent("Low Failure Guardrail")); - expect(screen.getAllByRole("row")[3]).toHaveTextContent("High Failure Guardrail"); + await user.click(screen.getByRole("button", { name: /Cost/ })); + await waitFor(() => expect(rowNames()[0]).toContain("High Failure Guardrail")); + expect(rowNames()[1]).toContain("Free Bedrock Guardrail"); + expect(rowNames()[2]).toContain("Low Failure Guardrail"); }); it("renders the page header and the export action", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx index 67630fef13a..0bbda6015b4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx @@ -112,11 +112,12 @@ export function GuardrailsOverview({ }, [guardrailsData, activeData]); const chartData = guardrailsData?.chart; const sorted = useMemo(() => { + const mult = sortDir === "desc" ? -1 : 1; return [...activeData].sort((a, b) => { - const mult = sortDir === "desc" ? -1 : 1; - const aVal = a[sortBy] ?? 0; - const bVal = b[sortBy] ?? 0; - return (Number(aVal) - Number(bVal)) * mult; + const aVal = a[sortBy]; + const bVal = b[sortBy]; + if (aVal == null || bVal == null) return Number(aVal == null) - Number(bVal == null); + return (aVal - bVal) * mult; }); }, [activeData, sortBy, sortDir]); const isLoading = guardrailsLoading; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 427e5deb555..c1f65299c52 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -38461,6 +38461,18 @@ export interface components { untracked_usage_units: { [key: string]: number; }; + /** Untracked Usage Units By Key */ + untracked_usage_units_by_key: { + [key: string]: { + [key: string]: number; + }; + }; + /** Untracked Usage Units By Team */ + untracked_usage_units_by_team: { + [key: string]: { + [key: string]: number; + }; + }; /** Usage Units */ usage_units: { [key: string]: number;