scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

fix(guardrails): flag unpriced units per team and key, sort unknown cost last

Browse source 
The detail endpoint now returns untracked_usage_units_by_team and
untracked_usage_units_by_key next to the cost breakdowns, and the By team and
By key tables show them in an Unpriced Units column, so a row that pairs its
total units with a partial cost says how many units that cost leaves out.

The overview comparator no longer treats a missing cost as zero: guardrails
with no known cost sort last in both directions instead of mixing in with
genuinely free ones.

Refs LIT-5652
This commit is contained in:
ryan-crabbe-berri 2026-09-04 19:46:14 -07:00
parent bde3f6ae46
commit 1d375d8ada
9 changed files with 114 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -13218,6 +13218,26 @@
"title": "Untracked Usage Units",
"type": "object"
},
"untracked_usage_units_by_key": {
"additionalProperties": {
"additionalProperties": {
"type": "integer"
},
"type": "object"
},
"title": "Untracked Usage Units By Key",
"type": "object"
},
"untracked_usage_units_by_team": {
"additionalProperties": {
"additionalProperties": {
"type": "integer"
},
"type": "object"
},
"title": "Untracked Usage Units By Team",
"type": "object"
},
"usage_units": {
"additionalProperties": {
"type": "integer"
@ -13274,7 +13294,9 @@
"cost_by_unit",
"cost_by_team",
"cost_by_key",
"untracked_usage_units"
"untracked_usage_units",
"untracked_usage_units_by_team",
"untracked_usage_units_by_key"
],
"title": "UsageDetailResponse",
"type": "object"

20
litellm/proxy/guardrails/usage_endpoints.py
View file

@ -156,6 +156,14 @@ def _counter_name(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> str:
return row.usage_unit
def _team_of(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> str:
return row.team_id
def _key_of(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> str:
return row.api_key
def _row_untracked_units(row: "prisma_models.LiteLLM_DailyGuardrailUsageUnits") -> int:
"""A row written before the cost column carries NULL cost and is untracked in full."""
return int(row.units) if row.cost is None else int(row.untracked_units)
@ -308,6 +316,8 @@ class UsageDetailResponse(BaseModel):
cost_by_team: Mapping[str, float | None]
cost_by_key: Mapping[str, float | None]
untracked_usage_units: Mapping[str, int]
untracked_usage_units_by_team: Mapping[str, Mapping[str, int]]
untracked_usage_units_by_key: Mapping[str, Mapping[str, int]]
class UsageLogEntry(BaseModel):
@ -705,13 +715,15 @@ async def guardrails_usage_detail(
time_series=time_series,
usage_units=_sum_counter_units(units_rows),
usage_units_daily=units_daily,
usage_units_by_team=_by(units_rows, lambda r: r.team_id, _sum_counter_units),
usage_units_by_key=_by(units_rows, lambda r: r.api_key, _sum_counter_units),
usage_units_by_team=_by(units_rows, _team_of, _sum_counter_units),
usage_units_by_key=_by(units_rows, _key_of, _sum_counter_units),
cost=_sum_tracked_cost(units_rows),
cost_by_unit=_by(units_rows, _counter_name, _sum_tracked_cost),
cost_by_team=_by(units_rows, lambda r: r.team_id, _sum_tracked_cost),
cost_by_key=_by(units_rows, lambda r: r.api_key, _sum_tracked_cost),
cost_by_team=_by(units_rows, _team_of, _sum_tracked_cost),
cost_by_key=_by(units_rows, _key_of, _sum_tracked_cost),
untracked_usage_units=_sum_untracked_units(units_rows),
untracked_usage_units_by_team=_by(units_rows, _team_of, _sum_untracked_units),
untracked_usage_units_by_key=_by(units_rows, _key_of, _sum_untracked_units),
)

8
tests/test_litellm/proxy/guardrails/test_usage_endpoints.py
View file

@ -430,6 +430,13 @@ async def test_detail_breaks_cost_down_by_unit_day_team_and_key():
assert resp.cost_by_team.keys() == resp.usage_units_by_team.keys()
assert resp.cost_by_key.keys() == resp.usage_units_by_key.keys()
assert resp.untracked_usage_units == {"contentPolicyUnits": 50, "topicPolicyUnits": 10}
assert resp.untracked_usage_units_by_team == {"team-a": {"topicPolicyUnits": 10}, "": {"contentPolicyUnits": 50}}
assert resp.untracked_usage_units_by_key == {
"hash-1": {"topicPolicyUnits": 10},
"hash-2": {"contentPolicyUnits": 50},
}
assert resp.untracked_usage_units_by_team.keys() == resp.usage_units_by_team.keys()
assert resp.untracked_usage_units_by_key.keys() == resp.usage_units_by_key.keys()
@pytest.mark.asyncio
@ -451,6 +458,7 @@ async def test_detail_degrades_units_to_empty_when_units_table_is_missing():
)
assert (resp.cost, resp.cost_by_unit, resp.cost_by_team, resp.cost_by_key) == (None, {}, {}, {})
assert resp.untracked_usage_units == {}
assert (resp.untracked_usage_units_by_team, resp.untracked_usage_units_by_key) == ({}, {})
# ---- logs -------------------------------------------------------------------

2
ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.test.tsx
View file

@ -45,6 +45,8 @@ const detail: GuardrailUsageDetail = {
cost_by_team: { "": 0.0004 },
cost_by_key: { "hash-1": 0.0004 },
untracked_usage_units: {},
untracked_usage_units_by_team: {},
untracked_usage_units_by_key: {},
};
const loaded = (data: GuardrailUsageDetail | undefined) => ({ data, isLoading: false, error: null });

11
ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx
View file

@ -31,6 +31,8 @@ const detail: GuardrailUsageDetail = {
cost_by_team: { "team-a": 0.165, "": 0.015 },
cost_by_key: { "hash-1": 0.18, "hash-2": null },
untracked_usage_units: { someFutureCounter: 7 },
untracked_usage_units_by_team: { "team-a": {}, "": { someFutureCounter: 7 } },
untracked_usage_units_by_key: { "hash-1": {}, "hash-2": { someFutureCounter: 7 } },
};
const rowNamed = (name: string) => screen.getByRole("row", { name: new RegExp(name) });
@ -61,7 +63,7 @@ describe("GuardrailUsageBreakdown", () => {
expect(within(future).getByText("—")).toBeInTheDocument();
});
it("breaks units and cost down by team and by key, naming the rows without one", () => {
it("breaks units and cost down by team and by key, flagging the unpriced share of each row", () => {
render(<GuardrailUsageBreakdown detail={detail} />);
expect(screen.getByRole("heading", { name: "By team" })).toBeInTheDocument();
@ -69,14 +71,17 @@ describe("GuardrailUsageBreakdown", () => {
const teamA = rowNamed("team-a");
expect(within(teamA).getByText("1,200")).toBeInTheDocument();
expect(within(teamA).getByText("$0.1650")).toBeInTheDocument();
expect(within(teamA).getByText("—")).toBeInTheDocument();
expect(within(teamA).queryByText("7")).not.toBeInTheDocument();
const noTeam = rowNamed("No team");
expect(within(noTeam).getByText("107")).toBeInTheDocument();
expect(within(noTeam).getByText("$0.0150")).toBeInTheDocument();
expect(within(noTeam).getByText("7", { selector: ".text-warning" })).toBeInTheDocument();
const unpricedKey = rowNamed("hash-2");
expect(within(unpricedKey).getByText("7")).toBeInTheDocument();
expect(within(unpricedKey).getByText("—")).toBeInTheDocument();
expect(within(unpricedKey).getByText("7", { selector: ".text-warning" })).toBeInTheDocument();
});
it("orders teams and keys by units, largest first", () => {
@ -104,6 +109,8 @@ describe("GuardrailUsageBreakdown", () => {
cost_by_team: {},
cost_by_key: {},
untracked_usage_units: {},
untracked_usage_units_by_team: {},
untracked_usage_units_by_key: {},
}}
/>,
);

40
ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx
View file

@ -19,6 +19,7 @@ interface GroupRow {
id: string;
units: number;
cost: number | null;
unpriced: number;
}
const counterRows = (detail: GuardrailUsageDetail): CounterRow[] =>
@ -32,11 +33,31 @@ const counterRows = (detail: GuardrailUsageDetail): CounterRow[] =>
const groupRows = (
unitsByGroup: GuardrailUsageDetail["usage_units_by_team"],
costByGroup: GuardrailUsageDetail["cost_by_team"],
untrackedByGroup: GuardrailUsageDetail["untracked_usage_units_by_team"],
): GroupRow[] =>
Object.entries(unitsByGroup)
.map(([id, units]) => ({ id, units: totalUnits(units), cost: costByGroup[id] ?? null }))
.map(([id, units]) => ({
id,
units: totalUnits(units),
cost: costByGroup[id] ?? null,
unpriced: totalUnits(untrackedByGroup[id] ?? {}),
}))
.sort((a, b) => b.units - a.units);
const UnpricedUnitsCell = ({ unpriced }: { unpriced: number }) =>
unpriced > 0 ? (
<span className="text-warning">{unpriced.toLocaleString()}</span>
) : (
<span className="text-muted-foreground">—</span>
);
const unpricedColumn = <TRow extends { unpriced: number }>(): ColumnDef<TRow> => ({
header: "Unpriced Units",
accessorKey: "unpriced",
meta: { numeric: true },
cell: ({ row }) => <UnpricedUnitsCell unpriced={row.original.unpriced} />,
});
const counterColumns: ColumnDef<CounterRow>[] = [
{ header: "Counter", accessorKey: "counter", cell: ({ row }) => counterLabel(row.original.counter) },
{
@ -51,17 +72,7 @@ const counterColumns: ColumnDef<CounterRow>[] = [
meta: { numeric: true },
cell: ({ row }) => <MoneyCell value={row.original.cost} emptyText="—" showZero />,
},
{
header: "Unpriced Units",
accessorKey: "unpriced",
meta: { numeric: true },
cell: ({ row }) =>
row.original.unpriced > 0 ? (
<span className="text-warning">{row.original.unpriced.toLocaleString()}</span>
) : (
<span className="text-muted-foreground">—</span>
),
},
unpricedColumn<CounterRow>(),
];
const groupColumns = (label: string, emptyLabel: string): ColumnDef<GroupRow>[] => [
@ -87,6 +98,7 @@ const groupColumns = (label: string, emptyLabel: string): ColumnDef<GroupRow>[]
meta: { numeric: true },
cell: ({ row }) => <MoneyCell value={row.original.cost} emptyText="—" showZero />,
},
unpricedColumn<GroupRow>(),
];
const teamColumns = groupColumns("Team", "No team");
@ -139,14 +151,14 @@ export function GuardrailUsageBreakdown({ detail }: { detail: GuardrailUsageDeta
<div className="grid gap-4 lg:grid-cols-2">
<DataTable
columns={teamColumns}
data={groupRows(detail.usage_units_by_team, detail.cost_by_team)}
data={groupRows(detail.usage_units_by_team, detail.cost_by_team, detail.untracked_usage_units_by_team)}
getRowId={(row) => row.id || "no-team"}
size="compact"
toolbar={() => <TableHeading title="By team" />}
/>
<DataTable
columns={keyColumns}
data={groupRows(detail.usage_units_by_key, detail.cost_by_key)}
data={groupRows(detail.usage_units_by_key, detail.cost_by_key, detail.untracked_usage_units_by_key)}
getRowId={(row) => row.id || "no-key"}
size="compact"
toolbar={() => <TableHeading title="By key" />}

16
ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx
View file

@ -160,14 +160,24 @@ describe("GuardrailsOverview", () => {
expect(screen.getByText("Sensitive Information Policy: 250")).toBeInTheDocument();
});
it("sorts by cost when its header is clicked", async () => {
it("sorts by cost when its header is clicked, keeping guardrails with no known cost last either way", async () => {
const user = userEvent.setup();
renderOverview();
const rowNames = () =>
screen
.getAllByRole("row")
.slice(1)
.map((r) => r.textContent ?? "");
await user.click(await screen.findByRole("button", { name: /Cost/ }));
await waitFor(() => expect(rowNames()[0]).toContain("Free Bedrock Guardrail"));
expect(rowNames()[1]).toContain("High Failure Guardrail");
expect(rowNames()[2]).toContain("Low Failure Guardrail");
await waitFor(() => expect(screen.getAllByRole("row")[1]).toHaveTextContent("Low Failure Guardrail"));
expect(screen.getAllByRole("row")[3]).toHaveTextContent("High Failure Guardrail");
await user.click(screen.getByRole("button", { name: /Cost/ }));
await waitFor(() => expect(rowNames()[0]).toContain("High Failure Guardrail"));
expect(rowNames()[1]).toContain("Free Bedrock Guardrail");
expect(rowNames()[2]).toContain("Low Failure Guardrail");
});
it("renders the page header and the export action", async () => {

9
ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx
View file

@ -112,11 +112,12 @@ export function GuardrailsOverview({
}, [guardrailsData, activeData]);
const chartData = guardrailsData?.chart;
const sorted = useMemo(() => {
const mult = sortDir === "desc" ? -1 : 1;
return [...activeData].sort((a, b) => {
const mult = sortDir === "desc" ? -1 : 1;
const aVal = a[sortBy] ?? 0;
const bVal = b[sortBy] ?? 0;
return (Number(aVal) - Number(bVal)) * mult;
const aVal = a[sortBy];
const bVal = b[sortBy];
if (aVal == null || bVal == null) return Number(aVal == null) - Number(bVal == null);
return (aVal - bVal) * mult;
});
}, [activeData, sortBy, sortDir]);
const isLoading = guardrailsLoading;

12
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -38461,6 +38461,18 @@ export interface components {
untracked_usage_units: {
[key: string]: number;
};
/** Untracked Usage Units By Key */
untracked_usage_units_by_key: {
[key: string]: {
[key: string]: number;
};
};
/** Untracked Usage Units By Team */
untracked_usage_units_by_team: {
[key: string]: {
[key: string]: number;
};
};
/** Usage Units */
usage_units: {
[key: string]: number;