diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx index 7929b97b000..ba90ca8e6ff 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx @@ -1,4 +1,5 @@ import { render, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { describe, expect, it } from "vitest"; import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; import { GuardrailUsageBreakdown } from "./GuardrailUsageBreakdown"; @@ -84,6 +85,37 @@ describe("GuardrailUsageBreakdown", () => { expect(within(unpricedKey).getByText("7", { selector: ".text-warning" })).toBeInTheDocument(); }); + it("explains the cost math per counter on hover", async () => { + const user = userEvent.setup(); + render(); + + await user.hover( + within(screen.getByRole("group", { name: "Cost" })).getByRole("button", { name: /How is this calculated/ }), + ); + + expect(await screen.findByText("Content Policy: 1,000 × $0.00015 = $0.1500")).toBeInTheDocument(); + expect(screen.getByText("Sensitive Information Policy: 300 × $0.0001 = $0.0300")).toBeInTheDocument(); + expect(screen.getByText("Some Future Counter: 7 units with no known price, left out")).toBeInTheDocument(); + expect(screen.getByText("Total: $0.1800")).toBeInTheDocument(); + }); + + it("explains the units sum on hover", async () => { + const user = userEvent.setup(); + render(); + + await user.hover( + within(screen.getByRole("group", { name: "Usage Units" })).getByRole("button", { + name: /How is this calculated/, + }), + ); + + expect( + await screen.findByText( + "Content Policy 1,000 + Sensitive Information Policy 300 + Some Future Counter 7 = 1,307", + ), + ).toBeInTheDocument(); + }); + it("orders teams and keys by units, largest first", () => { render(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx index 6e8b725aa2e..01dd8f79ce4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx @@ -3,7 +3,14 @@ import { CircleDollarSign } from "lucide-react"; import React from "react"; import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard"; -import { counterLabel, formatCost, totalUnits, unpricedSummary } from "@/components/GuardrailsMonitor/usageUnits"; +import { + counterLabel, + counterMathLine, + formatCost, + totalUnits, + unitsSumLine, + unpricedSummary, +} from "@/components/GuardrailsMonitor/usageUnits"; import { DataTable } from "@/components/shared/DataTable"; import { IdCell } from "@/components/shared/table_cells/id_cell"; import { MoneyCell } from "@/components/shared/table_cells/money_cell"; @@ -104,6 +111,26 @@ const groupColumns = (label: string, emptyLabel: string): ColumnDef[] const teamColumns = groupColumns("Team", "No team"); const keyColumns = groupColumns("Key", "No key"); +const CostMath = ({ counters, total }: { counters: CounterRow[]; total: number | null }) => ( +
+ {counters.map((row) => ( +
{counterMathLine(row)}
+ ))} +
Total: {formatCost(total)}
+
Per-unit prices come from the bedrock/guardrails entry in the cost map.
+
+); + +const UnitsMath = ({ units }: { units: GuardrailUsageDetail["usage_units"] }) => ( +
+
{unitsSumLine(units)}
+
+ Bedrock reports one unit per 1,000 characters of the message for each policy the guardrail has on, on every call, + blocked or not. +
+
+); + const TableHeading = ({ title }: { title: string }) => (
{title}
); @@ -132,11 +159,13 @@ export function GuardrailUsageBreakdown({ detail }: { detail: GuardrailUsageDeta valueColor={detail.cost != null ? "text-foreground" : "text-muted-foreground"} icon={} subtitle={unpriced ?? undefined} + hint={} /> } /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx index c52645def70..14e070afc0d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx @@ -211,6 +211,20 @@ describe("GuardrailsOverview", () => { expect(card).toHaveTextContent("250 units unpriced"); }); + it("explains the guardrail cost total on hover", async () => { + const user = userEvent.setup(); + renderOverview(); + + const card = await screen.findByRole("group", { name: "Guardrail Cost" }); + await user.hover(within(card).getByRole("button", { name: /How is this calculated/ })); + + expect(await screen.findByText("High Failure Guardrail: $0.1500")).toBeInTheDocument(); + expect(screen.getByText("Free Bedrock Guardrail: $0.0000")).toBeInTheDocument(); + expect(screen.queryByText(/Low Failure Guardrail: /)).not.toBeInTheDocument(); + expect(screen.getByText("Total: $0.1500")).toBeInTheDocument(); + expect(screen.getByText(/250 units unpriced had no known price and are left out/)).toBeInTheDocument(); + }); + it("shows a dash for guardrail cost when nothing in the window was priced", async () => { useGuardrailsUsageOverviewMock.mockReturnValue({ data: { ...overview, totalCost: null, totalUntrackedUsageUnits: {} }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx index 0bbda6015b4..3df7058baba 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx @@ -63,6 +63,34 @@ function UsageUnitsCell({ units }: { units: GuardrailUsageOverviewRow["usageUnit ); } +function TotalCostMath({ + rows, + total, + unpriced, +}: { + rows: GuardrailUsageOverviewRow[]; + total: number | null; + unpriced: string | null; +}) { + return ( +
+ {rows + .filter((row) => row.cost != null) + .map((row) => ( +
+ {row.name}: {formatCost(row.cost)} +
+ ))} +
Total: {formatCost(total)}
+
+ {`Each guardrail's cost is its units per policy × that policy's per-unit price from the cost map, added up${ + unpriced ? `; ${unpriced} had no known price and are left out` : "" + }. Open a guardrail for its per-policy math.`} +
+
+ ); +} + function CostCell({ row }: { row: GuardrailUsageOverviewRow }) { const unpriced = unpricedSummary(row.untrackedUsageUnits); return ( @@ -124,6 +152,25 @@ export function GuardrailsOverview({ const error = guardrailsError; const columns: ColumnDef[] = [ + { + header: "Status", + accessorKey: "status", + enableSorting: false, + cell: ({ row }) => ( + + + {row.original.status} + + ), + }, { header: "Guardrail", accessorKey: "name", @@ -215,25 +262,6 @@ export function GuardrailsOverview({ sortDescFirst: false, cell: ({ row }) => , }, - { - header: "Status", - accessorKey: "status", - enableSorting: false, - cell: ({ row }) => ( - - - {row.original.status} - - ), - }, ]; const sortableKeys: SortKey[] = ["failRate", "requestsEvaluated", "avgLatency", "cost"]; @@ -291,6 +319,7 @@ export function GuardrailsOverview({ valueColor={metrics.totalCost != null ? "text-foreground" : "text-muted-foreground"} icon={} subtitle={metrics.unpriced ?? undefined} + hint={} /> diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx index c0b5e0a50d1..1805dc797e4 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx @@ -1,4 +1,6 @@ +import { CircleHelp } from "lucide-react"; import React, { type ReactNode } from "react"; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; interface MetricCardProps { label: string; @@ -6,9 +8,10 @@ interface MetricCardProps { valueColor?: string; icon?: ReactNode; subtitle?: string; + hint?: ReactNode; } -export function MetricCard({ label, value, valueColor = "text-foreground", icon, subtitle }: MetricCardProps) { +export function MetricCard({ label, value, valueColor = "text-foreground", icon, subtitle, hint }: MetricCardProps) { return (
@@ -17,6 +20,26 @@ export function MetricCard({ label, value, valueColor = "text-foreground", icon,
{value}
{subtitle &&

{subtitle}

} + {hint && ( + + + + + How is this calculated? + + } + /> + + {hint} + + + + )}
); } diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts index 29362cc3701..560010bd852 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts @@ -1,5 +1,14 @@ import { describe, expect, it } from "vitest"; -import { counterLabel, formatCost, totalUnits, unpricedSummary } from "./usageUnits"; +import { + counterLabel, + counterMathLine, + formatCost, + formatUnitPrice, + totalUnits, + unitPrice, + unitsSumLine, + unpricedSummary, +} from "./usageUnits"; describe("formatCost", () => { it("renders a dash when nothing was priced", () => { @@ -53,3 +62,63 @@ describe("unpricedSummary", () => { expect(unpricedSummary({ someFutureCounter: 1 })).toBe("1 unit unpriced"); }); }); + +describe("unitPrice", () => { + it("backs the per-unit price out of the priced share only", () => { + expect(unitPrice({ counter: "contentPolicyUnits", units: 1200, unpriced: 200, cost: 0.15 })).toBeCloseTo( + 0.00015, + 10, + ); + }); + + it("is null when nothing was priced", () => { + expect(unitPrice({ counter: "someFutureCounter", units: 7, unpriced: 7, cost: null })).toBeNull(); + expect(unitPrice({ counter: "someFutureCounter", units: 7, unpriced: 7, cost: 0 })).toBeNull(); + }); +}); + +describe("formatUnitPrice", () => { + it("keeps the significant decimals and drops trailing zeros", () => { + expect(formatUnitPrice(0.0001)).toBe("$0.0001"); + expect(formatUnitPrice(0.00015)).toBe("$0.00015"); + expect(formatUnitPrice(0)).toBe("$0"); + expect(formatUnitPrice(1)).toBe("$1"); + }); +}); + +describe("counterMathLine", () => { + it("shows units × price = cost for a fully priced counter", () => { + expect(counterMathLine({ counter: "contentPolicyUnits", units: 1000, unpriced: 0, cost: 0.15 })).toBe( + "Content Policy: 1,000 × $0.00015 = $0.1500", + ); + }); + + it("prices only the priced share and calls out the rest", () => { + expect(counterMathLine({ counter: "sensitiveInformationPolicyUnits", units: 8, unpriced: 2, cost: 0.0006 })).toBe( + "Sensitive Information Policy: 6 × $0.0001 = $0.0006 (2 unpriced left out)", + ); + }); + + it("says so when a counter has no known price at all", () => { + expect(counterMathLine({ counter: "someFutureCounter", units: 7, unpriced: 7, cost: null })).toBe( + "Some Future Counter: 7 units with no known price, left out", + ); + expect(counterMathLine({ counter: "someFutureCounter", units: 1, unpriced: 1, cost: null })).toBe( + "Some Future Counter: 1 unit with no known price, left out", + ); + }); + + it("shows a free counter as × $0", () => { + expect(counterMathLine({ counter: "wordPolicyUnits", units: 2, unpriced: 0, cost: 0 })).toBe( + "Word Policy: 2 × $0 = $0.0000", + ); + }); +}); + +describe("unitsSumLine", () => { + it("adds the counters up in order", () => { + expect(unitsSumLine({ contentPolicyUnits: 2, topicPolicyUnits: 2, wordPolicyUnits: 1200 })).toBe( + "Content Policy 2 + Topic Policy 2 + Word Policy 1,200 = 1,204", + ); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts index f3a5e9d7140..05e046aaace 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts @@ -19,3 +19,35 @@ export const unpricedSummary = (untracked: UsageUnits): string | null => { const total = totalUnits(untracked); return total > 0 ? `${total.toLocaleString()} ${total === 1 ? "unit" : "units"} unpriced` : null; }; + +export interface CounterMath { + readonly counter: string; + readonly units: number; + readonly unpriced: number; + readonly cost: number | null; +} + +export const pricedUnits = ({ units, unpriced }: Pick): number => + Math.max(units - unpriced, 0); + +export const unitPrice = (row: CounterMath): number | null => { + const priced = pricedUnits(row); + return row.cost != null && priced > 0 ? row.cost / priced : null; +}; + +export const formatUnitPrice = (price: number): string => `$${price.toFixed(6).replace(/\.?0+$/, "")}`; + +export const counterMathLine = (row: CounterMath): string => { + const label = counterLabel(row.counter); + const price = unitPrice(row); + if (price == null) { + return `${label}: ${row.units.toLocaleString()} ${row.units === 1 ? "unit" : "units"} with no known price, left out`; + } + const line = `${label}: ${pricedUnits(row).toLocaleString()} × ${formatUnitPrice(price)} = ${formatCost(row.cost)}`; + return row.unpriced > 0 ? `${line} (${row.unpriced.toLocaleString()} unpriced left out)` : line; +}; + +export const unitsSumLine = (units: UsageUnits): string => + `${Object.entries(units) + .map(([counter, n]) => `${counterLabel(counter)} ${n.toLocaleString()}`) + .join(" + ")} = ${totalUnits(units).toLocaleString()}`;