From 29b93b57aaa0cef7660da3636df377422a2cd704 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 5 Sep 2026 12:58:18 -0700 Subject: [PATCH] feat(ui): show the guardrail cost math in a popover table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "How is this calculated?" hover was a plain-text tooltip. It is now a popover (opens on hover or click) with a title, the formula, a table of one row per counter or guardrail (units, × price, = cost, with unpriced units called out under the row) and a total row, so the math reads as a worked sum instead of a sentence. Refs LIT-5652 --- .../GuardrailUsageBreakdown.test.tsx | 55 ++++++++++----- .../_components/GuardrailUsageBreakdown.tsx | 26 +++---- .../_components/GuardrailsOverview.test.tsx | 25 ++++--- .../_components/GuardrailsOverview.tsx | 25 ++++--- .../GuardrailsMonitor/CalcPopover.tsx | 67 +++++++++++++++++++ .../GuardrailsMonitor/MetricCard.tsx | 23 +------ .../GuardrailsMonitor/UnpricedNote.tsx | 4 +- .../GuardrailsMonitor/usageUnits.test.ts | 55 +++++++++------ .../GuardrailsMonitor/usageUnits.ts | 30 ++++++--- 9 files changed, 204 insertions(+), 106 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/GuardrailsMonitor/CalcPopover.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx index 3a0a4c38ecb..db7855ab0d5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx @@ -85,20 +85,33 @@ describe("GuardrailUsageBreakdown", () => { expect(within(unpricedKey).getByText("7", { selector: ".text-warning" })).toBeInTheDocument(); }); - it("explains the cost math per counter on hover", async () => { + const cellsOf = (dialog: HTMLElement): string[][] => + within(dialog) + .getAllByRole("row") + .map((row) => + within(row) + .getAllByRole("cell") + .map((cell) => cell.textContent ?? ""), + ); + + it("lays the cost math out per counter as units × price = cost", async () => { const user = userEvent.setup(); render(); - await user.hover( + await user.click( within(screen.getByRole("group", { name: "Cost" })).getByRole("button", { name: /How is this calculated/ }), ); - expect(await screen.findByText("Content Policy: 1,000 × $0.00015 = $0.1500")).toBeInTheDocument(); - expect(screen.getByText("Sensitive Information Policy: 300 × $0.0001 = $0.0300")).toBeInTheDocument(); - expect(screen.getByText("Some Future Counter: 7 units with no known price, left out")).toBeInTheDocument(); - expect(screen.getByText("Total: $0.1800")).toBeInTheDocument(); - expect(screen.getByText(/7 units with no known price are left out of the cost/)).toBeInTheDocument(); - const issueLink = screen.getByRole("link", { name: "Request pricing on GitHub" }); + const dialog = await screen.findByRole("dialog", { name: "How this cost is calculated" }); + expect(cellsOf(dialog)).toEqual([ + ["Content Policy", "1,000", "× $0.00015", "= $0.1500"], + ["Sensitive Information Policy", "300", "× $0.0001", "= $0.0300"], + ["Some Future Counter", "7", "× —", "= —"], + ["no known price, left out"], + ["Total", "$0.1800"], + ]); + expect(within(dialog).getByText(/7 units with no known price are left out of the cost/)).toBeInTheDocument(); + const issueLink = within(dialog).getByRole("link", { name: "Request pricing on GitHub" }); expect(issueLink).toHaveAttribute("target", "_blank"); const issueUrl = new URL(issueLink.getAttribute("href") ?? ""); expect(issueUrl.searchParams.get("title")).toBe("[Feature]: add Bedrock guardrail pricing to the cost map"); @@ -119,29 +132,35 @@ describe("GuardrailUsageBreakdown", () => { />, ); - await user.hover( + await user.click( within(screen.getByRole("group", { name: "Cost" })).getByRole("button", { name: /How is this calculated/ }), ); - expect(await screen.findByText("Total: $0.1500")).toBeInTheDocument(); - expect(screen.queryByRole("link", { name: "Request pricing on GitHub" })).not.toBeInTheDocument(); + const dialog = await screen.findByRole("dialog", { name: "How this cost is calculated" }); + expect(cellsOf(dialog)).toEqual([ + ["Content Policy", "1,000", "× $0.00015", "= $0.1500"], + ["Total", "$0.1500"], + ]); + expect(within(dialog).queryByRole("link", { name: "Request pricing on GitHub" })).not.toBeInTheDocument(); }); - it("explains the units sum on hover", async () => { + it("lays the units sum out per counter", async () => { const user = userEvent.setup(); render(); - await user.hover( + await user.click( within(screen.getByRole("group", { name: "Usage Units" })).getByRole("button", { name: /How is this calculated/, }), ); - expect( - await screen.findByText( - "Content Policy 1,000 + Sensitive Information Policy 300 + Some Future Counter 7 = 1,307", - ), - ).toBeInTheDocument(); + const dialog = await screen.findByRole("dialog", { name: "How usage units add up" }); + expect(cellsOf(dialog)).toEqual([ + ["Content Policy", "1,000"], + ["Sensitive Information Policy", "300"], + ["Some Future Counter", "7"], + ["Total", "1,307"], + ]); }); it("orders teams and keys by units, largest first", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx index e67425adb10..27d9ba5162f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx @@ -2,14 +2,15 @@ import type { ColumnDef } from "@tanstack/react-table"; import { CircleDollarSign } from "lucide-react"; import React from "react"; import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; +import { CalcPopover, MathTable } from "@/components/GuardrailsMonitor/CalcPopover"; import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard"; import { UnpricedNote } from "@/components/GuardrailsMonitor/UnpricedNote"; import { counterLabel, - counterMathLine, + counterMathRow, formatCost, totalUnits, - unitsSumLine, + unitsMathRows, unpricedSummary, } from "@/components/GuardrailsMonitor/usageUnits"; import { DataTable } from "@/components/shared/DataTable"; @@ -113,21 +114,20 @@ const teamColumns = groupColumns("Team", "No team"); const keyColumns = groupColumns("Key", "No key"); const CostMath = ({ counters, detail }: { counters: CounterRow[]; detail: GuardrailUsageDetail }) => ( -
- {counters.map((row) => ( -
{counterMathLine(row)}
- ))} -
Total: {formatCost(detail.cost)}
-
Each counter is its priced units × the per-unit price LiteLLM has for it in the cost map.
+ + +

Per-unit prices come from the cost map LiteLLM ships with.

-
+ ); const UnitsMath = ({ units }: { units: GuardrailUsageDetail["usage_units"] }) => ( -
-
{unitsSumLine(units)}
-
Units are the billable counters the provider reported for this guardrail, added up over every call.
-
+ + +

+ Units are the billable counters the provider reported for this guardrail, added up over every call. +

+ ); const TableHeading = ({ title }: { title: string }) => ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx index 16361bdec27..959ed8b172e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.test.tsx @@ -211,19 +211,28 @@ describe("GuardrailsOverview", () => { expect(card).toHaveTextContent("250 units unpriced"); }); - it("explains the guardrail cost total on hover", async () => { + it("lays the guardrail cost total out per guardrail", async () => { const user = userEvent.setup(); renderOverview(); const card = await screen.findByRole("group", { name: "Guardrail Cost" }); - await user.hover(within(card).getByRole("button", { name: /How is this calculated/ })); + await user.click(within(card).getByRole("button", { name: /How is this calculated/ })); - expect(await screen.findByText("High Failure Guardrail: $0.1500")).toBeInTheDocument(); - expect(screen.getByText("Free Bedrock Guardrail: $0.0000")).toBeInTheDocument(); - expect(screen.queryByText(/Low Failure Guardrail: /)).not.toBeInTheDocument(); - expect(screen.getByText("Total: $0.1500")).toBeInTheDocument(); - expect(screen.getByText(/250 units with no known price are left out of the cost/)).toBeInTheDocument(); - const issueLink = screen.getByRole("link", { name: "Request pricing on GitHub" }); + const dialog = await screen.findByRole("dialog", { name: "How this cost is calculated" }); + const cells = within(dialog) + .getAllByRole("row") + .map((row) => + within(row) + .getAllByRole("cell") + .map((cell) => cell.textContent ?? ""), + ); + expect(cells).toEqual([ + ["High Failure Guardrail", "$0.1500"], + ["Free Bedrock Guardrail", "$0.0000"], + ["Total", "$0.1500"], + ]); + expect(within(dialog).getByText(/250 units with no known price are left out of the cost/)).toBeInTheDocument(); + const issueLink = within(dialog).getByRole("link", { name: "Request pricing on GitHub" }); const issueUrl = new URL(issueLink.getAttribute("href") ?? ""); expect(issueUrl.searchParams.get("template")).toBe("feature_request.yml"); expect(issueUrl.searchParams.get("the-feature")).toContain("sensitiveInformationPolicyUnits"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx index 33be85f3c81..468e6967d81 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx @@ -8,6 +8,7 @@ import { type GuardrailUsageOverviewRow, useGuardrailsUsageOverview, } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; +import { CalcPopover, MathTable } from "@/components/GuardrailsMonitor/CalcPopover"; import { UnpricedNote } from "@/components/GuardrailsMonitor/UnpricedNote"; import { counterLabel, @@ -80,20 +81,18 @@ function TotalCostMath({ untracked: UsageUnits; }) { return ( -
- {rows - .filter((row) => row.cost != null) - .map((row) => ( -
- {row.name}: {formatCost(row.cost)} -
- ))} -
Total: {formatCost(total)}
-
- {`Each guardrail's cost is its units per counter × that counter's per-unit price from the cost map, added up. Open a guardrail for its per-counter math.`} -
+ + row.cost != null) + .map((row) => ({ label: row.name, parts: [formatCost(row.cost)], note: null }))} + total={formatCost(total)} + /> +

+ {`Each guardrail's cost is its units per counter × that counter's per-unit price from the cost map. Open a guardrail for its per-counter math.`} +

-
+ ); } diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/CalcPopover.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/CalcPopover.tsx new file mode 100644 index 00000000000..686992a6fb4 --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/CalcPopover.tsx @@ -0,0 +1,67 @@ +import { CircleHelp } from "lucide-react"; +import React, { type ReactNode } from "react"; +import { Popover, PopoverContent, PopoverTitle, PopoverTrigger } from "@/components/ui/popover"; +import type { MathRow } from "./usageUnits"; + +export function CalcPopover({ title, formula, children }: { title: string; formula: string; children: ReactNode }) { + return ( + + + } + > + + How is this calculated? + + + {title} + {formula} + {children} + + + ); +} + +export function MathTable({ rows, total }: { rows: readonly MathRow[]; total: string }) { + const width = 1 + Math.max(...rows.map((row) => row.parts.length), 1); + return ( + + + {rows.map((row) => ( + + + + {row.parts.map((part, i) => ( + + ))} + + {row.note && ( + + + + )} + + ))} + + + + + + + +
{row.label} + {part} +
+ {row.note} +
+ Total + {total}
+ ); +} diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx index 1805dc797e4..008dc279f13 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/MetricCard.tsx @@ -1,6 +1,4 @@ -import { CircleHelp } from "lucide-react"; import React, { type ReactNode } from "react"; -import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; interface MetricCardProps { label: string; @@ -20,26 +18,7 @@ export function MetricCard({ label, value, valueColor = "text-foreground", icon,
{value}
{subtitle &&

{subtitle}

} - {hint && ( - - - - - How is this calculated? - - } - /> - - {hint} - - - - )} + {hint} ); } diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/UnpricedNote.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/UnpricedNote.tsx index 174124d18aa..b43d9d18841 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/UnpricedNote.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/UnpricedNote.tsx @@ -6,7 +6,7 @@ export function UnpricedNote({ unpriced, provider }: { unpriced: UsageUnits; pro if (total === 0) return null; const [noun, verb] = total === 1 ? ["unit", "is"] : ["units", "are"]; return ( -
+

{`${total.toLocaleString()} ${noun} with no known price ${verb} left out of the cost. `} Request pricing on GitHub -

+

); } diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts index 9b45eefaf54..8e2baaaa3c5 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts @@ -1,13 +1,13 @@ import { describe, expect, it } from "vitest"; import { counterLabel, - counterMathLine, + counterMathRow, formatCost, formatUnitPrice, pricingIssueUrl, totalUnits, unitPrice, - unitsSumLine, + unitsMathRows, unpricedSummary, } from "./usageUnits"; @@ -91,40 +91,51 @@ describe("formatUnitPrice", () => { }); }); -describe("counterMathLine", () => { +describe("counterMathRow", () => { it("shows units × price = cost for a fully priced counter", () => { - expect(counterMathLine({ counter: "contentPolicyUnits", units: 1000, unpriced: 0, cost: 0.15 })).toBe( - "Content Policy: 1,000 × $0.00015 = $0.1500", - ); + expect(counterMathRow({ counter: "contentPolicyUnits", units: 1000, unpriced: 0, cost: 0.15 })).toEqual({ + label: "Content Policy", + parts: ["1,000", "× $0.00015", "= $0.1500"], + note: null, + }); }); it("prices only the priced share and calls out the rest", () => { - expect(counterMathLine({ counter: "sensitiveInformationPolicyUnits", units: 8, unpriced: 2, cost: 0.0006 })).toBe( - "Sensitive Information Policy: 6 × $0.0001 = $0.0006 (2 unpriced left out)", + expect(counterMathRow({ counter: "sensitiveInformationPolicyUnits", units: 8, unpriced: 2, cost: 0.0006 })).toEqual( + { + label: "Sensitive Information Policy", + parts: ["6", "× $0.0001", "= $0.0006"], + note: "2 unpriced units left out", + }, ); + expect( + counterMathRow({ counter: "sensitiveInformationPolicyUnits", units: 8, unpriced: 1, cost: 0.0007 }).note, + ).toBe("1 unpriced unit left out"); }); it("says so when a counter has no known price at all", () => { - expect(counterMathLine({ counter: "someFutureCounter", units: 7, unpriced: 7, cost: null })).toBe( - "Some Future Counter: 7 units with no known price, left out", - ); - expect(counterMathLine({ counter: "someFutureCounter", units: 1, unpriced: 1, cost: null })).toBe( - "Some Future Counter: 1 unit with no known price, left out", - ); + expect(counterMathRow({ counter: "someFutureCounter", units: 7, unpriced: 7, cost: null })).toEqual({ + label: "Some Future Counter", + parts: ["7", "× —", "= —"], + note: "no known price, left out", + }); }); it("shows a free counter as × $0", () => { - expect(counterMathLine({ counter: "wordPolicyUnits", units: 2, unpriced: 0, cost: 0 })).toBe( - "Word Policy: 2 × $0 = $0.0000", - ); + expect(counterMathRow({ counter: "wordPolicyUnits", units: 2, unpriced: 0, cost: 0 }).parts).toEqual([ + "2", + "× $0", + "= $0.0000", + ]); }); }); -describe("unitsSumLine", () => { - it("adds the counters up in order", () => { - expect(unitsSumLine({ contentPolicyUnits: 2, topicPolicyUnits: 2, wordPolicyUnits: 1200 })).toBe( - "Content Policy 2 + Topic Policy 2 + Word Policy 1,200 = 1,204", - ); +describe("unitsMathRows", () => { + it("lists the counters in order with their counts", () => { + expect(unitsMathRows({ contentPolicyUnits: 2, wordPolicyUnits: 1200 })).toEqual([ + { label: "Content Policy", parts: ["2"], note: null }, + { label: "Word Policy", parts: ["1,200"], note: null }, + ]); }); }); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts index f914a3e9698..c47442de200 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts @@ -40,20 +40,34 @@ export const formatUnitPrice = (price: number): string => { return price > 0 && Number(fixed) === 0 ? "< $0.000001" : `$${fixed}`; }; -export const counterMathLine = (row: CounterMath): string => { +export interface MathRow { + readonly label: string; + readonly parts: readonly string[]; + readonly note: string | null; +} + +export const counterMathRow = (row: CounterMath): MathRow => { const label = counterLabel(row.counter); const price = unitPrice(row); if (price == null) { - return `${label}: ${row.units.toLocaleString()} ${row.units === 1 ? "unit" : "units"} with no known price, left out`; + return { label, parts: [row.units.toLocaleString(), "× —", "= —"], note: "no known price, left out" }; } - const line = `${label}: ${pricedUnits(row).toLocaleString()} × ${formatUnitPrice(price)} = ${formatCost(row.cost)}`; - return row.unpriced > 0 ? `${line} (${row.unpriced.toLocaleString()} unpriced left out)` : line; + return { + label, + parts: [pricedUnits(row).toLocaleString(), `× ${formatUnitPrice(price)}`, `= ${formatCost(row.cost)}`], + note: + row.unpriced > 0 + ? `${row.unpriced.toLocaleString()} unpriced ${row.unpriced === 1 ? "unit" : "units"} left out` + : null, + }; }; -export const unitsSumLine = (units: UsageUnits): string => - `${Object.entries(units) - .map(([counter, n]) => `${counterLabel(counter)} ${n.toLocaleString()}`) - .join(" + ")} = ${totalUnits(units).toLocaleString()}`; +export const unitsMathRows = (units: UsageUnits): readonly MathRow[] => + Object.entries(units).map(([counter, n]) => ({ + label: counterLabel(counter), + parts: [n.toLocaleString()], + note: null, + })); export const pricingIssueUrl = (unpriced: UsageUnits, provider?: string): string => { const subject = provider ? `${provider} guardrail` : "guardrail";