From e66ba0533fe43a626b8a157fae59f507894aa13b Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 5 Sep 2026 12:23:07 -0700 Subject: [PATCH] fix(ui): keep guardrail cost hints provider neutral and link to a pricing request The hint copy described Bedrock's unit semantics and cost map entry even though any provider's units reach this view, so it now explains the math in provider-neutral terms. When units have no known price, the hint says so and links to a prefilled GitHub feature request (provider and counter names filled in) so the reader can ask for pricing. Per-unit prices below $0.000001 now read "< $0.000001" instead of "$0". Refs LIT-5652 --- .../GuardrailUsageBreakdown.test.tsx | 28 +++++++++++++++++++ .../_components/GuardrailUsageBreakdown.tsx | 15 +++++----- .../_components/GuardrailsOverview.test.tsx | 6 +++- .../_components/GuardrailsOverview.tsx | 26 ++++++++++------- .../GuardrailsMonitor/UnpricedNote.tsx | 21 ++++++++++++++ .../GuardrailsMonitor/usageUnits.test.ts | 22 +++++++++++++++ .../GuardrailsMonitor/usageUnits.ts | 15 +++++++++- 7 files changed, 113 insertions(+), 20 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/GuardrailsMonitor/UnpricedNote.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx index ba90ca8e6ff..3a0a4c38ecb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.test.tsx @@ -97,6 +97,34 @@ describe("GuardrailUsageBreakdown", () => { expect(screen.getByText("Sensitive Information Policy: 300 × $0.0001 = $0.0300")).toBeInTheDocument(); expect(screen.getByText("Some Future Counter: 7 units with no known price, left out")).toBeInTheDocument(); expect(screen.getByText("Total: $0.1800")).toBeInTheDocument(); + expect(screen.getByText(/7 units with no known price are left out of the cost/)).toBeInTheDocument(); + const issueLink = screen.getByRole("link", { name: "Request pricing on GitHub" }); + expect(issueLink).toHaveAttribute("target", "_blank"); + const issueUrl = new URL(issueLink.getAttribute("href") ?? ""); + expect(issueUrl.searchParams.get("title")).toBe("[Feature]: add Bedrock guardrail pricing to the cost map"); + expect(issueUrl.searchParams.get("the-feature")).toContain("someFutureCounter"); + }); + + it("does not ask for pricing when every unit was priced", async () => { + const user = userEvent.setup(); + render( + , + ); + + await user.hover( + within(screen.getByRole("group", { name: "Cost" })).getByRole("button", { name: /How is this calculated/ }), + ); + + expect(await screen.findByText("Total: $0.1500")).toBeInTheDocument(); + expect(screen.queryByRole("link", { name: "Request pricing on GitHub" })).not.toBeInTheDocument(); }); it("explains the units sum on hover", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx index 01dd8f79ce4..e67425adb10 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailUsageBreakdown.tsx @@ -3,6 +3,7 @@ import { CircleDollarSign } from "lucide-react"; import React from "react"; import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard"; +import { UnpricedNote } from "@/components/GuardrailsMonitor/UnpricedNote"; import { counterLabel, counterMathLine, @@ -111,23 +112,21 @@ const groupColumns = (label: string, emptyLabel: string): ColumnDef[] const teamColumns = groupColumns("Team", "No team"); const keyColumns = groupColumns("Key", "No key"); -const CostMath = ({ counters, total }: { counters: CounterRow[]; total: number | null }) => ( +const CostMath = ({ counters, detail }: { counters: CounterRow[]; detail: GuardrailUsageDetail }) => (
{counters.map((row) => (
{counterMathLine(row)}
))} -
Total: {formatCost(total)}
-
Per-unit prices come from the bedrock/guardrails entry in the cost map.
+
Total: {formatCost(detail.cost)}
+
Each counter is its priced units × the per-unit price LiteLLM has for it in the cost map.
+
); const UnitsMath = ({ units }: { units: GuardrailUsageDetail["usage_units"] }) => (
{unitsSumLine(units)}
-
- Bedrock reports one unit per 1,000 characters of the message for each policy the guardrail has on, on every call, - blocked or not. -
+
Units are the billable counters the provider reported for this guardrail, added up over every call.
); @@ -159,7 +158,7 @@ export function GuardrailUsageBreakdown({ detail }: { detail: GuardrailUsageDeta valueColor={detail.cost != null ? "text-foreground" : "text-muted-foreground"} icon={} subtitle={unpriced ?? undefined} - hint={} + hint={} /> { expect(screen.getByText("Free Bedrock Guardrail: $0.0000")).toBeInTheDocument(); expect(screen.queryByText(/Low Failure Guardrail: /)).not.toBeInTheDocument(); expect(screen.getByText("Total: $0.1500")).toBeInTheDocument(); - expect(screen.getByText(/250 units unpriced had no known price and are left out/)).toBeInTheDocument(); + expect(screen.getByText(/250 units with no known price are left out of the cost/)).toBeInTheDocument(); + const issueLink = screen.getByRole("link", { name: "Request pricing on GitHub" }); + const issueUrl = new URL(issueLink.getAttribute("href") ?? ""); + expect(issueUrl.searchParams.get("template")).toBe("feature_request.yml"); + expect(issueUrl.searchParams.get("the-feature")).toContain("sensitiveInformationPolicyUnits"); }); it("shows a dash for guardrail cost when nothing in the window was priced", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx index 3df7058baba..33be85f3c81 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx @@ -8,7 +8,14 @@ import { type GuardrailUsageOverviewRow, useGuardrailsUsageOverview, } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage"; -import { counterLabel, formatCost, totalUnits, unpricedSummary } from "@/components/GuardrailsMonitor/usageUnits"; +import { UnpricedNote } from "@/components/GuardrailsMonitor/UnpricedNote"; +import { + counterLabel, + formatCost, + totalUnits, + unpricedSummary, + type UsageUnits, +} from "@/components/GuardrailsMonitor/usageUnits"; import { Button } from "@/components/ui/button"; import { PageHeader } from "@/components/shared/PageHeader"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; @@ -41,7 +48,7 @@ const EMPTY_METRICS = { avgLatency: 0, count: 0, totalCost: null as number | null, - unpriced: null as string | null, + untracked: {} as UsageUnits, }; function UsageUnitsCell({ units }: { units: GuardrailUsageOverviewRow["usageUnits"] }) { @@ -66,11 +73,11 @@ function UsageUnitsCell({ units }: { units: GuardrailUsageOverviewRow["usageUnit function TotalCostMath({ rows, total, - unpriced, + untracked, }: { rows: GuardrailUsageOverviewRow[]; total: number | null; - unpriced: string | null; + untracked: UsageUnits; }) { return (
@@ -83,10 +90,9 @@ function TotalCostMath({ ))}
Total: {formatCost(total)}
- {`Each guardrail's cost is its units per policy × that policy's per-unit price from the cost map, added up${ - unpriced ? `; ${unpriced} had no known price and are left out` : "" - }. Open a guardrail for its per-policy math.`} + {`Each guardrail's cost is its units per counter × that counter's per-unit price from the cost map, added up. Open a guardrail for its per-counter math.`}
+
); } @@ -135,7 +141,7 @@ export function GuardrailsOverview({ : 0, count: activeData.length, totalCost: guardrailsData.totalCost, - unpriced: unpricedSummary(guardrailsData.totalUntrackedUsageUnits), + untracked: guardrailsData.totalUntrackedUsageUnits, }; }, [guardrailsData, activeData]); const chartData = guardrailsData?.chart; @@ -318,8 +324,8 @@ export function GuardrailsOverview({ value={formatCost(metrics.totalCost)} valueColor={metrics.totalCost != null ? "text-foreground" : "text-muted-foreground"} icon={} - subtitle={metrics.unpriced ?? undefined} - hint={} + subtitle={unpricedSummary(metrics.untracked) ?? undefined} + hint={} /> diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/UnpricedNote.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/UnpricedNote.tsx new file mode 100644 index 00000000000..174124d18aa --- /dev/null +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/UnpricedNote.tsx @@ -0,0 +1,21 @@ +import React from "react"; +import { pricingIssueUrl, totalUnits, type UsageUnits } from "./usageUnits"; + +export function UnpricedNote({ unpriced, provider }: { unpriced: UsageUnits; provider?: string }) { + const total = totalUnits(unpriced); + if (total === 0) return null; + const [noun, verb] = total === 1 ? ["unit", "is"] : ["units", "are"]; + return ( +
+ {`${total.toLocaleString()} ${noun} with no known price ${verb} left out of the cost. `} + + Request pricing on GitHub + +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts index 560010bd852..9b45eefaf54 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.test.ts @@ -4,6 +4,7 @@ import { counterMathLine, formatCost, formatUnitPrice, + pricingIssueUrl, totalUnits, unitPrice, unitsSumLine, @@ -84,6 +85,10 @@ describe("formatUnitPrice", () => { expect(formatUnitPrice(0)).toBe("$0"); expect(formatUnitPrice(1)).toBe("$1"); }); + + it("never shows a positive price as free", () => { + expect(formatUnitPrice(0.0000002)).toBe("< $0.000001"); + }); }); describe("counterMathLine", () => { @@ -122,3 +127,20 @@ describe("unitsSumLine", () => { ); }); }); + +describe("pricingIssueUrl", () => { + it("prefills the feature request with the provider and the unpriced counters", () => { + const url = new URL(pricingIssueUrl({ text_records: 5, someFutureCounter: 7 }, "azure/prompt_shield")); + + expect(url.origin + url.pathname).toBe("https://github.com/BerriAI/litellm/issues/new"); + expect(url.searchParams.get("template")).toBe("feature_request.yml"); + expect(url.searchParams.get("title")).toBe("[Feature]: add azure/prompt_shield guardrail pricing to the cost map"); + expect(url.searchParams.get("the-feature")).toContain("text_records, someFutureCounter"); + }); + + it("stays generic when no provider is known", () => { + const url = new URL(pricingIssueUrl({ text_records: 5 })); + + expect(url.searchParams.get("title")).toBe("[Feature]: add guardrail pricing to the cost map"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts index 05e046aaace..f914a3e9698 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/usageUnits.ts @@ -35,7 +35,10 @@ export const unitPrice = (row: CounterMath): number | null => { return row.cost != null && priced > 0 ? row.cost / priced : null; }; -export const formatUnitPrice = (price: number): string => `$${price.toFixed(6).replace(/\.?0+$/, "")}`; +export const formatUnitPrice = (price: number): string => { + const fixed = price.toFixed(6).replace(/\.?0+$/, ""); + return price > 0 && Number(fixed) === 0 ? "< $0.000001" : `$${fixed}`; +}; export const counterMathLine = (row: CounterMath): string => { const label = counterLabel(row.counter); @@ -51,3 +54,13 @@ export const unitsSumLine = (units: UsageUnits): string => `${Object.entries(units) .map(([counter, n]) => `${counterLabel(counter)} ${n.toLocaleString()}`) .join(" + ")} = ${totalUnits(units).toLocaleString()}`; + +export const pricingIssueUrl = (unpriced: UsageUnits, provider?: string): string => { + const subject = provider ? `${provider} guardrail` : "guardrail"; + const params = new URLSearchParams({ + template: "feature_request.yml", + title: `[Feature]: add ${subject} pricing to the cost map`, + "the-feature": `LiteLLM has no price for these ${subject} usage units, so the Guardrails Monitor leaves them out of the cost: ${Object.keys(unpriced).join(", ")}`, + }); + return `https://github.com/BerriAI/litellm/issues/new?${params.toString()}`; +};