mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-08 22:21:35 +00:00
feat(ui): explain how guardrail usage and cost are calculated
Adds a "How is this calculated?" hover to the Guardrail Cost card on the overview and to the Cost and Usage Units cards on the detail page. The overview hint lists each guardrail's cost and the total; the detail cost hint shows units x per-unit price per counter with unpriced units called out, and the units hint shows the per-counter sum. Also moves the Status column to the front of the overview table. Refs LIT-5652
This commit is contained in:
parent
f73e683800
commit
5a22edb6c3
7 changed files with 250 additions and 22 deletions
|
|
@ -1,4 +1,5 @@
|
|||
import { render, screen, within } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage";
|
||||
import { GuardrailUsageBreakdown } from "./GuardrailUsageBreakdown";
|
||||
|
|
@ -84,6 +85,37 @@ describe("GuardrailUsageBreakdown", () => {
|
|||
expect(within(unpricedKey).getByText("7", { selector: ".text-warning" })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("explains the cost math per counter on hover", async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<GuardrailUsageBreakdown detail={detail} />);
|
||||
|
||||
await user.hover(
|
||||
within(screen.getByRole("group", { name: "Cost" })).getByRole("button", { name: /How is this calculated/ }),
|
||||
);
|
||||
|
||||
expect(await screen.findByText("Content Policy: 1,000 × $0.00015 = $0.1500")).toBeInTheDocument();
|
||||
expect(screen.getByText("Sensitive Information Policy: 300 × $0.0001 = $0.0300")).toBeInTheDocument();
|
||||
expect(screen.getByText("Some Future Counter: 7 units with no known price, left out")).toBeInTheDocument();
|
||||
expect(screen.getByText("Total: $0.1800")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("explains the units sum on hover", async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<GuardrailUsageBreakdown detail={detail} />);
|
||||
|
||||
await user.hover(
|
||||
within(screen.getByRole("group", { name: "Usage Units" })).getByRole("button", {
|
||||
name: /How is this calculated/,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(
|
||||
await screen.findByText(
|
||||
"Content Policy 1,000 + Sensitive Information Policy 300 + Some Future Counter 7 = 1,307",
|
||||
),
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("orders teams and keys by units, largest first", () => {
|
||||
render(<GuardrailUsageBreakdown detail={detail} />);
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,14 @@ import { CircleDollarSign } from "lucide-react";
|
|||
import React from "react";
|
||||
import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage";
|
||||
import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard";
|
||||
import { counterLabel, formatCost, totalUnits, unpricedSummary } from "@/components/GuardrailsMonitor/usageUnits";
|
||||
import {
|
||||
counterLabel,
|
||||
counterMathLine,
|
||||
formatCost,
|
||||
totalUnits,
|
||||
unitsSumLine,
|
||||
unpricedSummary,
|
||||
} from "@/components/GuardrailsMonitor/usageUnits";
|
||||
import { DataTable } from "@/components/shared/DataTable";
|
||||
import { IdCell } from "@/components/shared/table_cells/id_cell";
|
||||
import { MoneyCell } from "@/components/shared/table_cells/money_cell";
|
||||
|
|
@ -104,6 +111,26 @@ const groupColumns = (label: string, emptyLabel: string): ColumnDef<GroupRow>[]
|
|||
const teamColumns = groupColumns("Team", "No team");
|
||||
const keyColumns = groupColumns("Key", "No key");
|
||||
|
||||
const CostMath = ({ counters, total }: { counters: CounterRow[]; total: number | null }) => (
|
||||
<div className="space-y-1">
|
||||
{counters.map((row) => (
|
||||
<div key={row.counter}>{counterMathLine(row)}</div>
|
||||
))}
|
||||
<div className="font-medium">Total: {formatCost(total)}</div>
|
||||
<div>Per-unit prices come from the bedrock/guardrails entry in the cost map.</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
const UnitsMath = ({ units }: { units: GuardrailUsageDetail["usage_units"] }) => (
|
||||
<div className="space-y-1">
|
||||
<div>{unitsSumLine(units)}</div>
|
||||
<div>
|
||||
Bedrock reports one unit per 1,000 characters of the message for each policy the guardrail has on, on every call,
|
||||
blocked or not.
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
const TableHeading = ({ title }: { title: string }) => (
|
||||
<h6 className="text-sm font-semibold text-foreground">{title}</h6>
|
||||
);
|
||||
|
|
@ -132,11 +159,13 @@ export function GuardrailUsageBreakdown({ detail }: { detail: GuardrailUsageDeta
|
|||
valueColor={detail.cost != null ? "text-foreground" : "text-muted-foreground"}
|
||||
icon={<CircleDollarSign className="size-4" />}
|
||||
subtitle={unpriced ?? undefined}
|
||||
hint={<CostMath counters={counters} total={detail.cost} />}
|
||||
/>
|
||||
<MetricCard
|
||||
label="Usage Units"
|
||||
value={totalUnits(detail.usage_units).toLocaleString()}
|
||||
subtitle={`${counters.length} ${counters.length === 1 ? "counter" : "counters"}`}
|
||||
hint={<UnitsMath units={detail.usage_units} />}
|
||||
/>
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -211,6 +211,20 @@ describe("GuardrailsOverview", () => {
|
|||
expect(card).toHaveTextContent("250 units unpriced");
|
||||
});
|
||||
|
||||
it("explains the guardrail cost total on hover", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderOverview();
|
||||
|
||||
const card = await screen.findByRole("group", { name: "Guardrail Cost" });
|
||||
await user.hover(within(card).getByRole("button", { name: /How is this calculated/ }));
|
||||
|
||||
expect(await screen.findByText("High Failure Guardrail: $0.1500")).toBeInTheDocument();
|
||||
expect(screen.getByText("Free Bedrock Guardrail: $0.0000")).toBeInTheDocument();
|
||||
expect(screen.queryByText(/Low Failure Guardrail: /)).not.toBeInTheDocument();
|
||||
expect(screen.getByText("Total: $0.1500")).toBeInTheDocument();
|
||||
expect(screen.getByText(/250 units unpriced had no known price and are left out/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows a dash for guardrail cost when nothing in the window was priced", async () => {
|
||||
useGuardrailsUsageOverviewMock.mockReturnValue({
|
||||
data: { ...overview, totalCost: null, totalUntrackedUsageUnits: {} },
|
||||
|
|
|
|||
|
|
@ -63,6 +63,34 @@ function UsageUnitsCell({ units }: { units: GuardrailUsageOverviewRow["usageUnit
|
|||
);
|
||||
}
|
||||
|
||||
function TotalCostMath({
|
||||
rows,
|
||||
total,
|
||||
unpriced,
|
||||
}: {
|
||||
rows: GuardrailUsageOverviewRow[];
|
||||
total: number | null;
|
||||
unpriced: string | null;
|
||||
}) {
|
||||
return (
|
||||
<div className="space-y-1">
|
||||
{rows
|
||||
.filter((row) => row.cost != null)
|
||||
.map((row) => (
|
||||
<div key={row.id}>
|
||||
{row.name}: {formatCost(row.cost)}
|
||||
</div>
|
||||
))}
|
||||
<div className="font-medium">Total: {formatCost(total)}</div>
|
||||
<div>
|
||||
{`Each guardrail's cost is its units per policy × that policy's per-unit price from the cost map, added up${
|
||||
unpriced ? `; ${unpriced} had no known price and are left out` : ""
|
||||
}. Open a guardrail for its per-policy math.`}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function CostCell({ row }: { row: GuardrailUsageOverviewRow }) {
|
||||
const unpriced = unpricedSummary(row.untrackedUsageUnits);
|
||||
return (
|
||||
|
|
@ -124,6 +152,25 @@ export function GuardrailsOverview({
|
|||
const error = guardrailsError;
|
||||
|
||||
const columns: ColumnDef<GuardrailUsageOverviewRow>[] = [
|
||||
{
|
||||
header: "Status",
|
||||
accessorKey: "status",
|
||||
enableSorting: false,
|
||||
cell: ({ row }) => (
|
||||
<span className="inline-flex items-center gap-1.5">
|
||||
<span
|
||||
className={`w-2 h-2 rounded-full ${
|
||||
row.original.status === "healthy"
|
||||
? "bg-success"
|
||||
: row.original.status === "warning"
|
||||
? "bg-warning"
|
||||
: "bg-destructive"
|
||||
}`}
|
||||
/>
|
||||
<span className="text-xs text-muted-foreground capitalize">{row.original.status}</span>
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
header: "Guardrail",
|
||||
accessorKey: "name",
|
||||
|
|
@ -215,25 +262,6 @@ export function GuardrailsOverview({
|
|||
sortDescFirst: false,
|
||||
cell: ({ row }) => <CostCell row={row.original} />,
|
||||
},
|
||||
{
|
||||
header: "Status",
|
||||
accessorKey: "status",
|
||||
enableSorting: false,
|
||||
cell: ({ row }) => (
|
||||
<span className="inline-flex items-center gap-1.5">
|
||||
<span
|
||||
className={`w-2 h-2 rounded-full ${
|
||||
row.original.status === "healthy"
|
||||
? "bg-success"
|
||||
: row.original.status === "warning"
|
||||
? "bg-warning"
|
||||
: "bg-destructive"
|
||||
}`}
|
||||
/>
|
||||
<span className="text-xs text-muted-foreground capitalize">{row.original.status}</span>
|
||||
</span>
|
||||
),
|
||||
},
|
||||
];
|
||||
|
||||
const sortableKeys: SortKey[] = ["failRate", "requestsEvaluated", "avgLatency", "cost"];
|
||||
|
|
@ -291,6 +319,7 @@ export function GuardrailsOverview({
|
|||
valueColor={metrics.totalCost != null ? "text-foreground" : "text-muted-foreground"}
|
||||
icon={<CircleDollarSign className="size-4" />}
|
||||
subtitle={metrics.unpriced ?? undefined}
|
||||
hint={<TotalCostMath rows={activeData} total={metrics.totalCost} unpriced={metrics.unpriced} />}
|
||||
/>
|
||||
<MetricCard label="Active Guardrails" value={metrics.count} />
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
import { CircleHelp } from "lucide-react";
|
||||
import React, { type ReactNode } from "react";
|
||||
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
|
||||
|
||||
interface MetricCardProps {
|
||||
label: string;
|
||||
|
|
@ -6,9 +8,10 @@ interface MetricCardProps {
|
|||
valueColor?: string;
|
||||
icon?: ReactNode;
|
||||
subtitle?: string;
|
||||
hint?: ReactNode;
|
||||
}
|
||||
|
||||
export function MetricCard({ label, value, valueColor = "text-foreground", icon, subtitle }: MetricCardProps) {
|
||||
export function MetricCard({ label, value, valueColor = "text-foreground", icon, subtitle, hint }: MetricCardProps) {
|
||||
return (
|
||||
<div role="group" aria-label={label} className="h-full bg-card border border-border rounded-lg p-5 flex flex-col">
|
||||
<div className="flex items-center justify-between mb-1">
|
||||
|
|
@ -17,6 +20,26 @@ export function MetricCard({ label, value, valueColor = "text-foreground", icon,
|
|||
</div>
|
||||
<div className={`text-3xl font-semibold ${valueColor} tracking-tight`}>{value}</div>
|
||||
{subtitle && <p className="text-xs text-muted-foreground mt-1">{subtitle}</p>}
|
||||
{hint && (
|
||||
<TooltipProvider delay={300}>
|
||||
<Tooltip>
|
||||
<TooltipTrigger
|
||||
render={
|
||||
<button
|
||||
type="button"
|
||||
className="mt-2 inline-flex w-fit cursor-help items-start gap-1 text-left text-xs text-muted-foreground hover:text-foreground"
|
||||
>
|
||||
<CircleHelp className="mt-px size-3.5 shrink-0" />
|
||||
How is this calculated?
|
||||
</button>
|
||||
}
|
||||
/>
|
||||
<TooltipContent side="bottom" align="start" className="max-w-sm">
|
||||
{hint}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</TooltipProvider>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,14 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { counterLabel, formatCost, totalUnits, unpricedSummary } from "./usageUnits";
|
||||
import {
|
||||
counterLabel,
|
||||
counterMathLine,
|
||||
formatCost,
|
||||
formatUnitPrice,
|
||||
totalUnits,
|
||||
unitPrice,
|
||||
unitsSumLine,
|
||||
unpricedSummary,
|
||||
} from "./usageUnits";
|
||||
|
||||
describe("formatCost", () => {
|
||||
it("renders a dash when nothing was priced", () => {
|
||||
|
|
@ -53,3 +62,63 @@ describe("unpricedSummary", () => {
|
|||
expect(unpricedSummary({ someFutureCounter: 1 })).toBe("1 unit unpriced");
|
||||
});
|
||||
});
|
||||
|
||||
describe("unitPrice", () => {
|
||||
it("backs the per-unit price out of the priced share only", () => {
|
||||
expect(unitPrice({ counter: "contentPolicyUnits", units: 1200, unpriced: 200, cost: 0.15 })).toBeCloseTo(
|
||||
0.00015,
|
||||
10,
|
||||
);
|
||||
});
|
||||
|
||||
it("is null when nothing was priced", () => {
|
||||
expect(unitPrice({ counter: "someFutureCounter", units: 7, unpriced: 7, cost: null })).toBeNull();
|
||||
expect(unitPrice({ counter: "someFutureCounter", units: 7, unpriced: 7, cost: 0 })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatUnitPrice", () => {
|
||||
it("keeps the significant decimals and drops trailing zeros", () => {
|
||||
expect(formatUnitPrice(0.0001)).toBe("$0.0001");
|
||||
expect(formatUnitPrice(0.00015)).toBe("$0.00015");
|
||||
expect(formatUnitPrice(0)).toBe("$0");
|
||||
expect(formatUnitPrice(1)).toBe("$1");
|
||||
});
|
||||
});
|
||||
|
||||
describe("counterMathLine", () => {
|
||||
it("shows units × price = cost for a fully priced counter", () => {
|
||||
expect(counterMathLine({ counter: "contentPolicyUnits", units: 1000, unpriced: 0, cost: 0.15 })).toBe(
|
||||
"Content Policy: 1,000 × $0.00015 = $0.1500",
|
||||
);
|
||||
});
|
||||
|
||||
it("prices only the priced share and calls out the rest", () => {
|
||||
expect(counterMathLine({ counter: "sensitiveInformationPolicyUnits", units: 8, unpriced: 2, cost: 0.0006 })).toBe(
|
||||
"Sensitive Information Policy: 6 × $0.0001 = $0.0006 (2 unpriced left out)",
|
||||
);
|
||||
});
|
||||
|
||||
it("says so when a counter has no known price at all", () => {
|
||||
expect(counterMathLine({ counter: "someFutureCounter", units: 7, unpriced: 7, cost: null })).toBe(
|
||||
"Some Future Counter: 7 units with no known price, left out",
|
||||
);
|
||||
expect(counterMathLine({ counter: "someFutureCounter", units: 1, unpriced: 1, cost: null })).toBe(
|
||||
"Some Future Counter: 1 unit with no known price, left out",
|
||||
);
|
||||
});
|
||||
|
||||
it("shows a free counter as × $0", () => {
|
||||
expect(counterMathLine({ counter: "wordPolicyUnits", units: 2, unpriced: 0, cost: 0 })).toBe(
|
||||
"Word Policy: 2 × $0 = $0.0000",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("unitsSumLine", () => {
|
||||
it("adds the counters up in order", () => {
|
||||
expect(unitsSumLine({ contentPolicyUnits: 2, topicPolicyUnits: 2, wordPolicyUnits: 1200 })).toBe(
|
||||
"Content Policy 2 + Topic Policy 2 + Word Policy 1,200 = 1,204",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -19,3 +19,35 @@ export const unpricedSummary = (untracked: UsageUnits): string | null => {
|
|||
const total = totalUnits(untracked);
|
||||
return total > 0 ? `${total.toLocaleString()} ${total === 1 ? "unit" : "units"} unpriced` : null;
|
||||
};
|
||||
|
||||
export interface CounterMath {
|
||||
readonly counter: string;
|
||||
readonly units: number;
|
||||
readonly unpriced: number;
|
||||
readonly cost: number | null;
|
||||
}
|
||||
|
||||
export const pricedUnits = ({ units, unpriced }: Pick<CounterMath, "units" | "unpriced">): number =>
|
||||
Math.max(units - unpriced, 0);
|
||||
|
||||
export const unitPrice = (row: CounterMath): number | null => {
|
||||
const priced = pricedUnits(row);
|
||||
return row.cost != null && priced > 0 ? row.cost / priced : null;
|
||||
};
|
||||
|
||||
export const formatUnitPrice = (price: number): string => `$${price.toFixed(6).replace(/\.?0+$/, "")}`;
|
||||
|
||||
export const counterMathLine = (row: CounterMath): string => {
|
||||
const label = counterLabel(row.counter);
|
||||
const price = unitPrice(row);
|
||||
if (price == null) {
|
||||
return `${label}: ${row.units.toLocaleString()} ${row.units === 1 ? "unit" : "units"} with no known price, left out`;
|
||||
}
|
||||
const line = `${label}: ${pricedUnits(row).toLocaleString()} × ${formatUnitPrice(price)} = ${formatCost(row.cost)}`;
|
||||
return row.unpriced > 0 ? `${line} (${row.unpriced.toLocaleString()} unpriced left out)` : line;
|
||||
};
|
||||
|
||||
export const unitsSumLine = (units: UsageUnits): string =>
|
||||
`${Object.entries(units)
|
||||
.map(([counter, n]) => `${counterLabel(counter)} ${n.toLocaleString()}`)
|
||||
.join(" + ")} = ${totalUnits(units).toLocaleString()}`;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue