mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-06 02:48:13 +00:00
fix(ui): keep guardrail cost hints provider neutral and link to a pricing request
The hint copy described Bedrock's unit semantics and cost map entry even though any provider's units reach this view, so it now explains the math in provider-neutral terms. When units have no known price, the hint says so and links to a prefilled GitHub feature request (provider and counter names filled in) so the reader can ask for pricing. Per-unit prices below $0.000001 now read "< $0.000001" instead of "$0". Refs LIT-5652
This commit is contained in:
parent
5a22edb6c3
commit
e66ba0533f
7 changed files with 113 additions and 20 deletions
|
|
@ -97,6 +97,34 @@ describe("GuardrailUsageBreakdown", () => {
|
|||
expect(screen.getByText("Sensitive Information Policy: 300 × $0.0001 = $0.0300")).toBeInTheDocument();
|
||||
expect(screen.getByText("Some Future Counter: 7 units with no known price, left out")).toBeInTheDocument();
|
||||
expect(screen.getByText("Total: $0.1800")).toBeInTheDocument();
|
||||
expect(screen.getByText(/7 units with no known price are left out of the cost/)).toBeInTheDocument();
|
||||
const issueLink = screen.getByRole("link", { name: "Request pricing on GitHub" });
|
||||
expect(issueLink).toHaveAttribute("target", "_blank");
|
||||
const issueUrl = new URL(issueLink.getAttribute("href") ?? "");
|
||||
expect(issueUrl.searchParams.get("title")).toBe("[Feature]: add Bedrock guardrail pricing to the cost map");
|
||||
expect(issueUrl.searchParams.get("the-feature")).toContain("someFutureCounter");
|
||||
});
|
||||
|
||||
it("does not ask for pricing when every unit was priced", async () => {
|
||||
const user = userEvent.setup();
|
||||
render(
|
||||
<GuardrailUsageBreakdown
|
||||
detail={{
|
||||
...detail,
|
||||
cost: 0.15,
|
||||
usage_units: { contentPolicyUnits: 1000 },
|
||||
cost_by_unit: { contentPolicyUnits: 0.15 },
|
||||
untracked_usage_units: {},
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
await user.hover(
|
||||
within(screen.getByRole("group", { name: "Cost" })).getByRole("button", { name: /How is this calculated/ }),
|
||||
);
|
||||
|
||||
expect(await screen.findByText("Total: $0.1500")).toBeInTheDocument();
|
||||
expect(screen.queryByRole("link", { name: "Request pricing on GitHub" })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("explains the units sum on hover", async () => {
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { CircleDollarSign } from "lucide-react";
|
|||
import React from "react";
|
||||
import type { GuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage";
|
||||
import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard";
|
||||
import { UnpricedNote } from "@/components/GuardrailsMonitor/UnpricedNote";
|
||||
import {
|
||||
counterLabel,
|
||||
counterMathLine,
|
||||
|
|
@ -111,23 +112,21 @@ const groupColumns = (label: string, emptyLabel: string): ColumnDef<GroupRow>[]
|
|||
const teamColumns = groupColumns("Team", "No team");
|
||||
const keyColumns = groupColumns("Key", "No key");
|
||||
|
||||
const CostMath = ({ counters, total }: { counters: CounterRow[]; total: number | null }) => (
|
||||
const CostMath = ({ counters, detail }: { counters: CounterRow[]; detail: GuardrailUsageDetail }) => (
|
||||
<div className="space-y-1">
|
||||
{counters.map((row) => (
|
||||
<div key={row.counter}>{counterMathLine(row)}</div>
|
||||
))}
|
||||
<div className="font-medium">Total: {formatCost(total)}</div>
|
||||
<div>Per-unit prices come from the bedrock/guardrails entry in the cost map.</div>
|
||||
<div className="font-medium">Total: {formatCost(detail.cost)}</div>
|
||||
<div>Each counter is its priced units × the per-unit price LiteLLM has for it in the cost map.</div>
|
||||
<UnpricedNote unpriced={detail.untracked_usage_units} provider={detail.provider} />
|
||||
</div>
|
||||
);
|
||||
|
||||
const UnitsMath = ({ units }: { units: GuardrailUsageDetail["usage_units"] }) => (
|
||||
<div className="space-y-1">
|
||||
<div>{unitsSumLine(units)}</div>
|
||||
<div>
|
||||
Bedrock reports one unit per 1,000 characters of the message for each policy the guardrail has on, on every call,
|
||||
blocked or not.
|
||||
</div>
|
||||
<div>Units are the billable counters the provider reported for this guardrail, added up over every call.</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
|
|
@ -159,7 +158,7 @@ export function GuardrailUsageBreakdown({ detail }: { detail: GuardrailUsageDeta
|
|||
valueColor={detail.cost != null ? "text-foreground" : "text-muted-foreground"}
|
||||
icon={<CircleDollarSign className="size-4" />}
|
||||
subtitle={unpriced ?? undefined}
|
||||
hint={<CostMath counters={counters} total={detail.cost} />}
|
||||
hint={<CostMath counters={counters} detail={detail} />}
|
||||
/>
|
||||
<MetricCard
|
||||
label="Usage Units"
|
||||
|
|
|
|||
|
|
@ -222,7 +222,11 @@ describe("GuardrailsOverview", () => {
|
|||
expect(screen.getByText("Free Bedrock Guardrail: $0.0000")).toBeInTheDocument();
|
||||
expect(screen.queryByText(/Low Failure Guardrail: /)).not.toBeInTheDocument();
|
||||
expect(screen.getByText("Total: $0.1500")).toBeInTheDocument();
|
||||
expect(screen.getByText(/250 units unpriced had no known price and are left out/)).toBeInTheDocument();
|
||||
expect(screen.getByText(/250 units with no known price are left out of the cost/)).toBeInTheDocument();
|
||||
const issueLink = screen.getByRole("link", { name: "Request pricing on GitHub" });
|
||||
const issueUrl = new URL(issueLink.getAttribute("href") ?? "");
|
||||
expect(issueUrl.searchParams.get("template")).toBe("feature_request.yml");
|
||||
expect(issueUrl.searchParams.get("the-feature")).toContain("sensitiveInformationPolicyUnits");
|
||||
});
|
||||
|
||||
it("shows a dash for guardrail cost when nothing in the window was priced", async () => {
|
||||
|
|
|
|||
|
|
@ -8,7 +8,14 @@ import {
|
|||
type GuardrailUsageOverviewRow,
|
||||
useGuardrailsUsageOverview,
|
||||
} from "@/app/(dashboard)/hooks/guardrails/useGuardrailsUsage";
|
||||
import { counterLabel, formatCost, totalUnits, unpricedSummary } from "@/components/GuardrailsMonitor/usageUnits";
|
||||
import { UnpricedNote } from "@/components/GuardrailsMonitor/UnpricedNote";
|
||||
import {
|
||||
counterLabel,
|
||||
formatCost,
|
||||
totalUnits,
|
||||
unpricedSummary,
|
||||
type UsageUnits,
|
||||
} from "@/components/GuardrailsMonitor/usageUnits";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { PageHeader } from "@/components/shared/PageHeader";
|
||||
import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
|
||||
|
|
@ -41,7 +48,7 @@ const EMPTY_METRICS = {
|
|||
avgLatency: 0,
|
||||
count: 0,
|
||||
totalCost: null as number | null,
|
||||
unpriced: null as string | null,
|
||||
untracked: {} as UsageUnits,
|
||||
};
|
||||
|
||||
function UsageUnitsCell({ units }: { units: GuardrailUsageOverviewRow["usageUnits"] }) {
|
||||
|
|
@ -66,11 +73,11 @@ function UsageUnitsCell({ units }: { units: GuardrailUsageOverviewRow["usageUnit
|
|||
function TotalCostMath({
|
||||
rows,
|
||||
total,
|
||||
unpriced,
|
||||
untracked,
|
||||
}: {
|
||||
rows: GuardrailUsageOverviewRow[];
|
||||
total: number | null;
|
||||
unpriced: string | null;
|
||||
untracked: UsageUnits;
|
||||
}) {
|
||||
return (
|
||||
<div className="space-y-1">
|
||||
|
|
@ -83,10 +90,9 @@ function TotalCostMath({
|
|||
))}
|
||||
<div className="font-medium">Total: {formatCost(total)}</div>
|
||||
<div>
|
||||
{`Each guardrail's cost is its units per policy × that policy's per-unit price from the cost map, added up${
|
||||
unpriced ? `; ${unpriced} had no known price and are left out` : ""
|
||||
}. Open a guardrail for its per-policy math.`}
|
||||
{`Each guardrail's cost is its units per counter × that counter's per-unit price from the cost map, added up. Open a guardrail for its per-counter math.`}
|
||||
</div>
|
||||
<UnpricedNote unpriced={untracked} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -135,7 +141,7 @@ export function GuardrailsOverview({
|
|||
: 0,
|
||||
count: activeData.length,
|
||||
totalCost: guardrailsData.totalCost,
|
||||
unpriced: unpricedSummary(guardrailsData.totalUntrackedUsageUnits),
|
||||
untracked: guardrailsData.totalUntrackedUsageUnits,
|
||||
};
|
||||
}, [guardrailsData, activeData]);
|
||||
const chartData = guardrailsData?.chart;
|
||||
|
|
@ -318,8 +324,8 @@ export function GuardrailsOverview({
|
|||
value={formatCost(metrics.totalCost)}
|
||||
valueColor={metrics.totalCost != null ? "text-foreground" : "text-muted-foreground"}
|
||||
icon={<CircleDollarSign className="size-4" />}
|
||||
subtitle={metrics.unpriced ?? undefined}
|
||||
hint={<TotalCostMath rows={activeData} total={metrics.totalCost} unpriced={metrics.unpriced} />}
|
||||
subtitle={unpricedSummary(metrics.untracked) ?? undefined}
|
||||
hint={<TotalCostMath rows={activeData} total={metrics.totalCost} untracked={metrics.untracked} />}
|
||||
/>
|
||||
<MetricCard label="Active Guardrails" value={metrics.count} />
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
import React from "react";
|
||||
import { pricingIssueUrl, totalUnits, type UsageUnits } from "./usageUnits";
|
||||
|
||||
export function UnpricedNote({ unpriced, provider }: { unpriced: UsageUnits; provider?: string }) {
|
||||
const total = totalUnits(unpriced);
|
||||
if (total === 0) return null;
|
||||
const [noun, verb] = total === 1 ? ["unit", "is"] : ["units", "are"];
|
||||
return (
|
||||
<div>
|
||||
{`${total.toLocaleString()} ${noun} with no known price ${verb} left out of the cost. `}
|
||||
<a
|
||||
href={pricingIssueUrl(unpriced, provider)}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
className="underline underline-offset-2"
|
||||
>
|
||||
Request pricing on GitHub
|
||||
</a>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@ import {
|
|||
counterMathLine,
|
||||
formatCost,
|
||||
formatUnitPrice,
|
||||
pricingIssueUrl,
|
||||
totalUnits,
|
||||
unitPrice,
|
||||
unitsSumLine,
|
||||
|
|
@ -84,6 +85,10 @@ describe("formatUnitPrice", () => {
|
|||
expect(formatUnitPrice(0)).toBe("$0");
|
||||
expect(formatUnitPrice(1)).toBe("$1");
|
||||
});
|
||||
|
||||
it("never shows a positive price as free", () => {
|
||||
expect(formatUnitPrice(0.0000002)).toBe("< $0.000001");
|
||||
});
|
||||
});
|
||||
|
||||
describe("counterMathLine", () => {
|
||||
|
|
@ -122,3 +127,20 @@ describe("unitsSumLine", () => {
|
|||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("pricingIssueUrl", () => {
|
||||
it("prefills the feature request with the provider and the unpriced counters", () => {
|
||||
const url = new URL(pricingIssueUrl({ text_records: 5, someFutureCounter: 7 }, "azure/prompt_shield"));
|
||||
|
||||
expect(url.origin + url.pathname).toBe("https://github.com/BerriAI/litellm/issues/new");
|
||||
expect(url.searchParams.get("template")).toBe("feature_request.yml");
|
||||
expect(url.searchParams.get("title")).toBe("[Feature]: add azure/prompt_shield guardrail pricing to the cost map");
|
||||
expect(url.searchParams.get("the-feature")).toContain("text_records, someFutureCounter");
|
||||
});
|
||||
|
||||
it("stays generic when no provider is known", () => {
|
||||
const url = new URL(pricingIssueUrl({ text_records: 5 }));
|
||||
|
||||
expect(url.searchParams.get("title")).toBe("[Feature]: add guardrail pricing to the cost map");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -35,7 +35,10 @@ export const unitPrice = (row: CounterMath): number | null => {
|
|||
return row.cost != null && priced > 0 ? row.cost / priced : null;
|
||||
};
|
||||
|
||||
export const formatUnitPrice = (price: number): string => `$${price.toFixed(6).replace(/\.?0+$/, "")}`;
|
||||
export const formatUnitPrice = (price: number): string => {
|
||||
const fixed = price.toFixed(6).replace(/\.?0+$/, "");
|
||||
return price > 0 && Number(fixed) === 0 ? "< $0.000001" : `$${fixed}`;
|
||||
};
|
||||
|
||||
export const counterMathLine = (row: CounterMath): string => {
|
||||
const label = counterLabel(row.counter);
|
||||
|
|
@ -51,3 +54,13 @@ export const unitsSumLine = (units: UsageUnits): string =>
|
|||
`${Object.entries(units)
|
||||
.map(([counter, n]) => `${counterLabel(counter)} ${n.toLocaleString()}`)
|
||||
.join(" + ")} = ${totalUnits(units).toLocaleString()}`;
|
||||
|
||||
export const pricingIssueUrl = (unpriced: UsageUnits, provider?: string): string => {
|
||||
const subject = provider ? `${provider} guardrail` : "guardrail";
|
||||
const params = new URLSearchParams({
|
||||
template: "feature_request.yml",
|
||||
title: `[Feature]: add ${subject} pricing to the cost map`,
|
||||
"the-feature": `LiteLLM has no price for these ${subject} usage units, so the Guardrails Monitor leaves them out of the cost: ${Object.keys(unpriced).join(", ")}`,
|
||||
});
|
||||
return `https://github.com/BerriAI/litellm/issues/new?${params.toString()}`;
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue