From 9a9b4c4c2538bf0df6d68eaabe48cefbb4824e7d Mon Sep 17 00:00:00 2001 From: tin-berri Date: Mon, 7 Sep 2026 23:37:42 -0700 Subject: [PATCH] feat(ui): show auto-router classification rate (#40192) --- .../AutoRouterBenchmarksTab.test.tsx | 12 ++++----- .../_components/AutoRouterBenchmarksTab.tsx | 26 ++++++++++++++----- .../_components/costOptimizationUtils.test.ts | 18 +++++++++++++ .../_components/costOptimizationUtils.ts | 7 +++++ ...KeyAutoRouterUsageTab.integration.test.tsx | 2 +- 5 files changed, 52 insertions(+), 13 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx index 006da4f2725..2820a9dce83 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx @@ -187,10 +187,10 @@ describe("AutoRouterBenchmarksTab", () => { }); it.each([ - { spend: 20665.28, classifier_cost: 342.18, turns: 140815, llm: "$20,323.10", cost: "$342.18" }, - { spend: 0, classifier_cost: 0, turns: 0, llm: "$0.00", cost: "$0.00" }, - { spend: 0.002, classifier_cost: 0.0004, turns: 100, llm: "$0.0016", cost: "$0.0004" }, - ])("shows total classification cost across $turns turns without a per-turn rate", ({ llm, cost, ...values }) => { + { spend: 20665.28, classifier_cost: 342.18, turns: 140815, llm: "$20,323.10", cost: "$342.18", rate: "$2.43" }, + { spend: 0, classifier_cost: 0, turns: 0, llm: "$0.00", cost: "$0.00", rate: "$0.00" }, + { spend: 0.002, classifier_cost: 0.0004, turns: 100, llm: "$0.0016", cost: "$0.0004", rate: "$0.0040" }, + ])("shows total classification cost and its rate across $turns turns", ({ llm, cost, rate, ...values }) => { const stats = totals({ ...values, saved_spend: 10126.28, baseline_spend: values.spend + 10126.28 }); mockHook({ data: response([group(stats)], stats) }); renderTab(); @@ -201,7 +201,7 @@ describe("AutoRouterBenchmarksTab", () => { .map((node) => node.textContent) .slice(1, 3), ).toEqual([llm, cost]); - expect(screen.queryByText(/1K turns/)).not.toBeInTheDocument(); + expect(screen.getByText(`(${rate} / 1K turns)`)).toBeInTheDocument(); expect(screen.getAllByText("$10,126.28").length).toBeGreaterThan(0); }); @@ -237,7 +237,7 @@ describe("AutoRouterBenchmarksTab", () => { expect(terms).toEqual([ "Actual auto-router spend", "LLM spend", - "Classification cost", + "Classification cost($2.00 / 1K turns)", "Estimated spend at highest-tier model", ]); expect(values).toEqual(["$359.86", "$353.71", "$6.15", "$2,534.45"]); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx index 33ad1bfe555..ce5ab1c6776 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx @@ -30,7 +30,7 @@ import { type BenchmarkView, type BucketRow, } from "./autoRouterBenchmarks"; -import { formatRangeLabel, usd } from "./costOptimizationUtils"; +import { classificationRatePer1kTurns, formatRangeLabel, usd } from "./costOptimizationUtils"; import ShadowEvalSection from "./ShadowEvalSection"; import TierTurnsChart from "./TierTurnsChart"; import { useAutoRouterBenchmarks } from "./useAutoRouterBenchmarks"; @@ -52,9 +52,17 @@ const Metric: React.FC<{ label: string; value: string; hint?: string }> = ({ lab ); -const SpendRow: React.FC<{ label: string; value: string; subdued?: boolean }> = ({ label, value, subdued }) => ( +const SpendRow: React.FC<{ label: string; value: string; hint?: string; subdued?: boolean }> = ({ + label, + value, + hint, + subdued, +}) => (
-
{label}
+
+ {label} + {hint && {hint}} +
@@ -99,6 +107,11 @@ const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => { subdued label="Classification cost" value={stats.classifier_cost == null ? "Unavailable" : usd(stats.classifier_cost)} + hint={ + stats.classifier_cost == null + ? undefined + : classificationRatePer1kTurns(stats.classifier_cost, stats.turns) + } /> {stats.classifier_cost == null && ( @@ -277,9 +290,10 @@ const BenchmarksBody: React.FC = ({ isPending, error, data,

Compares your actual routed spend with the estimated cost of using only the most expensive model configured in the auto-router. It accounts for both the cache savings from staying on one model and the added cache costs from - switching models. Savings are net of recorded LLM classification cost, which is included in actual spend. The - range counts whole sessions that overlap it, so totals can differ slightly from the Overall tab, which buckets - savings by UTC day. + switching models. Savings are net of recorded LLM classification cost, which is included in actual spend. + Classification cost per 1K turns is averaged over all auto-router turns, including those that skip + classification. The range counts whole sessions that overlap it, so totals can differ slightly from the Overall + tab, which buckets savings by UTC day.

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts index 9a2d7a0b0ec..5d2c48e6440 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts @@ -7,6 +7,7 @@ import { SAVINGS_DRIVERS, SAVINGS_SERIES, buildDailyToolSeries, + classificationRatePer1kTurns, computeCacheLeakage, formatRangeLabel, isAnthropicModel, @@ -401,6 +402,23 @@ describe("usd", () => { }); }); +describe("classificationRatePer1kTurns", () => { + it("normalizes total classification cost to one thousand turns", () => { + expect(classificationRatePer1kTurns(342.18, 140815)).toBe("($2.43 / 1K turns)"); + expect(classificationRatePer1kTurns(0.0004, 100)).toBe("($0.0040 / 1K turns)"); + }); + + it("shows a floor instead of rounding a real cost down to zero", () => { + expect(classificationRatePer1kTurns(0.00001, 1000)).toBe("(<$0.0001 / 1K turns)"); + expect(classificationRatePer1kTurns(0.0001, 1000)).toBe("($0.0001 / 1K turns)"); + }); + + it("reports zero when there are no turns or no classification cost", () => { + expect(classificationRatePer1kTurns(0, 0)).toBe("($0.00 / 1K turns)"); + expect(classificationRatePer1kTurns(0, 100)).toBe("($0.00 / 1K turns)"); + }); +}); + describe("savings driver colours", () => { it("keeps a driver's colour when a driver above it is filtered out", () => { // Charts colour by position in the data they are given, and the donut is given diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts index 7019b0d3301..464c779aa2b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts @@ -10,6 +10,13 @@ export const usd = (value: number): string => { return `${value < 0 ? "-" : ""}$${formatNumberWithCommas(magnitude, decimals)}`; }; +export const classificationRatePer1kTurns = (classifierCost: number, turns: number): string => { + if (turns <= 0) return `(${usd(0)} / 1K turns)`; + const rate = (classifierCost * 1000) / turns; + if (rate > 0 && rate < 0.0001) return "(<$0.0001 / 1K turns)"; + return `(${usd(rate)} / 1K turns)`; +}; + export const pct = (ratio: number): string => `${formatNumberWithCommas(ratio * 100, 1)}%`; export const shortDate = (iso: string): string => diff --git a/ui/litellm-dashboard/src/components/templates/KeyAutoRouterUsageTab.integration.test.tsx b/ui/litellm-dashboard/src/components/templates/KeyAutoRouterUsageTab.integration.test.tsx index be95c0e600d..1f6a67b27ac 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyAutoRouterUsageTab.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/KeyAutoRouterUsageTab.integration.test.tsx @@ -89,7 +89,7 @@ describe("KeyAutoRouterUsageTab", () => { expect(screen.getByText("$1.00")).toBeInTheDocument(); expect(screen.getByText("Classification cost")).toBeInTheDocument(); expect(screen.getByText("$0.2500")).toBeInTheDocument(); - expect(screen.queryByText(/1K turns/)).not.toBeInTheDocument(); + expect(screen.getByText("($62.50 / 1K turns)")).toBeInTheDocument(); expect(screen.getByText("Estimated spend at highest-tier model")).toBeInTheDocument(); expect(screen.getByText("$10.00")).toBeInTheDocument(); expect(screen.getByText("Auto-router prompt caching")).toBeInTheDocument();