From f96bfdd66dbe3e2643dbbe2da422d0e40c14d961 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Fri, 25 Sep 2026 19:57:49 -0700 Subject: [PATCH] fix(autorouter): show only eligible spend in savings comparisons --- .../AutoRouterBenchmarksTab.test.tsx | 8 ++- .../_components/AutoRouterBenchmarksTab.tsx | 54 ++++++++++--------- .../user_info_view.integration.test.tsx | 7 +-- ...KeyAutoRouterUsageTab.integration.test.tsx | 24 +++------ 4 files changed, 40 insertions(+), 53 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx index 48501c93f55..868d41f061e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx @@ -179,8 +179,7 @@ describe("AutoRouterBenchmarksTab", () => { expect(screen.getByText("Total estimated savings")).toBeInTheDocument(); expect(screen.getByText("$2,174.59")).toBeInTheDocument(); expect(screen.queryByText("-86%")).not.toBeInTheDocument(); - expect(screen.getByText("$359.86")).toBeInTheDocument(); - expect(screen.getByText("Unavailable")).toBeInTheDocument(); + expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual(["Unavailable", "Unavailable"]); expect(screen.getByText("$23.13")).toBeInTheDocument(); expect(screen.getByText("Estimated baseline spend")).toBeInTheDocument(); }); @@ -237,7 +236,7 @@ describe("AutoRouterBenchmarksTab", () => { expect(screen.getAllByText("$10,126.28").length).toBeGreaterThan(0); }); - it.each([null, undefined])("keeps totals when the classification breakdown is %s", (classifier_cost) => { + it.each([null, undefined])("keeps recorded savings when the comparison is unavailable: %s", (classifier_cost) => { const partialCosts: Partial = { classifier_cost, cost_coverage: "partial", @@ -250,8 +249,7 @@ describe("AutoRouterBenchmarksTab", () => { expect(screen.getAllByText("Unavailable")).toHaveLength(2); expect(screen.queryByText(/\/ 1K turns/)).not.toBeInTheDocument(); - expect(screen.getByText("$359.86")).toBeInTheDocument(); - expect(screen.getByText("$353.71")).toBeInTheDocument(); + expect(screen.queryByText("LLM spend")).not.toBeInTheDocument(); expect(screen.getByText("$2,174.59")).toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx index 6842af45c75..d8828374530 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx @@ -11,7 +11,7 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@ import { Separator } from "@/components/ui/separator"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; +import { SimpleTooltip, Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { ApiError } from "@/lib/http/client"; import { formatNumberWithCommas } from "@/utils/dataUtils"; @@ -53,15 +53,17 @@ const Metric: React.FC<{ label: string; value: string; hint?: string }> = ({ lab ); -const SpendRow: React.FC<{ label: string; value: string; hint?: string; subdued?: boolean }> = ({ +const SpendRow: React.FC<{ label: string; value: string; hint?: string; subdued?: boolean; tooltip?: string }> = ({ label, value, hint, subdued, + tooltip, }) => (
{label} + {tooltip && } {hint && {hint}}
); -const CostBreakdown: React.FC<{ view: BenchmarkView; partialComparison: boolean }> = ({ view, partialComparison }) => { +const CostBreakdown: React.FC<{ view: BenchmarkView }> = ({ view }) => { const stats = view.stats; const costTurns = viewGroup(view) ? stats.turns : stats.cost_requests; + const eligibleSpend = stats.savings_estimated_turns > 0 ? stats.savings_estimated_actual_spend : stats.spend; + const actualSpend = stats.baseline_spend == null ? null : eligibleSpend; + const fullComparison = + actualSpend != null && (stats.savings_estimated_turns === 0 || stats.savings_estimated_turns === costTurns); const llmSpend = stats.llm_spend === undefined && stats.spend != null && stats.classifier_cost != null ? stats.spend - stats.classifier_cost @@ -83,29 +89,25 @@ const CostBreakdown: React.FC<{ view: BenchmarkView; partialComparison: boolean
-
- - -
- {stats.cost_coverage === "partial" && ( -

Some request costs are unavailable

+ {fullComparison && ( +
+ + +
)} - {partialComparison && stats.baseline_spend != null && ( - - )} = ({ view }) => {
{partialComparison && (

- {stats.savings_estimated_turns.toLocaleString()} of {comparisonRequests?.toLocaleString()} requests have - savings estimates + Savings based on {stats.savings_estimated_turns.toLocaleString()} of{" "} + {comparisonRequests?.toLocaleString()} requests

)} - + ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx index f734de07e4c..5890a9291ec 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx @@ -413,12 +413,7 @@ describe("UserInfoView auto-router usage", () => { ).toBeInTheDocument(); expect(screen.getByText("$3.00")).toBeInTheDocument(); expect(screen.getByText("Estimated baseline spend")).toBeInTheDocument(); - expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual([ - "Unavailable", - "Unavailable", - "Unavailable", - "Unavailable", - ]); + expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual(["Unavailable", "Unavailable"]); }, ); diff --git a/ui/litellm-dashboard/src/components/templates/KeyAutoRouterUsageTab.integration.test.tsx b/ui/litellm-dashboard/src/components/templates/KeyAutoRouterUsageTab.integration.test.tsx index cde9b329df5..c65a59d42cf 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyAutoRouterUsageTab.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/KeyAutoRouterUsageTab.integration.test.tsx @@ -92,7 +92,7 @@ describe("KeyAutoRouterUsageTab", () => { vi.stubGlobal("fetch", fetchMock); }); - it("renders this key's daily savings and selected-router session costs", async () => { + it("compares only eligible request costs and explains the excluded usage", async () => { const user = userEvent.setup(); const activity = { dateValue: { from: new Date(2025, 0, 1), to: new Date(2025, 0, 31) }, @@ -101,12 +101,7 @@ describe("KeyAutoRouterUsageTab", () => { renderWithProviders(); expect(await screen.findByText("$12.75")).toBeInTheDocument(); - expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual([ - "Unavailable", - "Unavailable", - "Unavailable", - "Unavailable", - ]); + expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual(["Unavailable", "Unavailable"]); expect(screen.getByText("$0.5000")).toBeInTheDocument(); expect(screen.queryByText("-50%")).not.toBeInTheDocument(); expect(screen.getByText("All auto-routers")).toBeInTheDocument(); @@ -123,15 +118,12 @@ describe("KeyAutoRouterUsageTab", () => { expect(screen.getByText("Whole sessions overlapping the selected dates")).toBeInTheDocument(); expect(screen.queryByText("$12.75")).not.toBeInTheDocument(); expect(screen.getByText("-50%")).toBeInTheDocument(); - expect(screen.getByText("1 of 2 requests have savings estimates")).toBeInTheDocument(); - expect(screen.getByText("Actual spend on estimated requests")).toBeInTheDocument(); - expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual([ - "$100.00", - "$99.75", - "$0.2500", - "$1.00", - "$2.00", - ]); + expect(screen.getByText("Savings based on 1 of 2 requests")).toBeInTheDocument(); + expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual(["$1.00", "$2.00"]); + await user.hover(screen.getByLabelText("question-circle")); + expect( + await screen.findByText(/Requests without an estimate are excluded from the savings comparison/), + ).toBeVisible(); expect(screen.getByText("Auto-router prompt caching")).toBeInTheDocument(); expect(screen.getAllByText("50.0%").length).toBeGreaterThan(0); });