diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.integration.test.tsx new file mode 100644 index 00000000000..f65d5600302 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.integration.test.tsx @@ -0,0 +1,232 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen, waitFor } from "@testing-library/react"; +import React from "react"; +import { describe, expect, it, vi } from "vitest"; + +const mockCall = vi.fn(); + +vi.mock("@/components/networking", () => ({ + autoRouterBenchmarksCall: (...args: unknown[]) => mockCall(...args), +})); + +import AutoRouterBenchmarksTab from "./AutoRouterBenchmarksTab"; +import type { AutoRouterCacheBenchmark, AutoRouterGroupBenchmark } from "./autoRouterBenchmarks"; + +const cache = (overrides: Partial = {}): AutoRouterCacheBenchmark => ({ + ttl_seconds: 3600, + usage_coverage_pct: 99.6, + hit_rate_pct: 93.3, + turns: 818, + hits: 763, + same_model_turns: 400, + same_model_hits: 391, + first_visit_turns: 37, + first_visit_hits: 9, + return_turns: 381, + return_hits: 311, + same_model_hit_rate_pct: 97.7, + first_visit_hit_rate_pct: 24.3, + return_hit_rate_pct: 81.6, + stale_miss_share_pct: 27.1, + warming_savable_miss_pct: 4.0, + warming_break_even_pct: 5.0, + stale_return_misses: 19, + savable_return_misses: 2, + warming_rescued_spend: 6.76, + warming_replay_spend: 3.91, + warming_net_spend: 2.85, + ...overrides, +}); + +const group = (overrides: Partial = {}): AutoRouterGroupBenchmark => ({ + model_group: "claude-auto", + router_kind: "complexity", + baseline_model: "anthropic/claude-opus-4-8", + sessions: 94, + turns: 3074, + avg_turns_per_session: 32.7, + avg_session_length_seconds: 7560, + total_tokens: 498_200_000, + avg_tokens_per_session: 5_300_000, + actual_spend: 359.86, + baseline_spend: 2534.45, + savings: 2174.59, + savings_pct: 85.8, + cache: cache(), + ...overrides, +}); + +const renderTab = () => { + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return render( + + + , + ); +}; + +describe("AutoRouterBenchmarksTab", () => { + it("leads with total estimated savings, before the three session-shape metrics", async () => { + mockCall.mockResolvedValue({ start_date: "2026-07-02", end_date: "2026-08-01", groups: [group()] }); + renderTab(); + + await waitFor(() => expect(screen.getByText("Total estimated savings")).toBeInTheDocument()); + const labels = screen + .getAllByText(/Total estimated savings|Avg turns per session|Avg session length|Avg tokens per session/) + .map((node) => node.textContent); + expect(labels).toEqual([ + "Total estimated savings", + "Avg turns per session", + "Avg session length", + "Avg tokens per session", + ]); + }); + + it("pairs the savings with the session count it was earned over", async () => { + mockCall.mockResolvedValue({ start_date: "2026-07-02", end_date: "2026-08-01", groups: [group()] }); + renderTab(); + + await waitFor(() => expect(screen.getByText("Sessions on auto-router")).toBeInTheDocument()); + expect(screen.getByText("94")).toBeInTheDocument(); + expect(screen.getByText("3,074 turns")).toBeInTheDocument(); + expect(screen.getByText("Saved per session")).toBeInTheDocument(); + expect(screen.getByText("$23.13")).toBeInTheDocument(); + expect(screen.getByText("Auto-routers in scope")).toBeInTheDocument(); + }); + + it("says n/a for saved-per-session when there is no baseline to divide", async () => { + mockCall.mockResolvedValue({ + start_date: "2026-07-02", + end_date: "2026-08-01", + groups: [group({ baseline_model: null, actual_spend: 325.21, baseline_spend: 325.21, savings: 0 })], + }); + renderTab(); + + await waitFor(() => expect(screen.getByText("Saved per session")).toBeInTheDocument()); + expect(screen.getByText("n/a")).toBeInTheDocument(); + }); + + it("renders the headline numbers the tiles exist for", async () => { + mockCall.mockResolvedValue({ start_date: "2026-07-02", end_date: "2026-08-01", groups: [group()] }); + renderTab(); + + await waitFor(() => expect(screen.getByText("$2,174.59")).toBeInTheDocument()); + expect(screen.getByText("32.7")).toBeInTheDocument(); + expect(screen.getByText("2.1h")).toBeInTheDocument(); + expect(screen.getByText("5.3M")).toBeInTheDocument(); + expect(screen.getByText("-86%")).toBeInTheDocument(); + expect(screen.getByText(/\$359\.86 routed/)).toBeInTheDocument(); + expect(screen.getByText(/\$2,534\.45 all-claude-opus-4-8 baseline/)).toBeInTheDocument(); + }); + + it("shows a cost increase as a positive delta rather than a saving", async () => { + mockCall.mockResolvedValue({ + start_date: "2026-07-02", + end_date: "2026-08-01", + groups: [group({ actual_spend: 120, baseline_spend: 100, savings: -20, savings_pct: -20 })], + }); + renderTab(); + + await waitFor(() => expect(screen.getByText("-$20.00")).toBeInTheDocument()); + expect(screen.getByText("+20%")).toBeInTheDocument(); + }); + + it("says the savings were not measured when no router declares a baseline", async () => { + mockCall.mockResolvedValue({ + start_date: "2026-07-02", + end_date: "2026-08-01", + groups: [group({ baseline_model: null, actual_spend: 325.21, baseline_spend: 325.21, savings: 0 })], + }); + renderTab(); + + await waitFor(() => expect(screen.getByText("Not measured")).toBeInTheDocument()); + expect(screen.queryByText("$0.00")).not.toBeInTheDocument(); + }); + + it("renders all three cache buckets with their turn counts and hit rates", async () => { + mockCall.mockResolvedValue({ start_date: "2026-07-02", end_date: "2026-08-01", groups: [group()] }); + renderTab(); + + await waitFor(() => expect(screen.getByText("Same model")).toBeInTheDocument()); + expect(screen.getByText("First visit")).toBeInTheDocument(); + expect(screen.getByText("Return to tier")).toBeInTheDocument(); + expect(screen.getByText("400")).toBeInTheDocument(); + expect(screen.getByText("37")).toBeInTheDocument(); + expect(screen.getByText("381")).toBeInTheDocument(); + expect(screen.getByText("97.8%")).toBeInTheDocument(); + expect(screen.getByText("24.3%")).toBeInTheDocument(); + expect(screen.getByText("81.6%")).toBeInTheDocument(); + }); + + it("recomputes each bucket rate from its counts instead of trusting the payload's rate", async () => { + mockCall.mockResolvedValue({ + start_date: "2026-07-02", + end_date: "2026-08-01", + groups: [group({ cache: cache({ return_turns: 200, return_hits: 100, return_hit_rate_pct: 99.9 }) })], + }); + renderTab(); + + await waitFor(() => expect(screen.getByText("Return to tier")).toBeInTheDocument()); + expect(screen.getByText("50.0%")).toBeInTheDocument(); + expect(screen.queryByText("99.9%")).not.toBeInTheDocument(); + }); + + it("shows the warming estimate with its two sides and the break-even marker", async () => { + mockCall.mockResolvedValue({ start_date: "2026-07-02", end_date: "2026-08-01", groups: [group()] }); + renderTab(); + + await waitFor(() => expect(screen.getByText("Cache writes rescued")).toBeInTheDocument()); + expect(screen.getByText("$6.76")).toBeInTheDocument(); + expect(screen.getByText("-$3.91")).toBeInTheDocument(); + expect(screen.getByText("$2.85")).toBeInTheDocument(); + expect(screen.getByText(/break-even ≈ 5% at 1h/)).toBeInTheDocument(); + }); + + it("hides the caching section when no router reported cache usage", async () => { + mockCall.mockResolvedValue({ + start_date: "2026-07-02", + end_date: "2026-08-01", + groups: [group({ cache: null })], + }); + renderTab(); + + await waitFor(() => expect(screen.getByText("Total estimated savings")).toBeInTheDocument()); + expect(screen.queryByText("Auto-router prompt caching")).not.toBeInTheDocument(); + }); + + it("says so when there are no auto-router sessions at all", async () => { + mockCall.mockResolvedValue({ start_date: "2026-07-02", end_date: "2026-08-01", groups: [] }); + renderTab(); + + await waitFor(() => + expect(screen.getByText(/No auto-router sessions in the last 30 days yet/)).toBeInTheDocument(), + ); + }); + + it("degrades to a message when the endpoint is unavailable", async () => { + mockCall.mockRejectedValue(new Error("403")); + renderTab(); + + await waitFor(() => + expect(screen.getByText(/Auto-router benchmarks are unavailable right now/)).toBeInTheDocument(), + ); + }); + + it("labels the default selection instead of leaking the __all__ sentinel", async () => { + mockCall.mockResolvedValue({ start_date: "2026-07-02", end_date: "2026-08-01", groups: [group()] }); + renderTab(); + + await waitFor(() => expect(screen.getByText("All auto-routers")).toBeInTheDocument()); + expect(screen.queryByText("__all__")).not.toBeInTheDocument(); + }); + + it("requests exactly a thirty day window", async () => { + mockCall.mockResolvedValue({ start_date: "2026-07-02", end_date: "2026-08-01", groups: [group()] }); + renderTab(); + + await waitFor(() => expect(mockCall).toHaveBeenCalled()); + const [, start, end] = mockCall.mock.calls[0] as [string, string, string]; + const days = (Date.parse(end) - Date.parse(start)) / (24 * 60 * 60 * 1000); + expect(days).toBe(30); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx new file mode 100644 index 00000000000..c403f6b78a3 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx @@ -0,0 +1,388 @@ +"use client"; + +import React, { useMemo, useState } from "react"; + +import { Badge } from "@/components/ui/badge"; +import { Card, CardAction, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Separator } from "@/components/ui/separator"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; + +import { + ALL_ROUTERS, + compactNumber, + durationLabel, + pct, + toView, + ttlLabel, + usd, + type AutoRouterCacheBenchmark, + type AutoRouterGroupBenchmark, + type BenchmarkView, +} from "./autoRouterBenchmarks"; +import { benchmarksWindow, useAutoRouterBenchmarks } from "./useAutoRouterBenchmarks"; + +interface Bucket { + key: "same_model" | "first_visit" | "return"; + label: string; + turns: number; + hitRate: number; + fill: string; +} + +const bucketsOf = (cache: AutoRouterCacheBenchmark): Bucket[] => [ + { + key: "same_model", + label: "Same model", + turns: cache.same_model_turns, + hitRate: cache.same_model_hit_rate_pct, + fill: "bg-foreground", + }, + { + key: "first_visit", + label: "First visit", + turns: cache.first_visit_turns, + hitRate: cache.first_visit_hit_rate_pct, + fill: "bg-foreground/30", + }, + { + key: "return", + label: "Return to tier", + turns: cache.return_turns, + hitRate: cache.return_hit_rate_pct, + fill: "bg-foreground/60", + }, +]; + +const Metric: React.FC<{ label: string; value: string; muted?: boolean }> = ({ label, value, muted = false }) => ( + + + {label} + + +

{value}

+ + +); + +/** + * The one hero figure on the view. Savings leads because it is the number the tab + * exists to prove; the session shape beside it is the denominator that number is + * earned over, which is why they share a card rather than sitting in a tile row. + */ +const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => { + const measured = view.baselineLabel !== null; + const cheaper = view.savings >= 0; + return ( + +
+
+

Total estimated savings

+ {measured ? ( + <> +
+

{usd(view.savings)}

+ + {cheaper ? "-" : "+"} + {Math.abs(view.savings_pct).toFixed(0)}% + +
+

+ {usd(view.actual_spend)} routed + / + {usd(view.baseline_spend)} all-{view.baselineLabel?.split("/").pop()} baseline +

+ + ) : ( + <> +

Not measured

+

+ These routers declare no counterfactual baseline, so there is nothing to compare the routed mix against +

+ + )} +
+ +
+

Sessions on auto-router

+
+

{view.sessions.toLocaleString()}

+

{view.turns.toLocaleString()} turns

+
+ +
+
+
Saved per session
+
+ {measured ? usd(view.saved_per_session) : "n/a"} +
+
+
+
Auto-routers in scope
+
{view.routers}
+
+
+
+
+ + ); +}; + +/** + * Segments are separated by a 2px gap in the surface colour rather than a stroke, + * so neighbouring steps of the same ramp stay distinct without adding ink that + * isn't data. + */ +const StackedTurnBar: React.FC<{ buckets: Bucket[]; total: number }> = ({ buckets, total }) => ( +
+ {buckets + .filter((b) => b.turns > 0) + .map((b) => ( +
0 ? (100 * b.turns) / total : 0}%` }} + title={`${b.label}: ${b.turns.toLocaleString()} turns (${pct(total > 0 ? (100 * b.turns) / total : 0)})`} + /> + ))} +
+); + +const BucketTable: React.FC<{ buckets: Bucket[] }> = ({ buckets }) => ( + + + + Bucket + Turns + + Hit + + + + {buckets.map((b) => ( + + + + + {b.label} + + + {b.turns.toLocaleString()} + +
+
+
+ + {pct(b.hitRate)} + + ))} + +
+); + +const HitRateCard: React.FC<{ cache: AutoRouterCacheBenchmark; mixedTtl: boolean }> = ({ cache, mixedTtl }) => { + const buckets = bucketsOf(cache); + return ( + + + Cache hit rate + + + {pct(cache.usage_coverage_pct)} coverage + + + + +
+

{pct(cache.hit_rate_pct)}

+

+ across {cache.turns.toLocaleString()} turns + {mixedTtl ? "; mixed TTLs" : ` · ${ttlLabel(cache.ttl_seconds)} TTL`} +

+
+
+ +
+ share of turns + {cache.turns.toLocaleString()} turns total +
+
+ + + + +

+ Mutually exclusive buckets; every turn lands in exactly one, by what the router did. The headline is{" "} + weighted by turn count, not averaged, which is why{" "} + {pct(cache.hit_rate_pct)} sits near same-model's {pct(cache.same_model_hit_rate_pct)} rather than in the + middle. A first visit to a tier is cold by design +

+ + + ); +}; + +const WarmingCard: React.FC<{ cache: AutoRouterCacheBenchmark; routers: number }> = ({ cache, routers }) => { + const paysOff = cache.warming_savable_miss_pct >= cache.warming_break_even_pct; + return ( + + + Savable by warming + + + estimate + + + + +
+

{pct(cache.warming_savable_miss_pct)}

+

of every cache miss

+
+

+ {pct(cache.stale_miss_share_pct)} of return-to-tier misses expired past the TTL; the rest missed because the + prefix changed, which warming cannot fix +

+
+
+
+
+
+ + break-even ≈ {pct(cache.warming_break_even_pct, 0)} at {ttlLabel(cache.ttl_seconds)} + +
+ + + +

If warming were running

+
+
+
Cache writes rescued
+
{usd(cache.warming_rescued_spend)}
+
+
+
+
Cache warming costs
+

replays, one per idle window

+
+
{usd(-cache.warming_replay_spend)}
+
+
+
+
Warming estimate
+

+ net, across {routers} {routers === 1 ? "router" : "routers"} +

+
+
= 0 ? "text-foreground" : "text-destructive" + }`} + > + {usd(cache.warming_net_spend)} +
+
+
+ + + ); +}; + +const Message: React.FC<{ children: React.ReactNode }> = ({ children }) => ( +

{children}

+); + +interface AutoRouterBenchmarksTabProps { + accessToken: string | null; +} + +const AutoRouterBenchmarksTab: React.FC = ({ accessToken }) => { + const window = useMemo(() => benchmarksWindow(new Date()), []); + const { data, isPending, isError } = useAutoRouterBenchmarks(accessToken, window); + const [selected, setSelected] = useState(ALL_ROUTERS); + + const groups: AutoRouterGroupBenchmark[] = useMemo(() => data?.groups ?? [], [data]); + const shown = useMemo( + () => (selected === ALL_ROUTERS ? groups : groups.filter((g) => g.model_group === selected)), + [groups, selected], + ); + const view = useMemo(() => toView(shown), [shown]); + + if (isPending) return Loading auto-router benchmarks...; + if (isError) return Auto-router benchmarks are unavailable right now; + if (groups.length === 0) return No auto-router sessions in the last 30 days yet; + if (view === null) return No sessions for this auto-router in the last 30 days; + + return ( +
+
+
+

Auto-router benchmarks

+

Last 30 days

+
+
+ +
+
+ + + +
+ + + +
+ + {view.baselineLabel !== null && ( +

+ Compares the routed model mix against sending every request to{" "} + {view.baselineLabel} at list prices, pricing that baseline with + a warm single-model cache. A router that thrashes the prompt cache can therefore show a loss, which is a real + cost rather than a rounding artefact +

+ )} + + {view.cache && ( +
+
+

Auto-router prompt caching

+

+ per router; tier ladders differ, and a blended rate would hide who is paying for cold writes +

+
+
+ + +
+
+ )} +
+ ); +}; + +export default AutoRouterBenchmarksTab; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx index bb2cb865877..3bc5443b2ea 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx @@ -114,10 +114,7 @@ const CacheLeakageCard: React.FC = ({ activity }) => {
- setDimension(value === "model" ? "model" : "key")} - > + setDimension(value === "model" ? "model" : "key")}> By virtual key By model diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx index f6593e80999..4073521f3b7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx @@ -7,6 +7,7 @@ import { Alert, Tabs } from "antd"; import UsageTab from "./UsageTab"; import PromptCompressionTab from "./PromptCompressionTab"; import PromptCachingTab from "./PromptCachingTab"; +import AutoRouterBenchmarksTab from "./AutoRouterBenchmarksTab"; import { useDailyActivityRange } from "./useDailyActivityRange"; interface CostOptimizationViewProps { @@ -34,6 +35,11 @@ const CostOptimizationView: React.FC = ({ accessToken label: "Prompt Caching", children: , }, + { + key: "autorouter-benchmarks", + label: "Auto-Router Benchmarks", + children: , + }, ]; return ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks.test.ts new file mode 100644 index 00000000000..d7eb0bc08fe --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks.test.ts @@ -0,0 +1,188 @@ +import { describe, expect, it } from "vitest"; + +import { toView, usd, type AutoRouterCacheBenchmark, type AutoRouterGroupBenchmark } from "./autoRouterBenchmarks"; + +const cache = (overrides: Partial = {}): AutoRouterCacheBenchmark => ({ + ttl_seconds: 3600, + usage_coverage_pct: 100, + hit_rate_pct: 90, + turns: 100, + hits: 90, + same_model_turns: 70, + same_model_hits: 68, + first_visit_turns: 10, + first_visit_hits: 2, + return_turns: 20, + return_hits: 20, + same_model_hit_rate_pct: 97.1, + first_visit_hit_rate_pct: 20, + return_hit_rate_pct: 100, + stale_miss_share_pct: 0, + warming_savable_miss_pct: 4, + warming_break_even_pct: 5, + stale_return_misses: 0, + savable_return_misses: 0, + warming_rescued_spend: 6.76, + warming_replay_spend: 3.91, + warming_net_spend: 2.85, + ...overrides, +}); + +const group = (overrides: Partial = {}): AutoRouterGroupBenchmark => ({ + model_group: "claude-auto", + router_kind: "complexity", + baseline_model: "anthropic/claude-opus-4-8", + sessions: 10, + turns: 100, + avg_turns_per_session: 10, + avg_session_length_seconds: 3600, + total_tokens: 1_000_000, + avg_tokens_per_session: 100_000, + actual_spend: 10, + baseline_spend: 100, + savings: 90, + savings_pct: 90, + cache: cache(), + ...overrides, +}); + +describe("toView", () => { + it("returns null when there are no groups", () => { + expect(toView([])).toBeNull(); + }); + + it("returns null when the groups hold no sessions, rather than dividing by zero", () => { + expect(toView([group({ sessions: 0, turns: 0 })])).toBeNull(); + }); + + it("divides total turns by total sessions rather than averaging per-router averages", () => { + const view = toView([ + group({ model_group: "a", sessions: 1, turns: 100 }), + group({ model_group: "b", sessions: 99, turns: 99 }), + ]); + expect(view?.avg_turns_per_session).toBeCloseTo(199 / 100); + }); + + it("weights session length by session count", () => { + const view = toView([ + group({ model_group: "a", sessions: 1, avg_session_length_seconds: 100 }), + group({ model_group: "b", sessions: 9, avg_session_length_seconds: 1000 }), + ]); + expect(view?.avg_session_length_seconds).toBeCloseTo((100 * 1 + 1000 * 9) / 10); + }); + + it("recomputes savings from summed spend so it stays consistent with the tiles", () => { + const view = toView([ + group({ model_group: "a", actual_spend: 10, baseline_spend: 100 }), + group({ model_group: "b", actual_spend: 5, baseline_spend: 20 }), + ]); + expect(view?.actual_spend).toBe(15); + expect(view?.baseline_spend).toBe(120); + expect(view?.savings).toBe(105); + expect(view?.savings_pct).toBeCloseTo((100 * 105) / 120); + }); + + it("keeps the sign when routing cost more than the baseline", () => { + const view = toView([group({ actual_spend: 120, baseline_spend: 100 })]); + expect(view?.savings).toBe(-20); + expect(view?.savings_pct).toBeCloseTo(-20); + }); + + it("reports no percentage instead of dividing by a zero baseline", () => { + expect(toView([group({ actual_spend: 5, baseline_spend: 0 })])?.savings_pct).toBe(0); + }); +}); + +describe("toView cache combination", () => { + it("weights the headline hit rate by turns rather than averaging the routers", () => { + const view = toView([ + group({ model_group: "a", cache: cache({ turns: 1000, hits: 900 }) }), + group({ model_group: "b", cache: cache({ turns: 10, hits: 0 }) }), + ]); + expect(view?.cache?.hit_rate_pct).toBeCloseTo((100 * 900) / 1010); + }); + + it("keeps the three buckets summing to the combined turn count", () => { + const view = toView([ + group({ model_group: "a" }), + group({ + model_group: "b", + cache: cache({ same_model_turns: 5, first_visit_turns: 3, return_turns: 2, turns: 10 }), + }), + ]); + const c = view?.cache; + expect(c).toBeTruthy(); + expect((c?.same_model_turns ?? 0) + (c?.first_visit_turns ?? 0) + (c?.return_turns ?? 0)).toBe(c?.turns); + }); + + it("recomputes the savable share against every combined miss", () => { + const view = toView([ + group({ model_group: "a", cache: cache({ turns: 100, hits: 90, savable_return_misses: 5 }) }), + group({ model_group: "b", cache: cache({ turns: 100, hits: 80, savable_return_misses: 5 }) }), + ]); + expect(view?.cache?.warming_savable_miss_pct).toBeCloseTo((100 * 10) / 30); + }); + + it("nets the warming estimate from the summed sides", () => { + const view = toView([ + group({ model_group: "a", cache: cache({ warming_rescued_spend: 6, warming_replay_spend: 4 }) }), + group({ model_group: "b", cache: cache({ warming_rescued_spend: 1, warming_replay_spend: 5 }) }), + ]); + expect(view?.cache?.warming_net_spend).toBeCloseTo(-2); + }); + + it("flags mixed TTLs so the card does not claim one regime for all of them", () => { + const view = toView([ + group({ model_group: "a", cache: cache({ ttl_seconds: 300 }) }), + group({ model_group: "b", cache: cache({ ttl_seconds: 3600 }) }), + ]); + expect(view?.mixedTtl).toBe(true); + }); + + it("omits the cache entirely when no router reported one", () => { + expect(toView([group({ cache: null })])?.cache).toBeNull(); + }); +}); + +describe("toView baseline label", () => { + it("names the shared baseline when every router used the same one", () => { + const view = toView([group({ model_group: "a" }), group({ model_group: "b" })]); + expect(view?.baselineLabel).toBe("anthropic/claude-opus-4-8"); + }); + + it("refuses to name one baseline when the routers disagree", () => { + const view = toView([ + group({ model_group: "a", baseline_model: "anthropic/claude-opus-4-8" }), + group({ model_group: "b", baseline_model: "openai/gpt-5" }), + ]); + expect(view?.baselineLabel).toBe("each router's own baseline"); + }); + + it("reports no baseline when the routers declare none, so savings is not shown as zero", () => { + expect(toView([group({ baseline_model: null })])?.baselineLabel).toBeNull(); + }); +}); + +describe("usd", () => { + it("sizes and signs off the magnitude so a small loss does not render as -$0.00", () => { + expect(usd(-0.004)).toBe("-$0.00"); + expect(usd(-12.5)).toBe("-$12.50"); + expect(usd(2174.59)).toBe("$2,174.59"); + }); +}); + +describe("toView saved per session", () => { + it("divides total savings by total sessions", () => { + const view = toView([ + group({ model_group: "a", sessions: 60, actual_spend: 10, baseline_spend: 100 }), + group({ model_group: "b", sessions: 40, actual_spend: 5, baseline_spend: 20 }), + ]); + expect(view?.saved_per_session).toBeCloseTo(105 / 100); + }); + + it("goes negative when routing cost more than the baseline", () => { + expect(toView([group({ sessions: 10, actual_spend: 120, baseline_spend: 100 })])?.saved_per_session).toBeCloseTo( + -2, + ); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks.ts new file mode 100644 index 00000000000..1fb4a34dfd1 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks.ts @@ -0,0 +1,187 @@ +export interface AutoRouterCacheBenchmark { + ttl_seconds: number; + usage_coverage_pct: number; + hit_rate_pct: number; + turns: number; + hits: number; + same_model_turns: number; + same_model_hits: number; + first_visit_turns: number; + first_visit_hits: number; + return_turns: number; + return_hits: number; + same_model_hit_rate_pct: number; + first_visit_hit_rate_pct: number; + return_hit_rate_pct: number; + stale_miss_share_pct: number; + warming_savable_miss_pct: number; + warming_break_even_pct: number; + stale_return_misses: number; + savable_return_misses: number; + warming_rescued_spend: number; + warming_replay_spend: number; + warming_net_spend: number; +} + +export interface AutoRouterGroupBenchmark { + model_group: string; + router_kind: string; + baseline_model: string | null; + sessions: number; + turns: number; + avg_turns_per_session: number; + avg_session_length_seconds: number; + total_tokens: number; + avg_tokens_per_session: number; + actual_spend: number; + baseline_spend: number; + savings: number; + savings_pct: number; + cache: AutoRouterCacheBenchmark | null; +} + +export interface AutoRouterBenchmarksResponse { + start_date: string; + end_date: string; + groups: AutoRouterGroupBenchmark[]; +} + +export interface BenchmarkView { + routers: number; + sessions: number; + turns: number; + avg_turns_per_session: number; + avg_session_length_seconds: number; + avg_tokens_per_session: number; + actual_spend: number; + baseline_spend: number; + savings: number; + savings_pct: number; + saved_per_session: number; + baselineLabel: string | null; + cache: AutoRouterCacheBenchmark | null; + mixedTtl: boolean; +} + +export const ALL_ROUTERS = "__all__"; + +const rate = (part: number, whole: number): number => (whole > 0 ? (100 * part) / whole : 0); + +const sum = (rows: readonly T[], pick: (row: T) => number): number => + rows.reduce((total, row) => total + pick(row), 0); + +/** + * Every rate is recomputed from summed counts rather than averaged across routers. + * Averaging rates weights a router with three turns the same as one with three + * thousand, which is how a blended hit rate ends up sitting nowhere near either. + */ +const combineCache = (caches: readonly AutoRouterCacheBenchmark[]): AutoRouterCacheBenchmark | null => { + if (caches.length === 0) return null; + + const turns = sum(caches, (c) => c.turns); + const hits = sum(caches, (c) => c.hits); + const sameTurns = sum(caches, (c) => c.same_model_turns); + const firstTurns = sum(caches, (c) => c.first_visit_turns); + const returnTurns = sum(caches, (c) => c.return_turns); + const returnHits = sum(caches, (c) => c.return_hits); + const staleMisses = sum(caches, (c) => c.stale_return_misses); + const savableMisses = sum(caches, (c) => c.savable_return_misses); + const rescued = sum(caches, (c) => c.warming_rescued_spend); + const replay = sum(caches, (c) => c.warming_replay_spend); + + return { + ttl_seconds: Math.max(...caches.map((c) => c.ttl_seconds)), + usage_coverage_pct: rate( + sum(caches, (c) => (c.usage_coverage_pct * c.turns) / 100), + turns, + ), + hit_rate_pct: rate(hits, turns), + turns, + hits, + same_model_turns: sameTurns, + same_model_hits: sum(caches, (c) => c.same_model_hits), + first_visit_turns: firstTurns, + first_visit_hits: sum(caches, (c) => c.first_visit_hits), + return_turns: returnTurns, + return_hits: returnHits, + same_model_hit_rate_pct: rate( + sum(caches, (c) => c.same_model_hits), + sameTurns, + ), + first_visit_hit_rate_pct: rate( + sum(caches, (c) => c.first_visit_hits), + firstTurns, + ), + return_hit_rate_pct: rate(returnHits, returnTurns), + stale_miss_share_pct: rate(staleMisses, returnTurns - returnHits), + warming_savable_miss_pct: rate(savableMisses, turns - hits), + warming_break_even_pct: Math.max(...caches.map((c) => c.warming_break_even_pct)), + stale_return_misses: staleMisses, + savable_return_misses: savableMisses, + warming_rescued_spend: rescued, + warming_replay_spend: replay, + warming_net_spend: rescued - replay, + }; +}; + +/** + * The baseline only reads as a single model when every router measured itself + * against the same one; otherwise naming one of them would misattribute the rest. + */ +const combineBaselineLabel = (groups: readonly AutoRouterGroupBenchmark[]): string | null => { + const baselines = new Set(groups.map((g) => g.baseline_model).filter((m): m is string => Boolean(m))); + if (baselines.size === 0) return null; + if (baselines.size === 1) return [...baselines][0]; + return "each router's own baseline"; +}; + +export const toView = (groups: readonly AutoRouterGroupBenchmark[]): BenchmarkView | null => { + if (groups.length === 0) return null; + + const sessions = sum(groups, (g) => g.sessions); + if (sessions === 0) return null; + + const turns = sum(groups, (g) => g.turns); + const actualSpend = sum(groups, (g) => g.actual_spend); + const baselineSpend = sum(groups, (g) => g.baseline_spend); + const savings = baselineSpend - actualSpend; + const caches = groups.map((g) => g.cache).filter((c): c is AutoRouterCacheBenchmark => c !== null); + + return { + routers: groups.length, + sessions, + turns, + avg_turns_per_session: turns / sessions, + avg_session_length_seconds: sum(groups, (g) => g.avg_session_length_seconds * g.sessions) / sessions, + avg_tokens_per_session: sum(groups, (g) => g.total_tokens) / sessions, + actual_spend: actualSpend, + baseline_spend: baselineSpend, + savings, + savings_pct: baselineSpend > 0 ? (100 * savings) / baselineSpend : 0, + saved_per_session: savings / sessions, + baselineLabel: combineBaselineLabel(groups), + cache: combineCache(caches), + mixedTtl: new Set(caches.map((c) => c.ttl_seconds)).size > 1, + }; +}; + +export const compactNumber = (n: number): string => + new Intl.NumberFormat("en-US", { notation: "compact", maximumFractionDigits: 1 }).format(n); + +export const usd = (n: number): string => + `${n < 0 ? "-" : ""}${new Intl.NumberFormat("en-US", { + style: "currency", + currency: "USD", + maximumFractionDigits: 2, + }).format(Math.abs(n))}`; + +export const durationLabel = (seconds: number): string => { + if (seconds < 60) return `${Math.round(seconds)}s`; + if (seconds < 3600) return `${(seconds / 60).toFixed(1)}m`; + return `${(seconds / 3600).toFixed(1)}h`; +}; + +export const ttlLabel = (seconds: number): string => + seconds >= 3600 ? `${Math.round(seconds / 3600)}h` : `${Math.round(seconds / 60)}m`; + +export const pct = (n: number, digits = 1): string => `${n.toFixed(digits)}%`; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterBenchmarks.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterBenchmarks.ts new file mode 100644 index 00000000000..2c6b6a578aa --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterBenchmarks.ts @@ -0,0 +1,26 @@ +import { useQuery } from "@tanstack/react-query"; + +import { autoRouterBenchmarksCall } from "@/components/networking"; + +import type { AutoRouterBenchmarksResponse } from "./autoRouterBenchmarks"; + +export const BENCHMARKS_WINDOW_DAYS = 30; + +const isoDate = (date: Date): string => date.toISOString().slice(0, 10); + +/** + * The window is computed once per mount rather than per render, so the query key + * stays stable and the request is not refired every time the parent re-renders. + */ +export const benchmarksWindow = (now: Date): { start: string; end: string } => ({ + start: isoDate(new Date(now.getTime() - BENCHMARKS_WINDOW_DAYS * 24 * 60 * 60 * 1000)), + end: isoDate(now), +}); + +export const useAutoRouterBenchmarks = (accessToken: string | null, window: { start: string; end: string }) => + useQuery({ + queryKey: ["autoRouterBenchmarks", window.start, window.end], + queryFn: async () => await autoRouterBenchmarksCall(accessToken!, window.start, window.end), + enabled: Boolean(accessToken), + retry: false, + }); diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 5a2d33ee4bd..d634fe2acd9 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -7922,3 +7922,19 @@ export const deleteMemory = async (accessToken: string, key: string): Promise { + /** + * Session-level benchmarks for every configured auto-router, read from the + * per-session rollup. Admin-only; 404s when no auto-router is configured. + */ + try { + return await apiClient.get(`/auto_router/benchmarks`, { + accessToken, + query: { start_date: startDate, end_date: endDate }, + }); + } catch (error) { + console.error("Failed to get auto-router benchmarks:", error); + throw error; + } +};