diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx index 1d82fbb48ea..bb40c341786 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx @@ -1,4 +1,4 @@ -import { render } from "@testing-library/react"; +import { fireEvent, render } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; import type { DailyData, KeyMetricWithMetadata, SpendMetrics } from "@/components/UsagePage/types"; @@ -41,6 +41,24 @@ const dayWithKeys = (date: string, apiKeys: Record>): DailyData => ({ + date, + metrics: baseMetrics({}), + breakdown: { + models: Object.fromEntries( + Object.entries(models).map(([name, m]) => [ + name, + { metrics: baseMetrics(m), metadata: {}, api_key_breakdown: {} }, + ]), + ), + model_groups: {}, + mcp_servers: {}, + providers: {}, + api_keys: {}, + entities: {}, + }, +}); + const renderWith = (results: DailyData[]) => render( { expect(getByText("0.0%")).toBeInTheDocument(); expect(getByText("90.0%")).toBeInTheDocument(); [ - "Input tokens in the selected range that were neither read from nor written to the prompt cache", - "Share of this key's total input tokens that were served from the prompt cache", - "Dollars this key actually saved because cached input was billed at the discounted cache-read rate", - "Approximate dollars this key could still save if its uncached input had hit the cache at the portfolio's realized discount", + "Input tokens you sent in this range that weren't served from or written to the cache", + "Share of your input tokens that were served from the cache", + "About how much you'd save if this uncached input used prompt caching. Estimated as uncached input tokens times the per-token discount your cached traffic already gets (realized cache savings ÷ cache-read tokens).", ].forEach((info) => expect(getByLabelText(info)).toBeInTheDocument()); }); + it("switches to the model view and lists only Anthropic models", () => { + const { getByText, queryByText } = renderWith([ + dayWithModels("2026-07-12", { + "claude-sonnet-5": { prompt_tokens: 5000, cache_read_input_tokens: 0 }, + "gpt-4o": { prompt_tokens: 8000, cache_read_input_tokens: 0 }, + }), + ]); + + fireEvent.click(getByText("By model")); + + expect(getByText("Cache leakage by model")).toBeInTheDocument(); + expect(getByText("claude-sonnet-5")).toBeInTheDocument(); + expect(queryByText("gpt-4o")).not.toBeInTheDocument(); + }); + it("shows an empty state when no key used tokens in the range", () => { const { getByText, queryByRole } = renderWith([dayWithKeys("2026-07-12", {})]); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx index 359ce502b07..ad1ff303dbc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.tsx @@ -1,14 +1,15 @@ "use client"; -import React, { useMemo } from "react"; +import React, { useMemo, useState } from "react"; import { Info } from "lucide-react"; import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { formatNumberWithCommas } from "@/utils/dataUtils"; -import { computeCacheLeakage, pct, usd } from "./costOptimizationUtils"; +import { CacheLeakageDimension, computeCacheLeakage, pct, usd } from "./costOptimizationUtils"; import { DailyActivityRange } from "./useDailyActivityRange"; interface CacheLeakageCardProps { @@ -19,19 +20,22 @@ const HeadWithInfo = ({ label, info }: { label: string; info: string }) => ( {label} - - - - + }> + - {info} + {info} ); const CacheLeakageCard: React.FC = ({ activity }) => { const { dateValue, onDateChange, results, loading, isFetchingMore } = activity; - const leakage = useMemo(() => computeCacheLeakage(results), [results]); + const [dimension, setDimension] = useState("key"); + const leakage = useMemo(() => computeCacheLeakage(results, dimension), [results, dimension]); + + const subject = dimension === "model" ? "Models" : "Keys"; + const firstColumn = dimension === "model" ? "Model" : "Key"; + const emptyNoun = dimension === "model" ? "model" : "key"; return ( @@ -39,64 +43,67 @@ const CacheLeakageCard: React.FC = ({ activity }) => {
- Cache leakage by virtual key + Cache leakage by {dimension === "model" ? "model" : "virtual key"}

- Keys sending large volumes of uncached prompt tokens with a low cache-hit ratio are likely missing - prompt caching. Estimated savings left is approximate: uncached prompt tokens priced at the - portfolio's realized cache-read discount. + {subject} sending large volumes of uncached input with a low cache hit rate are likely missing prompt + caching. Potential savings is approximate: uncached input priced at the realized cache-read discount. + {dimension === "model" ? " Limited to Anthropic (Claude) models, which support prompt caching." : ""}

+ setDimension(value === "model" ? "model" : "key")} + className="mt-3" + > + + By virtual key + By model + + {leakage.rows.length === 0 ? (

- {loading || isFetchingMore ? "Loading..." : "No key usage in this range."} + {loading || isFetchingMore ? "Loading..." : `No ${emptyNoun} usage in this range.`}

) : ( - Key + {firstColumn} - - - {leakage.rows.map((row) => ( - + - {row.keyAlias || `${row.apiKey.slice(0, 8)}...`} - {row.teamId && ({row.teamId})} + {row.label} + {row.sublabel && ({row.sublabel})} {formatNumberWithCommas(row.uncachedPromptTokens)} {pct(row.cacheHitRatio)} - {usd(row.realizedCachingSavings)} - {row.estSavingsLeft == null ? "—" : usd(row.estSavingsLeft)} + {row.potentialSavings == null ? "—" : usd(row.potentialSavings)} ))} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts index 562552ffb50..2f1558465d5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; import type { ToolSpendDailyEntry, ToolSpendEntry } from "@/components/networking"; -import { buildDailyToolSeries, computeCacheLeakage, topToolsBySpend } from "./costOptimizationUtils"; +import { buildDailyToolSeries, computeCacheLeakage, isAnthropicModel, topToolsBySpend } from "./costOptimizationUtils"; const metrics = (overrides: Partial): SpendMetrics => ({ spend: 0, @@ -38,6 +38,21 @@ const day = ( }, }); +const modelDay = (date: string, models: Record>): DailyData => ({ + date, + metrics: metrics({}), + breakdown: { + models: Object.fromEntries( + Object.entries(models).map(([name, m]) => [name, { metrics: metrics(m), metadata: {}, api_key_breakdown: {} }]), + ), + model_groups: {}, + mcp_servers: {}, + providers: {}, + entities: {}, + api_keys: {}, + }, +}); + describe("computeCacheLeakage", () => { it("aggregates a key's tokens and savings across multiple days", () => { const results = [ @@ -76,8 +91,8 @@ describe("computeCacheLeakage", () => { ]; const { rows, discountPerToken } = computeCacheLeakage(results); expect(discountPerToken).toBeCloseTo(0.002, 6); - expect(rows.map((r) => r.keyAlias)).toEqual(["leaker"]); - expect(rows[0].estSavingsLeft).toBeCloseTo(1.0, 6); + expect(rows.map((r) => r.label)).toEqual(["leaker"]); + expect(rows[0].potentialSavings).toBeCloseTo(1.0, 6); }); it("returns null estimate and ranks by uncached tokens when nobody used caching", () => { @@ -89,8 +104,8 @@ describe("computeCacheLeakage", () => { ]; const { rows, discountPerToken } = computeCacheLeakage(results); expect(discountPerToken).toBeNull(); - expect(rows.map((r) => r.keyAlias)).toEqual(["big", "small"]); - expect(rows.every((r) => r.estSavingsLeft === null)).toBe(true); + expect(rows.map((r) => r.label)).toEqual(["big", "small"]); + expect(rows.every((r) => r.potentialSavings === null)).toBe(true); }); it("computes cache hit ratio against total prompt tokens and clamps inconsistent data at zero", () => { @@ -101,7 +116,7 @@ describe("computeCacheLeakage", () => { }), ]; const { rows } = computeCacheLeakage(results); - expect(rows.map((r) => r.keyAlias)).toEqual(["mixed"]); + expect(rows.map((r) => r.label)).toEqual(["mixed"]); expect(rows[0].cacheHitRatio).toBeCloseTo(0.75, 6); expect(rows[0].uncachedPromptTokens).toBe(250); }); @@ -110,11 +125,63 @@ describe("computeCacheLeakage", () => { const keys = Object.fromEntries( Array.from({ length: 15 }, (_, i) => [`h${i}`, { alias: `k${i}`, metrics: { prompt_tokens: i + 1 } }]), ); - const { rows } = computeCacheLeakage([day("2026-07-01", keys)], 5); + const { rows } = computeCacheLeakage([day("2026-07-01", keys)], "key", 5); expect(rows).toHaveLength(5); }); }); +describe("computeCacheLeakage by model", () => { + it("aggregates only Anthropic models and ignores other providers", () => { + const models: Record> = { + "claude-sonnet-5": { prompt_tokens: 10000, cache_read_input_tokens: 0 }, + "anthropic/claude-haiku-4-5": { prompt_tokens: 4000, cache_read_input_tokens: 0 }, + "bedrock/anthropic.claude-3-5-sonnet": { prompt_tokens: 2000, cache_read_input_tokens: 0 }, + "gpt-4o": { prompt_tokens: 9000, cache_read_input_tokens: 0 }, + "deepseek-chat": { prompt_tokens: 8000, cache_read_input_tokens: 0 }, + }; + const { rows } = computeCacheLeakage([modelDay("2026-07-01", models)], "model"); + expect(rows.map((r) => r.id)).toEqual([ + "claude-sonnet-5", + "anthropic/claude-haiku-4-5", + "bedrock/anthropic.claude-3-5-sonnet", + ]); + }); + + it("labels model rows by model name with no sublabel", () => { + const results = [modelDay("2026-07-01", { "claude-sonnet-5": { prompt_tokens: 1000 } })]; + const { rows } = computeCacheLeakage(results, "model"); + expect(rows[0].label).toBe("claude-sonnet-5"); + expect(rows[0].sublabel).toBeNull(); + }); + + it("prices model leakage at the Anthropic realized cache-read discount", () => { + const results = [ + modelDay("2026-07-01", { + "claude-sonnet-5": { prompt_tokens: 1000, cache_read_input_tokens: 1000, prompt_caching_savings_spend: 2.0 }, + "claude-haiku-4-5": { prompt_tokens: 500 }, + }), + ]; + const { rows, discountPerToken } = computeCacheLeakage(results, "model"); + expect(discountPerToken).toBeCloseTo(0.002, 6); + expect(rows.map((r) => r.id)).toEqual(["claude-haiku-4-5"]); + expect(rows[0].potentialSavings).toBeCloseTo(1.0, 6); + }); +}); + +describe("isAnthropicModel", () => { + it("matches Claude-family models across providers and rejects others", () => { + const anthropic = [ + "claude-sonnet-5", + "anthropic/claude-haiku-4-5", + "bedrock/anthropic.claude-3-5-sonnet", + "vertex_ai/claude-opus-4-8", + ]; + const others = ["gpt-4o", "deepseek-chat", "gemini-2.5-pro", "mistral-large"]; + expect(anthropic.every(isAnthropicModel)).toBe(true); + expect(others.some(isAnthropicModel)).toBe(false); + }); +}); + describe("buildDailyToolSeries", () => { const daily: ToolSpendDailyEntry[] = [ { date: "2026-07-01", tool_name: "search", spend: 1.0, call_count: 1 }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts index 2868bdac880..30f851bfeca 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts @@ -1,4 +1,4 @@ -import { DailyData } from "@/components/UsagePage/types"; +import { DailyData, SpendMetrics } from "@/components/UsagePage/types"; import { ToolSpendDailyEntry, ToolSpendEntry } from "@/components/networking"; import { formatNumberWithCommas } from "@/utils/dataUtils"; @@ -9,15 +9,15 @@ export const usd = (value: number): string => { export const pct = (ratio: number): string => `${formatNumberWithCommas(ratio * 100, 1)}%`; +export type CacheLeakageDimension = "key" | "model"; + export interface CacheLeakageRow { - apiKey: string; - keyAlias: string | null; - teamId: string | null; + id: string; + label: string; + sublabel: string | null; uncachedPromptTokens: number; - cacheReadTokens: number; cacheHitRatio: number; - realizedCachingSavings: number; - estSavingsLeft: number | null; + potentialSavings: number | null; } export interface CacheLeakageResult { @@ -25,8 +25,10 @@ export interface CacheLeakageResult { discountPerToken: number | null; } -interface KeyAccumulator { - keyAlias: string | null; +export const isAnthropicModel = (model: string): boolean => /claude|anthropic/i.test(model); + +interface LeakageAccumulator { + alias: string | null; teamId: string | null; promptTokens: number; cacheReadTokens: number; @@ -34,8 +36,8 @@ interface KeyAccumulator { realizedCachingSavings: number; } -const emptyAccumulator = (): KeyAccumulator => ({ - keyAlias: null, +const emptyAccumulator = (): LeakageAccumulator => ({ + alias: null, teamId: null, promptTokens: 0, cacheReadTokens: 0, @@ -43,26 +45,54 @@ const emptyAccumulator = (): KeyAccumulator => ({ realizedCachingSavings: 0, }); -export const computeCacheLeakage = (results: readonly DailyData[], limit = 10): CacheLeakageResult => { - const byKey = new Map(); +const addMetrics = ( + acc: LeakageAccumulator, + m: SpendMetrics, + alias: string | null, + teamId: string | null, +): LeakageAccumulator => ({ + alias: acc.alias ?? alias, + teamId: acc.teamId ?? teamId, + promptTokens: acc.promptTokens + (m.prompt_tokens ?? 0), + cacheReadTokens: acc.cacheReadTokens + (m.cache_read_input_tokens ?? 0), + cacheCreationTokens: acc.cacheCreationTokens + (m.cache_creation_input_tokens ?? 0), + realizedCachingSavings: acc.realizedCachingSavings + (m.prompt_caching_savings_spend ?? 0), +}); + +const aggregateByKey = (results: readonly DailyData[]): Map => { + const byKey = new Map(); for (const day of results) { - const apiKeys = day.breakdown?.api_keys ?? {}; - for (const [apiKey, entry] of Object.entries(apiKeys)) { + for (const [apiKey, entry] of Object.entries(day.breakdown?.api_keys ?? {})) { const acc = byKey.get(apiKey) ?? emptyAccumulator(); - const m = entry.metrics; - const next: KeyAccumulator = { - keyAlias: acc.keyAlias ?? entry.metadata?.key_alias ?? null, - teamId: acc.teamId ?? entry.metadata?.team_id ?? null, - promptTokens: acc.promptTokens + (m.prompt_tokens ?? 0), - cacheReadTokens: acc.cacheReadTokens + (m.cache_read_input_tokens ?? 0), - cacheCreationTokens: acc.cacheCreationTokens + (m.cache_creation_input_tokens ?? 0), - realizedCachingSavings: acc.realizedCachingSavings + (m.prompt_caching_savings_spend ?? 0), - }; - byKey.set(apiKey, next); + byKey.set( + apiKey, + addMetrics(acc, entry.metrics, entry.metadata?.key_alias ?? null, entry.metadata?.team_id ?? null), + ); } } + return byKey; +}; - const totals = [...byKey.values()].reduce( +const aggregateByModel = (results: readonly DailyData[]): Map => { + const byModel = new Map(); + for (const day of results) { + for (const [model, entry] of Object.entries(day.breakdown?.models ?? {})) { + if (!isAnthropicModel(model)) continue; + const acc = byModel.get(model) ?? emptyAccumulator(); + byModel.set(model, addMetrics(acc, entry.metrics, null, null)); + } + } + return byModel; +}; + +export const computeCacheLeakage = ( + results: readonly DailyData[], + dimension: CacheLeakageDimension = "key", + limit = 10, +): CacheLeakageResult => { + const byEntity = dimension === "model" ? aggregateByModel(results) : aggregateByKey(results); + + const totals = [...byEntity.values()].reduce( (agg, a) => ({ cacheReadTokens: agg.cacheReadTokens + a.cacheReadTokens, realizedCachingSavings: agg.realizedCachingSavings + a.realizedCachingSavings, @@ -71,25 +101,23 @@ export const computeCacheLeakage = (results: readonly DailyData[], limit = 10): ); const discountPerToken = totals.cacheReadTokens > 0 ? totals.realizedCachingSavings / totals.cacheReadTokens : null; - const rows: CacheLeakageRow[] = [...byKey.entries()] - .map(([apiKey, a]) => { + const rows: CacheLeakageRow[] = [...byEntity.entries()] + .map(([id, a]) => { const uncachedPromptTokens = Math.max(0, a.promptTokens - a.cacheReadTokens - a.cacheCreationTokens); return { - apiKey, - keyAlias: a.keyAlias, - teamId: a.teamId, + id, + label: dimension === "model" ? id : a.alias ?? `${id.slice(0, 8)}...`, + sublabel: dimension === "model" ? null : a.teamId, uncachedPromptTokens, - cacheReadTokens: a.cacheReadTokens, cacheHitRatio: a.promptTokens > 0 ? a.cacheReadTokens / a.promptTokens : 0, - realizedCachingSavings: a.realizedCachingSavings, - estSavingsLeft: discountPerToken != null ? uncachedPromptTokens * discountPerToken : null, + potentialSavings: discountPerToken != null ? uncachedPromptTokens * discountPerToken : null, }; }) .filter((row) => row.uncachedPromptTokens > 0); const sorted = rows.sort((x, y) => discountPerToken != null - ? (y.estSavingsLeft ?? 0) - (x.estSavingsLeft ?? 0) + ? (y.potentialSavings ?? 0) - (x.potentialSavings ?? 0) : y.uncachedPromptTokens - x.uncachedPromptTokens, );