diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 818f5bf7847..505b5e0974d 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -13,7 +13,7 @@ from typing import NamedTuple import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token -from litellm.types.utils import Usage +from litellm.types.utils import PromptTokensDetailsWrapper, Usage class SavingsSpend(NamedTuple): @@ -61,6 +61,40 @@ def _cost_of_usage(model: str, custom_llm_provider: str | None, usage: Usage) -> return prompt_cost + completion_cost +def _cache_token_split(usage: Usage) -> tuple[int, int]: + """``(cache_read_tokens, cache_creation_tokens)`` for a request.""" + details = usage.prompt_tokens_details + if details is None: + return 0, 0 + read = getattr(details, "cached_tokens", 0) or 0 + created = (getattr(details, "cache_creation_tokens", 0) or 0) or (getattr(details, "cache_write_tokens", 0) or 0) + return int(read), int(created) + + +def _baseline_usage(usage: Usage) -> Usage: + """The same request as a single-model baseline would have met it. + + Staying on one model, the cache is written once and read from thereafter, so the + tokens the router forced a cold model to re-write would already have been cached. + A request that read nothing from cache is a genuine cold start that the baseline + would have paid to write too, so it is left alone. + """ + cache_read, cache_creation = _cache_token_split(usage) + if cache_read <= 0 or cache_creation <= 0: + return usage + return Usage( + prompt_tokens=usage.prompt_tokens, + completion_tokens=usage.completion_tokens, + total_tokens=usage.total_tokens, + completion_tokens_details=usage.completion_tokens_details, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=cache_read + cache_creation, + cache_creation_tokens=0, + text_tokens=max(usage.prompt_tokens - cache_read - cache_creation, 0), + ), + ) + + def compute_autorouter_savings( baseline_model: str | None, selected_model: str | None, @@ -68,19 +102,20 @@ def compute_autorouter_savings( selected_provider: str | None, usage: Usage, ) -> float: - """Net dollars saved by serving this request on ``selected_model`` rather than ``baseline_model``. + """Net dollars the router saved, or cost, by serving this request on ``selected_model``. - Both arms price the same usage, so each token is charged once in its own - dimension; ``prompt_tokens`` already includes the cache tokens. Zero when the - model is unchanged or unpriced, and floored at zero on an escalation. + Signed on purpose. Switching models leaves the new one with a cold cache, so the + request pays a cache-creation charge that staying on one model would not have + incurred; when that charge outweighs the cheaper rates, routing lost money and the + dashboard has to be able to say so. Zero when the model is unchanged or unpriced. """ if not baseline_model or not selected_model or baseline_model == selected_model: return 0.0 - baseline_cost = _cost_of_usage(baseline_model, baseline_provider, usage) + baseline_cost = _cost_of_usage(baseline_model, baseline_provider, _baseline_usage(usage)) selected_cost = _cost_of_usage(selected_model, selected_provider, usage) if baseline_cost is None or selected_cost is None: return 0.0 - return max(baseline_cost - selected_cost, 0.0) + return baseline_cost - selected_cost def _usage_from_spend_log(usage_object: dict | None) -> Usage | None: diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 23d3c19832f..11bea5cc429 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -116,107 +116,101 @@ def test_negative_token_counts_clamp_to_zero(): assert result.prompt_caching == 0.0 -def test_autorouter_savings_does_not_double_charge_cache_tokens(): - """`prompt_tokens` already includes cache-read and cache-creation tokens. - - Charging those tokens again at the full input rate, or subtracting a separate - cache-write penalty on top of them, prices the same tokens twice. Both arms go - through litellm's cost engine on the identical usage, so each token is priced - exactly once, in its own dimension. - """ - usage_object = _cached_usage_object() - result = compute_autorouter_savings( - baseline_model="claude-opus-5", - selected_model="claude-haiku-4-5", - baseline_provider="anthropic", - selected_provider="anthropic", - usage=Usage(**usage_object), +def _usage(fresh: int, cached: int, written: int, out: int) -> Usage: + """Usage as the spend log records it; `prompt_tokens` is the inclusive total.""" + return Usage( + prompt_tokens=fresh + cached + written, + completion_tokens=out, + total_tokens=fresh + cached + written + out, + prompt_tokens_details={"cached_tokens": cached, "cache_creation_tokens": written, "text_tokens": fresh}, + cache_read_input_tokens=cached, + cache_creation_input_tokens=written, ) - expected = _cost_on("claude-opus-5", usage_object) - _cost_on("claude-haiku-4-5", usage_object) - assert result == pytest.approx(expected) + +def _savings(baseline: str, selected: str, usage: Usage) -> float: + return compute_autorouter_savings( + baseline_model=baseline, + selected_model=selected, + baseline_provider="anthropic", + selected_provider="anthropic", + usage=usage, + ) + + +def test_switching_models_mid_conversation_charges_the_cold_cache_write(): + """Staying on one model writes the cache once and reads it thereafter. Switching + leaves the new model cold, so it pays to write the whole prompt again; when that + charge outweighs the cheaper rates the route lost money and must report a loss. + + Pricing the baseline as if it too re-wrote the cache credits a charge it never + paid, which is how a losing switch used to read as the largest saving on the page. + """ + usage = _usage(fresh=3, cached=500, written=12304, out=500) + result = _savings("claude-sonnet-5", "claude-haiku-4-5", usage) + + sonnet = litellm.get_model_info("claude-sonnet-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + warm_baseline = ( + 3 * sonnet["input_cost_per_token"] + + 12804 * sonnet["cache_read_input_token_cost"] + + 500 * sonnet["output_cost_per_token"] + ) + actually_paid = ( + 3 * haiku["input_cost_per_token"] + + 500 * haiku["cache_read_input_token_cost"] + + 12304 * haiku["cache_creation_input_token_cost"] + + 500 * haiku["output_cost_per_token"] + ) + assert result == pytest.approx(warm_baseline - actually_paid) + assert result < 0, "a cache-thrashing switch must report a loss, not a saving" + + phantom = 12304 * sonnet["cache_creation_input_token_cost"] + assert result != pytest.approx(warm_baseline + phantom - actually_paid) + + +def test_cold_start_prices_a_cache_write_on_both_models(): + """With nothing read from cache the request is a first turn: the baseline would + have paid to write too, so charging only the selected model would invent a loss.""" + usage = _usage(fresh=3, cached=0, written=12304, out=500) + result = _savings("claude-sonnet-5", "claude-haiku-4-5", usage) + + sonnet = litellm.get_model_info("claude-sonnet-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + assert result == pytest.approx( + (3 * sonnet["input_cost_per_token"] + 12304 * sonnet["cache_creation_input_token_cost"]) + - (3 * haiku["input_cost_per_token"] + 12304 * haiku["cache_creation_input_token_cost"]) + + 500 * (sonnet["output_cost_per_token"] - haiku["output_cost_per_token"]) + ) assert result > 0 - # The double-counting formula this replaced: every prompt token (cache reads - # and cache writes included) charged at the flat input rate on both sides, - # minus a cache-write penalty already accounted for inside the selected arm. - base_in, base_out, base_write = _flat_rates("claude-opus-5") - sel_in, sel_out, sel_write = _flat_rates("claude-haiku-4-5") - prompt_tokens = usage_object["prompt_tokens"] - completion_tokens = usage_object["completion_tokens"] - double_counted = max( - (prompt_tokens * base_in + completion_tokens * base_out) - - (prompt_tokens * sel_in + completion_tokens * sel_out) - - usage_object["cache_creation_input_tokens"] * sel_write, - 0.0, - ) - assert result != pytest.approx(double_counted) - -def test_autorouter_savings_charges_cache_reads_at_the_cache_read_rate(): - """A request served almost entirely from cache is cheap on both models, so the - routed saving must be far smaller than the same token count would suggest at - full input price.""" - usage_object = { - "prompt_tokens": 10_000, - "completion_tokens": 0, - "total_tokens": 10_000, - "prompt_tokens_details": {"cached_tokens": 10_000, "text_tokens": 0}, - "cache_read_input_tokens": 10_000, - } - result = compute_autorouter_savings( - baseline_model="claude-opus-5", - selected_model="claude-haiku-4-5", - baseline_provider="anthropic", - selected_provider="anthropic", - usage=Usage(**usage_object), +def test_uncached_request_is_the_plain_rate_difference(): + usage = _usage(fresh=2000, cached=0, written=0, out=500) + sonnet = litellm.get_model_info("claude-sonnet-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + assert _savings("claude-sonnet-5", "claude-haiku-4-5", usage) == pytest.approx( + 2000 * (sonnet["input_cost_per_token"] - haiku["input_cost_per_token"]) + + 500 * (sonnet["output_cost_per_token"] - haiku["output_cost_per_token"]) ) - base_read = litellm.get_model_info("claude-opus-5", "anthropic")["cache_read_input_token_cost"] - sel_read = litellm.get_model_info("claude-haiku-4-5", "anthropic")["cache_read_input_token_cost"] - assert result == pytest.approx(10_000 * (base_read - sel_read)) - base_in = litellm.get_model_info("claude-opus-5", "anthropic")["input_cost_per_token"] - sel_in = litellm.get_model_info("claude-haiku-4-5", "anthropic")["input_cost_per_token"] - assert result < 10_000 * (base_in - sel_in) +def test_escalation_reports_its_real_cost(): + """Routing up to a pricier model is a real cost; hiding it behind a zero floor + would let the dashboard only ever move in one direction.""" + usage = _usage(fresh=2000, cached=0, written=0, out=500) + assert _savings("claude-haiku-4-5", "claude-sonnet-5", usage) < 0 def test_autorouter_savings_zero_when_model_unchanged(): - result = compute_autorouter_savings( - baseline_model="claude-opus-5", - selected_model="claude-opus-5", - baseline_provider="anthropic", - selected_provider="anthropic", - usage=Usage(**_cached_usage_object()), - ) - assert result == 0.0 - - -def test_autorouter_savings_floored_at_zero_on_escalation(): - # Routing UP to a pricier model must never show as negative savings. - result = compute_autorouter_savings( - baseline_model="claude-haiku-4-5", - selected_model="claude-opus-5", - baseline_provider="anthropic", - selected_provider="anthropic", - usage=Usage(**_cached_usage_object()), - ) - assert result == 0.0 + assert _savings("claude-opus-5", "claude-opus-5", _usage(3, 500, 12304, 500)) == 0.0 def test_autorouter_savings_unknown_baseline_fails_open_to_zero(): - result = compute_autorouter_savings( - baseline_model="totally-made-up-model-xyz", - selected_model="claude-haiku-4-5", - baseline_provider="anthropic", - selected_provider="anthropic", - usage=Usage(**_cached_usage_object()), - ) - assert result == 0.0 + assert _savings("totally-made-up-model-xyz", "claude-haiku-4-5", _usage(3, 500, 12304, 500)) == 0.0 def test_autorouter_savings_zero_without_baseline(): - # No configured/produced baseline -> the driver contributes nothing. result = compute_savings_spend( model="claude-haiku-4-5", custom_llm_provider="anthropic", @@ -228,36 +222,19 @@ def test_autorouter_savings_zero_without_baseline(): assert result.autorouter == 0.0 -def test_compute_savings_spend_includes_autorouter_driver(): - usage_object = _cached_usage_object() +def test_compute_savings_spend_carries_a_losing_switch_through(): + """The signed value must survive into SavingsSpend; clamping it here would put the + dashboard back to only ever showing gains.""" result = compute_savings_spend( model="claude-haiku-4-5", custom_llm_provider="anthropic", compression_saved_tokens=0, cache_read_input_tokens=0, - baseline_model="claude-opus-5", + baseline_model="claude-sonnet-5", baseline_provider="anthropic", - usage_object=usage_object, + usage_object=_cached_usage_object(), ) - expected = _cost_on("claude-opus-5", usage_object) - _cost_on("claude-haiku-4-5", usage_object) - assert result.autorouter == pytest.approx(expected) - assert result.autorouter > 0 - - -def test_compute_savings_spend_without_usage_object_keeps_other_drivers(): - """A row with no recorded usage still prices compression and caching; only the - counterfactual driver needs the usage breakdown.""" - input_cost, _ = _anthropic_costs("claude-sonnet-5") - result = compute_savings_spend( - model="claude-sonnet-5", - custom_llm_provider="anthropic", - compression_saved_tokens=1000, - cache_read_input_tokens=0, - baseline_model="claude-opus-5", - usage_object=None, - ) - assert result.compression == pytest.approx(1000 * input_cost) - assert result.autorouter == 0.0 + assert result.autorouter < 0 def test_malformed_usage_object_does_not_fail_the_spend_write(): diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx index 36126b5dfb8..07b84c7fac4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx @@ -228,6 +228,26 @@ describe("UsageTab", () => { expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]); }); + it("subtracts a losing auto-router route from the total and keeps it out of the donut", () => { + // Switching models leaves the new one with a cold cache, so a route can cost more + // than the baseline would have. A negative slice is meaningless in a donut, but the + // total has to keep the loss or the page can only ever report good news. + const { getByText, getByTestId } = renderWith([ + day("2026-07-12", { + compression_savings_spend: 0.1, + prompt_caching_savings_spend: 0.02, + autorouter_savings_spend: -0.05, + }), + ]); + + expect(getByText("$0.0700")).toBeInTheDocument(); + expect(getByText("-$0.0500")).toBeInTheDocument(); + + const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); + expect(slices.map((d: { driver: string }) => d.driver)).toEqual(["Compression", "Prompt caching"]); + expect(getByTestId("donut-chart").getAttribute("data-label")).toBe("$0.1200"); + }); + it("carries auto-router savings into the summary card, donut slice, and cumulative series", () => { const { getByText, getByTestId } = renderWith([ day("2026-07-12", { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx index ab767f5e508..cb516890653 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx @@ -146,6 +146,9 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { .filter(Boolean) .join(" \u00b7 "); + // A driver can come out negative (auto-router pays a cold-cache write on every + // model switch), and a negative slice has no meaning in a donut, so only drivers + // that actually saved are plotted; the range total keeps the signed truth. const byDriver = useMemo( () => [ @@ -155,6 +158,7 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { ].filter((d) => d.usd > 0), [compressionTotal, cachingTotal, autorouterTotal], ); + const plottedDriverTotal = useMemo(() => byDriver.reduce((sum, d) => sum + d.usd, 0), [byDriver]); const topTools = useMemo(() => topToolsBySpend(toolSpend?.by_tool ?? []), [toolSpend]); const topToolNames = useMemo(() => topTools.map((t) => t.tool_name), [topTools]); @@ -200,7 +204,7 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { label="Auto-router savings" value={usd(autorouterTotal)} hint="vs. the router's baseline model" - info="Cost of the auto-router's configured baseline model minus the cost of the model it actually routed to, net of any cache-write cost from switching models." + info="What this traffic would have cost on the router's baseline model, minus what it actually cost. Switching models leaves the new one with a cold cache, so the cache-write it pays counts against the saving; a negative total means routing cost more than staying on the baseline would have." /> @@ -260,7 +264,7 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { colors={SAVINGS_COLORS} valueFormatter={usd} showLabel - label={usd(totalSaved)} + label={usd(plottedDriverTotal)} /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts index c088e07cb91..5c73f0fc5dc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts @@ -10,6 +10,7 @@ import { localIsoDay, toCumulative, topToolsBySpend, + usd, withStartAnchor, } from "./costOptimizationUtils"; @@ -316,3 +317,19 @@ describe("formatRangeLabel", () => { expect(formatRangeLabel(new Date(2026, 6, 23), undefined)).toBe(""); }); }); + +describe("usd", () => { + it("keeps four decimals for sub-dollar amounts so small savings stay visible", () => { + expect(usd(0.05)).toBe("$0.0500"); + expect(usd(1.5)).toBe("$1.50"); + expect(usd(0)).toBe("$0.00"); + }); + + it("signs a loss ahead of the symbol and keeps its precision", () => { + // A driver can be negative once a model switch is charged for its cold cache. + // Sizing decimals off the raw value would render this as "$-0.00". + expect(usd(-0.05)).toBe("-$0.0500"); + expect(usd(-0.0004)).toBe("-$0.0004"); + expect(usd(-12.4)).toBe("-$12.40"); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts index 18aa8f1a3ef..43ebdea82d4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts @@ -3,8 +3,11 @@ import { ToolSpendDailyEntry, ToolSpendEntry } from "@/components/networking"; import { formatNumberWithCommas } from "@/utils/dataUtils"; export const usd = (value: number): string => { - const decimals = value > 0 && value < 1 ? 4 : 2; - return `$${formatNumberWithCommas(value, decimals)}`; + // Sized and signed off the magnitude: a driver can come out negative, and a small + // loss rendered at two decimals would read as "$-0.00" + const magnitude = Math.abs(value); + const decimals = magnitude > 0 && magnitude < 1 ? 4 : 2; + return `${value < 0 ? "-" : ""}$${formatNumberWithCommas(magnitude, decimals)}`; }; export const pct = (ratio: number): string => `${formatNumberWithCommas(ratio * 100, 1)}%`;