diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 505b5e0974d..b291ac6cf02 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -47,15 +47,42 @@ def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | Non return input_cost, float(cache_read_cost) -def _cost_of_usage(model: str, custom_llm_provider: str | None, usage: Usage) -> float | None: +class _ModelIdentity(NamedTuple): + model: str + provider: str + + +def _resolve_model(model: str | None, custom_llm_provider: str | None) -> _ModelIdentity | None: + """Canonical ``(model, provider)``, or ``None`` when the model cannot be resolved. + + The two sides of the comparison arrive spelled differently: the spend log records a + normalized model name alongside its provider, while the baseline arrives as the + operator wrote it in config, with the provider prefixed, implied, or absent. Raw + string equality therefore reads `anthropic/claude-opus-5` as a switch away from + `claude-opus-5`, and pricing a bare name with no provider can resolve it to a + different vendor's rates than the deployment it names. + """ + if not model: + return None + try: + resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + except Exception as e: # noqa: BLE001 # get_llm_provider raises for unroutable names; degrade to zero savings + verbose_proxy_logger.debug( + "savings: cannot resolve provider for model=%s custom_llm_provider=%s (%s)", model, custom_llm_provider, e + ) + return None + return _ModelIdentity(model=resolved_model, provider=provider) + + +def _cost_of_usage(model: _ModelIdentity, usage: Usage) -> float | None: """What ``usage`` costs on ``model``, or ``None`` when the model has no pricing.""" try: prompt_cost, completion_cost = generic_cost_per_token( - model=model, usage=usage, custom_llm_provider=custom_llm_provider or "" + model=model.model, usage=usage, custom_llm_provider=model.provider ) except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings verbose_proxy_logger.debug( - "savings: cannot price usage for provider=%s model=%s (%s)", custom_llm_provider, model, e + "savings: cannot price usage for provider=%s model=%s (%s)", model.provider, model.model, e ) return None return prompt_cost + completion_cost @@ -107,12 +134,15 @@ def compute_autorouter_savings( Signed on purpose. Switching models leaves the new one with a cold cache, so the request pays a cache-creation charge that staying on one model would not have incurred; when that charge outweighs the cheaper rates, routing lost money and the - dashboard has to be able to say so. Zero when the model is unchanged or unpriced. + dashboard has to be able to say so. Zero when both sides resolve to the same + deployment, or when either cannot be resolved or priced. """ - if not baseline_model or not selected_model or baseline_model == selected_model: + baseline = _resolve_model(baseline_model, baseline_provider) + selected = _resolve_model(selected_model, selected_provider) + if baseline is None or selected is None or baseline == selected: return 0.0 - baseline_cost = _cost_of_usage(baseline_model, baseline_provider, _baseline_usage(usage)) - selected_cost = _cost_of_usage(selected_model, selected_provider, usage) + baseline_cost = _cost_of_usage(baseline, _baseline_usage(usage)) + selected_cost = _cost_of_usage(selected, usage) if baseline_cost is None or selected_cost is None: return 0.0 return baseline_cost - selected_cost diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 11bea5cc429..49e0273dd15 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -264,3 +264,45 @@ def test_model_without_cache_read_pricing_yields_no_caching_savings(): cache_read_input_tokens=5000, ) assert result.prompt_caching == 0.0 + + +def test_the_same_deployment_spelled_two_ways_is_not_a_switch(): + """The spend log records a normalized model name while the baseline arrives as the + operator wrote it in config. Comparing the raw strings makes a request that never + changed model look like a switch, and prices one deployment against itself.""" + # Must be a cached request: the baseline arm is priced against a warm cache and the + # selected arm against what was actually paid, so treating one deployment as two + # charges it a cold-cache write it never took, inventing a loss on a request that + # never changed model. An uncached request prices identically either way and would + # make this assertion vacuous. + usage = _usage(fresh=3, cached=500, written=12304, out=500) + assert _savings("anthropic/claude-opus-5", "claude-opus-5", usage) == 0.0 + assert _savings("claude-opus-5", "anthropic/claude-opus-5", usage) == 0.0 + + +def test_baseline_is_priced_under_its_own_provider(): + """Two providers can serve the same bare model name at different rates, so dropping + the provider prices the baseline against a vendor the operator never named. Here it + decides whether routing reads as a saving or a loss.""" + usage = Usage(prompt_tokens=100_000, completion_tokens=10_000, total_tokens=110_000) + azure = compute_autorouter_savings( + baseline_model="azure_ai/deepseek-r1", + selected_model="claude-haiku-4-5", + baseline_provider=None, + selected_provider="anthropic", + usage=usage, + ) + deepseek = compute_autorouter_savings( + baseline_model="deepseek/deepseek-r1", + selected_model="claude-haiku-4-5", + baseline_provider=None, + selected_provider="anthropic", + usage=usage, + ) + assert azure != pytest.approx(deepseek) + assert azure > 0 > deepseek + + +def test_unresolvable_baseline_fails_open_to_zero(): + usage = _usage(fresh=2000, cached=0, written=0, out=500) + assert _savings("no-such-provider-xyz/no-such-model", "claude-haiku-4-5", usage) == 0.0 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx index 07b84c7fac4..58aef24dddc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx @@ -29,12 +29,14 @@ vi.mock("@/components/shared/charts", () => ({ colors, showLegend, maxBarSize, + stack, }: { data: unknown; categories: string[]; colors?: readonly string[]; showLegend?: boolean; maxBarSize?: number; + stack?: boolean; }) => (
({ data-colors={(colors ?? []).join(",")} data-show-legend={String(showLegend ?? true)} data-max-bar-size={maxBarSize === undefined ? "" : String(maxBarSize)} + data-stack={String(stack ?? false)} data-series={JSON.stringify(data)} /> ), @@ -228,6 +231,24 @@ describe("UsageTab", () => { expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]); }); + it("does not stack the per-day drivers, because one of them can be negative", async () => { + // Stacking sums the series into one bar. Auto-router savings go negative when a + // model switch pays for a cold cache, and that segment would be drawn below the + // axis while the rest of the bar still read as the day's total. + const { getByRole, getByTestId } = renderWith([ + day("2026-07-12", { + compression_savings_spend: 0.1, + prompt_caching_savings_spend: 0.02, + autorouter_savings_spend: -0.05, + }), + ]); + + await userEvent.click(getByRole("tab", { name: "Per day" })); + const bars = getByTestId("bar-chart"); + expect(bars.getAttribute("data-stack")).toBe("false"); + expect(readSeries(bars)[0]).toMatchObject({ "Auto-router": -0.05 }); + }); + it("subtracts a losing auto-router route from the total and keeps it out of the donut", () => { // Switching models leaves the new one with a cold cache, so a route can cost more // than the baseline would have. A negative slice is meaningless in a donut, but the diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx index cb516890653..1eb3802c216 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx @@ -239,12 +239,14 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { showDots={overTime.length <= MAX_POINTS_WITH_DOTS} /> ) : ( + // Not stacked: a driver can be negative once a model switch is charged + // for its cold cache, and stacking would draw that segment below the axis + // while the remaining bar still read as the day's total