From c46e96a6a991f7bfedbdbbcb2b4630ac9fe255de Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Fri, 31 Jul 2026 20:37:43 -0700 Subject: [PATCH] fix(spend): compare and price auto-router models as resolved identities The two sides of the comparison arrived spelled differently. The spend log records a normalized model name alongside its provider, while the baseline arrives as the operator wrote it in config, with the provider prefixed, implied or absent, so the raw strings were never comparable. Read as a switch, `anthropic/claude-opus-5` against a served `claude-opus-5` priced one deployment against itself, and because the baseline arm is priced warm while the served arm pays its cold-cache write, a request that never changed model reported a $0.0707 loss. That mis-comparison was harmless until the cache fix made the two arms asymmetric, so the identity check has to land with it. Pricing had the same root cause: a baseline resolved without a provider takes whichever vendor owns the bare name. `azure_ai/deepseek-r1` and `deepseek/deepseek-r1` are the same bare model at different rates, and the difference decides the sign, +$0.039 against -$0.0731 on the same request. Both now resolve through `get_llm_provider` to a canonical (model, provider) before being compared or priced, so one deployment spelled two ways is not a switch and every arm is priced under the vendor that serves it. The per-day chart no longer stacks its drivers. Stacking sums the series into one bar, and a driver that goes negative would be drawn below the axis while the rest of the bar still read as the day's total. --- litellm/proxy/spend_tracking/savings.py | 44 ++++++++++++++++--- .../proxy/spend_tracking/test_savings.py | 42 ++++++++++++++++++ .../_components/UsageTab.test.tsx | 21 +++++++++ .../_components/UsageTab.tsx | 4 +- 4 files changed, 103 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 505b5e0974d..b291ac6cf02 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -47,15 +47,42 @@ def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | Non return input_cost, float(cache_read_cost) -def _cost_of_usage(model: str, custom_llm_provider: str | None, usage: Usage) -> float | None: +class _ModelIdentity(NamedTuple): + model: str + provider: str + + +def _resolve_model(model: str | None, custom_llm_provider: str | None) -> _ModelIdentity | None: + """Canonical ``(model, provider)``, or ``None`` when the model cannot be resolved. + + The two sides of the comparison arrive spelled differently: the spend log records a + normalized model name alongside its provider, while the baseline arrives as the + operator wrote it in config, with the provider prefixed, implied, or absent. Raw + string equality therefore reads `anthropic/claude-opus-5` as a switch away from + `claude-opus-5`, and pricing a bare name with no provider can resolve it to a + different vendor's rates than the deployment it names. + """ + if not model: + return None + try: + resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + except Exception as e: # noqa: BLE001 # get_llm_provider raises for unroutable names; degrade to zero savings + verbose_proxy_logger.debug( + "savings: cannot resolve provider for model=%s custom_llm_provider=%s (%s)", model, custom_llm_provider, e + ) + return None + return _ModelIdentity(model=resolved_model, provider=provider) + + +def _cost_of_usage(model: _ModelIdentity, usage: Usage) -> float | None: """What ``usage`` costs on ``model``, or ``None`` when the model has no pricing.""" try: prompt_cost, completion_cost = generic_cost_per_token( - model=model, usage=usage, custom_llm_provider=custom_llm_provider or "" + model=model.model, usage=usage, custom_llm_provider=model.provider ) except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings verbose_proxy_logger.debug( - "savings: cannot price usage for provider=%s model=%s (%s)", custom_llm_provider, model, e + "savings: cannot price usage for provider=%s model=%s (%s)", model.provider, model.model, e ) return None return prompt_cost + completion_cost @@ -107,12 +134,15 @@ def compute_autorouter_savings( Signed on purpose. Switching models leaves the new one with a cold cache, so the request pays a cache-creation charge that staying on one model would not have incurred; when that charge outweighs the cheaper rates, routing lost money and the - dashboard has to be able to say so. Zero when the model is unchanged or unpriced. + dashboard has to be able to say so. Zero when both sides resolve to the same + deployment, or when either cannot be resolved or priced. """ - if not baseline_model or not selected_model or baseline_model == selected_model: + baseline = _resolve_model(baseline_model, baseline_provider) + selected = _resolve_model(selected_model, selected_provider) + if baseline is None or selected is None or baseline == selected: return 0.0 - baseline_cost = _cost_of_usage(baseline_model, baseline_provider, _baseline_usage(usage)) - selected_cost = _cost_of_usage(selected_model, selected_provider, usage) + baseline_cost = _cost_of_usage(baseline, _baseline_usage(usage)) + selected_cost = _cost_of_usage(selected, usage) if baseline_cost is None or selected_cost is None: return 0.0 return baseline_cost - selected_cost diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 11bea5cc429..49e0273dd15 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -264,3 +264,45 @@ def test_model_without_cache_read_pricing_yields_no_caching_savings(): cache_read_input_tokens=5000, ) assert result.prompt_caching == 0.0 + + +def test_the_same_deployment_spelled_two_ways_is_not_a_switch(): + """The spend log records a normalized model name while the baseline arrives as the + operator wrote it in config. Comparing the raw strings makes a request that never + changed model look like a switch, and prices one deployment against itself.""" + # Must be a cached request: the baseline arm is priced against a warm cache and the + # selected arm against what was actually paid, so treating one deployment as two + # charges it a cold-cache write it never took, inventing a loss on a request that + # never changed model. An uncached request prices identically either way and would + # make this assertion vacuous. + usage = _usage(fresh=3, cached=500, written=12304, out=500) + assert _savings("anthropic/claude-opus-5", "claude-opus-5", usage) == 0.0 + assert _savings("claude-opus-5", "anthropic/claude-opus-5", usage) == 0.0 + + +def test_baseline_is_priced_under_its_own_provider(): + """Two providers can serve the same bare model name at different rates, so dropping + the provider prices the baseline against a vendor the operator never named. Here it + decides whether routing reads as a saving or a loss.""" + usage = Usage(prompt_tokens=100_000, completion_tokens=10_000, total_tokens=110_000) + azure = compute_autorouter_savings( + baseline_model="azure_ai/deepseek-r1", + selected_model="claude-haiku-4-5", + baseline_provider=None, + selected_provider="anthropic", + usage=usage, + ) + deepseek = compute_autorouter_savings( + baseline_model="deepseek/deepseek-r1", + selected_model="claude-haiku-4-5", + baseline_provider=None, + selected_provider="anthropic", + usage=usage, + ) + assert azure != pytest.approx(deepseek) + assert azure > 0 > deepseek + + +def test_unresolvable_baseline_fails_open_to_zero(): + usage = _usage(fresh=2000, cached=0, written=0, out=500) + assert _savings("no-such-provider-xyz/no-such-model", "claude-haiku-4-5", usage) == 0.0 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx index 07b84c7fac4..58aef24dddc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx @@ -29,12 +29,14 @@ vi.mock("@/components/shared/charts", () => ({ colors, showLegend, maxBarSize, + stack, }: { data: unknown; categories: string[]; colors?: readonly string[]; showLegend?: boolean; maxBarSize?: number; + stack?: boolean; }) => (
({ data-colors={(colors ?? []).join(",")} data-show-legend={String(showLegend ?? true)} data-max-bar-size={maxBarSize === undefined ? "" : String(maxBarSize)} + data-stack={String(stack ?? false)} data-series={JSON.stringify(data)} /> ), @@ -228,6 +231,24 @@ describe("UsageTab", () => { expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]); }); + it("does not stack the per-day drivers, because one of them can be negative", async () => { + // Stacking sums the series into one bar. Auto-router savings go negative when a + // model switch pays for a cold cache, and that segment would be drawn below the + // axis while the rest of the bar still read as the day's total. + const { getByRole, getByTestId } = renderWith([ + day("2026-07-12", { + compression_savings_spend: 0.1, + prompt_caching_savings_spend: 0.02, + autorouter_savings_spend: -0.05, + }), + ]); + + await userEvent.click(getByRole("tab", { name: "Per day" })); + const bars = getByTestId("bar-chart"); + expect(bars.getAttribute("data-stack")).toBe("false"); + expect(readSeries(bars)[0]).toMatchObject({ "Auto-router": -0.05 }); + }); + it("subtracts a losing auto-router route from the total and keeps it out of the donut", () => { // Switching models leaves the new one with a cold cache, so a route can cost more // than the baseline would have. A negative slice is meaningless in a donut, but the diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx index cb516890653..1eb3802c216 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx @@ -239,12 +239,14 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { showDots={overTime.length <= MAX_POINTS_WITH_DOTS} /> ) : ( + // Not stacked: a driver can be negative once a model switch is charged + // for its cold cache, and stacking would draw that segment below the axis + // while the remaining bar still read as the day's total