From d3db7cebca06f6c990086d8b09c9f48efd04e2a9 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Sat, 29 Aug 2026 16:09:35 -0700 Subject: [PATCH] fix(proxy): count auto-router classifier cost in savings and benchmarks The LLM classifier's cost was recorded on the routing decision but never reached any savings surface: per-request autorouter_savings stayed gross and the session rollup recorded only the served request's spend, so /auto_router/benchmarks overstated savings and understated routed spend. Net the classifier cost into the savings figure at its one computation owner and fold it into the rollup turn's spend, keeping baseline_spend = spend + saved_spend. The response header's numeric guard now shares the same reader. Fixes #38816 --- litellm/proxy/common_request_processing.py | 8 ++- litellm/proxy/db/autorouter_session_rollup.py | 11 +++- litellm/proxy/spend_tracking/savings.py | 26 ++++++++- .../db/test_autorouter_session_rollup.py | 22 ++++++++ .../proxy/spend_tracking/test_savings.py | 55 +++++++++++++++++++ 5 files changed, 114 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index ff6c8d1b1f8..21abbfa40c2 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1362,6 +1362,8 @@ def _classifier_cost_from_request_data(request_data: Mapping[str, object] | None routes and in `metadata` on chat-style routes, so both buckets are consulted, in the same precedence `get_or_create_metadata_bucket` writes them. """ + from litellm.proxy.spend_tracking.savings import classifier_cost_from_decision + data: Final = request_data or {} for metadata_key in ("litellm_metadata", "metadata"): metadata = data.get(metadata_key) @@ -1370,10 +1372,10 @@ def _classifier_cost_from_request_data(request_data: Mapping[str, object] | None decision = metadata.get("routing_decision") if not isinstance(decision, dict): continue - cost = decision.get("classifier_cost") - if isinstance(cost, bool) or not isinstance(cost, (int, float)): + cost = classifier_cost_from_decision(decision) + if cost is None: continue - return float(cost) + return cost return None diff --git a/litellm/proxy/db/autorouter_session_rollup.py b/litellm/proxy/db/autorouter_session_rollup.py index 96192b884d8..9c637a62dc1 100644 --- a/litellm/proxy/db/autorouter_session_rollup.py +++ b/litellm/proxy/db/autorouter_session_rollup.py @@ -185,8 +185,10 @@ def build_autorouter_turn_transaction( of a request through the router) are excluded by their internal_call_origin stamp: they are not traffic a user sent, so counting them would manufacture sessions and savings in the adoption metrics. Failed requests served nothing and are excluded. - Cache facts are derived from the payload's own usage record through the savings - owner, never handed in beside it. + The classifier's charge still lands here exactly once, via the decision's own + classifier_cost folded into this turn's spend: the excluded classifier row is how + it was billed, the decision is how it is attributed. Cache facts are derived from + the payload's own usage record through the savings owner, never handed in beside it. """ if payload.get("status") != "success": return None @@ -204,9 +206,12 @@ def build_autorouter_turn_transaction( turn_at: Final = _turn_time_utc(str(payload.get("startTime") or "")) if turn_at is None: return None + from litellm.proxy.spend_tracking.savings import classifier_cost_from_decision + usage_object_raw: Final = metadata.get("usage_object") cache: Final = turn_cache_facts(usage_object_raw if isinstance(usage_object_raw, Mapping) else None) tier_raw: Final = routing_decision.get("tier") + classifier_cost: Final = classifier_cost_from_decision(routing_decision) return AutoRouterTurnTransaction( api_key=api_key, session_id=_bounded_session_id(session_id), @@ -216,7 +221,7 @@ def build_autorouter_turn_transaction( model=model, turn_at=turn_at, total_tokens=int(payload.get("prompt_tokens") or 0) + int(payload.get("completion_tokens") or 0), - spend=float(payload.get("spend") or 0.0), + spend=float(payload.get("spend") or 0.0) + (classifier_cost or 0.0), saved_spend=saved_spend, covered=cache.covered, cache_hit=cache.read_tokens > 0, diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 7d20aeeebac..1d0eb12da75 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -481,6 +481,20 @@ def _numeric_savings(value: object) -> float | None: return float(value) +def classifier_cost_from_decision(routing_decision: Mapping[str, object] | None) -> float | None: + """The LLM-classifier cost a routing decision recorded, or ``None`` when it holds none. + + ``None`` covers the decision-less request, the heuristic short-circuit that never + called a classifier, the unpriced classifier model, and a malformed value alike: + in every one of those cases there is no dollar figure to move, so callers treat + ``None`` as zero rather than as an error. The one owner of that reading, shared by + the savings netting, the session rollup and the response header, so the three can + never disagree about what counts as a classifier charge. + """ + decision: Final = routing_decision if isinstance(routing_decision, Mapping) else {} + return _numeric_savings(decision.get("classifier_cost")) + + def autorouter_savings_for_request( model: str | None, custom_llm_provider: str | None, @@ -490,7 +504,8 @@ def autorouter_savings_for_request( llm_router: "Callable[[], Router | None] | None" = None, cost_breakdown: Mapping[str, object] | None = None, ) -> float | None: - """Auto-router savings for one request, or ``None`` when the driver is off. + """Auto-router savings for one request, net of the classifier call that routed it, + or ``None`` when the driver is off. ``None`` and ``0.0`` are different facts: ``None`` means this request cannot carry a figure at all (no routing decision, no baseline, unusable usage), while ``0.0`` is a @@ -498,6 +513,11 @@ def autorouter_savings_for_request( Never raises: pricing failures inside degrade to zero, and the driver-off cases return ``None``, so this is safe on the logging path where a raise would fail the request's logging. + + The classifier deduction lives here, at the figure's one computation owner, rather + than in any reader: the stamped ``autorouter_savings`` is then already net, so the + session rollup, the daily tables and every logging consumer agree without each + re-deriving the deduction, and the recorded-figure-wins path cannot deduct twice. """ usage: Final = _usage_from_spend_log(usage_object) if usage is None or not model: @@ -510,7 +530,7 @@ def autorouter_savings_for_request( if not decision or not baseline_model: return None router_instance: Final = llm_router() if llm_router else None - return compute_autorouter_savings( + gross: Final = compute_autorouter_savings( baseline_model=baseline_model, selected_model=model, selected_provider=custom_llm_provider, @@ -522,6 +542,8 @@ def autorouter_savings_for_request( baseline_info=_effective_model_info(router_instance, baseline_id, baseline_model or ""), cost_breakdown=cost_breakdown, ) + classifier_cost: Final = classifier_cost_from_decision(decision) + return gross if classifier_cost is None else gross - classifier_cost def autorouter_savings_for_logging_payload( diff --git a/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py b/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py index cb4687ef370..2ed4f843711 100644 --- a/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py +++ b/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py @@ -106,6 +106,28 @@ class TestBuildTransaction: transaction = _build() assert transaction is not None and transaction.tier is None + def test_a_priced_classifier_rides_the_turns_spend(self): + """The classifier row is excluded from the rollup, so its charge lands here, + folded once into the turn that paid for it (GH #38816).""" + transaction = _build(metadata=_metadata(routing_decision={**ROUTING_DECISION, "classifier_cost": 0.005})) + assert transaction is not None and transaction.spend == pytest.approx(0.015) + + @pytest.mark.parametrize( + "decision_extra", [{}, {"classifier_cost": 0.0}, {"classifier_cost": "bogus"}, {"classifier_cost": True}] + ) + def test_an_unpriced_classifier_leaves_the_spend_alone(self, decision_extra: dict): + transaction = _build(metadata=_metadata(routing_decision={**ROUTING_DECISION, **decision_extra})) + assert transaction is not None and transaction.spend == pytest.approx(0.01) + + def test_every_turn_carries_its_own_classifier_charge(self): + first = _build(metadata=_metadata(routing_decision={**ROUTING_DECISION, "classifier_cost": 0.005})) + second = _build( + payload=_payload(startTime="2026-08-01T12:01:00", spend=0.02), + metadata=_metadata(routing_decision={**ROUTING_DECISION, "classifier_cost": 0.007}), + ) + assert first is not None and first.spend == pytest.approx(0.015) + assert second is not None and second.spend == pytest.approx(0.027) + def test_router_name_falls_back_to_the_payload_model_group(self): transaction = _build(metadata=_metadata(routing_decision={"router_type": "complexity"})) assert transaction is not None and transaction.router_name == "live-auto" diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index e8ca569763d..5fda4fb20b5 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -1151,6 +1151,61 @@ def test_logging_payload_never_stamps_internal_calls(): assert internal is None +def test_savings_are_net_of_a_priced_classifier(): + """The classifier call is part of what routing cost, so the per-request figure + deducts it; a charge big enough to outweigh the model saving goes negative, + since the figure is signed on purpose (GH #38816).""" + from litellm.proxy.spend_tracking.savings import autorouter_savings_for_request + + gross = autorouter_savings_for_request( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + routing_decision=_routed_decision(), + usage_object=_cached_usage_object(), + ) + net = autorouter_savings_for_request( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + routing_decision={**_routed_decision(), "classifier_cost": 0.005}, + usage_object=_cached_usage_object(), + ) + assert gross is not None and net == pytest.approx(gross - 0.005) + + +@pytest.mark.parametrize("classifier_cost", [0.0, "bogus", True]) +def test_an_unpriced_classifier_deducts_nothing(classifier_cost: object): + from litellm.proxy.spend_tracking.savings import autorouter_savings_for_request + + gross = autorouter_savings_for_request( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + routing_decision=_routed_decision(), + usage_object=_cached_usage_object(), + ) + with_cost_field = autorouter_savings_for_request( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + routing_decision={**_routed_decision(), "classifier_cost": classifier_cost}, + usage_object=_cached_usage_object(), + ) + assert with_cost_field == gross + + +def test_recorded_savings_are_already_net_and_not_deducted_again(): + """The deduction lives at the figure's computation owner, so a stamped figure is + net by construction; the recorded-wins path must not subtract a second time.""" + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + gateway_injected_cache=False, + routing_decision={**_routed_decision(), "classifier_cost": 0.005}, + usage_object=_cached_usage_object(), + recorded_autorouter_savings=0.5, + ) + assert result.autorouter == 0.5 + + def test_caching_savings_require_a_gateway_injected_breakpoint(): """The same cached usage is attributed to the gateway only when it added a breakpoint.