scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-13 23:11:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 101

feat(spend): add net auto-router savings to the cost-optimization dashboard

Browse source 
Adds auto-router as a third savings driver alongside compression and prompt
caching: a summary card, a donut segment, and a series in the savings graph.

Savings are the net dollars from routing a request to the selected model
instead of a counterfactual baseline. The delta prices prompt tokens at each
model's input rate and completion tokens at each model's output rate, then
subtracts the cache-write penalty the selected deployment incurs on a cold
cache. Cache-read discounts stay attributed to the prompt-caching driver to
avoid double-counting. The result is floored at zero so an escalation to a
pricier model never reads as negative savings.

The baseline model defaults to claude-opus-5 and is operator-configurable per
deployment via the auto_router_savings_baseline_model litellm_param. It flows
AutoRouter to PreRoutingHookResponse to the metadata bucket to SpendLogsMetadata
to the daily spend writer, mirroring the routing_decision path, and is stripped
from untrusted caller metadata so it cannot be spoofed.

Savings accrue into a new autorouter_savings_spend column on the six
LiteLLM_Daily*Spend rollup tables; no LiteLLM_SpendLogs queries are added.
This commit is contained in:
Tin Chi Lo 2026-07-31 15:03:05 -07:00
parent fa56283806
commit e30b7f2f1f
20 changed files with 455 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql Normal file
View file

@ -0,0 +1,17 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
-- AlterTable
ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

6
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)

1
litellm/proxy/_types.py
View file

@ -3321,6 +3321,7 @@ class SpendLogsMetadata(TypedDict):
max_retries: Optional[int] # Max retries configured for this request
cost_breakdown: Optional[CostBreakdown] # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)
compression_savings: CompressionSavingsMetadata | None
auto_router_savings_baseline_model: str | None # counterfactual model for the auto-router savings driver
class SpendLogsPayload(TypedDict):

19
litellm/proxy/db/db_spend_update_writer.py
View file

@ -1595,6 +1595,10 @@ class DBSpendUpdateWriter:
common_data["prompt_caching_savings_spend"] = transaction.get(
"prompt_caching_savings_spend", 0
)
if "autorouter_savings_spend" in transaction:
common_data["autorouter_savings_spend"] = transaction.get(
"autorouter_savings_spend", 0
)
if entity_type == "tag" and "request_id" in transaction:
common_data["request_id"] = transaction.get("request_id")
@ -1630,6 +1634,12 @@ class DBSpendUpdateWriter:
update_data["prompt_caching_savings_spend"] = {
"increment": transaction.get("prompt_caching_savings_spend", 0)
}
if "autorouter_savings_spend" in transaction:
update_data[
"autorouter_savings_spend"
] = { # mutable-ok: extends preset dict following compression_savings_spend pattern
"increment": transaction.get("autorouter_savings_spend", 0)
}
if entity_type == "tag" and "request_id" in transaction:
update_data["request_id"] = transaction.get("request_id")
@ -1881,11 +1891,17 @@ class DBSpendUpdateWriter:
cache_read_input_tokens = _extract_cache_read_tokens(usage_obj)
compression_saved_tokens = extract_compression_saved_tokens(_metadata)
cache_creation_input_tokens = _extract_cache_creation_tokens(usage_obj)
savings_spend = compute_savings_spend(
model=payload.get("model", None),
custom_llm_provider=payload.get("custom_llm_provider", None),
compression_saved_tokens=compression_saved_tokens,
cache_read_input_tokens=cache_read_input_tokens,
baseline_model=_metadata.get("auto_router_savings_baseline_model"),
prompt_tokens=payload.get("prompt_tokens", 0),
completion_tokens=payload.get("completion_tokens", 0),
cache_creation_input_tokens=cache_creation_input_tokens,
)
daily_transaction = BaseDailySpendTransaction(
@ -1903,10 +1919,11 @@ class DBSpendUpdateWriter:
successful_requests=1 if request_status == "success" else 0,
failed_requests=1 if request_status != "success" else 0,
cache_read_input_tokens=cache_read_input_tokens,
cache_creation_input_tokens=_extract_cache_creation_tokens(usage_obj),
cache_creation_input_tokens=cache_creation_input_tokens,
compression_saved_tokens=compression_saved_tokens,
compression_savings_spend=savings_spend.compression,
prompt_caching_savings_spend=savings_spend.prompt_caching,
autorouter_savings_spend=savings_spend.autorouter,
)
return daily_transaction
except Exception as e:

1
litellm/proxy/litellm_pre_call_utils.py
View file

@ -203,6 +203,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS = (
"applied_policies",
"policy_sources",
"routing_decision",
"auto_router_savings_baseline_model",
INTERNAL_CALL_ORIGIN_METADATA_KEY,
"standard_logging_object",
"proxy_server_request",

6
litellm/proxy/schema.prisma
View file

@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)

98
litellm/proxy/spend_tracking/savings.py
View file

@ -17,31 +17,81 @@ from litellm._logging import verbose_proxy_logger
class SavingsSpend(NamedTuple):
compression: float
prompt_caching: float
autorouter: float = 0.0
def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | None) -> tuple[float, float]:
class _ModelRates(NamedTuple):
input: float
output: float
cache_read: float
cache_write: float
def _model_rates(model: str | None, custom_llm_provider: str | None) -> _ModelRates:
"""
Return ``(input_cost_per_token, cache_read_cost_per_token)`` for a model.
Per-token prices for a model, in its four billed dimensions.
Falls open to ``(0.0, 0.0)`` when the model is unknown so savings degrade to
zero rather than raising inside the spend writer. When a model has no
separate cache-read price the cache-read cost mirrors the input cost, which
yields zero caching savings.
Falls open to all-zero rates when the model is unknown so savings degrade to
zero rather than raising inside the spend writer. When a model has no separate
cache-read price the cache-read rate mirrors the input rate, which yields zero
caching savings; a missing cache-write price falls back to the input rate,
which is what providers without a distinct cache-creation charge bill.
"""
if not model:
return 0.0, 0.0
return _ModelRates(0.0, 0.0, 0.0, 0.0)
try:
info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings
verbose_proxy_logger.debug(
"savings: no model info for provider=%s model=%s (%s)", custom_llm_provider, model, e
)
return 0.0, 0.0
return _ModelRates(0.0, 0.0, 0.0, 0.0)
input_cost = float(info.get("input_cost_per_token") or 0.0)
cache_read_cost = info.get("cache_read_input_token_cost")
if cache_read_cost is None:
return input_cost, input_cost
return input_cost, float(cache_read_cost)
output_cost = float(info.get("output_cost_per_token") or 0.0)
cache_read = info.get("cache_read_input_token_cost")
cache_write = info.get("cache_creation_input_token_cost")
return _ModelRates(
input=input_cost,
output=output_cost,
cache_read=input_cost if cache_read is None else float(cache_read),
cache_write=input_cost if cache_write is None else float(cache_write),
)
def compute_autorouter_savings(
baseline_model: str | None,
selected_model: str | None,
baseline_provider: str | None,
selected_provider: str | None,
prompt_tokens: int,
completion_tokens: int,
cache_creation_input_tokens: int,
) -> float:
"""
Net dollars saved by routing this request to ``selected_model`` instead of the
counterfactual ``baseline_model``.
The model-switch delta prices prompt tokens at each model's input rate and
completion tokens at each model's output rate (output is typically several
times the input rate, so pricing completions at the input rate materially
understates the gap). The cache-write penalty is the cost of switching: a
cold cache on the selected deployment forces a cache-creation charge that the
baseline would not have incurred, priced at the selected model's cache-write
rate. Cache-read discounts are deliberately excluded here; they are attributed
to the prompt-caching driver, so folding them in would double-count.
Returns zero when routing did not change the model or when pricing is unknown.
Floored at zero so an escalation to a pricier model never reads as negative
savings on the dashboard.
"""
if not baseline_model or not selected_model or baseline_model == selected_model:
return 0.0
baseline = _model_rates(baseline_model, baseline_provider)
selected = _model_rates(selected_model, selected_provider)
baseline_cost = (prompt_tokens * baseline.input) + (completion_tokens * baseline.output)
selected_cost = (prompt_tokens * selected.input) + (completion_tokens * selected.output)
cache_write_penalty = max(cache_creation_input_tokens, 0) * selected.cache_write
return max((baseline_cost - selected_cost) - cache_write_penalty, 0.0)
def compute_savings_spend(
@ -49,6 +99,11 @@ def compute_savings_spend(
custom_llm_provider: str | None,
compression_saved_tokens: int,
cache_read_input_tokens: int,
baseline_model: str | None = None,
baseline_provider: str | None = None,
prompt_tokens: int = 0,
completion_tokens: int = 0,
cache_creation_input_tokens: int = 0,
) -> SavingsSpend:
"""
Dollar savings for one request, split by optimization driver.
@ -56,8 +111,19 @@ def compute_savings_spend(
Compression savings price the tokens compression removed at the model's
input rate. Prompt-caching savings price the cache-read tokens at the
difference between the input rate and the discounted cache-read rate.
Auto-router savings compare the served ``model`` against the counterfactual
``baseline_model`` and are zero unless the two differ.
"""
input_cost, cache_read_cost = _input_and_cache_read_cost(model, custom_llm_provider)
compression = max(compression_saved_tokens, 0) * input_cost
prompt_caching = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0)
return SavingsSpend(compression=compression, prompt_caching=prompt_caching)
rates = _model_rates(model, custom_llm_provider)
compression = max(compression_saved_tokens, 0) * rates.input
prompt_caching = max(cache_read_input_tokens, 0) * max(rates.input - rates.cache_read, 0.0)
autorouter = compute_autorouter_savings(
baseline_model=baseline_model,
selected_model=model,
baseline_provider=baseline_provider,
selected_provider=custom_llm_provider,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
cache_creation_input_tokens=cache_creation_input_tokens,
)
return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter)

1
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -117,6 +117,7 @@ def _get_spend_logs_metadata(
max_retries=None,
cost_breakdown=None,
compression_savings=None,
auto_router_savings_baseline_model=None,
litellm_call_id=litellm_call_id,
)
verbose_proxy_logger.debug(

25
litellm/router.py
View file

@ -7670,6 +7670,7 @@ class Router:
default_model=default_model,
embedding_model=embedding_model,
litellm_router_instance=self,
savings_baseline_model=deployment.litellm_params.auto_router_savings_baseline_model,
)
self._register_pre_routing_strategy(
registry=self.auto_routers,
@ -11175,6 +11176,10 @@ class Router:
request_kwargs=request_kwargs,
routing_decision=(pre_routing_hook_response.routing_decision if pre_routing_hook_response else None),
)
self._record_savings_baseline_model(
request_kwargs=request_kwargs,
baseline_model=(pre_routing_hook_response.savings_baseline_model if pre_routing_hook_response else None),
)
# `model` (the alias, e.g. "smart-router") is never the deployment actually
# called - apply the alias's own litellm_params (besides `model` itself,
@ -11222,6 +11227,26 @@ class Router:
request_kwargs=request_kwargs, routing_decision=routing_decision
)
@staticmethod
def _record_savings_baseline_model(
request_kwargs: dict,
baseline_model: str | None,
) -> None:
"""Stash the auto-router's savings baseline for this attempt, same write-or-clear
discipline as `_record_routing_decision`: a fallback that re-enters this hook
without an auto-router strategy must clear a prior attempt's baseline, or the
spend writer would price savings against a model this attempt never routed
against.
"""
if baseline_model is None:
for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")):
if isinstance(bucket, dict):
bucket.pop("auto_router_savings_baseline_model", None)
return
_, metadata_bucket = get_or_create_metadata_bucket(request_kwargs)
metadata_bucket["auto_router_savings_baseline_model"] = baseline_model
@staticmethod
def _redact_prompt_text_if_needed(
request_kwargs: Mapping[str, Any],

8
litellm/router_strategy/auto_router/auto_router.py
View file

@ -20,6 +20,10 @@ else:
class AutoRouter(CustomLogger):
DEFAULT_AUTO_SYNC_VALUE = "local"
# Flagship the auto-router's savings are measured against when the deployment
# does not configure `auto_router_savings_baseline_model`. Bare key (no
# provider prefix) to match the pricing map's canonical entry.
DEFAULT_SAVINGS_BASELINE_MODEL = "claude-opus-5"
def __init__(
self,
@ -29,6 +33,7 @@ class AutoRouter(CustomLogger):
litellm_router_instance: "Router",
auto_router_config_path: Optional[str] = None,
auto_router_config: Optional[str] = None,
savings_baseline_model: str | None = None,
):
"""
Auto-Router class that uses a semantic router to route requests to the appropriate model.
@ -40,6 +45,7 @@ class AutoRouter(CustomLogger):
default_model: The default model to use if no route is found.
embedding_model: The embedding model to use for the auto-router.
litellm_router_instance: The instance of the LiteLLM Router.
savings_baseline_model: The counterfactual model the dashboard measures savings against; falls back to DEFAULT_SAVINGS_BASELINE_MODEL.
"""
from semantic_router.routers import SemanticRouter
@ -51,6 +57,7 @@ class AutoRouter(CustomLogger):
self.default_model = default_model
self.embedding_model: str = embedding_model
self.litellm_router_instance: "Router" = litellm_router_instance
self.savings_baseline_model: str = savings_baseline_model or self.DEFAULT_SAVINGS_BASELINE_MODEL
def _load_semantic_routing_routes(self) -> List[Route]:
from semantic_router.routers import SemanticRouter
@ -156,4 +163,5 @@ class AutoRouter(CustomLogger):
return PreRoutingHookResponse(
model=model,
messages=messages,
savings_baseline_model=self.savings_baseline_model,
)

7
litellm/types/router.py
View file

@ -272,6 +272,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
auto_router_config: Optional[str] = None
auto_router_default_model: Optional[str] = None
auto_router_embedding_model: Optional[str] = None
# counterfactual model the auto-router's savings are measured against on the
# cost-optimization dashboard; unset falls back to AutoRouter.DEFAULT_SAVINGS_BASELINE_MODEL
auto_router_savings_baseline_model: Optional[str] = None
# complexity-router params
complexity_router_config: Optional[Dict] = None
@ -840,6 +843,10 @@ class PreRoutingHookResponse(BaseModel):
model: str
messages: Optional[List[Dict[str, Any]]]
routing_decision: StandardLoggingRoutingDecision | None = None
# counterfactual model for the cost-optimization dashboard's auto-router savings
# card; only auto-router populates this today, so it is None for every other
# pre-routing strategy
savings_baseline_model: Optional[str] = None
_PreRoutingStrategyT_co = TypeVar("_PreRoutingStrategyT_co", covariant=True)

6
schema.prisma
View file

@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend {
compression_saved_tokens BigInt @default(0)
compression_savings_spend Float @default(0.0)
prompt_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)

143
tests/test_litellm/proxy/spend_tracking/test_savings.py
View file

@ -6,7 +6,10 @@ sys.path.insert(0, os.path.abspath("../../../.."))
import pytest
import litellm
from litellm.proxy.spend_tracking.savings import compute_savings_spend
from litellm.proxy.spend_tracking.savings import (
compute_autorouter_savings,
compute_savings_spend,
)
def _anthropic_costs(model: str) -> tuple[float, float]:
@ -16,6 +19,14 @@ def _anthropic_costs(model: str) -> tuple[float, float]:
return input_cost, cache_read_cost
def _rates(model: str) -> tuple[float, float, float]:
info = litellm.get_model_info(model=model, custom_llm_provider="anthropic")
input_cost = info["input_cost_per_token"] or 0.0
output_cost = info["output_cost_per_token"] or 0.0
cache_write_cost = info.get("cache_creation_input_token_cost") or input_cost
return input_cost, output_cost, cache_write_cost
def test_compression_savings_priced_at_input_rate():
input_cost, _ = _anthropic_costs("claude-sonnet-5")
result = compute_savings_spend(
@ -76,3 +87,133 @@ def test_negative_token_counts_clamp_to_zero():
)
assert result.compression == 0.0
assert result.prompt_caching == 0.0
def test_autorouter_savings_prices_completion_at_output_rate():
# Completion tokens must be priced at each model's OUTPUT rate, not its input
# rate. Output is several times input on flagship models, so pricing
# completions at the input rate materially understates the routed savings.
base_in, base_out, _ = _rates("claude-opus-5")
sel_in, sel_out, sel_cache_write = _rates("claude-haiku-4-5")
assert base_out > base_in # otherwise this test asserts nothing
prompt_tokens, completion_tokens, cache_creation = 1000, 500, 200
result = compute_autorouter_savings(
baseline_model="claude-opus-5",
selected_model="claude-haiku-4-5",
baseline_provider="anthropic",
selected_provider="anthropic",
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
cache_creation_input_tokens=cache_creation,
)
baseline_cost = prompt_tokens * base_in + completion_tokens * base_out
selected_cost = prompt_tokens * sel_in + completion_tokens * sel_out
penalty = cache_creation * sel_cache_write
assert result == pytest.approx(baseline_cost - selected_cost - penalty)
# A mutant that priced completions at the input rate would land here instead.
wrong = (prompt_tokens * base_in + completion_tokens * base_in) - (
prompt_tokens * sel_in + completion_tokens * sel_in
) - cache_creation * base_in
assert result != pytest.approx(wrong)
def test_autorouter_cache_write_penalty_uses_selected_model_rate():
# The switch penalty is a cache-creation charge on the deployment actually
# written to (the selected model), priced at its cache-creation rate, not the
# baseline's input rate.
base_in, base_out, _ = _rates("claude-opus-5")
sel_in, sel_out, sel_cache_write = _rates("claude-haiku-4-5")
prompt_tokens, completion_tokens, cache_creation = 0, 0, 1000
result = compute_autorouter_savings(
baseline_model="claude-opus-5",
selected_model="claude-haiku-4-5",
baseline_provider="anthropic",
selected_provider="anthropic",
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
cache_creation_input_tokens=cache_creation,
)
# With no prompt/completion tokens, savings is purely the negative penalty,
# floored at zero, so a pure-penalty request never reads as savings.
assert result == 0.0
# Confirm the penalty magnitude uses the SELECTED model's cache-write rate by
# giving enough token delta to stay positive, then isolating the penalty.
prompt_tokens = 100_000
with_penalty = compute_autorouter_savings(
"claude-opus-5", "claude-haiku-4-5", "anthropic", "anthropic",
prompt_tokens, 0, cache_creation,
)
without_penalty = compute_autorouter_savings(
"claude-opus-5", "claude-haiku-4-5", "anthropic", "anthropic",
prompt_tokens, 0, 0,
)
assert without_penalty - with_penalty == pytest.approx(cache_creation * sel_cache_write)
def test_autorouter_savings_zero_when_model_unchanged():
result = compute_autorouter_savings(
baseline_model="claude-opus-5",
selected_model="claude-opus-5",
baseline_provider="anthropic",
selected_provider="anthropic",
prompt_tokens=1000,
completion_tokens=500,
cache_creation_input_tokens=0,
)
assert result == 0.0
def test_autorouter_savings_floored_at_zero_on_escalation():
# Routing UP to a pricier model must never show as negative savings.
result = compute_autorouter_savings(
baseline_model="claude-haiku-4-5",
selected_model="claude-opus-5",
baseline_provider="anthropic",
selected_provider="anthropic",
prompt_tokens=1000,
completion_tokens=500,
cache_creation_input_tokens=0,
)
assert result == 0.0
def test_autorouter_savings_zero_without_baseline():
# No configured/produced baseline -> the driver contributes nothing.
result = compute_savings_spend(
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
cache_read_input_tokens=0,
baseline_model=None,
prompt_tokens=1000,
completion_tokens=500,
)
assert result.autorouter == 0.0
def test_compute_savings_spend_includes_autorouter_driver():
base_in, base_out, _ = _rates("claude-opus-5")
sel_in, sel_out, _ = _rates("claude-haiku-4-5")
prompt_tokens, completion_tokens = 2000, 800
result = compute_savings_spend(
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
cache_read_input_tokens=0,
baseline_model="claude-opus-5",
baseline_provider="anthropic",
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
cache_creation_input_tokens=0,
)
expected = (prompt_tokens * base_in + completion_tokens * base_out) - (
prompt_tokens * sel_in + completion_tokens * sel_out
)
assert result.autorouter == pytest.approx(expected)
assert result.autorouter > 0

67
tests/test_litellm/router_strategy/test_auto_router.py
View file

@ -316,3 +316,70 @@ class TestAutoRouter:
# Assert
assert result is None
@patch("semantic_router.routers.SemanticRouter")
def test_init_defaults_savings_baseline_model(self, mock_semantic_router_class, mock_router_instance):
"""Unconfigured deployments fall back to the flagship default, not an empty baseline."""
mock_semantic_router_class.from_json.return_value = mock_semantic_router_class
auto_router = AutoRouter(
model_name="test-auto-router",
auto_router_config_path="test/path/router.json",
default_model="gpt-4o-mini",
embedding_model="text-embedding-model",
litellm_router_instance=mock_router_instance,
)
assert auto_router.savings_baseline_model == AutoRouter.DEFAULT_SAVINGS_BASELINE_MODEL
@patch("semantic_router.routers.SemanticRouter")
def test_init_honors_configured_savings_baseline_model(self, mock_semantic_router_class, mock_router_instance):
"""An operator-configured baseline overrides the flagship default."""
mock_semantic_router_class.from_json.return_value = mock_semantic_router_class
auto_router = AutoRouter(
model_name="test-auto-router",
auto_router_config_path="test/path/router.json",
default_model="gpt-4o-mini",
embedding_model="text-embedding-model",
litellm_router_instance=mock_router_instance,
savings_baseline_model="claude-sonnet-5",
)
assert auto_router.savings_baseline_model == "claude-sonnet-5"
@pytest.mark.asyncio
@patch("semantic_router.routers.SemanticRouter")
@patch("litellm.router_strategy.auto_router.litellm_encoder.LiteLLMRouterEncoder")
async def test_async_pre_routing_hook_carries_savings_baseline_model(
self,
mock_encoder_class,
mock_semantic_router_class,
mock_router_instance,
mock_route_choice,
):
"""The hook response must carry the baseline so the spend writer can price
auto-router savings; without it the dashboard's driver silently stays zero."""
mock_loaded_router = MagicMock()
mock_loaded_router.routes = ["route1", "route2"]
mock_semantic_router_class.from_json.return_value = mock_loaded_router
mock_routelayer = MagicMock()
mock_routelayer.return_value = mock_route_choice
mock_semantic_router_class.return_value = mock_routelayer
auto_router = AutoRouter(
model_name="test-auto-router",
auto_router_config_path="test/path/router.json",
default_model="gpt-4o-mini",
embedding_model="text-embedding-model",
litellm_router_instance=mock_router_instance,
savings_baseline_model="claude-opus-5",
)
result = await auto_router.async_pre_routing_hook(
model="test-model", request_kwargs={}, messages=[{"role": "user", "content": "hi"}]
)
assert result is not None
assert result.savings_baseline_model == "claude-opus-5"

31
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx
View file

@ -228,6 +228,37 @@ describe("UsageTab", () => {
expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]);
});
it("carries auto-router savings into the summary card, donut slice, and cumulative series", () => {
const { getByText, getByTestId } = renderWith([
day("2026-07-12", {
compression_savings_spend: 0.04,
prompt_caching_savings_spend: 0.006,
autorouter_savings_spend: 0.02,
}),
day("2026-07-13", {
compression_savings_spend: 0.1,
prompt_caching_savings_spend: 0.01,
autorouter_savings_spend: 0.05,
}),
]);
// Total saved now sums three drivers, and the auto-router card carries its own total.
expect(getByText("$0.2260")).toBeInTheDocument();
expect(getByText("$0.0700")).toBeInTheDocument();
// The driver donut gains a third slice priced from the range totals.
const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]");
expect(slices).toEqual([
{ driver: "Compression", usd: expect.closeTo(0.14, 5) },
{ driver: "Prompt caching", usd: expect.closeTo(0.016, 5) },
{ driver: "Auto-router", usd: expect.closeTo(0.07, 5) },
]);
// And the cumulative line accumulates the auto-router series alongside the others.
const series = readSeries(getByTestId("area-chart"));
expect(series[2]["Auto-router"]).toBeCloseTo(0.07, 5);
});
it("renders spend-by-tool bars from the tool spend endpoint", async () => {
const toolSpend = {
by_tool: [

22
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx
View file

@ -38,7 +38,7 @@ const EMPTY_TOOL_SPEND: ToolSpendResponse = {
end_date: null,
};
const SAVINGS_COLORS = ["emerald", "blue"] as const;
const SAVINGS_COLORS = ["emerald", "blue", "amber"] as const;
const shortDate = (iso: string): string =>
new Date(`${iso}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" });
@ -47,6 +47,7 @@ const isoDay = (d: Date): string => d.toISOString().slice(0, 10);
const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0;
const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0;
const autorouterOf = (m: SpendMetrics): number => m.autorouter_savings_spend ?? 0;
const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0;
const SummaryCard = ({ label, value, hint, info }: { label: string; value: string; hint?: string; info?: string }) => (
@ -105,8 +106,9 @@ const UsageTab: React.FC<UsageTabProps> = ({ accessToken, activity }) => {
const compressionTotal = useMemo(() => results.reduce((sum, d) => sum + compressionOf(d.metrics), 0), [results]);
const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]);
const autorouterTotal = useMemo(() => results.reduce((sum, d) => sum + autorouterOf(d.metrics), 0), [results]);
const savedTokensTotal = useMemo(() => results.reduce((sum, d) => sum + savedTokensOf(d.metrics), 0), [results]);
const totalSaved = compressionTotal + cachingTotal;
const totalSaved = compressionTotal + cachingTotal + autorouterTotal;
const [accumulation, setAccumulation] = useState<SavingsAccumulation>("cumulative");
@ -122,6 +124,7 @@ const UsageTab: React.FC<UsageTabProps> = ({ accessToken, activity }) => {
date: shortDate(d.date),
Compression: compressionOf(d.metrics),
"Prompt caching": cachingOf(d.metrics),
"Auto-router": autorouterOf(d.metrics),
})),
[results],
);
@ -148,8 +151,9 @@ const UsageTab: React.FC<UsageTabProps> = ({ accessToken, activity }) => {
[
{ driver: "Compression", usd: compressionTotal },
{ driver: "Prompt caching", usd: cachingTotal },
{ driver: "Auto-router", usd: autorouterTotal },
].filter((d) => d.usd > 0),
[compressionTotal, cachingTotal],
[compressionTotal, cachingTotal, autorouterTotal],
);
const topTools = useMemo(() => topToolsBySpend(toolSpend?.by_tool ?? []), [toolSpend]);
@ -174,11 +178,11 @@ const UsageTab: React.FC<UsageTabProps> = ({ accessToken, activity }) => {
<AdvancedDatePicker value={dateValue} onValueChange={onDateChange} />
</div>
<div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-3">
<div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-4">
<SummaryCard
label="Total saved"
value={usd(totalSaved)}
hint={loading || isFetchingMore ? "Loading..." : "Compression + prompt caching"}
hint={loading || isFetchingMore ? "Loading..." : "Compression + prompt caching + auto-router"}
/>
<SummaryCard
label="Compression savings"
@ -192,6 +196,12 @@ const UsageTab: React.FC<UsageTabProps> = ({ accessToken, activity }) => {
hint="Cache read discount"
info="Tokens the provider served from cache, priced at the discount between the input and cache-read rates."
/>
<SummaryCard
label="Auto-router savings"
value={usd(autorouterTotal)}
hint="vs. the router's baseline model"
info="Cost of the auto-router's configured baseline model minus the cost of the model it actually routed to, net of any cache-write cost from switching models."
/>
</div>
<div className="grid grid-cols-1 gap-6 lg:grid-cols-3">
@ -247,7 +257,7 @@ const UsageTab: React.FC<UsageTabProps> = ({ accessToken, activity }) => {
data={byDriver}
index="driver"
category="usd"
colors={["emerald", "blue"]}
colors={SAVINGS_COLORS}
valueFormatter={usd}
showLabel
label={usd(totalSaved)}

14
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts
View file

@ -239,22 +239,25 @@ describe("localIsoDay", () => {
});
describe("toCumulative", () => {
const point = (date: string, compression: number, caching: number) => ({
const point = (date: string, compression: number, caching: number, autorouter: number = 0) => ({
date,
Compression: compression,
"Prompt caching": caching,
"Auto-router": autorouter,
});
it("turns each reading into everything saved up to that point", () => {
const running = toCumulative([point("Jul 1", 1, 10), point("Jul 2", 2, 20), point("Jul 3", 3, 30)]);
expect(running.map((p) => p.Compression)).toEqual([1, 3, 6]);
expect(running.map((p) => p["Prompt caching"])).toEqual([10, 30, 60]);
expect(running.map((p) => p["Auto-router"])).toEqual([0, 0, 0]);
});
it("accumulates each driver on its own, so one flat series cannot lift the other", () => {
const running = toCumulative([point("Jul 1", 0, 5), point("Jul 2", 0, 5)]);
expect(running.map((p) => p.Compression)).toEqual([0, 0]);
expect(running.map((p) => p["Prompt caching"])).toEqual([5, 10]);
expect(running.map((p) => p["Auto-router"])).toEqual([0, 0]);
});
it("never falls, even across a quiet interval", () => {
@ -267,13 +270,19 @@ describe("toCumulative", () => {
expect(running.map((p) => p.date)).toEqual(["9am", "10am"]);
expect(toCumulative([])).toEqual([]);
});
it("accumulates auto-router savings like other drivers", () => {
const running = toCumulative([point("Jul 1", 1, 1, 5), point("Jul 2", 1, 1, 10)]);
expect(running.map((p) => p["Auto-router"])).toEqual([5, 15]);
});
});
describe("withStartAnchor", () => {
const point = (date: string, compression: number, caching: number) => ({
const point = (date: string, compression: number, caching: number, autorouter: number = 0) => ({
date,
Compression: compression,
"Prompt caching": caching,
"Auto-router": autorouter,
});
it("lifts a single-day cumulative off a floating dot by prepending a $0 origin", () => {
@ -285,6 +294,7 @@ describe("withStartAnchor", () => {
const anchored = withStartAnchor([point("Jul 16", 5, 1), point("Jul 17", 9, 4)], "Jul 16");
expect(anchored.map((p) => p.Compression)).toEqual([0, 5, 9]);
expect(anchored.map((p) => p["Prompt caching"])).toEqual([0, 1, 4]);
expect(anchored.map((p) => p["Auto-router"])).toEqual([0, 0, 0]);
});
it("leaves an empty series alone so the chart's own no-data state can show", () => {

6
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts
View file

@ -161,9 +161,10 @@ export type SavingsPoint = {
date: string;
Compression: number;
"Prompt caching": number;
"Auto-router": number;
};
export const SAVINGS_SERIES = ["Compression", "Prompt caching"] as const;
export const SAVINGS_SERIES = ["Compression", "Prompt caching", "Auto-router"] as const;
/**
* Running total of each series across the selected window. The total restarts
@ -179,6 +180,7 @@ export const toCumulative = (points: readonly SavingsPoint[]): SavingsPoint[] =>
date: point.date,
Compression: (previous?.Compression ?? 0) + point.Compression,
"Prompt caching": (previous?.["Prompt caching"] ?? 0) + point["Prompt caching"],
"Auto-router": (previous?.["Auto-router"] ?? 0) + point["Auto-router"],
},
];
}, []);
@ -193,7 +195,7 @@ export const toCumulative = (points: readonly SavingsPoint[]): SavingsPoint[] =>
export const withStartAnchor = (cumulative: readonly SavingsPoint[], startLabel: string): SavingsPoint[] =>
cumulative.length === 0
? [...cumulative]
: [{ date: startLabel, Compression: 0, "Prompt caching": 0 }, ...cumulative];
: [{ date: startLabel, Compression: 0, "Prompt caching": 0, "Auto-router": 0 }, ...cumulative];
/** "Jul 16 – Jul 23", collapsing to a single date when the range is one day. */
export const formatRangeLabel = (from: Date | undefined, to: Date | undefined): string => {

1
ui/litellm-dashboard/src/components/UsagePage/types.ts
View file

@ -11,6 +11,7 @@ export interface SpendMetrics {
compression_saved_tokens?: number;
compression_savings_spend?: number;
prompt_caching_savings_spend?: number;
autorouter_savings_spend?: number;
}
export type DailyData = {

4
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -25936,6 +25936,8 @@ export interface components {
auto_router_default_model?: string | null;
/** Auto Router Embedding Model */
auto_router_embedding_model?: string | null;
/** Auto Router Savings Baseline Model */
auto_router_savings_baseline_model?: string | null;
/** Aws Access Key Id */
aws_access_key_id?: string | null;
/** Aws Bedrock Project Id */
@ -34068,6 +34070,8 @@ export interface components {
auto_router_default_model?: string | null;
/** Auto Router Embedding Model */
auto_router_embedding_model?: string | null;
/** Auto Router Savings Baseline Model */
auto_router_savings_baseline_model?: string | null;
/** Aws Access Key Id */
aws_access_key_id?: string | null;
/** Aws Bedrock Project Id */