diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql new file mode 100644 index 00000000000..a7efd444fdb --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql @@ -0,0 +1,17 @@ +-- AlterTable +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 37ea55f8c13..0d7fa8692c8 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 23fe7af9994..c1a3684415a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3321,6 +3321,7 @@ class SpendLogsMetadata(TypedDict): max_retries: Optional[int] # Max retries configured for this request cost_breakdown: Optional[CostBreakdown] # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.) compression_savings: CompressionSavingsMetadata | None + auto_router_savings_baseline_model: str | None # counterfactual model for the auto-router savings driver class SpendLogsPayload(TypedDict): diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index fd8132fef22..2930dbc279b 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -1595,6 +1595,10 @@ class DBSpendUpdateWriter: common_data["prompt_caching_savings_spend"] = transaction.get( "prompt_caching_savings_spend", 0 ) + if "autorouter_savings_spend" in transaction: + common_data["autorouter_savings_spend"] = transaction.get( + "autorouter_savings_spend", 0 + ) if entity_type == "tag" and "request_id" in transaction: common_data["request_id"] = transaction.get("request_id") @@ -1630,6 +1634,12 @@ class DBSpendUpdateWriter: update_data["prompt_caching_savings_spend"] = { "increment": transaction.get("prompt_caching_savings_spend", 0) } + if "autorouter_savings_spend" in transaction: + update_data[ + "autorouter_savings_spend" + ] = { # mutable-ok: extends preset dict following compression_savings_spend pattern + "increment": transaction.get("autorouter_savings_spend", 0) + } if entity_type == "tag" and "request_id" in transaction: update_data["request_id"] = transaction.get("request_id") @@ -1881,11 +1891,17 @@ class DBSpendUpdateWriter: cache_read_input_tokens = _extract_cache_read_tokens(usage_obj) compression_saved_tokens = extract_compression_saved_tokens(_metadata) + cache_creation_input_tokens = _extract_cache_creation_tokens(usage_obj) + savings_spend = compute_savings_spend( model=payload.get("model", None), custom_llm_provider=payload.get("custom_llm_provider", None), compression_saved_tokens=compression_saved_tokens, cache_read_input_tokens=cache_read_input_tokens, + baseline_model=_metadata.get("auto_router_savings_baseline_model"), + prompt_tokens=payload.get("prompt_tokens", 0), + completion_tokens=payload.get("completion_tokens", 0), + cache_creation_input_tokens=cache_creation_input_tokens, ) daily_transaction = BaseDailySpendTransaction( @@ -1903,10 +1919,11 @@ class DBSpendUpdateWriter: successful_requests=1 if request_status == "success" else 0, failed_requests=1 if request_status != "success" else 0, cache_read_input_tokens=cache_read_input_tokens, - cache_creation_input_tokens=_extract_cache_creation_tokens(usage_obj), + cache_creation_input_tokens=cache_creation_input_tokens, compression_saved_tokens=compression_saved_tokens, compression_savings_spend=savings_spend.compression, prompt_caching_savings_spend=savings_spend.prompt_caching, + autorouter_savings_spend=savings_spend.autorouter, ) return daily_transaction except Exception as e: diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 1fad1954dc4..9d412eca799 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -203,6 +203,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS = ( "applied_policies", "policy_sources", "routing_decision", + "auto_router_savings_baseline_model", INTERNAL_CALL_ORIGIN_METADATA_KEY, "standard_logging_object", "proxy_server_request", diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 37ea55f8c13..0d7fa8692c8 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index ad9d02052bb..6a08570bd5c 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -17,31 +17,81 @@ from litellm._logging import verbose_proxy_logger class SavingsSpend(NamedTuple): compression: float prompt_caching: float + autorouter: float = 0.0 -def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | None) -> tuple[float, float]: +class _ModelRates(NamedTuple): + input: float + output: float + cache_read: float + cache_write: float + + +def _model_rates(model: str | None, custom_llm_provider: str | None) -> _ModelRates: """ - Return ``(input_cost_per_token, cache_read_cost_per_token)`` for a model. + Per-token prices for a model, in its four billed dimensions. - Falls open to ``(0.0, 0.0)`` when the model is unknown so savings degrade to - zero rather than raising inside the spend writer. When a model has no - separate cache-read price the cache-read cost mirrors the input cost, which - yields zero caching savings. + Falls open to all-zero rates when the model is unknown so savings degrade to + zero rather than raising inside the spend writer. When a model has no separate + cache-read price the cache-read rate mirrors the input rate, which yields zero + caching savings; a missing cache-write price falls back to the input rate, + which is what providers without a distinct cache-creation charge bill. """ if not model: - return 0.0, 0.0 + return _ModelRates(0.0, 0.0, 0.0, 0.0) try: info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings verbose_proxy_logger.debug( "savings: no model info for provider=%s model=%s (%s)", custom_llm_provider, model, e ) - return 0.0, 0.0 + return _ModelRates(0.0, 0.0, 0.0, 0.0) input_cost = float(info.get("input_cost_per_token") or 0.0) - cache_read_cost = info.get("cache_read_input_token_cost") - if cache_read_cost is None: - return input_cost, input_cost - return input_cost, float(cache_read_cost) + output_cost = float(info.get("output_cost_per_token") or 0.0) + cache_read = info.get("cache_read_input_token_cost") + cache_write = info.get("cache_creation_input_token_cost") + return _ModelRates( + input=input_cost, + output=output_cost, + cache_read=input_cost if cache_read is None else float(cache_read), + cache_write=input_cost if cache_write is None else float(cache_write), + ) + + +def compute_autorouter_savings( + baseline_model: str | None, + selected_model: str | None, + baseline_provider: str | None, + selected_provider: str | None, + prompt_tokens: int, + completion_tokens: int, + cache_creation_input_tokens: int, +) -> float: + """ + Net dollars saved by routing this request to ``selected_model`` instead of the + counterfactual ``baseline_model``. + + The model-switch delta prices prompt tokens at each model's input rate and + completion tokens at each model's output rate (output is typically several + times the input rate, so pricing completions at the input rate materially + understates the gap). The cache-write penalty is the cost of switching: a + cold cache on the selected deployment forces a cache-creation charge that the + baseline would not have incurred, priced at the selected model's cache-write + rate. Cache-read discounts are deliberately excluded here; they are attributed + to the prompt-caching driver, so folding them in would double-count. + + Returns zero when routing did not change the model or when pricing is unknown. + Floored at zero so an escalation to a pricier model never reads as negative + savings on the dashboard. + """ + if not baseline_model or not selected_model or baseline_model == selected_model: + return 0.0 + baseline = _model_rates(baseline_model, baseline_provider) + selected = _model_rates(selected_model, selected_provider) + baseline_cost = (prompt_tokens * baseline.input) + (completion_tokens * baseline.output) + selected_cost = (prompt_tokens * selected.input) + (completion_tokens * selected.output) + cache_write_penalty = max(cache_creation_input_tokens, 0) * selected.cache_write + return max((baseline_cost - selected_cost) - cache_write_penalty, 0.0) def compute_savings_spend( @@ -49,6 +99,11 @@ def compute_savings_spend( custom_llm_provider: str | None, compression_saved_tokens: int, cache_read_input_tokens: int, + baseline_model: str | None = None, + baseline_provider: str | None = None, + prompt_tokens: int = 0, + completion_tokens: int = 0, + cache_creation_input_tokens: int = 0, ) -> SavingsSpend: """ Dollar savings for one request, split by optimization driver. @@ -56,8 +111,19 @@ def compute_savings_spend( Compression savings price the tokens compression removed at the model's input rate. Prompt-caching savings price the cache-read tokens at the difference between the input rate and the discounted cache-read rate. + Auto-router savings compare the served ``model`` against the counterfactual + ``baseline_model`` and are zero unless the two differ. """ - input_cost, cache_read_cost = _input_and_cache_read_cost(model, custom_llm_provider) - compression = max(compression_saved_tokens, 0) * input_cost - prompt_caching = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0) - return SavingsSpend(compression=compression, prompt_caching=prompt_caching) + rates = _model_rates(model, custom_llm_provider) + compression = max(compression_saved_tokens, 0) * rates.input + prompt_caching = max(cache_read_input_tokens, 0) * max(rates.input - rates.cache_read, 0.0) + autorouter = compute_autorouter_savings( + baseline_model=baseline_model, + selected_model=model, + baseline_provider=baseline_provider, + selected_provider=custom_llm_provider, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cache_creation_input_tokens=cache_creation_input_tokens, + ) + return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index a6a67d57582..20dbd075438 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -117,6 +117,7 @@ def _get_spend_logs_metadata( max_retries=None, cost_breakdown=None, compression_savings=None, + auto_router_savings_baseline_model=None, litellm_call_id=litellm_call_id, ) verbose_proxy_logger.debug( diff --git a/litellm/router.py b/litellm/router.py index ac00cdbc2b0..2103e7e662c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7670,6 +7670,7 @@ class Router: default_model=default_model, embedding_model=embedding_model, litellm_router_instance=self, + savings_baseline_model=deployment.litellm_params.auto_router_savings_baseline_model, ) self._register_pre_routing_strategy( registry=self.auto_routers, @@ -11175,6 +11176,10 @@ class Router: request_kwargs=request_kwargs, routing_decision=(pre_routing_hook_response.routing_decision if pre_routing_hook_response else None), ) + self._record_savings_baseline_model( + request_kwargs=request_kwargs, + baseline_model=(pre_routing_hook_response.savings_baseline_model if pre_routing_hook_response else None), + ) # `model` (the alias, e.g. "smart-router") is never the deployment actually # called - apply the alias's own litellm_params (besides `model` itself, @@ -11222,6 +11227,26 @@ class Router: request_kwargs=request_kwargs, routing_decision=routing_decision ) + @staticmethod + def _record_savings_baseline_model( + request_kwargs: dict, + baseline_model: str | None, + ) -> None: + """Stash the auto-router's savings baseline for this attempt, same write-or-clear + discipline as `_record_routing_decision`: a fallback that re-enters this hook + without an auto-router strategy must clear a prior attempt's baseline, or the + spend writer would price savings against a model this attempt never routed + against. + """ + if baseline_model is None: + for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")): + if isinstance(bucket, dict): + bucket.pop("auto_router_savings_baseline_model", None) + return + + _, metadata_bucket = get_or_create_metadata_bucket(request_kwargs) + metadata_bucket["auto_router_savings_baseline_model"] = baseline_model + @staticmethod def _redact_prompt_text_if_needed( request_kwargs: Mapping[str, Any], diff --git a/litellm/router_strategy/auto_router/auto_router.py b/litellm/router_strategy/auto_router/auto_router.py index c01e2f10d2c..cc837f89f56 100644 --- a/litellm/router_strategy/auto_router/auto_router.py +++ b/litellm/router_strategy/auto_router/auto_router.py @@ -20,6 +20,10 @@ else: class AutoRouter(CustomLogger): DEFAULT_AUTO_SYNC_VALUE = "local" + # Flagship the auto-router's savings are measured against when the deployment + # does not configure `auto_router_savings_baseline_model`. Bare key (no + # provider prefix) to match the pricing map's canonical entry. + DEFAULT_SAVINGS_BASELINE_MODEL = "claude-opus-5" def __init__( self, @@ -29,6 +33,7 @@ class AutoRouter(CustomLogger): litellm_router_instance: "Router", auto_router_config_path: Optional[str] = None, auto_router_config: Optional[str] = None, + savings_baseline_model: str | None = None, ): """ Auto-Router class that uses a semantic router to route requests to the appropriate model. @@ -40,6 +45,7 @@ class AutoRouter(CustomLogger): default_model: The default model to use if no route is found. embedding_model: The embedding model to use for the auto-router. litellm_router_instance: The instance of the LiteLLM Router. + savings_baseline_model: The counterfactual model the dashboard measures savings against; falls back to DEFAULT_SAVINGS_BASELINE_MODEL. """ from semantic_router.routers import SemanticRouter @@ -51,6 +57,7 @@ class AutoRouter(CustomLogger): self.default_model = default_model self.embedding_model: str = embedding_model self.litellm_router_instance: "Router" = litellm_router_instance + self.savings_baseline_model: str = savings_baseline_model or self.DEFAULT_SAVINGS_BASELINE_MODEL def _load_semantic_routing_routes(self) -> List[Route]: from semantic_router.routers import SemanticRouter @@ -156,4 +163,5 @@ class AutoRouter(CustomLogger): return PreRoutingHookResponse( model=model, messages=messages, + savings_baseline_model=self.savings_baseline_model, ) diff --git a/litellm/types/router.py b/litellm/types/router.py index 837a93367a2..4deb241d635 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -272,6 +272,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): auto_router_config: Optional[str] = None auto_router_default_model: Optional[str] = None auto_router_embedding_model: Optional[str] = None + # counterfactual model the auto-router's savings are measured against on the + # cost-optimization dashboard; unset falls back to AutoRouter.DEFAULT_SAVINGS_BASELINE_MODEL + auto_router_savings_baseline_model: Optional[str] = None # complexity-router params complexity_router_config: Optional[Dict] = None @@ -840,6 +843,10 @@ class PreRoutingHookResponse(BaseModel): model: str messages: Optional[List[Dict[str, Any]]] routing_decision: StandardLoggingRoutingDecision | None = None + # counterfactual model for the cost-optimization dashboard's auto-router savings + # card; only auto-router populates this today, so it is None for every other + # pre-routing strategy + savings_baseline_model: Optional[str] = None _PreRoutingStrategyT_co = TypeVar("_PreRoutingStrategyT_co", covariant=True) diff --git a/schema.prisma b/schema.prisma index 37ea55f8c13..0d7fa8692c8 100644 --- a/schema.prisma +++ b/schema.prisma @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 704cf7a63dd..167f21fcce1 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -6,7 +6,10 @@ sys.path.insert(0, os.path.abspath("../../../..")) import pytest import litellm -from litellm.proxy.spend_tracking.savings import compute_savings_spend +from litellm.proxy.spend_tracking.savings import ( + compute_autorouter_savings, + compute_savings_spend, +) def _anthropic_costs(model: str) -> tuple[float, float]: @@ -16,6 +19,14 @@ def _anthropic_costs(model: str) -> tuple[float, float]: return input_cost, cache_read_cost +def _rates(model: str) -> tuple[float, float, float]: + info = litellm.get_model_info(model=model, custom_llm_provider="anthropic") + input_cost = info["input_cost_per_token"] or 0.0 + output_cost = info["output_cost_per_token"] or 0.0 + cache_write_cost = info.get("cache_creation_input_token_cost") or input_cost + return input_cost, output_cost, cache_write_cost + + def test_compression_savings_priced_at_input_rate(): input_cost, _ = _anthropic_costs("claude-sonnet-5") result = compute_savings_spend( @@ -76,3 +87,133 @@ def test_negative_token_counts_clamp_to_zero(): ) assert result.compression == 0.0 assert result.prompt_caching == 0.0 + + +def test_autorouter_savings_prices_completion_at_output_rate(): + # Completion tokens must be priced at each model's OUTPUT rate, not its input + # rate. Output is several times input on flagship models, so pricing + # completions at the input rate materially understates the routed savings. + base_in, base_out, _ = _rates("claude-opus-5") + sel_in, sel_out, sel_cache_write = _rates("claude-haiku-4-5") + assert base_out > base_in # otherwise this test asserts nothing + prompt_tokens, completion_tokens, cache_creation = 1000, 500, 200 + + result = compute_autorouter_savings( + baseline_model="claude-opus-5", + selected_model="claude-haiku-4-5", + baseline_provider="anthropic", + selected_provider="anthropic", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cache_creation_input_tokens=cache_creation, + ) + + baseline_cost = prompt_tokens * base_in + completion_tokens * base_out + selected_cost = prompt_tokens * sel_in + completion_tokens * sel_out + penalty = cache_creation * sel_cache_write + assert result == pytest.approx(baseline_cost - selected_cost - penalty) + + # A mutant that priced completions at the input rate would land here instead. + wrong = (prompt_tokens * base_in + completion_tokens * base_in) - ( + prompt_tokens * sel_in + completion_tokens * sel_in + ) - cache_creation * base_in + assert result != pytest.approx(wrong) + + +def test_autorouter_cache_write_penalty_uses_selected_model_rate(): + # The switch penalty is a cache-creation charge on the deployment actually + # written to (the selected model), priced at its cache-creation rate, not the + # baseline's input rate. + base_in, base_out, _ = _rates("claude-opus-5") + sel_in, sel_out, sel_cache_write = _rates("claude-haiku-4-5") + prompt_tokens, completion_tokens, cache_creation = 0, 0, 1000 + + result = compute_autorouter_savings( + baseline_model="claude-opus-5", + selected_model="claude-haiku-4-5", + baseline_provider="anthropic", + selected_provider="anthropic", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cache_creation_input_tokens=cache_creation, + ) + # With no prompt/completion tokens, savings is purely the negative penalty, + # floored at zero, so a pure-penalty request never reads as savings. + assert result == 0.0 + + # Confirm the penalty magnitude uses the SELECTED model's cache-write rate by + # giving enough token delta to stay positive, then isolating the penalty. + prompt_tokens = 100_000 + with_penalty = compute_autorouter_savings( + "claude-opus-5", "claude-haiku-4-5", "anthropic", "anthropic", + prompt_tokens, 0, cache_creation, + ) + without_penalty = compute_autorouter_savings( + "claude-opus-5", "claude-haiku-4-5", "anthropic", "anthropic", + prompt_tokens, 0, 0, + ) + assert without_penalty - with_penalty == pytest.approx(cache_creation * sel_cache_write) + + +def test_autorouter_savings_zero_when_model_unchanged(): + result = compute_autorouter_savings( + baseline_model="claude-opus-5", + selected_model="claude-opus-5", + baseline_provider="anthropic", + selected_provider="anthropic", + prompt_tokens=1000, + completion_tokens=500, + cache_creation_input_tokens=0, + ) + assert result == 0.0 + + +def test_autorouter_savings_floored_at_zero_on_escalation(): + # Routing UP to a pricier model must never show as negative savings. + result = compute_autorouter_savings( + baseline_model="claude-haiku-4-5", + selected_model="claude-opus-5", + baseline_provider="anthropic", + selected_provider="anthropic", + prompt_tokens=1000, + completion_tokens=500, + cache_creation_input_tokens=0, + ) + assert result == 0.0 + + +def test_autorouter_savings_zero_without_baseline(): + # No configured/produced baseline -> the driver contributes nothing. + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + cache_read_input_tokens=0, + baseline_model=None, + prompt_tokens=1000, + completion_tokens=500, + ) + assert result.autorouter == 0.0 + + +def test_compute_savings_spend_includes_autorouter_driver(): + base_in, base_out, _ = _rates("claude-opus-5") + sel_in, sel_out, _ = _rates("claude-haiku-4-5") + prompt_tokens, completion_tokens = 2000, 800 + + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + cache_read_input_tokens=0, + baseline_model="claude-opus-5", + baseline_provider="anthropic", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cache_creation_input_tokens=0, + ) + expected = (prompt_tokens * base_in + completion_tokens * base_out) - ( + prompt_tokens * sel_in + completion_tokens * sel_out + ) + assert result.autorouter == pytest.approx(expected) + assert result.autorouter > 0 diff --git a/tests/test_litellm/router_strategy/test_auto_router.py b/tests/test_litellm/router_strategy/test_auto_router.py index cb46a4ae553..3cc23cc6f9e 100644 --- a/tests/test_litellm/router_strategy/test_auto_router.py +++ b/tests/test_litellm/router_strategy/test_auto_router.py @@ -316,3 +316,70 @@ class TestAutoRouter: # Assert assert result is None + + @patch("semantic_router.routers.SemanticRouter") + def test_init_defaults_savings_baseline_model(self, mock_semantic_router_class, mock_router_instance): + """Unconfigured deployments fall back to the flagship default, not an empty baseline.""" + mock_semantic_router_class.from_json.return_value = mock_semantic_router_class + + auto_router = AutoRouter( + model_name="test-auto-router", + auto_router_config_path="test/path/router.json", + default_model="gpt-4o-mini", + embedding_model="text-embedding-model", + litellm_router_instance=mock_router_instance, + ) + + assert auto_router.savings_baseline_model == AutoRouter.DEFAULT_SAVINGS_BASELINE_MODEL + + @patch("semantic_router.routers.SemanticRouter") + def test_init_honors_configured_savings_baseline_model(self, mock_semantic_router_class, mock_router_instance): + """An operator-configured baseline overrides the flagship default.""" + mock_semantic_router_class.from_json.return_value = mock_semantic_router_class + + auto_router = AutoRouter( + model_name="test-auto-router", + auto_router_config_path="test/path/router.json", + default_model="gpt-4o-mini", + embedding_model="text-embedding-model", + litellm_router_instance=mock_router_instance, + savings_baseline_model="claude-sonnet-5", + ) + + assert auto_router.savings_baseline_model == "claude-sonnet-5" + + @pytest.mark.asyncio + @patch("semantic_router.routers.SemanticRouter") + @patch("litellm.router_strategy.auto_router.litellm_encoder.LiteLLMRouterEncoder") + async def test_async_pre_routing_hook_carries_savings_baseline_model( + self, + mock_encoder_class, + mock_semantic_router_class, + mock_router_instance, + mock_route_choice, + ): + """The hook response must carry the baseline so the spend writer can price + auto-router savings; without it the dashboard's driver silently stays zero.""" + mock_loaded_router = MagicMock() + mock_loaded_router.routes = ["route1", "route2"] + mock_semantic_router_class.from_json.return_value = mock_loaded_router + + mock_routelayer = MagicMock() + mock_routelayer.return_value = mock_route_choice + mock_semantic_router_class.return_value = mock_routelayer + + auto_router = AutoRouter( + model_name="test-auto-router", + auto_router_config_path="test/path/router.json", + default_model="gpt-4o-mini", + embedding_model="text-embedding-model", + litellm_router_instance=mock_router_instance, + savings_baseline_model="claude-opus-5", + ) + + result = await auto_router.async_pre_routing_hook( + model="test-model", request_kwargs={}, messages=[{"role": "user", "content": "hi"}] + ) + + assert result is not None + assert result.savings_baseline_model == "claude-opus-5" diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx index 5c26ac30477..36126b5dfb8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx @@ -228,6 +228,37 @@ describe("UsageTab", () => { expect(slices).toEqual([{ driver: "Compression", usd: expect.closeTo(0.04, 5) }]); }); + it("carries auto-router savings into the summary card, donut slice, and cumulative series", () => { + const { getByText, getByTestId } = renderWith([ + day("2026-07-12", { + compression_savings_spend: 0.04, + prompt_caching_savings_spend: 0.006, + autorouter_savings_spend: 0.02, + }), + day("2026-07-13", { + compression_savings_spend: 0.1, + prompt_caching_savings_spend: 0.01, + autorouter_savings_spend: 0.05, + }), + ]); + + // Total saved now sums three drivers, and the auto-router card carries its own total. + expect(getByText("$0.2260")).toBeInTheDocument(); + expect(getByText("$0.0700")).toBeInTheDocument(); + + // The driver donut gains a third slice priced from the range totals. + const slices = JSON.parse(getByTestId("donut-chart").getAttribute("data-slices") ?? "[]"); + expect(slices).toEqual([ + { driver: "Compression", usd: expect.closeTo(0.14, 5) }, + { driver: "Prompt caching", usd: expect.closeTo(0.016, 5) }, + { driver: "Auto-router", usd: expect.closeTo(0.07, 5) }, + ]); + + // And the cumulative line accumulates the auto-router series alongside the others. + const series = readSeries(getByTestId("area-chart")); + expect(series[2]["Auto-router"]).toBeCloseTo(0.07, 5); + }); + it("renders spend-by-tool bars from the tool spend endpoint", async () => { const toolSpend = { by_tool: [ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx index ec37418e0b5..ab767f5e508 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.tsx @@ -38,7 +38,7 @@ const EMPTY_TOOL_SPEND: ToolSpendResponse = { end_date: null, }; -const SAVINGS_COLORS = ["emerald", "blue"] as const; +const SAVINGS_COLORS = ["emerald", "blue", "amber"] as const; const shortDate = (iso: string): string => new Date(`${iso}T00:00:00`).toLocaleDateString("en-US", { month: "short", day: "numeric" }); @@ -47,6 +47,7 @@ const isoDay = (d: Date): string => d.toISOString().slice(0, 10); const compressionOf = (m: SpendMetrics): number => m.compression_savings_spend ?? 0; const cachingOf = (m: SpendMetrics): number => m.prompt_caching_savings_spend ?? 0; +const autorouterOf = (m: SpendMetrics): number => m.autorouter_savings_spend ?? 0; const savedTokensOf = (m: SpendMetrics): number => m.compression_saved_tokens ?? 0; const SummaryCard = ({ label, value, hint, info }: { label: string; value: string; hint?: string; info?: string }) => ( @@ -105,8 +106,9 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { const compressionTotal = useMemo(() => results.reduce((sum, d) => sum + compressionOf(d.metrics), 0), [results]); const cachingTotal = useMemo(() => results.reduce((sum, d) => sum + cachingOf(d.metrics), 0), [results]); + const autorouterTotal = useMemo(() => results.reduce((sum, d) => sum + autorouterOf(d.metrics), 0), [results]); const savedTokensTotal = useMemo(() => results.reduce((sum, d) => sum + savedTokensOf(d.metrics), 0), [results]); - const totalSaved = compressionTotal + cachingTotal; + const totalSaved = compressionTotal + cachingTotal + autorouterTotal; const [accumulation, setAccumulation] = useState("cumulative"); @@ -122,6 +124,7 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { date: shortDate(d.date), Compression: compressionOf(d.metrics), "Prompt caching": cachingOf(d.metrics), + "Auto-router": autorouterOf(d.metrics), })), [results], ); @@ -148,8 +151,9 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { [ { driver: "Compression", usd: compressionTotal }, { driver: "Prompt caching", usd: cachingTotal }, + { driver: "Auto-router", usd: autorouterTotal }, ].filter((d) => d.usd > 0), - [compressionTotal, cachingTotal], + [compressionTotal, cachingTotal, autorouterTotal], ); const topTools = useMemo(() => topToolsBySpend(toolSpend?.by_tool ?? []), [toolSpend]); @@ -174,11 +178,11 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { -
+
= ({ accessToken, activity }) => { hint="Cache read discount" info="Tokens the provider served from cache, priced at the discount between the input and cache-read rates." /> +
@@ -247,7 +257,7 @@ const UsageTab: React.FC = ({ accessToken, activity }) => { data={byDriver} index="driver" category="usd" - colors={["emerald", "blue"]} + colors={SAVINGS_COLORS} valueFormatter={usd} showLabel label={usd(totalSaved)} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts index dc08799a3c4..c088e07cb91 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts @@ -239,22 +239,25 @@ describe("localIsoDay", () => { }); describe("toCumulative", () => { - const point = (date: string, compression: number, caching: number) => ({ + const point = (date: string, compression: number, caching: number, autorouter: number = 0) => ({ date, Compression: compression, "Prompt caching": caching, + "Auto-router": autorouter, }); it("turns each reading into everything saved up to that point", () => { const running = toCumulative([point("Jul 1", 1, 10), point("Jul 2", 2, 20), point("Jul 3", 3, 30)]); expect(running.map((p) => p.Compression)).toEqual([1, 3, 6]); expect(running.map((p) => p["Prompt caching"])).toEqual([10, 30, 60]); + expect(running.map((p) => p["Auto-router"])).toEqual([0, 0, 0]); }); it("accumulates each driver on its own, so one flat series cannot lift the other", () => { const running = toCumulative([point("Jul 1", 0, 5), point("Jul 2", 0, 5)]); expect(running.map((p) => p.Compression)).toEqual([0, 0]); expect(running.map((p) => p["Prompt caching"])).toEqual([5, 10]); + expect(running.map((p) => p["Auto-router"])).toEqual([0, 0]); }); it("never falls, even across a quiet interval", () => { @@ -267,13 +270,19 @@ describe("toCumulative", () => { expect(running.map((p) => p.date)).toEqual(["9am", "10am"]); expect(toCumulative([])).toEqual([]); }); + + it("accumulates auto-router savings like other drivers", () => { + const running = toCumulative([point("Jul 1", 1, 1, 5), point("Jul 2", 1, 1, 10)]); + expect(running.map((p) => p["Auto-router"])).toEqual([5, 15]); + }); }); describe("withStartAnchor", () => { - const point = (date: string, compression: number, caching: number) => ({ + const point = (date: string, compression: number, caching: number, autorouter: number = 0) => ({ date, Compression: compression, "Prompt caching": caching, + "Auto-router": autorouter, }); it("lifts a single-day cumulative off a floating dot by prepending a $0 origin", () => { @@ -285,6 +294,7 @@ describe("withStartAnchor", () => { const anchored = withStartAnchor([point("Jul 16", 5, 1), point("Jul 17", 9, 4)], "Jul 16"); expect(anchored.map((p) => p.Compression)).toEqual([0, 5, 9]); expect(anchored.map((p) => p["Prompt caching"])).toEqual([0, 1, 4]); + expect(anchored.map((p) => p["Auto-router"])).toEqual([0, 0, 0]); }); it("leaves an empty series alone so the chart's own no-data state can show", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts index 32eb6ae198d..18aa8f1a3ef 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts @@ -161,9 +161,10 @@ export type SavingsPoint = { date: string; Compression: number; "Prompt caching": number; + "Auto-router": number; }; -export const SAVINGS_SERIES = ["Compression", "Prompt caching"] as const; +export const SAVINGS_SERIES = ["Compression", "Prompt caching", "Auto-router"] as const; /** * Running total of each series across the selected window. The total restarts @@ -179,6 +180,7 @@ export const toCumulative = (points: readonly SavingsPoint[]): SavingsPoint[] => date: point.date, Compression: (previous?.Compression ?? 0) + point.Compression, "Prompt caching": (previous?.["Prompt caching"] ?? 0) + point["Prompt caching"], + "Auto-router": (previous?.["Auto-router"] ?? 0) + point["Auto-router"], }, ]; }, []); @@ -193,7 +195,7 @@ export const toCumulative = (points: readonly SavingsPoint[]): SavingsPoint[] => export const withStartAnchor = (cumulative: readonly SavingsPoint[], startLabel: string): SavingsPoint[] => cumulative.length === 0 ? [...cumulative] - : [{ date: startLabel, Compression: 0, "Prompt caching": 0 }, ...cumulative]; + : [{ date: startLabel, Compression: 0, "Prompt caching": 0, "Auto-router": 0 }, ...cumulative]; /** "Jul 16 – Jul 23", collapsing to a single date when the range is one day. */ export const formatRangeLabel = (from: Date | undefined, to: Date | undefined): string => { diff --git a/ui/litellm-dashboard/src/components/UsagePage/types.ts b/ui/litellm-dashboard/src/components/UsagePage/types.ts index bf33fa37111..b10fc79be15 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/types.ts +++ b/ui/litellm-dashboard/src/components/UsagePage/types.ts @@ -11,6 +11,7 @@ export interface SpendMetrics { compression_saved_tokens?: number; compression_savings_spend?: number; prompt_caching_savings_spend?: number; + autorouter_savings_spend?: number; } export type DailyData = { diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 9a301f1c474..4a504a80417 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25936,6 +25936,8 @@ export interface components { auto_router_default_model?: string | null; /** Auto Router Embedding Model */ auto_router_embedding_model?: string | null; + /** Auto Router Savings Baseline Model */ + auto_router_savings_baseline_model?: string | null; /** Aws Access Key Id */ aws_access_key_id?: string | null; /** Aws Bedrock Project Id */ @@ -34068,6 +34070,8 @@ export interface components { auto_router_default_model?: string | null; /** Auto Router Embedding Model */ auto_router_embedding_model?: string | null; + /** Auto Router Savings Baseline Model */ + auto_router_savings_baseline_model?: string | null; /** Aws Access Key Id */ aws_access_key_id?: string | null; /** Aws Bedrock Project Id */