diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c1a3684415a..b24e075dd68 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -4560,6 +4560,7 @@ class BaseDailySpendTransaction(TypedDict): # cost-savings metrics (dollars, priced per request before aggregation) compression_savings_spend: float prompt_caching_savings_spend: float + autorouter_savings_spend: float # request level metrics spend: float diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 2930dbc279b..244dc99ebbf 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -1556,6 +1556,30 @@ class DBSpendUpdateWriter: # Get the table dynamically table = getattr(batcher, table_name) + # Additive metrics that older queued rows may omit; one + # enumeration feeds both the create and the increment below + optional_metrics = { + field: value + for field, value in ( + ("cache_read_input_tokens", transaction.get("cache_read_input_tokens")), + ( + "cache_creation_input_tokens", + transaction.get("cache_creation_input_tokens"), + ), + ("compression_saved_tokens", transaction.get("compression_saved_tokens")), + ( + "compression_savings_spend", + transaction.get("compression_savings_spend"), + ), + ( + "prompt_caching_savings_spend", + transaction.get("prompt_caching_savings_spend"), + ), + ("autorouter_savings_spend", transaction.get("autorouter_savings_spend")), + ) + if value is not None + } + # Common data structure for both create and update common_data = { entity_id_field: entity_id, @@ -1572,34 +1596,9 @@ class DBSpendUpdateWriter: "api_requests": transaction["api_requests"], "successful_requests": transaction["successful_requests"], "failed_requests": transaction["failed_requests"], + **optional_metrics, } - # Add cache-related fields if they exist - if "cache_read_input_tokens" in transaction: - common_data["cache_read_input_tokens"] = transaction.get( - "cache_read_input_tokens", 0 - ) - if "cache_creation_input_tokens" in transaction: - common_data["cache_creation_input_tokens"] = transaction.get( - "cache_creation_input_tokens", 0 - ) - if "compression_saved_tokens" in transaction: - common_data["compression_saved_tokens"] = transaction.get( - "compression_saved_tokens", 0 - ) - if "compression_savings_spend" in transaction: - common_data["compression_savings_spend"] = transaction.get( - "compression_savings_spend", 0 - ) - if "prompt_caching_savings_spend" in transaction: - common_data["prompt_caching_savings_spend"] = transaction.get( - "prompt_caching_savings_spend", 0 - ) - if "autorouter_savings_spend" in transaction: - common_data["autorouter_savings_spend"] = transaction.get( - "autorouter_savings_spend", 0 - ) - if entity_type == "tag" and "request_id" in transaction: common_data["request_id"] = transaction.get("request_id") @@ -1611,36 +1610,9 @@ class DBSpendUpdateWriter: "api_requests": {"increment": transaction["api_requests"]}, "successful_requests": {"increment": transaction["successful_requests"]}, "failed_requests": {"increment": transaction["failed_requests"]}, + **{field: {"increment": value} for field, value in optional_metrics.items()}, } - # Add cache-related fields to update if they exist - if "cache_read_input_tokens" in transaction: - update_data["cache_read_input_tokens"] = { - "increment": transaction.get("cache_read_input_tokens", 0) - } - if "cache_creation_input_tokens" in transaction: - update_data["cache_creation_input_tokens"] = { - "increment": transaction.get("cache_creation_input_tokens", 0) - } - if "compression_saved_tokens" in transaction: - update_data["compression_saved_tokens"] = { - "increment": transaction.get("compression_saved_tokens", 0) - } - if "compression_savings_spend" in transaction: - update_data["compression_savings_spend"] = { - "increment": transaction.get("compression_savings_spend", 0) - } - if "prompt_caching_savings_spend" in transaction: - update_data["prompt_caching_savings_spend"] = { - "increment": transaction.get("prompt_caching_savings_spend", 0) - } - if "autorouter_savings_spend" in transaction: - update_data[ - "autorouter_savings_spend" - ] = { # mutable-ok: extends preset dict following compression_savings_spend pattern - "increment": transaction.get("autorouter_savings_spend", 0) - } - if entity_type == "tag" and "request_id" in transaction: update_data["request_id"] = transaction.get("request_id") @@ -1891,17 +1863,13 @@ class DBSpendUpdateWriter: cache_read_input_tokens = _extract_cache_read_tokens(usage_obj) compression_saved_tokens = extract_compression_saved_tokens(_metadata) - cache_creation_input_tokens = _extract_cache_creation_tokens(usage_obj) - savings_spend = compute_savings_spend( model=payload.get("model", None), custom_llm_provider=payload.get("custom_llm_provider", None), compression_saved_tokens=compression_saved_tokens, cache_read_input_tokens=cache_read_input_tokens, baseline_model=_metadata.get("auto_router_savings_baseline_model"), - prompt_tokens=payload.get("prompt_tokens", 0), - completion_tokens=payload.get("completion_tokens", 0), - cache_creation_input_tokens=cache_creation_input_tokens, + usage_object=usage_obj, ) daily_transaction = BaseDailySpendTransaction( @@ -1919,7 +1887,7 @@ class DBSpendUpdateWriter: successful_requests=1 if request_status == "success" else 0, failed_requests=1 if request_status != "success" else 0, cache_read_input_tokens=cache_read_input_tokens, - cache_creation_input_tokens=cache_creation_input_tokens, + cache_creation_input_tokens=_extract_cache_creation_tokens(usage_obj), compression_saved_tokens=compression_saved_tokens, compression_savings_spend=savings_spend.compression, prompt_caching_savings_spend=savings_spend.prompt_caching, diff --git a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py index b6462636393..444685167e6 100644 --- a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py @@ -134,6 +134,10 @@ class DailySpendUpdateQueue(BaseUpdateQueue): payload.get("prompt_caching_savings_spend", 0) or 0 ) + daily_transaction.get("prompt_caching_savings_spend", 0) + daily_transaction["autorouter_savings_spend"] = ( + payload.get("autorouter_savings_spend", 0) or 0 + ) + daily_transaction.get("autorouter_savings_spend", 0) + else: aggregated_daily_spend_update_transactions[_key] = deepcopy(payload) return aggregated_daily_spend_update_transactions diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 8a5a31710cf..ea1fb800afa 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -95,6 +95,9 @@ class DailySpendRecord(Protocol): @property def prompt_caching_savings_spend(self) -> float: ... + @property + def autorouter_savings_spend(self) -> float: ... + @property def api_requests(self) -> int: ... @@ -135,6 +138,7 @@ class _GroupingSetsRow(SimpleNamespace): compression_saved_tokens: int | None compression_savings_spend: float | None prompt_caching_savings_spend: float | None + autorouter_savings_spend: float | None api_requests: int | None successful_requests: int | None failed_requests: int | None @@ -158,6 +162,7 @@ def update_metrics(existing_metrics: SpendMetrics, record: DailySpendRecord) -> existing_metrics.compression_saved_tokens += record.compression_saved_tokens or 0 existing_metrics.compression_savings_spend += record.compression_savings_spend or 0 existing_metrics.prompt_caching_savings_spend += record.prompt_caching_savings_spend or 0 + existing_metrics.autorouter_savings_spend += record.autorouter_savings_spend or 0 existing_metrics.api_requests += record.api_requests or 0 existing_metrics.successful_requests += record.successful_requests or 0 existing_metrics.failed_requests += record.failed_requests or 0 @@ -590,6 +595,7 @@ def _build_aggregated_sql_query( SUM(compression_saved_tokens)::bigint AS compression_saved_tokens, SUM(compression_savings_spend)::float AS compression_savings_spend, SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend, + SUM(autorouter_savings_spend)::float AS autorouter_savings_spend, SUM(api_requests)::bigint AS api_requests, SUM(successful_requests)::bigint AS successful_requests, SUM(failed_requests)::bigint AS failed_requests @@ -732,6 +738,7 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics: compression_saved_tokens=record.compression_saved_tokens or 0, compression_savings_spend=record.compression_savings_spend or 0, prompt_caching_savings_spend=record.prompt_caching_savings_spend or 0, + autorouter_savings_spend=record.autorouter_savings_spend or 0, api_requests=record.api_requests or 0, successful_requests=record.successful_requests or 0, failed_requests=record.failed_requests or 0, @@ -986,6 +993,7 @@ async def get_daily_activity( total_compression_saved_tokens=metadata_metrics.compression_saved_tokens, total_compression_savings_spend=metadata_metrics.compression_savings_spend, total_prompt_caching_savings_spend=metadata_metrics.prompt_caching_savings_spend, + total_autorouter_savings_spend=metadata_metrics.autorouter_savings_spend, page=page, total_pages=-(-total_count // page_size), # Ceiling division has_more=(page * page_size) < total_count, @@ -1075,6 +1083,7 @@ async def get_daily_activity_aggregated( total_compression_saved_tokens=aggregated["totals"].compression_saved_tokens, total_compression_savings_spend=aggregated["totals"].compression_savings_spend, total_prompt_caching_savings_spend=aggregated["totals"].prompt_caching_savings_spend, + total_autorouter_savings_spend=aggregated["totals"].autorouter_savings_spend, page=1, total_pages=1, has_more=False, diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 6a08570bd5c..972a9d32e52 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -12,6 +12,8 @@ from typing import NamedTuple import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.types.utils import Usage class SavingsSpend(NamedTuple): @@ -20,42 +22,51 @@ class SavingsSpend(NamedTuple): autorouter: float = 0.0 -class _ModelRates(NamedTuple): - input: float - output: float - cache_read: float - cache_write: float - - -def _model_rates(model: str | None, custom_llm_provider: str | None) -> _ModelRates: +def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | None) -> tuple[float, float]: """ - Per-token prices for a model, in its four billed dimensions. + Return ``(input_cost_per_token, cache_read_cost_per_token)`` for a model. - Falls open to all-zero rates when the model is unknown so savings degrade to - zero rather than raising inside the spend writer. When a model has no separate - cache-read price the cache-read rate mirrors the input rate, which yields zero - caching savings; a missing cache-write price falls back to the input rate, - which is what providers without a distinct cache-creation charge bill. + Falls open to ``(0.0, 0.0)`` when the model is unknown so savings degrade to + zero rather than raising inside the spend writer. When a model has no + separate cache-read price the cache-read cost mirrors the input cost, which + yields zero caching savings. """ if not model: - return _ModelRates(0.0, 0.0, 0.0, 0.0) + return 0.0, 0.0 try: info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings verbose_proxy_logger.debug( "savings: no model info for provider=%s model=%s (%s)", custom_llm_provider, model, e ) - return _ModelRates(0.0, 0.0, 0.0, 0.0) + return 0.0, 0.0 input_cost = float(info.get("input_cost_per_token") or 0.0) - output_cost = float(info.get("output_cost_per_token") or 0.0) - cache_read = info.get("cache_read_input_token_cost") - cache_write = info.get("cache_creation_input_token_cost") - return _ModelRates( - input=input_cost, - output=output_cost, - cache_read=input_cost if cache_read is None else float(cache_read), - cache_write=input_cost if cache_write is None else float(cache_write), - ) + cache_read_cost = info.get("cache_read_input_token_cost") + if cache_read_cost is None: + return input_cost, input_cost + return input_cost, float(cache_read_cost) + + +def _cost_of_usage(model: str, custom_llm_provider: str | None, usage: Usage) -> float | None: + """ + What ``usage`` costs on ``model``, or ``None`` when the model has no pricing. + + Delegates to litellm's own cost engine rather than re-deriving per-token + arithmetic, so cache-read and cache-creation tokens are split out of the + inclusive ``prompt_tokens`` total exactly once, and tiered rates, ephemeral + cache-write tiers and regional uplifts stay consistent with the spend the + request was actually billed. + """ + try: + prompt_cost, completion_cost = generic_cost_per_token( + model=model, usage=usage, custom_llm_provider=custom_llm_provider or "" + ) + except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings + verbose_proxy_logger.debug( + "savings: cannot price usage for provider=%s model=%s (%s)", custom_llm_provider, model, e + ) + return None + return prompt_cost + completion_cost def compute_autorouter_savings( @@ -63,35 +74,45 @@ def compute_autorouter_savings( selected_model: str | None, baseline_provider: str | None, selected_provider: str | None, - prompt_tokens: int, - completion_tokens: int, - cache_creation_input_tokens: int, + usage: Usage, ) -> float: """ - Net dollars saved by routing this request to ``selected_model`` instead of the + Net dollars saved by serving this request on ``selected_model`` instead of the counterfactual ``baseline_model``. - The model-switch delta prices prompt tokens at each model's input rate and - completion tokens at each model's output rate (output is typically several - times the input rate, so pricing completions at the input rate materially - understates the gap). The cache-write penalty is the cost of switching: a - cold cache on the selected deployment forces a cache-creation charge that the - baseline would not have incurred, priced at the selected model's cache-write - rate. Cache-read discounts are deliberately excluded here; they are attributed - to the prompt-caching driver, so folding them in would double-count. + Both arms price the same usage through litellm's cost engine, so the answer is + the honest difference between what the request cost and what it would have cost + on the baseline. Pricing the identical usage twice is what keeps the cache + dimensions right: ``prompt_tokens`` already includes cache-read and + cache-creation tokens, so charging them separately on top would count them + twice, and the cost of a cold cache on the selected deployment is already + inside its own arm at its own cache-creation rate. - Returns zero when routing did not change the model or when pricing is unknown. - Floored at zero so an escalation to a pricier model never reads as negative - savings on the dashboard. + Returns zero when routing did not change the model or when either model has no + pricing. Floored at zero so an escalation to a pricier model never reads as + negative savings on the dashboard. """ if not baseline_model or not selected_model or baseline_model == selected_model: return 0.0 - baseline = _model_rates(baseline_model, baseline_provider) - selected = _model_rates(selected_model, selected_provider) - baseline_cost = (prompt_tokens * baseline.input) + (completion_tokens * baseline.output) - selected_cost = (prompt_tokens * selected.input) + (completion_tokens * selected.output) - cache_write_penalty = max(cache_creation_input_tokens, 0) * selected.cache_write - return max((baseline_cost - selected_cost) - cache_write_penalty, 0.0) + baseline_cost = _cost_of_usage(baseline_model, baseline_provider, usage) + selected_cost = _cost_of_usage(selected_model, selected_provider, usage) + if baseline_cost is None or selected_cost is None: + return 0.0 + return max(baseline_cost - selected_cost, 0.0) + + +def _usage_from_spend_log(usage_object: dict | None) -> Usage | None: + """ + Rebuild the request's ``Usage`` from the copy the spend log recorded, or + ``None`` when there is nothing priceable to rebuild it from. + """ + if not usage_object: + return None + try: + return Usage(**usage_object) + except Exception as e: # noqa: BLE001 # a malformed usage_object must not fail the daily spend write + verbose_proxy_logger.debug("savings: unusable usage_object (%s)", e) + return None def compute_savings_spend( @@ -101,9 +122,7 @@ def compute_savings_spend( cache_read_input_tokens: int, baseline_model: str | None = None, baseline_provider: str | None = None, - prompt_tokens: int = 0, - completion_tokens: int = 0, - cache_creation_input_tokens: int = 0, + usage_object: dict | None = None, ) -> SavingsSpend: """ Dollar savings for one request, split by optimization driver. @@ -114,16 +133,19 @@ def compute_savings_spend( Auto-router savings compare the served ``model`` against the counterfactual ``baseline_model`` and are zero unless the two differ. """ - rates = _model_rates(model, custom_llm_provider) - compression = max(compression_saved_tokens, 0) * rates.input - prompt_caching = max(cache_read_input_tokens, 0) * max(rates.input - rates.cache_read, 0.0) + input_cost, cache_read_cost = _input_and_cache_read_cost(model, custom_llm_provider) + compression = max(compression_saved_tokens, 0) * input_cost + prompt_caching = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0) + + usage = _usage_from_spend_log(usage_object) + if usage is None or not model: + return SavingsSpend(compression=compression, prompt_caching=prompt_caching) + autorouter = compute_autorouter_savings( baseline_model=baseline_model, selected_model=model, baseline_provider=baseline_provider, selected_provider=custom_llm_provider, - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - cache_creation_input_tokens=cache_creation_input_tokens, + usage=usage, ) return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter) diff --git a/litellm/router.py b/litellm/router.py index 2103e7e662c..06407a75603 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11162,7 +11162,7 @@ class Router: router_strategy = self._select_pre_routing_strategy(model=model, request_kwargs=request_kwargs) if router_strategy is None: - self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) + self._record_routing_decision(request_kwargs=request_kwargs, pre_routing_hook_response=None) return None pre_routing_hook_response = await router_strategy.async_pre_routing_hook( @@ -11174,11 +11174,7 @@ class Router: ) self._record_routing_decision( request_kwargs=request_kwargs, - routing_decision=(pre_routing_hook_response.routing_decision if pre_routing_hook_response else None), - ) - self._record_savings_baseline_model( - request_kwargs=request_kwargs, - baseline_model=(pre_routing_hook_response.savings_baseline_model if pre_routing_hook_response else None), + pre_routing_hook_response=pre_routing_hook_response, ) # `model` (the alias, e.g. "smart-router") is never the deployment actually @@ -11201,51 +11197,45 @@ class Router: @staticmethod def _record_routing_decision( request_kwargs: dict, - routing_decision: StandardLoggingRoutingDecision | None, + pre_routing_hook_response: Optional[PreRoutingHookResponse], ) -> None: """Make the request's metadata describe THIS routing attempt, and only this one. Fallbacks re-enter the hook with the same `request_kwargs`, so an attempt that picks a plain model group after an auto-router group failed must clear the earlier decision; leaving it would attribute the first router's tier and cause - to the deployment that actually served the request. Every attempt therefore - writes or clears, never just writes. - """ - if routing_decision is None: - for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")): - if isinstance(bucket, dict): - bucket.pop("routing_decision", None) - return + to the deployment that actually served the request, and would price savings + against a baseline this attempt never routed against. Every fact the hook + records is therefore written or cleared here together, from one response, so + no exit path can clear one and leave the other behind. - # `get_or_create_metadata_bucket` is the single owner of "which dict holds - # proxy-internal metadata": it picks `litellm_metadata` when present (so the - # decision never lands in the `metadata` dict that routes like /v1/messages - # forward to the provider) and replaces a non-dict value rather than silently - # skipping the write. - _, metadata_bucket = get_or_create_metadata_bucket(request_kwargs) - metadata_bucket["routing_decision"] = Router._redact_prompt_text_if_needed( - request_kwargs=request_kwargs, routing_decision=routing_decision - ) - - @staticmethod - def _record_savings_baseline_model( - request_kwargs: dict, - baseline_model: str | None, - ) -> None: - """Stash the auto-router's savings baseline for this attempt, same write-or-clear - discipline as `_record_routing_decision`: a fallback that re-enters this hook - without an auto-router strategy must clear a prior attempt's baseline, or the - spend writer would price savings against a model this attempt never routed - against. + `get_or_create_metadata_bucket` is the single owner of "which dict holds + proxy-internal metadata": it picks `litellm_metadata` when present (so nothing + lands in the `metadata` dict that routes like /v1/messages forward to the + provider) and replaces a non-dict value rather than silently skipping the write. """ - if baseline_model is None: - for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")): - if isinstance(bucket, dict): - bucket.pop("auto_router_savings_baseline_model", None) + routing_decision = pre_routing_hook_response.routing_decision if pre_routing_hook_response else None + baseline_model = pre_routing_hook_response.savings_baseline_model if pre_routing_hook_response else None + + recorded: dict[str, Any] = {} + if routing_decision is not None: + recorded["routing_decision"] = Router._redact_prompt_text_if_needed( + request_kwargs=request_kwargs, routing_decision=routing_decision + ) + if baseline_model is not None: + recorded["auto_router_savings_baseline_model"] = baseline_model + + cleared = {"routing_decision", "auto_router_savings_baseline_model"} - recorded.keys() + for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")): + if isinstance(bucket, dict): + for key in cleared: + bucket.pop(key, None) + + if not recorded: return _, metadata_bucket = get_or_create_metadata_bucket(request_kwargs) - metadata_bucket["auto_router_savings_baseline_model"] = baseline_model + metadata_bucket.update(recorded) @staticmethod def _redact_prompt_text_if_needed( diff --git a/litellm/router_strategy/auto_router/auto_router.py b/litellm/router_strategy/auto_router/auto_router.py index cc837f89f56..3b8d4d134b0 100644 --- a/litellm/router_strategy/auto_router/auto_router.py +++ b/litellm/router_strategy/auto_router/auto_router.py @@ -20,9 +20,6 @@ else: class AutoRouter(CustomLogger): DEFAULT_AUTO_SYNC_VALUE = "local" - # Flagship the auto-router's savings are measured against when the deployment - # does not configure `auto_router_savings_baseline_model`. Bare key (no - # provider prefix) to match the pricing map's canonical entry. DEFAULT_SAVINGS_BASELINE_MODEL = "claude-opus-5" def __init__( diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 00f67d0f39c..21b7ffca3f2 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -25,6 +25,7 @@ class SpendMetrics(BaseModel): compression_saved_tokens: int = Field(default=0) compression_savings_spend: float = Field(default=0.0) prompt_caching_savings_spend: float = Field(default=0.0) + autorouter_savings_spend: float = Field(default=0.0) total_tokens: int = Field(default=0) successful_requests: int = Field(default=0) failed_requests: int = Field(default=0) @@ -85,6 +86,7 @@ class DailySpendMetadata(BaseModel): total_compression_saved_tokens: int = Field(default=0) total_compression_savings_spend: float = Field(default=0.0) total_prompt_caching_savings_spend: float = Field(default=0.0) + total_autorouter_savings_spend: float = Field(default=0.0) page: int = Field(default=1) total_pages: int = Field(default=1) has_more: bool = Field(default=False) @@ -111,6 +113,7 @@ class LiteLLM_DailyUserSpend(BaseModel): compression_saved_tokens: int = 0 compression_savings_spend: float = 0.0 prompt_caching_savings_spend: float = 0.0 + autorouter_savings_spend: float = 0.0 spend: float = 0.0 api_requests: int = 0 successful_requests: int = 0 diff --git a/litellm/types/router.py b/litellm/types/router.py index 4deb241d635..07b23b3701e 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -272,8 +272,6 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): auto_router_config: Optional[str] = None auto_router_default_model: Optional[str] = None auto_router_embedding_model: Optional[str] = None - # counterfactual model the auto-router's savings are measured against on the - # cost-optimization dashboard; unset falls back to AutoRouter.DEFAULT_SAVINGS_BASELINE_MODEL auto_router_savings_baseline_model: Optional[str] = None # complexity-router params @@ -843,9 +841,6 @@ class PreRoutingHookResponse(BaseModel): model: str messages: Optional[List[Dict[str, Any]]] routing_decision: StandardLoggingRoutingDecision | None = None - # counterfactual model for the cost-optimization dashboard's auto-router savings - # card; only auto-router populates this today, so it is None for every other - # pre-routing strategy savings_baseline_model: Optional[str] = None diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py index c7e7ef5d469..2ac22a6d1fd 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py @@ -16,6 +16,7 @@ from litellm.proxy._types import ( Litellm_EntityType, SpendUpdateQueueItem, ) +from litellm.proxy._types import BaseDailySpendTransaction from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import ( DailySpendUpdateQueue, ) @@ -209,6 +210,7 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key(): "compression_saved_tokens": 0, "compression_savings_spend": 0, "prompt_caching_savings_spend": 0, + "autorouter_savings_spend": 0, } updates = [{test_key: test_transaction1}, {test_key: test_transaction2}] @@ -259,6 +261,7 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions( "compression_saved_tokens": 0, "compression_savings_spend": 0, "prompt_caching_savings_spend": 0, + "autorouter_savings_spend": 0, } # Add updates to queue @@ -527,3 +530,52 @@ async def test_compression_saved_tokens_aggregation(daily_spend_update_queue): assert agg["cache_creation_input_tokens"] == 7 assert agg["compression_savings_spend"] == pytest.approx(0.0076) assert agg["prompt_caching_savings_spend"] == pytest.approx(0.0108) + + +@pytest.mark.asyncio +async def test_every_optional_daily_metric_aggregates(daily_spend_update_queue): + """Every additive metric must survive the merge, not just the ones wired by hand. + + Two requests landing on one rollup key before a flush is the common case under + load, and this same merge runs again on every cross-pod Redis drain. A metric + persisted by the database write but skipped here is silently dropped on both + paths, so the driver reads as zero on the dashboard however much it saved. + """ + test_key = "user1_2023-01-01_key123_claude-haiku-4-5_anthropic" + numeric_fields = [ + name for name, annotation in BaseDailySpendTransaction.__annotations__.items() if annotation in (int, float) + ] + assert "autorouter_savings_spend" in numeric_fields + increments = {field: index + 1 for index, field in enumerate(numeric_fields)} + + await daily_spend_update_queue.add_update({test_key: dict(increments)}) + await daily_spend_update_queue.add_update({test_key: dict(increments)}) + await daily_spend_update_queue.aggregate_queue_updates() + updates = await daily_spend_update_queue.flush_all_updates_from_in_memory_queue() + + agg = updates[0][test_key] + for field, value in increments.items(): + assert agg[field] == pytest.approx(value * 2), f"{field} did not accumulate" + + +@pytest.mark.asyncio +async def test_optional_metric_missing_from_an_older_payload_still_aggregates( + daily_spend_update_queue, +): + """A queued row written before a metric existed must not zero it out.""" + test_key = "user1_2023-01-01_key123_claude-haiku-4-5_anthropic" + base = { + "spend": 1.0, + "prompt_tokens": 10, + "completion_tokens": 5, + "api_requests": 1, + "successful_requests": 1, + "failed_requests": 0, + } + + await daily_spend_update_queue.add_update({test_key: dict(base)}) + await daily_spend_update_queue.add_update({test_key: {**base, "autorouter_savings_spend": 0.25}}) + await daily_spend_update_queue.aggregate_queue_updates() + updates = await daily_spend_update_queue.flush_all_updates_from_in_memory_queue() + + assert updates[0][test_key]["autorouter_savings_spend"] == pytest.approx(0.25) diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index c2a0d34a915..f2749be5d6e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -19,7 +19,10 @@ from litellm.proxy.management_endpoints.common_daily_activity import ( get_daily_activity_aggregated, update_metrics, ) -from litellm.types.proxy.management_endpoints.common_daily_activity import SpendMetrics +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + DailySpendMetadata, + SpendMetrics, +) @pytest.mark.asyncio @@ -153,6 +156,7 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown(): "compression_saved_tokens": 0, "compression_savings_spend": 0.0, "prompt_caching_savings_spend": 0.0, + "autorouter_savings_spend": 0.0, "failed_requests": 0, } mock_rows = [ @@ -498,6 +502,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_1.compression_saved_tokens = 0 mock_record_1.compression_savings_spend = 0.0 mock_record_1.prompt_caching_savings_spend = 0.0 + mock_record_1.autorouter_savings_spend = 0.0 mock_record_1.api_requests = 10 mock_record_1.successful_requests = 9 mock_record_1.failed_requests = 1 @@ -520,6 +525,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_2.compression_saved_tokens = 0 mock_record_2.compression_savings_spend = 0.0 mock_record_2.prompt_caching_savings_spend = 0.0 + mock_record_2.autorouter_savings_spend = 0.0 mock_record_2.api_requests = 5 mock_record_2.successful_requests = 5 mock_record_2.failed_requests = 0 @@ -582,6 +588,7 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys(): "compression_saved_tokens": 0, "compression_savings_spend": 0.0, "prompt_caching_savings_spend": 0.0, + "autorouter_savings_spend": 0.0, "failed_requests": 0, } mock_rows = [ @@ -669,6 +676,7 @@ def _daily_user_spend_record(*, user_id, api_key, spend, model="gpt-4", model_gr compression_saved_tokens=0, compression_savings_spend=0.0, prompt_caching_savings_spend=0.0, + autorouter_savings_spend=0.0, api_requests=1, successful_requests=1, failed_requests=0, @@ -973,6 +981,7 @@ async def test_get_daily_activity_aggregated_empty_result_set(): "compression_saved_tokens": None, "compression_savings_spend": None, "prompt_caching_savings_spend": None, + "autorouter_savings_spend": None, "api_requests": None, "successful_requests": None, "failed_requests": None, @@ -1016,6 +1025,7 @@ def _no_spend_record(): compression_saved_tokens=None, compression_savings_spend=None, prompt_caching_savings_spend=None, + autorouter_savings_spend=None, api_requests=None, successful_requests=None, failed_requests=None, @@ -1050,3 +1060,54 @@ def test_update_metrics_handles_none_values(): assert metrics.cache_read_input_tokens == 0 assert metrics.cache_creation_input_tokens == 0 assert metrics.compression_saved_tokens == 0 + + +class TestEverySavingsDriverSurvivesTheReadPath: + """A savings driver is only real if it survives the whole read path. + + The write path can price a driver correctly and persist it to all six rollup + tables, and the dashboard can still render a permanent $0.00 because the + aggregation query never summed the column or the response model never + declared it. That failure is silent: the card renders, the number is just + always zero, which is indistinguishable from having saved nothing. These + tests enumerate the drivers from the response model itself, so a driver added + later cannot be half-wired. + """ + + def _drivers(self) -> list[str]: + drivers = [field for field in SpendMetrics.model_fields if field.endswith("_savings_spend")] + assert drivers, "expected the dashboard response to expose at least one savings driver" + return drivers + + def test_every_driver_is_summed_by_the_rollup_query(self): + sql, _ = _build_aggregated_sql_query( + table_name="litellm_dailyuserspend", + entity_id_field="user_id", + entity_id="user-1", + start_date="2026-07-01", + end_date="2026-07-31", + model=None, + api_key=None, + timezone_offset_minutes=None, + ) + for driver in self._drivers(): + assert f"SUM({driver})" in sql, f"{driver} is never summed, so it reads as zero" + + def test_every_driver_is_accumulated_across_rows(self): + for driver in self._drivers(): + record = _no_spend_record() + setattr(record, driver, 1.25) + metrics = update_metrics(SpendMetrics(), record) + assert getattr(metrics, driver) == pytest.approx(1.25), f"{driver} is dropped when accumulating rows" + + def test_every_driver_is_carried_by_a_single_row_conversion(self): + for driver in self._drivers(): + record = _no_spend_record() + setattr(record, driver, 2.5) + assert getattr(_record_to_spend_metrics(record), driver) == pytest.approx(2.5) + + def test_every_driver_has_a_range_total(self): + for driver in self._drivers(): + assert f"total_{driver}" in DailySpendMetadata.model_fields, ( + f"total_{driver} is missing, so the range summary omits the driver" + ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 167f21fcce1..a7c6f861c80 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -6,10 +6,12 @@ sys.path.insert(0, os.path.abspath("../../../..")) import pytest import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.proxy.spend_tracking.savings import ( compute_autorouter_savings, compute_savings_spend, ) +from litellm.types.utils import Usage def _anthropic_costs(model: str) -> tuple[float, float]: @@ -19,12 +21,37 @@ def _anthropic_costs(model: str) -> tuple[float, float]: return input_cost, cache_read_cost -def _rates(model: str) -> tuple[float, float, float]: +def _cached_usage_object() -> dict: + """A cache-heavy Anthropic request, shaped as the spend log records it. + + `prompt_tokens` is the inclusive total: 3 uncached text tokens plus 500 read + from cache plus 12304 written to cache. + """ + return { + "prompt_tokens": 12807, + "completion_tokens": 500, + "total_tokens": 13307, + "prompt_tokens_details": {"cached_tokens": 500, "cache_creation_tokens": 12304, "text_tokens": 3}, + "cache_creation_input_tokens": 12304, + "cache_read_input_tokens": 500, + } + + +def _cost_on(model: str, usage_object: dict) -> float: + prompt_cost, completion_cost = generic_cost_per_token( + model=model, usage=Usage(**usage_object), custom_llm_provider="anthropic" + ) + return prompt_cost + completion_cost + + +def _flat_rates(model: str) -> tuple[float, float, float]: info = litellm.get_model_info(model=model, custom_llm_provider="anthropic") input_cost = info["input_cost_per_token"] or 0.0 - output_cost = info["output_cost_per_token"] or 0.0 - cache_write_cost = info.get("cache_creation_input_token_cost") or input_cost - return input_cost, output_cost, cache_write_cost + return ( + input_cost, + info["output_cost_per_token"] or 0.0, + info.get("cache_creation_input_token_cost") or input_cost, + ) def test_compression_savings_priced_at_input_rate(): @@ -89,70 +116,69 @@ def test_negative_token_counts_clamp_to_zero(): assert result.prompt_caching == 0.0 -def test_autorouter_savings_prices_completion_at_output_rate(): - # Completion tokens must be priced at each model's OUTPUT rate, not its input - # rate. Output is several times input on flagship models, so pricing - # completions at the input rate materially understates the routed savings. - base_in, base_out, _ = _rates("claude-opus-5") - sel_in, sel_out, sel_cache_write = _rates("claude-haiku-4-5") - assert base_out > base_in # otherwise this test asserts nothing - prompt_tokens, completion_tokens, cache_creation = 1000, 500, 200 +def test_autorouter_savings_does_not_double_charge_cache_tokens(): + """`prompt_tokens` already includes cache-read and cache-creation tokens. + Charging those tokens again at the full input rate, or subtracting a separate + cache-write penalty on top of them, prices the same tokens twice. Both arms go + through litellm's cost engine on the identical usage, so each token is priced + exactly once, in its own dimension. + """ + usage_object = _cached_usage_object() result = compute_autorouter_savings( baseline_model="claude-opus-5", selected_model="claude-haiku-4-5", baseline_provider="anthropic", selected_provider="anthropic", - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - cache_creation_input_tokens=cache_creation, + usage=Usage(**usage_object), ) - baseline_cost = prompt_tokens * base_in + completion_tokens * base_out - selected_cost = prompt_tokens * sel_in + completion_tokens * sel_out - penalty = cache_creation * sel_cache_write - assert result == pytest.approx(baseline_cost - selected_cost - penalty) + expected = _cost_on("claude-opus-5", usage_object) - _cost_on("claude-haiku-4-5", usage_object) + assert result == pytest.approx(expected) + assert result > 0 - # A mutant that priced completions at the input rate would land here instead. - wrong = (prompt_tokens * base_in + completion_tokens * base_in) - ( - prompt_tokens * sel_in + completion_tokens * sel_in - ) - cache_creation * base_in - assert result != pytest.approx(wrong) + # The double-counting formula this replaced: every prompt token (cache reads + # and cache writes included) charged at the flat input rate on both sides, + # minus a cache-write penalty already accounted for inside the selected arm. + base_in, base_out, base_write = _flat_rates("claude-opus-5") + sel_in, sel_out, sel_write = _flat_rates("claude-haiku-4-5") + prompt_tokens = usage_object["prompt_tokens"] + completion_tokens = usage_object["completion_tokens"] + double_counted = max( + (prompt_tokens * base_in + completion_tokens * base_out) + - (prompt_tokens * sel_in + completion_tokens * sel_out) + - usage_object["cache_creation_input_tokens"] * sel_write, + 0.0, + ) + assert result != pytest.approx(double_counted) -def test_autorouter_cache_write_penalty_uses_selected_model_rate(): - # The switch penalty is a cache-creation charge on the deployment actually - # written to (the selected model), priced at its cache-creation rate, not the - # baseline's input rate. - base_in, base_out, _ = _rates("claude-opus-5") - sel_in, sel_out, sel_cache_write = _rates("claude-haiku-4-5") - prompt_tokens, completion_tokens, cache_creation = 0, 0, 1000 - +def test_autorouter_savings_charges_cache_reads_at_the_cache_read_rate(): + """A request served almost entirely from cache is cheap on both models, so the + routed saving must be far smaller than the same token count would suggest at + full input price.""" + usage_object = { + "prompt_tokens": 10_000, + "completion_tokens": 0, + "total_tokens": 10_000, + "prompt_tokens_details": {"cached_tokens": 10_000, "text_tokens": 0}, + "cache_read_input_tokens": 10_000, + } result = compute_autorouter_savings( baseline_model="claude-opus-5", selected_model="claude-haiku-4-5", baseline_provider="anthropic", selected_provider="anthropic", - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - cache_creation_input_tokens=cache_creation, + usage=Usage(**usage_object), ) - # With no prompt/completion tokens, savings is purely the negative penalty, - # floored at zero, so a pure-penalty request never reads as savings. - assert result == 0.0 - # Confirm the penalty magnitude uses the SELECTED model's cache-write rate by - # giving enough token delta to stay positive, then isolating the penalty. - prompt_tokens = 100_000 - with_penalty = compute_autorouter_savings( - "claude-opus-5", "claude-haiku-4-5", "anthropic", "anthropic", - prompt_tokens, 0, cache_creation, - ) - without_penalty = compute_autorouter_savings( - "claude-opus-5", "claude-haiku-4-5", "anthropic", "anthropic", - prompt_tokens, 0, 0, - ) - assert without_penalty - with_penalty == pytest.approx(cache_creation * sel_cache_write) + base_read = litellm.get_model_info("claude-opus-5", "anthropic")["cache_read_input_token_cost"] + sel_read = litellm.get_model_info("claude-haiku-4-5", "anthropic")["cache_read_input_token_cost"] + assert result == pytest.approx(10_000 * (base_read - sel_read)) + + base_in = litellm.get_model_info("claude-opus-5", "anthropic")["input_cost_per_token"] + sel_in = litellm.get_model_info("claude-haiku-4-5", "anthropic")["input_cost_per_token"] + assert result < 10_000 * (base_in - sel_in) def test_autorouter_savings_zero_when_model_unchanged(): @@ -161,9 +187,7 @@ def test_autorouter_savings_zero_when_model_unchanged(): selected_model="claude-opus-5", baseline_provider="anthropic", selected_provider="anthropic", - prompt_tokens=1000, - completion_tokens=500, - cache_creation_input_tokens=0, + usage=Usage(**_cached_usage_object()), ) assert result == 0.0 @@ -175,9 +199,18 @@ def test_autorouter_savings_floored_at_zero_on_escalation(): selected_model="claude-opus-5", baseline_provider="anthropic", selected_provider="anthropic", - prompt_tokens=1000, - completion_tokens=500, - cache_creation_input_tokens=0, + usage=Usage(**_cached_usage_object()), + ) + assert result == 0.0 + + +def test_autorouter_savings_unknown_baseline_fails_open_to_zero(): + result = compute_autorouter_savings( + baseline_model="totally-made-up-model-xyz", + selected_model="claude-haiku-4-5", + baseline_provider="anthropic", + selected_provider="anthropic", + usage=Usage(**_cached_usage_object()), ) assert result == 0.0 @@ -190,17 +223,13 @@ def test_autorouter_savings_zero_without_baseline(): compression_saved_tokens=0, cache_read_input_tokens=0, baseline_model=None, - prompt_tokens=1000, - completion_tokens=500, + usage_object=_cached_usage_object(), ) assert result.autorouter == 0.0 def test_compute_savings_spend_includes_autorouter_driver(): - base_in, base_out, _ = _rates("claude-opus-5") - sel_in, sel_out, _ = _rates("claude-haiku-4-5") - prompt_tokens, completion_tokens = 2000, 800 - + usage_object = _cached_usage_object() result = compute_savings_spend( model="claude-haiku-4-5", custom_llm_provider="anthropic", @@ -208,12 +237,24 @@ def test_compute_savings_spend_includes_autorouter_driver(): cache_read_input_tokens=0, baseline_model="claude-opus-5", baseline_provider="anthropic", - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - cache_creation_input_tokens=0, - ) - expected = (prompt_tokens * base_in + completion_tokens * base_out) - ( - prompt_tokens * sel_in + completion_tokens * sel_out + usage_object=usage_object, ) + expected = _cost_on("claude-opus-5", usage_object) - _cost_on("claude-haiku-4-5", usage_object) assert result.autorouter == pytest.approx(expected) assert result.autorouter > 0 + + +def test_compute_savings_spend_without_usage_object_keeps_other_drivers(): + """A row with no recorded usage still prices compression and caching; only the + counterfactual driver needs the usage breakdown.""" + input_cost, _ = _anthropic_costs("claude-sonnet-5") + result = compute_savings_spend( + model="claude-sonnet-5", + custom_llm_provider="anthropic", + compression_saved_tokens=1000, + cache_read_input_tokens=0, + baseline_model="claude-opus-5", + usage_object=None, + ) + assert result.compression == pytest.approx(1000 * input_cost) + assert result.autorouter == 0.0 diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index f6216d1646e..68235c7e215 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -2437,7 +2437,7 @@ class TestSpendLogsPayload: "model": "gpt-4o", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', + "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "auto_router_savings_baseline_model": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, @@ -2533,7 +2533,7 @@ class TestSpendLogsPayload: "model": "claude-4-sonnet-20250514", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "auto_router_savings_baseline_model": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, @@ -2627,7 +2627,7 @@ class TestSpendLogsPayload: "model": "claude-4-sonnet-20250514", "user": "", "team_id": "", - "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', + "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "auto_router_savings_baseline_model": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}', "cache_key": "Cache OFF", "spend": 0.01383, "total_tokens": 2598, diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 2b4e882675f..78cf64aa9a8 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -35,6 +35,7 @@ from litellm.router_strategy.complexity_router.config import ( from litellm.types.router import ( Deployment, LiteLLM_Params, + PreRoutingHookResponse, TaggedPreRoutingStrategy, ) @@ -4081,22 +4082,61 @@ class TestRecordRoutingDecision: the request's metadata must describe the current attempt and nothing else.""" DECISION = {"router_model_name": "smart-router", "router_type": "complexity", "routed_model": "gpt-4o-mini"} + STALE_METADATA_KEYS = ("routing_decision", "auto_router_savings_baseline_model") def test_none_clears_a_previous_decision_from_both_buckets(self): request_kwargs: Dict = { "metadata": {"routing_decision": self.DECISION, "keep": 1}, "litellm_metadata": {"routing_decision": self.DECISION}, } - Router._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) + Router._record_routing_decision(request_kwargs=request_kwargs, pre_routing_hook_response=None) assert "routing_decision" not in request_kwargs["metadata"] assert "routing_decision" not in request_kwargs["litellm_metadata"] assert request_kwargs["metadata"]["keep"] == 1 def test_none_creates_no_bucket_on_a_request_that_had_none(self): request_kwargs: Dict = {} - Router._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) + Router._record_routing_decision(request_kwargs=request_kwargs, pre_routing_hook_response=None) assert request_kwargs == {} + @pytest.mark.parametrize("stale_key", STALE_METADATA_KEYS) + def test_an_attempt_without_a_strategy_clears_every_stale_fact(self, stale_key): + """A fallback to a plain model group re-enters the hook with the same + `request_kwargs`. Anything the auto-router attempt left behind would be + attributed to the deployment that actually served the request, letting a + caller who forces a router failure inflate the recorded savings.""" + request_kwargs: Dict = { + "metadata": {stale_key: "stale", "keep": 1}, + "litellm_metadata": {stale_key: "stale"}, + } + Router._record_routing_decision(request_kwargs=request_kwargs, pre_routing_hook_response=None) + assert stale_key not in request_kwargs["metadata"] + assert stale_key not in request_kwargs["litellm_metadata"] + assert request_kwargs["metadata"]["keep"] == 1 + + def test_a_response_without_a_baseline_clears_a_previous_one(self): + """Not every pre-routing strategy sets a savings baseline; one that does not + must not inherit the previous attempt's.""" + request_kwargs: Dict = {"litellm_metadata": {"auto_router_savings_baseline_model": "claude-opus-5"}} + Router._record_routing_decision( + request_kwargs=request_kwargs, + pre_routing_hook_response=PreRoutingHookResponse(model="gpt-4o-mini", messages=[]), + ) + assert "auto_router_savings_baseline_model" not in request_kwargs["litellm_metadata"] + + def test_baseline_is_recorded_on_the_internal_bucket(self): + """The baseline must land in `litellm_metadata`, never the `metadata` dict that + surfaces like /v1/messages forward verbatim to the provider.""" + request_kwargs: Dict = {"litellm_metadata": {}, "metadata": {}} + Router._record_routing_decision( + request_kwargs=request_kwargs, + pre_routing_hook_response=PreRoutingHookResponse( + model="claude-haiku-4-5", messages=[], savings_baseline_model="claude-opus-5" + ), + ) + assert request_kwargs["litellm_metadata"]["auto_router_savings_baseline_model"] == "claude-opus-5" + assert "auto_router_savings_baseline_model" not in request_kwargs["metadata"] + class TestEscalationIsRecordedConsistently: """An escalation keyword records two separate facts on every path: that the caller diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 4a504a80417..4e9e06f3787 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -23487,6 +23487,11 @@ export interface components { * @default 0 */ total_api_requests: number; + /** + * Total Autorouter Savings Spend + * @default 0 + */ + total_autorouter_savings_spend: number; /** * Total Cache Creation Input Tokens * @default 0 @@ -31390,6 +31395,11 @@ export interface components { * @default 0 */ api_requests: number; + /** + * Autorouter Savings Spend + * @default 0 + */ + autorouter_savings_spend: number; /** * Cache Creation Input Tokens * @default 0