diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql new file mode 100644 index 00000000000..a7efd444fdb --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260731000000_add_autorouter_savings_spend/migration.sql @@ -0,0 +1,17 @@ +-- AlterTable +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "autorouter_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 37ea55f8c13..0d7fa8692c8 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/litellm/__init__.py b/litellm/__init__.py index a9a78846fa1..62c41c2959b 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -264,6 +264,7 @@ databricks_key: Optional[str] = None openai_like_key: Optional[str] = None azure_key: Optional[str] = None anthropic_key: Optional[str] = None +autorouter_savings_baseline_model: Optional[str] = None replicate_key: Optional[str] = None bytez_key: Optional[str] = None gdc_key: Optional[str] = None diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index face1d1b49f..2eaac7cb1ca 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -683,6 +683,7 @@ def generic_cost_per_token( custom_llm_provider: str, service_tier: str | None = None, data_residency: str | None = None, + model_info: ModelInfo | None = None, ) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -700,7 +701,12 @@ def generic_cost_per_token( """ ## GET MODEL INFO - model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + # A caller that already resolved the deployment's effective rates passes them in + # rather than handing back a name for this to re-resolve. A name cannot express a + # per-deployment override: those are registered under the deployment id and kept off + # the shared model-name key, so resolving from the name here reads the public rate. + if model_info is None: + model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider) ## CALCULATE INPUT COST ### Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 268b08d0f2e..5063bc790b5 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -14,7 +14,7 @@ from pydantic import ( field_validator, model_validator, ) -from typing_extensions import Required, TypedDict +from typing_extensions import NotRequired, Required, TypedDict from litellm._uuid import uuid from litellm.constants import MCP_STDIO_ALLOWED_COMMANDS @@ -4566,6 +4566,11 @@ class BaseDailySpendTransaction(TypedDict): # cost-savings metrics (dollars, priced per request before aggregation) compression_savings_spend: float prompt_caching_savings_spend: float + # Not required: rows queued by a pod running the previous release, or replayed from + # the Redis buffer across an upgrade, carry no such key. Every reader coalesces a + # missing value to zero, so requiring it here would describe a shape the aggregation + # is explicitly tested against. + autorouter_savings_spend: NotRequired[float] # request level metrics spend: float diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 17410698aed..7c9ee96e809 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -12,7 +12,9 @@ import os import random import time import traceback +from collections.abc import Mapping from datetime import datetime, timedelta, timezone +from types import MappingProxyType from typing import ( TYPE_CHECKING, Any, @@ -68,6 +70,26 @@ else: ProxyLogging = Any +# Only tag rows carry a request_id, so the other entity types spread nothing. Built +# once here rather than as an empty literal per transaction, and read-only so it cannot +# be filled in by accident from one of the call sites that spreads it. +_NO_TAG_REQUEST_ID: Mapping[str, Any] = MappingProxyType({}) + + +def _get_llm_router(): + """The proxy's router, or None outside a running proxy. + + Injected rather than imported where it is used, so the savings computation stays + a pure function of its arguments and the caller owns where the router comes from. + """ + try: + from litellm.proxy.proxy_server import llm_router + + return llm_router + except Exception: # noqa: BLE001 # no proxy in scope; savings degrade to zero + return None + + def _extract_cache_read_tokens(usage_obj: dict) -> int: """ Anthropic: top-level cache_read_input_tokens field. @@ -1545,6 +1567,40 @@ class DBSpendUpdateWriter: # Get the table dynamically table = getattr(batcher, table_name) + # Additive metrics that older queued rows may omit; one + # enumeration feeds both the create and the increment below + optional_metrics = { + field: value + for field, value in ( + ("cache_read_input_tokens", transaction.get("cache_read_input_tokens")), + ( + "cache_creation_input_tokens", + transaction.get("cache_creation_input_tokens"), + ), + ("compression_saved_tokens", transaction.get("compression_saved_tokens")), + ( + "compression_savings_spend", + transaction.get("compression_savings_spend"), + ), + ( + "prompt_caching_savings_spend", + transaction.get("prompt_caching_savings_spend"), + ), + ("autorouter_savings_spend", transaction.get("autorouter_savings_spend")), + ) + if value is not None + } + + # Only tag rows carry a request_id. Resolved to a spreadable + # value here so both payloads are built in one shot: a dict + # appended to after construction is one nobody can reason about + # by reading its literal. + tag_request_id: Mapping[str, Any] = ( + MappingProxyType({"request_id": transaction["request_id"]}) + if entity_type == "tag" and "request_id" in transaction + else _NO_TAG_REQUEST_ID + ) + # Common data structure for both create and update common_data = { entity_id_field: entity_id, @@ -1561,34 +1617,10 @@ class DBSpendUpdateWriter: "api_requests": transaction["api_requests"], "successful_requests": transaction["successful_requests"], "failed_requests": transaction["failed_requests"], + **optional_metrics, + **tag_request_id, } - # Add cache-related fields if they exist - if "cache_read_input_tokens" in transaction: - common_data["cache_read_input_tokens"] = transaction.get( - "cache_read_input_tokens", 0 - ) - if "cache_creation_input_tokens" in transaction: - common_data["cache_creation_input_tokens"] = transaction.get( - "cache_creation_input_tokens", 0 - ) - if "compression_saved_tokens" in transaction: - common_data["compression_saved_tokens"] = transaction.get( - "compression_saved_tokens", 0 - ) - if "compression_savings_spend" in transaction: - common_data["compression_savings_spend"] = transaction.get( - "compression_savings_spend", 0 - ) - if "prompt_caching_savings_spend" in transaction: - common_data["prompt_caching_savings_spend"] = transaction.get( - "prompt_caching_savings_spend", 0 - ) - - if entity_type == "tag" and "request_id" in transaction: - common_data["request_id"] = transaction.get("request_id") - - # Create update data structure update_data = { "prompt_tokens": {"increment": transaction["prompt_tokens"]}, "completion_tokens": {"increment": transaction["completion_tokens"]}, @@ -1596,36 +1628,12 @@ class DBSpendUpdateWriter: "api_requests": {"increment": transaction["api_requests"]}, "successful_requests": {"increment": transaction["successful_requests"]}, "failed_requests": {"increment": transaction["failed_requests"]}, + **{field: {"increment": value} for field, value in optional_metrics.items()}, + # An existing row predating the endpoint column gets it filled in here + "endpoint": transaction.get("endpoint") or "", + **tag_request_id, } - # Add cache-related fields to update if they exist - if "cache_read_input_tokens" in transaction: - update_data["cache_read_input_tokens"] = { - "increment": transaction.get("cache_read_input_tokens", 0) - } - if "cache_creation_input_tokens" in transaction: - update_data["cache_creation_input_tokens"] = { - "increment": transaction.get("cache_creation_input_tokens", 0) - } - if "compression_saved_tokens" in transaction: - update_data["compression_saved_tokens"] = { - "increment": transaction.get("compression_saved_tokens", 0) - } - if "compression_savings_spend" in transaction: - update_data["compression_savings_spend"] = { - "increment": transaction.get("compression_savings_spend", 0) - } - if "prompt_caching_savings_spend" in transaction: - update_data["prompt_caching_savings_spend"] = { - "increment": transaction.get("prompt_caching_savings_spend", 0) - } - - if entity_type == "tag" and "request_id" in transaction: - update_data["request_id"] = transaction.get("request_id") - - # Add endpoint to update_data so existing rows get their endpoint field updated - update_data["endpoint"] = transaction.get("endpoint") or "" - table.upsert( where=where_clause, data={ @@ -1875,6 +1883,10 @@ class DBSpendUpdateWriter: custom_llm_provider=payload.get("custom_llm_provider", None), compression_saved_tokens=compression_saved_tokens, cache_read_input_tokens=cache_read_input_tokens, + routing_decision=_metadata.get("routing_decision"), + model_id=payload.get("model_id"), + llm_router=_get_llm_router(), + usage_object=usage_obj, ) daily_transaction = BaseDailySpendTransaction( @@ -1896,6 +1908,7 @@ class DBSpendUpdateWriter: compression_saved_tokens=compression_saved_tokens, compression_savings_spend=savings_spend.compression, prompt_caching_savings_spend=savings_spend.prompt_caching, + autorouter_savings_spend=savings_spend.autorouter, ) return daily_transaction except Exception as e: diff --git a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py index 18df25093f1..a61db128382 100644 --- a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py @@ -133,6 +133,10 @@ class DailySpendUpdateQueue(BaseUpdateQueue): payload.get("prompt_caching_savings_spend", 0) or 0 ) + daily_transaction.get("prompt_caching_savings_spend", 0) + daily_transaction["autorouter_savings_spend"] = ( + payload.get("autorouter_savings_spend", 0) or 0 + ) + daily_transaction.get("autorouter_savings_spend", 0) + else: aggregated_daily_spend_update_transactions[_key] = deepcopy(payload) return aggregated_daily_spend_update_transactions diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 0dc85f98786..132a8409e06 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -95,6 +95,9 @@ class DailySpendRecord(Protocol): @property def prompt_caching_savings_spend(self) -> float: ... + @property + def autorouter_savings_spend(self) -> float: ... + @property def api_requests(self) -> int: ... @@ -135,6 +138,7 @@ class _GroupingSetsRow(SimpleNamespace): compression_saved_tokens: int | None compression_savings_spend: float | None prompt_caching_savings_spend: float | None + autorouter_savings_spend: float | None api_requests: int | None successful_requests: int | None failed_requests: int | None @@ -158,6 +162,7 @@ def update_metrics(existing_metrics: SpendMetrics, record: DailySpendRecord) -> existing_metrics.compression_saved_tokens += record.compression_saved_tokens or 0 existing_metrics.compression_savings_spend += record.compression_savings_spend or 0 existing_metrics.prompt_caching_savings_spend += record.prompt_caching_savings_spend or 0 + existing_metrics.autorouter_savings_spend += record.autorouter_savings_spend or 0 existing_metrics.api_requests += record.api_requests or 0 existing_metrics.successful_requests += record.successful_requests or 0 existing_metrics.failed_requests += record.failed_requests or 0 @@ -590,6 +595,7 @@ def _build_aggregated_sql_query( SUM(compression_saved_tokens)::bigint AS compression_saved_tokens, SUM(compression_savings_spend)::float AS compression_savings_spend, SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend, + SUM(autorouter_savings_spend)::float AS autorouter_savings_spend, SUM(api_requests)::bigint AS api_requests, SUM(successful_requests)::bigint AS successful_requests, SUM(failed_requests)::bigint AS failed_requests @@ -732,6 +738,7 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics: compression_saved_tokens=record.compression_saved_tokens or 0, compression_savings_spend=record.compression_savings_spend or 0, prompt_caching_savings_spend=record.prompt_caching_savings_spend or 0, + autorouter_savings_spend=record.autorouter_savings_spend or 0, api_requests=record.api_requests or 0, successful_requests=record.successful_requests or 0, failed_requests=record.failed_requests or 0, @@ -986,6 +993,7 @@ async def get_daily_activity( total_compression_saved_tokens=metadata_metrics.compression_saved_tokens, total_compression_savings_spend=metadata_metrics.compression_savings_spend, total_prompt_caching_savings_spend=metadata_metrics.prompt_caching_savings_spend, + total_autorouter_savings_spend=metadata_metrics.autorouter_savings_spend, page=page, total_pages=-(-total_count // page_size), # Ceiling division has_more=(page * page_size) < total_count, @@ -1075,6 +1083,7 @@ async def get_daily_activity_aggregated( total_compression_saved_tokens=aggregated["totals"].compression_saved_tokens, total_compression_savings_spend=aggregated["totals"].compression_savings_spend, total_prompt_caching_savings_spend=aggregated["totals"].prompt_caching_savings_spend, + total_autorouter_savings_spend=aggregated["totals"].autorouter_savings_spend, page=1, total_pages=1, has_more=False, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 37ea55f8c13..0d7fa8692c8 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index ad9d02052bb..1a1c813d784 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -8,15 +8,22 @@ are known) and summed into the daily tables; tokens cannot be priced after they have been aggregated across models. """ -from typing import NamedTuple +from collections.abc import Mapping +from typing import TYPE_CHECKING, NamedTuple import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + +if TYPE_CHECKING: + from litellm.router import Router +from litellm.types.utils import ModelInfo, PromptTokensDetailsWrapper, Usage class SavingsSpend(NamedTuple): compression: float prompt_caching: float + autorouter: float = 0.0 def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | None) -> tuple[float, float]: @@ -44,11 +51,243 @@ def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | Non return input_cost, float(cache_read_cost) +class _ModelIdentity(NamedTuple): + model: str + provider: str + + +def _resolve_model(model: str | None, custom_llm_provider: str | None) -> _ModelIdentity | None: + """Canonical ``(model, provider)``, or ``None`` when the model cannot be resolved. + + The two sides of the comparison arrive spelled differently: the spend log records a + normalized model name alongside its provider, while the baseline arrives as the + operator wrote it in config, with the provider prefixed, implied, or absent. Raw + string equality therefore reads `anthropic/claude-opus-5` as a switch away from + `claude-opus-5`, and pricing a bare name with no provider can resolve it to a + different vendor's rates than the deployment it names. + """ + if not model: + return None + try: + resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + except Exception as e: # noqa: BLE001 # get_llm_provider raises for unroutable names; degrade to zero savings + verbose_proxy_logger.debug( + "savings: cannot resolve provider for model=%s custom_llm_provider=%s (%s)", model, custom_llm_provider, e + ) + return None + return _ModelIdentity(model=resolved_model, provider=provider) + + +def _effective_model_info(router: "Router | None", deployment_id: str | None, model: str) -> ModelInfo | None: + """What a deployment is actually charged, or ``None`` to price by name. + + `Router.get_deployment_model_info` owns this: it merges a deployment's configured + prices over the built-in map, folds in `base_model` defaults for deployments whose + name is not a model, and falls back to the model name when nothing is overridden. + Resolving a name here instead reads the public rate, which a deployment with a + negotiated price does not pay, and an Azure deployment name prices to nothing at all. + """ + if router is None or deployment_id is None: + return None + try: + return router.get_deployment_model_info(deployment_id, model) + except Exception as e: # noqa: BLE001 # a dashboard metric must not fail the spend write + verbose_proxy_logger.debug("savings: no deployment pricing for %s (%s)", model, e) + return None + + +def _model_info(model: _ModelIdentity) -> ModelInfo | None: + """The public rates for ``model``, or ``None`` when it has none.""" + try: + return litellm.get_model_info(model=model.model, custom_llm_provider=model.provider) + except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models + verbose_proxy_logger.debug("savings: no pricing for provider=%s model=%s (%s)", model.provider, model.model, e) + return None + + +def _cost_of_usage(model: _ModelIdentity, usage: Usage, model_info: ModelInfo | None = None) -> float | None: + """What ``usage`` costs on ``model``, or ``None`` when the model has no pricing.""" + try: + prompt_cost, completion_cost = generic_cost_per_token( + model=model.model, usage=usage, custom_llm_provider=model.provider, model_info=model_info + ) + except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings + verbose_proxy_logger.debug( + "savings: cannot price usage for provider=%s model=%s (%s)", model.provider, model.model, e + ) + return None + return prompt_cost + completion_cost + + +def _cache_token_split(usage: Usage) -> tuple[int, int]: + """``(cache_read_tokens, cache_creation_tokens)`` for a request.""" + details = usage.prompt_tokens_details + if details is None: + return 0, 0 + read = getattr(details, "cached_tokens", 0) or 0 + created = (getattr(details, "cache_creation_tokens", 0) or 0) or (getattr(details, "cache_write_tokens", 0) or 0) + return int(read), int(created) + + +_CACHE_SPLIT_FIELDS = frozenset( + ("cached_tokens", "cache_creation_tokens", "cache_write_tokens", "cache_creation_token_details", "text_tokens") +) + + +def _baseline_cache_rate_keys(baseline_info: ModelInfo | None) -> tuple[bool, bool]: + """Whether the baseline model has a ``(cache read, cache write)`` rate of its own. + + A missing rate is not a free bucket. `_get_token_base_cost` resolves an absent + `cache_read_input_token_cost` or `cache_creation_input_token_cost` to 0.0, so a + baseline whose provider prices caching implicitly, which is every OpenAI, Azure and + Gemini entry for cache writes, would carry the whole prompt for nothing and turn a + profitable route into a reported loss. Such a model pays its plain input rate for + those tokens, so the buckets it cannot price become ordinary input below. + """ + if baseline_info is None: + return True, True + return bool(baseline_info.get("cache_read_input_token_cost")), bool( + baseline_info.get("cache_creation_input_token_cost") + ) + + +def _baseline_usage(usage: Usage, conversation_continuing: bool, baseline_info: ModelInfo | None = None) -> Usage: + """The same request as a single-model baseline would have met it. + + The baseline is one model serving every turn, so whether it had this prompt cached + is simply whether the conversation was already underway. On a continuing + conversation it wrote the prompt on an earlier turn and would only read it now, so + the cache tokens move into the read bucket and whatever this request paid to write + counts against the saving; that write is what switching models costs. + + On a conversation's first turn nothing was cached anywhere, for any model. The + baseline would have written the same prompt, so the cache buckets stay where they are + and both arms carry the write at their own rates, unless the baseline has no rate for + a bucket, in which case those tokens are its plain input. Charging the write to this case + too, which is all a single rollup row can support, understates a first turn to a + few percent of its value and can render a profitable route as a loss. + + A continuing turn that mostly read from cache is the third case: the selected model + was already warm, so it is the one that has been serving this conversation and the + baseline's cache holds exactly what its does. The tokens written are the turn's own + growth, new to every model, and the baseline would have paid to write them too. + Moving them would forgive the baseline a write it really owes and shrink the + reported saving. "Mostly read" rather than "read anything" on purpose: a switch onto + a model holding a small prefix of this prompt still writes most of it, and must keep + counting that write against the saving. + + Only the cache buckets move. Every other field the request was priced on travels + through untouched, audio and image and video counts among them, because the baseline + is this same request served by a model that happened to be warm; naming the fields to + keep instead would price the baseline on a request that never ran, and would go stale + the next time a priced field is added. + """ + cache_read, cache_creation = _cache_token_split(usage) + details = usage.prompt_tokens_details + if details is None or (cache_read <= 0 and cache_creation <= 0): + return usage + + # The tokens this request paid to write move into the cached count and the creation + # charge is dropped: on one model that cache was already warm, so the baseline would + # have read them rather than paying to create them. The 5m/1h breakdown goes with + # them; left behind it re-charges the write. + warm = conversation_continuing and cache_creation > 0 and cache_read <= cache_creation + reads = cache_read + cache_creation if warm else cache_read + writes = 0 if warm else cache_creation + + prices_reads, prices_writes = _baseline_cache_rate_keys(baseline_info) + reads = reads if prices_reads else 0 + writes = writes if prices_writes else 0 + if (reads, writes) == (cache_read, cache_creation): + return usage + + other_modalities = sum( + (getattr(details, field, 0) or 0) for field in ("audio_tokens", "image_tokens", "video_tokens") + ) + return Usage( + prompt_tokens=usage.prompt_tokens, + completion_tokens=usage.completion_tokens, + total_tokens=usage.total_tokens, + completion_tokens_details=usage.completion_tokens_details, + prompt_tokens_details=PromptTokensDetailsWrapper( + **details.model_dump(exclude=_CACHE_SPLIT_FIELDS), + cached_tokens=reads, + cache_creation_tokens=writes, + cache_write_tokens=writes, + cache_creation_token_details=details.cache_creation_token_details if writes else None, + # Whatever no longer sits in a cache bucket is plain input on the baseline. + text_tokens=max(usage.prompt_tokens - reads - writes - other_modalities, 0), + ), + ) + + +def compute_autorouter_savings( + baseline_model: str | None, + selected_model: str | None, + selected_provider: str | None, + usage: Usage, + conversation_continuing: bool = True, + selected_info: ModelInfo | None = None, +) -> float: + """Net dollars the router saved, or cost, by serving this request on ``selected_model``. + + Signed on purpose. Switching models leaves the new one with a cold cache, so the + request pays a cache-creation charge that staying on one model would not have + incurred; when that charge outweighs the cheaper rates, routing lost money and the + dashboard has to be able to say so. Zero when both sides resolve to the same + deployment, or when either cannot be resolved or priced. + + ``conversation_continuing`` says whether the baseline would already have had this + prompt cached. It defaults to True because that is the conservative reading: a + request whose shape the router could not determine is charged the write and + under-claims rather than inflating a savings figure. + """ + # No provider argument for the baseline on purpose: it arrives from the routing + # metadata as a single self-describing string, already qualified by the auto-router, + # so there is no second field that could disagree with it. + baseline = _resolve_model(baseline_model, None) + selected = _resolve_model(selected_model, selected_provider) + if baseline is None or selected is None: + return 0.0 + # Same model is only the same cost when it is also the same deployment. Two + # deployments of one model can carry different negotiated rates, and routing from + # the dear one to the cheap one is a real saving that short-circuiting on the model + # name alone reports as zero. + if baseline == selected: + return 0.0 + baseline_info = _model_info(baseline) + baseline_cost = _cost_of_usage( + baseline, _baseline_usage(usage, conversation_continuing, baseline_info), baseline_info + ) + selected_cost = _cost_of_usage(selected, usage, selected_info) + if baseline_cost is None or selected_cost is None: + return 0.0 + return baseline_cost - selected_cost + + +def _usage_from_spend_log(usage_object: Mapping[str, object] | None) -> Usage | None: + """Rebuild the request's ``Usage`` from the copy the spend log recorded.""" + if not usage_object: + return None + try: + return Usage(**usage_object) + except Exception as e: # noqa: BLE001 # a malformed usage_object must not fail the daily spend write + # Warning, not debug: this silently zeroes the auto-router driver for every + # affected row, and a shape change in Usage would otherwise show up only as a + # dashboard that quietly reads $0.00. + verbose_proxy_logger.warning("savings: unusable usage_object, auto-router savings will read zero (%s)", e) + return None + + def compute_savings_spend( model: str | None, custom_llm_provider: str | None, compression_saved_tokens: int, cache_read_input_tokens: int, + routing_decision: Mapping[str, object] | None = None, + usage_object: Mapping[str, object] | None = None, + model_id: str | None = None, + llm_router: "Router | None" = None, ) -> SavingsSpend: """ Dollar savings for one request, split by optimization driver. @@ -56,8 +295,35 @@ def compute_savings_spend( Compression savings price the tokens compression removed at the model's input rate. Prompt-caching savings price the cache-read tokens at the difference between the input rate and the discounted cache-read rate. + Auto-router savings compare the served ``model`` against the counterfactual + baseline the router recorded on its ``routing_decision``, and are zero unless the + two differ. That record also says whether the conversation was already underway, + which is what tells a mid-conversation switch from a first turn. """ input_cost, cache_read_cost = _input_and_cache_read_cost(model, custom_llm_provider) compression = max(compression_saved_tokens, 0) * input_cost prompt_caching = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0) - return SavingsSpend(compression=compression, prompt_caching=prompt_caching) + + usage = _usage_from_spend_log(usage_object) + if usage is None or not model: + return SavingsSpend(compression=compression, prompt_caching=prompt_caching) + + # The counterfactual is one model an operator would have run instead of the router, + # configured once for the proxy rather than derived per request. Unset means the + # driver is off; a routing decision is what says this request was auto-routed at all. + decision = routing_decision if isinstance(routing_decision, Mapping) else {} + autorouter = ( + compute_autorouter_savings( + baseline_model=litellm.autorouter_savings_baseline_model, + selected_model=model, + selected_provider=custom_llm_provider, + usage=usage, + # Absent means the router never recorded a shape, which is the conservative + # reading: charge the cache write rather than claim a first turn's saving. + conversation_continuing=decision.get("conversation_continuing") is not False, + selected_info=_effective_model_info(llm_router, model_id, model or ""), + ) + if decision + else 0.0 + ) + return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 6f2bf61834b..ca2e7fcc038 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -224,6 +224,40 @@ def _iter_human_asks_newest_first(messages: Sequence[Mapping[str, object]]) -> I ) +def _conversation_is_continuing(messages: Sequence[Mapping[str, object]] | None) -> bool: + """Whether this request continues a conversation that was already underway. + + The counterfactual the savings driver prices against is one model serving every + turn, so whether that model had this prompt cached is just whether an earlier turn + exists. An assistant turn in the history is the direct evidence of one: something + answered before, so a single-model deployment wrote the prompt then and would only + read it now, and the write this request paid is what switching models cost. A + conversation's first turn has no assistant turn, nothing was cached for any model, + and the baseline would have paid the same write. + + Assistant turns rather than human asks, because an agent loop can run twenty turns + on one human ask: its tool traffic rides `tool_result` blocks on user turns that + flatten to empty text, and on `tool` roles, so counting asks reads a long + conversation as its own first turn and hands it the untouched-write arithmetic. That + is the one direction this must never fail in, since it inflates. + + Reading the conversation rather than remembering it keeps this free of a cache, a + session id and their failure modes, and it works for callers that send no session + header at all. A few-shot prompt's synthetic assistant turns read as prior + conversation, which charges the write and under-claims; that is the safe side. + + So is an unreadable request. No messages says nothing about whether a turn was + served, and a surface that carries its turns somewhere this cannot see, or a + genuinely single-turn call arriving with none, is treated as continuing: it pays the + cache write and under-claims rather than being handed a first turn's larger saving + on no evidence. That direction is deliberate in both cases and is the only one that + cannot inflate. + """ + if not messages: + return True + return any(message.get("role") == "assistant" for message in messages) + + def _newest_turn_ask(messages: Sequence[Mapping[str, object]]) -> str | None: """The human ask on the newest user turn, or None when that turn carries only plumbing. @@ -647,6 +681,7 @@ class ComplexityRouter(CustomLogger): escalation_keyword: str | None = None, escalated: bool = False, classifier_model: str | None = None, + conversation_continuing: bool = True, ) -> StandardLoggingRoutingDecision: """Assemble the per-request provenance record for this router's decision. @@ -660,6 +695,7 @@ class ComplexityRouter(CustomLogger): router_type="complexity", routed_model=routed_model, cause=cause, + conversation_continuing=conversation_continuing, ) if tier is not None: decision["tier"] = tier.value @@ -1392,6 +1428,8 @@ class ComplexityRouter(CustomLogger): if isinstance(metadata, dict): metadata[RETURN_RAW_MODEL_NAME_METADATA_KEY] = True + conversation_continuing = _conversation_is_continuing(self._resolve_messages(messages, request_kwargs)) + use_session_affinity = self.config.session_affinity and not self.config.plugins session_id = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None cache_key = self._get_session_affinity_cache_key(session_id, request_kwargs) if session_id is not None else None @@ -1438,6 +1476,7 @@ class ComplexityRouter(CustomLogger): cause=cause, escalation_keyword=pin_escalation_keyword, escalated=escalated, + conversation_continuing=conversation_continuing, ), ) @@ -1447,6 +1486,7 @@ class ComplexityRouter(CustomLogger): messages=messages, input=input, specific_deployment=specific_deployment, + conversation_continuing=conversation_continuing, ) if cache_key is not None and response is not None: await self.litellm_router_instance.cache.async_set_cache( @@ -1463,6 +1503,7 @@ class ComplexityRouter(CustomLogger): messages: list[dict[str, Any]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, + conversation_continuing: bool = True, ) -> PreRoutingHookResponse | None: """ Classifies the request by complexity and returns the appropriate model. @@ -1509,7 +1550,11 @@ class ComplexityRouter(CustomLogger): return PreRoutingHookResponse( model=routed_model, messages=messages if has_original_messages else None, - routing_decision=self._build_routing_decision(routed_model=routed_model, cause="default_fallback"), + routing_decision=self._build_routing_decision( + routed_model=routed_model, + cause="default_fallback", + conversation_continuing=conversation_continuing, + ), ) newest_ask = _newest_turn_ask(resolved_messages) @@ -1532,6 +1577,7 @@ class ComplexityRouter(CustomLogger): messages=messages if has_original_messages else None, routing_decision=self._build_routing_decision( routed_model=routed_model, + conversation_continuing=conversation_continuing, cause=keyword_cause, tier=routed_tier, matched_keyword=override.matched_keyword, @@ -1579,6 +1625,7 @@ class ComplexityRouter(CustomLogger): messages=messages if has_original_messages else None, routing_decision=self._build_routing_decision( routed_model=routed_model, + conversation_continuing=conversation_continuing, cause=outcome.cause, tier=tier, score=score, diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 00f67d0f39c..21b7ffca3f2 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -25,6 +25,7 @@ class SpendMetrics(BaseModel): compression_saved_tokens: int = Field(default=0) compression_savings_spend: float = Field(default=0.0) prompt_caching_savings_spend: float = Field(default=0.0) + autorouter_savings_spend: float = Field(default=0.0) total_tokens: int = Field(default=0) successful_requests: int = Field(default=0) failed_requests: int = Field(default=0) @@ -85,6 +86,7 @@ class DailySpendMetadata(BaseModel): total_compression_saved_tokens: int = Field(default=0) total_compression_savings_spend: float = Field(default=0.0) total_prompt_caching_savings_spend: float = Field(default=0.0) + total_autorouter_savings_spend: float = Field(default=0.0) page: int = Field(default=1) total_pages: int = Field(default=1) has_more: bool = Field(default=False) @@ -111,6 +113,7 @@ class LiteLLM_DailyUserSpend(BaseModel): compression_saved_tokens: int = 0 compression_savings_spend: float = 0.0 prompt_caching_savings_spend: float = 0.0 + autorouter_savings_spend: float = 0.0 spend: float = 0.0 api_requests: int = 0 successful_requests: int = 0 diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3539ac0f27a..21d69d9925a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2734,6 +2734,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): classifier_model: str escalated: bool tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries + conversation_continuing: bool # Fields whose values quote the caller's prompt. Dropped when an operator turns message @@ -2753,6 +2754,7 @@ DERIVED_ROUTING_DECISION_FIELDS: FrozenSet[str] = frozenset( "classifier_model", "escalated", "tier_boundaries", + "conversation_continuing", } ) diff --git a/schema.prisma b/schema.prisma index 37ea55f8c13..0d7fa8692c8 100644 --- a/schema.prisma +++ b/schema.prisma @@ -748,6 +748,7 @@ model LiteLLM_DailyUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -782,6 +783,7 @@ model LiteLLM_DailyOrganizationSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -816,6 +818,7 @@ model LiteLLM_DailyEndUserSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -849,6 +852,7 @@ model LiteLLM_DailyAgentSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -882,6 +886,7 @@ model LiteLLM_DailyTeamSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -917,6 +922,7 @@ model LiteLLM_DailyTagSpend { compression_saved_tokens BigInt @default(0) compression_savings_spend Float @default(0.0) prompt_caching_savings_spend Float @default(0.0) + autorouter_savings_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py index c7e7ef5d469..a55d4f0dcfd 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py @@ -16,6 +16,9 @@ from litellm.proxy._types import ( Litellm_EntityType, SpendUpdateQueueItem, ) +from typing import get_args + +from litellm.proxy._types import BaseDailySpendTransaction from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import ( DailySpendUpdateQueue, ) @@ -209,6 +212,7 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key(): "compression_saved_tokens": 0, "compression_savings_spend": 0, "prompt_caching_savings_spend": 0, + "autorouter_savings_spend": 0, } updates = [{test_key: test_transaction1}, {test_key: test_transaction2}] @@ -259,6 +263,7 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions( "compression_saved_tokens": 0, "compression_savings_spend": 0, "prompt_caching_savings_spend": 0, + "autorouter_savings_spend": 0, } # Add updates to queue @@ -527,3 +532,58 @@ async def test_compression_saved_tokens_aggregation(daily_spend_update_queue): assert agg["cache_creation_input_tokens"] == 7 assert agg["compression_savings_spend"] == pytest.approx(0.0076) assert agg["prompt_caching_savings_spend"] == pytest.approx(0.0108) + + +@pytest.mark.asyncio +async def test_every_optional_daily_metric_aggregates(daily_spend_update_queue): + """Every additive metric must survive the merge, not just the ones wired by hand. + + Two requests landing on one rollup key before a flush is the common case under + load, and this same merge runs again on every cross-pod Redis drain. A metric + persisted by the database write but skipped here is silently dropped on both + paths, so the driver reads as zero on the dashboard however much it saved. + """ + test_key = "user1_2023-01-01_key123_claude-haiku-4-5_anthropic" + def _numeric(annotation): + # additive metrics may be declared NotRequired[float] for rows queued by a pod + # running the previous release, so unwrap before matching + args = get_args(annotation) + return (args[0] if args else annotation) in (int, float) + + numeric_fields = [ + name for name, annotation in BaseDailySpendTransaction.__annotations__.items() if _numeric(annotation) + ] + assert "autorouter_savings_spend" in numeric_fields + increments = {field: index + 1 for index, field in enumerate(numeric_fields)} + + await daily_spend_update_queue.add_update({test_key: dict(increments)}) + await daily_spend_update_queue.add_update({test_key: dict(increments)}) + await daily_spend_update_queue.aggregate_queue_updates() + updates = await daily_spend_update_queue.flush_all_updates_from_in_memory_queue() + + agg = updates[0][test_key] + for field, value in increments.items(): + assert agg[field] == pytest.approx(value * 2), f"{field} did not accumulate" + + +@pytest.mark.asyncio +async def test_optional_metric_missing_from_an_older_payload_still_aggregates( + daily_spend_update_queue, +): + """A queued row written before a metric existed must not zero it out.""" + test_key = "user1_2023-01-01_key123_claude-haiku-4-5_anthropic" + base = { + "spend": 1.0, + "prompt_tokens": 10, + "completion_tokens": 5, + "api_requests": 1, + "successful_requests": 1, + "failed_requests": 0, + } + + await daily_spend_update_queue.add_update({test_key: dict(base)}) + await daily_spend_update_queue.add_update({test_key: {**base, "autorouter_savings_spend": 0.25}}) + await daily_spend_update_queue.aggregate_queue_updates() + updates = await daily_spend_update_queue.flush_all_updates_from_in_memory_queue() + + assert updates[0][test_key]["autorouter_savings_spend"] == pytest.approx(0.25) diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index c2a0d34a915..f2749be5d6e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -19,7 +19,10 @@ from litellm.proxy.management_endpoints.common_daily_activity import ( get_daily_activity_aggregated, update_metrics, ) -from litellm.types.proxy.management_endpoints.common_daily_activity import SpendMetrics +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + DailySpendMetadata, + SpendMetrics, +) @pytest.mark.asyncio @@ -153,6 +156,7 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown(): "compression_saved_tokens": 0, "compression_savings_spend": 0.0, "prompt_caching_savings_spend": 0.0, + "autorouter_savings_spend": 0.0, "failed_requests": 0, } mock_rows = [ @@ -498,6 +502,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_1.compression_saved_tokens = 0 mock_record_1.compression_savings_spend = 0.0 mock_record_1.prompt_caching_savings_spend = 0.0 + mock_record_1.autorouter_savings_spend = 0.0 mock_record_1.api_requests = 10 mock_record_1.successful_requests = 9 mock_record_1.failed_requests = 1 @@ -520,6 +525,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero(): mock_record_2.compression_saved_tokens = 0 mock_record_2.compression_savings_spend = 0.0 mock_record_2.prompt_caching_savings_spend = 0.0 + mock_record_2.autorouter_savings_spend = 0.0 mock_record_2.api_requests = 5 mock_record_2.successful_requests = 5 mock_record_2.failed_requests = 0 @@ -582,6 +588,7 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys(): "compression_saved_tokens": 0, "compression_savings_spend": 0.0, "prompt_caching_savings_spend": 0.0, + "autorouter_savings_spend": 0.0, "failed_requests": 0, } mock_rows = [ @@ -669,6 +676,7 @@ def _daily_user_spend_record(*, user_id, api_key, spend, model="gpt-4", model_gr compression_saved_tokens=0, compression_savings_spend=0.0, prompt_caching_savings_spend=0.0, + autorouter_savings_spend=0.0, api_requests=1, successful_requests=1, failed_requests=0, @@ -973,6 +981,7 @@ async def test_get_daily_activity_aggregated_empty_result_set(): "compression_saved_tokens": None, "compression_savings_spend": None, "prompt_caching_savings_spend": None, + "autorouter_savings_spend": None, "api_requests": None, "successful_requests": None, "failed_requests": None, @@ -1016,6 +1025,7 @@ def _no_spend_record(): compression_saved_tokens=None, compression_savings_spend=None, prompt_caching_savings_spend=None, + autorouter_savings_spend=None, api_requests=None, successful_requests=None, failed_requests=None, @@ -1050,3 +1060,54 @@ def test_update_metrics_handles_none_values(): assert metrics.cache_read_input_tokens == 0 assert metrics.cache_creation_input_tokens == 0 assert metrics.compression_saved_tokens == 0 + + +class TestEverySavingsDriverSurvivesTheReadPath: + """A savings driver is only real if it survives the whole read path. + + The write path can price a driver correctly and persist it to all six rollup + tables, and the dashboard can still render a permanent $0.00 because the + aggregation query never summed the column or the response model never + declared it. That failure is silent: the card renders, the number is just + always zero, which is indistinguishable from having saved nothing. These + tests enumerate the drivers from the response model itself, so a driver added + later cannot be half-wired. + """ + + def _drivers(self) -> list[str]: + drivers = [field for field in SpendMetrics.model_fields if field.endswith("_savings_spend")] + assert drivers, "expected the dashboard response to expose at least one savings driver" + return drivers + + def test_every_driver_is_summed_by_the_rollup_query(self): + sql, _ = _build_aggregated_sql_query( + table_name="litellm_dailyuserspend", + entity_id_field="user_id", + entity_id="user-1", + start_date="2026-07-01", + end_date="2026-07-31", + model=None, + api_key=None, + timezone_offset_minutes=None, + ) + for driver in self._drivers(): + assert f"SUM({driver})" in sql, f"{driver} is never summed, so it reads as zero" + + def test_every_driver_is_accumulated_across_rows(self): + for driver in self._drivers(): + record = _no_spend_record() + setattr(record, driver, 1.25) + metrics = update_metrics(SpendMetrics(), record) + assert getattr(metrics, driver) == pytest.approx(1.25), f"{driver} is dropped when accumulating rows" + + def test_every_driver_is_carried_by_a_single_row_conversion(self): + for driver in self._drivers(): + record = _no_spend_record() + setattr(record, driver, 2.5) + assert getattr(_record_to_spend_metrics(record), driver) == pytest.approx(2.5) + + def test_every_driver_has_a_range_total(self): + for driver in self._drivers(): + assert f"total_{driver}" in DailySpendMetadata.model_fields, ( + f"total_{driver} is missing, so the range summary omits the driver" + ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 704cf7a63dd..6d0195b632b 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -6,7 +6,14 @@ sys.path.insert(0, os.path.abspath("../../../..")) import pytest import litellm -from litellm.proxy.spend_tracking.savings import compute_savings_spend +from litellm.router import Router +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.proxy.spend_tracking.savings import ( + _baseline_usage, + compute_autorouter_savings, + compute_savings_spend, +) +from litellm.types.utils import Usage def _anthropic_costs(model: str) -> tuple[float, float]: @@ -16,6 +23,39 @@ def _anthropic_costs(model: str) -> tuple[float, float]: return input_cost, cache_read_cost +def _cached_usage_object() -> dict: + """A cache-heavy Anthropic request, shaped as the spend log records it. + + `prompt_tokens` is the inclusive total: 3 uncached text tokens plus 500 read + from cache plus 12304 written to cache. + """ + return { + "prompt_tokens": 12807, + "completion_tokens": 500, + "total_tokens": 13307, + "prompt_tokens_details": {"cached_tokens": 500, "cache_creation_tokens": 12304, "text_tokens": 3}, + "cache_creation_input_tokens": 12304, + "cache_read_input_tokens": 500, + } + + +def _cost_on(model: str, usage_object: dict) -> float: + prompt_cost, completion_cost = generic_cost_per_token( + model=model, usage=Usage(**usage_object), custom_llm_provider="anthropic" + ) + return prompt_cost + completion_cost + + +def _flat_rates(model: str) -> tuple[float, float, float]: + info = litellm.get_model_info(model=model, custom_llm_provider="anthropic") + input_cost = info["input_cost_per_token"] or 0.0 + return ( + input_cost, + info["output_cost_per_token"] or 0.0, + info.get("cache_creation_input_token_cost") or input_cost, + ) + + def test_compression_savings_priced_at_input_rate(): input_cost, _ = _anthropic_costs("claude-sonnet-5") result = compute_savings_spend( @@ -76,3 +116,421 @@ def test_negative_token_counts_clamp_to_zero(): ) assert result.compression == 0.0 assert result.prompt_caching == 0.0 + + +def _usage(fresh: int, cached: int, written: int, out: int) -> Usage: + """Usage as the spend log records it; `prompt_tokens` is the inclusive total.""" + return Usage( + prompt_tokens=fresh + cached + written, + completion_tokens=out, + total_tokens=fresh + cached + written + out, + prompt_tokens_details={"cached_tokens": cached, "cache_creation_tokens": written, "text_tokens": fresh}, + cache_read_input_tokens=cached, + cache_creation_input_tokens=written, + ) + + +def _savings(baseline: str, selected: str, usage: Usage, continuing: bool = True) -> float: + """Savings for a request, defaulting to a conversation already underway. + + `continuing=True` is the mid-conversation case, where the baseline had the prompt + cached and this request's write is what the switch cost. `continuing=False` is a + conversation's first turn, where nothing was cached for any model. + """ + return compute_autorouter_savings( + baseline_model=baseline, + selected_model=selected, + selected_provider="anthropic", + usage=usage, + conversation_continuing=continuing, + ) + + +def test_switching_models_mid_conversation_charges_the_cold_cache_write(): + """Staying on one model writes the cache once and reads it thereafter. Switching + leaves the new model cold, so it pays to write the whole prompt again; when that + charge outweighs the cheaper rates the route lost money and must report a loss. + + Pricing the baseline as if it too re-wrote the cache credits a charge it never + paid, which is how a losing switch used to read as the largest saving on the page. + """ + usage = _usage(fresh=3, cached=500, written=12304, out=500) + result = _savings("claude-sonnet-5", "claude-haiku-4-5", usage) + + sonnet = litellm.get_model_info("claude-sonnet-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + warm_baseline = ( + 3 * sonnet["input_cost_per_token"] + + 12804 * sonnet["cache_read_input_token_cost"] + + 500 * sonnet["output_cost_per_token"] + ) + actually_paid = ( + 3 * haiku["input_cost_per_token"] + + 500 * haiku["cache_read_input_token_cost"] + + 12304 * haiku["cache_creation_input_token_cost"] + + 500 * haiku["output_cost_per_token"] + ) + assert result == pytest.approx(warm_baseline - actually_paid) + assert result < 0, "a cache-thrashing switch must report a loss, not a saving" + + phantom = 12304 * sonnet["cache_creation_input_token_cost"] + assert result != pytest.approx(warm_baseline + phantom - actually_paid) + + +def test_a_cold_switch_never_beats_turning_caching_off(): + """Switching to a cold model makes it write the whole prompt again. That write is a + real cost of switching, so the same traffic must look worse than if caching were off + entirely. + + The baseline is priced as a warm cache even though this request read nothing: a + switch reads nothing precisely because the new model's cache is empty, and staying + on one model would have had the prompt cached already. Gating the warm baseline on + a read charged the baseline a write it would never repeat, which made a cold switch + report a larger saving than no caching at all. + """ + cold_switch = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 0, 20_000, 1_000)) + caching_off = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(20_000, 0, 0, 1_000)) + + assert cold_switch < caching_off + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + warm_baseline = 20_000 * opus["cache_read_input_token_cost"] + 1_000 * opus["output_cost_per_token"] + actually_paid = 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + assert cold_switch == pytest.approx(warm_baseline - actually_paid) + + +def test_moving_one_token_between_cache_buckets_does_not_move_the_answer(): + """A continuing conversation writes a few new tokens and reads the rest. Treating the + presence of a write as the signal for a switch made that ordinary increment flip the + result, so a request reading 19,999 and writing 1 landed somewhere entirely different + from one reading 20,000 and writing none. + """ + reads_nothing = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 0, 20_000, 1_000)) + reads_one = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 1, 19_999, 1_000)) + assert reads_one == pytest.approx(reads_nothing, abs=1e-4) + + +def test_multimodal_prompts_are_priced_on_the_baseline_too(): + """The baseline is this same request met by a warm cache, so every field it was + priced on has to survive. Rebuilding the details from the cache buckets alone + dropped the image and audio counts, which priced the baseline as a text-only + request that never ran and shrank the reported saving on multimodal traffic. + """ + details = {"cached_tokens": 0, "cache_creation_tokens": 16_000, "text_tokens": 0, "image_tokens": 4_000} + with_images = Usage( + prompt_tokens=20_000, + completion_tokens=1_000, + total_tokens=21_000, + prompt_tokens_details=details, + ) + baseline = _baseline_usage(with_images, conversation_continuing=True) + + assert baseline.prompt_tokens_details.image_tokens == 4_000, "image tokens must survive into the baseline" + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic") + text_only = 20_000 * opus["cache_read_input_token_cost"] + assert priced > text_only, "dropping the image tokens undercharges the baseline and hides the saving" + + +def test_the_baseline_is_never_charged_a_cache_write(): + """Carrying the details through must not carry the 5m/1h creation breakdown with + them. `generic_cost_per_token` charges a creation cost whenever that breakdown is + present, even against a zeroed creation count, which would put the phantom write + back on the baseline for every long-cache request. + """ + long_cache = Usage( + prompt_tokens=20_000, + completion_tokens=1_000, + total_tokens=21_000, + prompt_tokens_details={ + "cached_tokens": 0, + "cache_creation_tokens": 20_000, + "text_tokens": 0, + "cache_creation_token_details": {"ephemeral_1h_input_tokens": 20_000}, + }, + ) + baseline = _baseline_usage(long_cache, conversation_continuing=True) + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic") + assert priced == pytest.approx(20_000 * opus["cache_read_input_token_cost"]), ( + "the baseline reads a warm cache; it never pays to create one" + ) + + +def test_uncached_request_is_the_plain_rate_difference(): + usage = _usage(fresh=2000, cached=0, written=0, out=500) + sonnet = litellm.get_model_info("claude-sonnet-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + assert _savings("claude-sonnet-5", "claude-haiku-4-5", usage) == pytest.approx( + 2000 * (sonnet["input_cost_per_token"] - haiku["input_cost_per_token"]) + + 500 * (sonnet["output_cost_per_token"] - haiku["output_cost_per_token"]) + ) + + +def test_escalation_reports_its_real_cost(): + """Routing up to a pricier model is a real cost; hiding it behind a zero floor + would let the dashboard only ever move in one direction.""" + usage = _usage(fresh=2000, cached=0, written=0, out=500) + assert _savings("claude-haiku-4-5", "claude-sonnet-5", usage) < 0 + + +def test_autorouter_savings_zero_when_model_unchanged(): + assert _savings("claude-opus-5", "claude-opus-5", _usage(3, 500, 12304, 500)) == 0.0 + + +def test_autorouter_savings_unknown_baseline_fails_open_to_zero(): + assert _savings("totally-made-up-model-xyz", "claude-haiku-4-5", _usage(3, 500, 12304, 500)) == 0.0 + + +def test_autorouter_savings_zero_without_baseline(): + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + cache_read_input_tokens=0, + routing_decision=None, + usage_object=_cached_usage_object(), + ) + assert result.autorouter == 0.0 + + +def test_compute_savings_spend_carries_a_losing_switch_through(monkeypatch): + """The signed value must survive into SavingsSpend; clamping it here would put the + dashboard back to only ever showing gains.""" + monkeypatch.setattr(litellm, "autorouter_savings_baseline_model", "claude-sonnet-5") + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + cache_read_input_tokens=0, + routing_decision={"conversation_continuing": True}, + usage_object=_cached_usage_object(), + ) + assert result.autorouter < 0 + + +def test_the_driver_is_off_until_a_baseline_is_configured(): + """No configured counterfactual means there is nothing to measure against, so the + driver reports zero rather than inventing a model the operator never named.""" + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=1000, + cache_read_input_tokens=0, + routing_decision={"conversation_continuing": True}, + usage_object=_cached_usage_object(), + ) + assert result.autorouter == 0.0 + assert result.compression > 0, "the other drivers keep working" + + +def test_malformed_usage_object_does_not_fail_the_spend_write(): + """The daily spend write must survive an unusable usage_object; losing one row's + savings is recoverable, losing the row is not.""" + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=1000, + cache_read_input_tokens=0, + routing_decision={"conversation_continuing": True}, + usage_object={"prompt_tokens": ["not", "a", "number"]}, + ) + assert result.autorouter == 0.0 + assert result.compression > 0 + + +def test_model_without_cache_read_pricing_yields_no_caching_savings(): + """A model with no discounted cache-read rate cannot have saved anything by + reading from cache, so the driver must report zero rather than the full input rate.""" + model = "azure/gpt-3.5-turbo" + assert litellm.get_model_info(model=model).get("cache_read_input_token_cost") is None + result = compute_savings_spend( + model=model, + custom_llm_provider="azure", + compression_saved_tokens=0, + cache_read_input_tokens=5000, + ) + assert result.prompt_caching == 0.0 + + +def test_the_same_deployment_spelled_two_ways_is_not_a_switch(): + """The spend log records a normalized model name while the baseline arrives as the + operator wrote it in config. Comparing the raw strings makes a request that never + changed model look like a switch, and prices one deployment against itself.""" + # Must be a cached request: the baseline arm is priced against a warm cache and the + # selected arm against what was actually paid, so treating one deployment as two + # charges it a cold-cache write it never took, inventing a loss on a request that + # never changed model. An uncached request prices identically either way and would + # make this assertion vacuous. + usage = _usage(fresh=3, cached=500, written=12304, out=500) + assert _savings("anthropic/claude-opus-5", "claude-opus-5", usage) == 0.0 + assert _savings("claude-opus-5", "anthropic/claude-opus-5", usage) == 0.0 + + +def test_baseline_is_priced_under_its_own_provider(): + """Two providers can serve the same bare model name at different rates, so dropping + the provider prices the baseline against a vendor the operator never named. Here it + decides whether routing reads as a saving or a loss.""" + usage = Usage(prompt_tokens=100_000, completion_tokens=10_000, total_tokens=110_000) + azure = compute_autorouter_savings( + baseline_model="azure_ai/deepseek-r1", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + ) + deepseek = compute_autorouter_savings( + baseline_model="deepseek/deepseek-r1", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + ) + assert azure != pytest.approx(deepseek) + assert azure > 0 > deepseek + + +def test_unresolvable_baseline_fails_open_to_zero(): + usage = _usage(fresh=2000, cached=0, written=0, out=500) + assert _savings("no-such-provider-xyz/no-such-model", "claude-haiku-4-5", usage) == 0.0 + + +def test_a_first_turn_is_the_rate_difference_not_a_switch_penalty(): + """Nothing was cached anywhere on a conversation's first turn, so the baseline would + have paid the same cache write. Charging it to the selected arm alone reported a + fraction of the real saving; on this shape roughly 4% of it. + """ + usage = _usage(fresh=0, cached=0, written=20_000, out=1_000) + first_turn = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, continuing=False) + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + both_write = (20_000 * opus["cache_creation_input_token_cost"] + 1_000 * opus["output_cost_per_token"]) - ( + 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + ) + assert first_turn == pytest.approx(both_write) + + mid_conversation = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage) + assert first_turn > mid_conversation * 10, "a first turn must not be priced as a switch" + + +def test_a_first_turn_that_saves_money_never_reports_a_loss(): + """The write premium is fixed by prompt size while the saving grows with completion + length, so charging the write to a first turn made short answers over a large cached + prompt read as losses on requests that genuinely saved. That is the shape most likely + to be on the dashboard, and the sign has to be right. + """ + short_answer = _usage(fresh=0, cached=0, written=20_000, out=200) + assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", short_answer, continuing=False) > 0 + assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", short_answer) < 0 + + +def test_an_undetermined_conversation_shape_stays_conservative(): + """The default must charge the write. A caller that cannot be read, or a surface the + router never classified, has said nothing about whether the baseline was warm, and a + savings figure must not inflate on a guess. + """ + usage = _usage(fresh=0, cached=0, written=20_000, out=1_000) + defaulted = compute_autorouter_savings( + baseline_model="anthropic/claude-opus-5", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=usage, + ) + assert defaulted == pytest.approx(_savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage)) + assert defaulted < _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, continuing=False) + + +def test_a_continuing_turn_on_the_same_model_writes_its_growth_on_both_arms(): + """A conversation that grew by a few tokens writes those on whatever model serves + it, and they are new to every model, so the baseline would have written them too. + Moving them into the baseline's read bucket forgives it a write it really owes and + shrinks the reported saving on ordinary steady-state traffic. + """ + usage = _usage(fresh=0, cached=19_900, written=100, out=1_000) + opus = litellm.get_model_info("claude-opus-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + + def cost(info: dict) -> float: + return ( + 19_900 * info["cache_read_input_token_cost"] + + 100 * info["cache_creation_input_token_cost"] + + 1_000 * info["output_cost_per_token"] + ) + + both_write_the_growth = cost(opus) - cost(haiku) + assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage) == pytest.approx(both_write_the_growth) + + +def test_a_switch_onto_a_partly_cached_model_still_pays_for_the_write(): + """A model holding a small prefix of this prompt still has to write the rest, and + that write is the switch's cost. Keying the same-model case off reading *anything* + rather than reading *most of it* would hand this request the full rate gap and + inflate the saving by an order of magnitude. + """ + mostly_written = _usage(fresh=0, cached=500, written=19_500, out=1_000) + reported = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", mostly_written) + + opus = litellm.get_model_info("claude-opus-5", "anthropic") + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + if_treated_as_same_model = ( + 500 * opus["cache_read_input_token_cost"] + + 19_500 * opus["cache_creation_input_token_cost"] + + 1_000 * opus["output_cost_per_token"] + ) - ( + 500 * haiku["cache_read_input_token_cost"] + + 19_500 * haiku["cache_creation_input_token_cost"] + + 1_000 * haiku["output_cost_per_token"] + ) + assert reported < if_treated_as_same_model / 10, "a mostly-cold switch must not be priced as a continuation" + +def test_a_baseline_that_prices_caching_implicitly_still_pays_for_its_prompt(): + """OpenAI, Azure and Gemini entries carry no `cache_creation_input_token_cost`, + because those providers cache implicitly and charge nothing to write. Leaving this + request's written tokens in the creation bucket priced them at the 0.0 the cost + resolver falls back to, so the baseline carried a 20k prompt for free and a first + turn that saved money reported a loss. Those tokens are plain input on such a model. + """ + first_turn = _usage(fresh=0, cached=0, written=20_000, out=1_000) + reported = compute_autorouter_savings( + baseline_model="gpt-5", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=first_turn, + conversation_continuing=False, + ) + + gpt5 = litellm.get_model_info("gpt-5", "openai") + assert gpt5.get("cache_creation_input_token_cost") is None, "pick a baseline with no cache-write rate" + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + baseline_pays_input = 20_000 * gpt5["input_cost_per_token"] + 1_000 * gpt5["output_cost_per_token"] + actually_paid = ( + 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + ) + assert reported == pytest.approx(baseline_pays_input - actually_paid) + assert reported > 0, "routing a cold first turn onto a cheaper model is a saving, not a loss" + + +def test_a_baseline_with_no_cache_read_rate_is_charged_its_input_rate(): + """The same hole on the other bucket. A baseline whose entry has no + `cache_read_input_token_cost` reads for 0.0, so a continuing turn priced the whole + prompt at nothing and every switch away from it reported a loss. + """ + continuing = _usage(fresh=0, cached=0, written=20_000, out=1_000) + reported = compute_autorouter_savings( + baseline_model="xai/grok-4", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=continuing, + conversation_continuing=True, + ) + + grok = litellm.get_model_info("grok-4", "xai") + assert grok.get("cache_read_input_token_cost") is None, "pick a baseline with no cache-read rate" + haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") + baseline_pays_input = 20_000 * grok["input_cost_per_token"] + 1_000 * grok["output_cost_per_token"] + actually_paid = ( + 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] + ) + assert reported == pytest.approx(baseline_pays_input - actually_paid) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 3a94b1e0f85..2f249241f21 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -4094,6 +4094,23 @@ class TestRecordRoutingDecision: assert request_kwargs == {} + def test_clearing_the_decision_takes_the_savings_facts_with_it(self): + """A fallback to a plain model group re-enters the hook with the same + `request_kwargs`. The baseline and the conversation shape ride inside the + decision rather than beside it, so one clear cannot leave either behind and + attribute an auto-router saving to a deployment that never routed.""" + decision = { + "router_model_name": "smart-router", + "router_type": "complexity", + "routed_model": "gpt-4o-mini", + "savings_baseline_model": "anthropic/claude-opus-5", + "conversation_continuing": False, + } + request_kwargs: Dict = {"litellm_metadata": {"routing_decision": decision}} + Router._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) + assert request_kwargs["litellm_metadata"] == {} + + class TestEscalationIsRecordedConsistently: """An escalation keyword records two separate facts on every path: that the caller asked, and whether the tier actually moved. Dropping the ask when there is nowhere @@ -5039,3 +5056,142 @@ class TestClassifierTrustBoundary: assert "Classify only the current message" not in system_prompt assert "using the earlier turns quoted above it as context" in system_prompt assert "rate the work it approves rather than the reply itself" in system_prompt + + +class TestConversationShapeDiscriminator: + """Whether the counterfactual single model would already have had the prompt cached.""" + + @staticmethod + def _router(mock_router_instance, basic_config) -> ComplexityRouter: + return ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={**basic_config, "session_affinity": False}, + ) + + @pytest.mark.asyncio + async def test_a_single_ask_is_a_first_turn(self, mock_router_instance, basic_config): + """Nothing is cached for any model yet, so the baseline would have paid the same + cache write and the saving is the plain rate difference.""" + mock_router_instance.cache = DualCache() + result = await self._router(mock_router_instance, basic_config).async_pre_routing_hook( + model="test-model", + request_kwargs={"metadata": {}}, + messages=[{"role": "user", "content": "Hello!"}], + ) + assert result.routing_decision["conversation_continuing"] is False + + @pytest.mark.asyncio + async def test_a_second_ask_means_the_baseline_was_already_warm(self, mock_router_instance, basic_config): + """An earlier turn was served, so a single-model deployment wrote the prompt then + and would only read it now; this request's write is what switching cost.""" + mock_router_instance.cache = DualCache() + result = await self._router(mock_router_instance, basic_config).async_pre_routing_hook( + model="test-model", + request_kwargs={"metadata": {}}, + messages=[ + {"role": "user", "content": "First question about the codebase"}, + {"role": "assistant", "content": "Here is the answer"}, + {"role": "user", "content": "Hello!"}, + ], + ) + assert result.routing_decision["conversation_continuing"] is True + + @pytest.mark.asyncio + async def test_it_needs_no_session_id(self, mock_router_instance, basic_config): + """The whole point of reading the conversation rather than remembering it: a + caller that sends no session header is still classified correctly.""" + mock_router_instance.cache = DualCache() + router = self._router(mock_router_instance, basic_config) + first = await router.async_pre_routing_hook( + model="test-model", request_kwargs={}, messages=[{"role": "user", "content": "Hello!"}] + ) + later = await router.async_pre_routing_hook( + model="test-model", + request_kwargs={}, + messages=[ + {"role": "user", "content": "First question"}, + {"role": "assistant", "content": "Answer"}, + {"role": "user", "content": "Hello!"}, + ], + ) + assert first.routing_decision["conversation_continuing"] is False + assert later.routing_decision["conversation_continuing"] is True + + @pytest.mark.asyncio + async def test_it_touches_no_cache(self, mock_router_instance, basic_config): + """Reading the request instead of remembering it is what removes the routing-path + round-trip, and with it a cache failure that would read as a first turn.""" + cache = AsyncMock() + cache.async_get_cache = AsyncMock(return_value=None) + mock_router_instance.cache = cache + result = await self._router(mock_router_instance, basic_config).async_pre_routing_hook( + model="test-model", + request_kwargs={"metadata": {}}, + messages=[{"role": "user", "content": "Hello!"}], + ) + assert result.routing_decision["conversation_continuing"] is False + assert cache.async_get_cache.await_count == 0 + assert cache.async_set_cache.await_count == 0 + + @pytest.mark.parametrize( + "history", + [ + pytest.param( + [ + {"role": "user", "content": "do X"}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "1", "name": "t", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "1", "content": "r"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "2", "name": "t", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "2", "content": "r"}]}, + ], + id="messages-api-tool-result-blocks", + ), + pytest.param( + [ + {"role": "user", "content": "do X"}, + {"role": "assistant", "tool_calls": [{"id": "1"}]}, + {"role": "tool", "tool_call_id": "1", "content": "r"}, + ], + id="chat-completions-tool-role", + ), + ], + ) + def test_an_agent_loop_on_one_human_ask_is_not_a_first_turn(self, history): + """An agent can run twenty turns on a single human ask: its tool traffic rides + `tool_result` blocks that flatten to empty text and `tool` roles. Counting human + asks read that as a first turn and handed it the untouched-write arithmetic, + which is the one direction this must never fail in, because it inflates.""" + from litellm.router_strategy.complexity_router.complexity_router import _conversation_is_continuing + + assert _conversation_is_continuing(history) is True + + def test_a_system_prompt_does_not_make_a_first_turn_look_continued(self): + from litellm.router_strategy.complexity_router.complexity_router import _conversation_is_continuing + + assert _conversation_is_continuing([{"role": "system", "content": "s"}, {"role": "user", "content": "hi"}]) is False + + def test_unreadable_messages_stay_conservative(self): + """No messages says nothing about the baseline's cache, so it keeps charging the + write and under-claims rather than inflating.""" + from litellm.router_strategy.complexity_router.complexity_router import _conversation_is_continuing + + assert _conversation_is_continuing(None) is True + assert _conversation_is_continuing([]) is True + assert _conversation_is_continuing([{"role": "user", "content": ""}]) is False + + @pytest.mark.asyncio + async def test_the_shape_travels_on_every_pre_routing_response(self): + """A response without it defaults to charging the write, silently undoing the fix + for whichever routing path forgot it.""" + import inspect + + from litellm.router_strategy.complexity_router import complexity_router as module + + source = inspect.getsource(module.ComplexityRouter.async_pre_routing_hook) + inspect.getsource( + module.ComplexityRouter._classify_and_route + ) + builds = source.split("self._build_routing_decision(")[1:] + assert builds + missing = [i for i, block in enumerate(builds) if "conversation_continuing=conversation_continuing" not in block.split("),")[0]] + assert not missing, f"routing decisions {missing} do not carry the conversation shape" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 0f4eacc820c..1ea4113b821 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -23521,6 +23521,11 @@ export interface components { * @default 0 */ total_api_requests: number; + /** + * Total Autorouter Savings Spend + * @default 0 + */ + total_autorouter_savings_spend: number; /** * Total Cache Creation Input Tokens * @default 0 @@ -31427,6 +31432,11 @@ export interface components { * @default 0 */ api_requests: number; + /** + * Autorouter Savings Spend + * @default 0 + */ + autorouter_savings_spend: number; /** * Cache Creation Input Tokens * @default 0